From 08e785eb8d475a6906bc5135d6379c0ced78054b Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:37:23 +0100 Subject: [PATCH 01/53] Show the live reading of threshold alerts held below trigger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A threshold alert stays open after its reading drops below the trigger: it holds until the reading reaches the clear level, then must stay there for the recovery delay. While it held, the evaluator returned without touching the alert, so Value, Message and LastSeen stayed at the last breach. On 2026-10-06 minipc's row and guests said "Node temperature at 80.0°C" while the node table read a green 72°C, with nothing explaining why the warning was still there (also #2068: a disk alert that stayed open after usage dropped, with no sign of why). The evaluator now records a volatile MetricStatus on every evaluation: the value it compared (the average for rolling-average rules, with the latest sample beside it), the phase (breaching, latched or recovering), the trigger and clear levels, the recovery delay and the reducer's monotonic recovery progress. Value, Message and LastSeen keep meaning the last breach, and the hold branch still never notifies. The status is stripped from the active-state checkpoint, the recovery mirror and the degraded recovery projection, so a held alert adds no durable writes. It reaches the websocket state through the active-alert projection. Stale-alert cleanup and the retention TTLs now use the latest observation. Before, an alert held below its trigger for over a day had an old LastSeen and was auto-resolved as "resource no longer being monitored" while it was still being evaluated. Surfaces format the status through one helper and never re-derive the phase. Node, guest and Mail Gateway drawer rows lead with the reading now and say what clears the alert, with the last breach on hover. The Proxmox TEMP cell keeps its own open temperature alert's tone and explains it on hover instead of reading green beside a warning row. Storage rows stop saying "Over N%" when usage is back under the limit, and the Assistant handoff no longer labels the breach value as current. Refs #2068 --- .../v6/internal/subsystems/agent-lifecycle.md | 8 + .../v6/internal/subsystems/alerts.md | 17 ++ .../v6/internal/subsystems/api-contracts.md | 13 ++ .../subsystems/frontend-primitives.md | 8 + .../v6/internal/subsystems/monitoring.md | 8 + .../subsystems/performance-and-scalability.md | 7 + .../internal/subsystems/storage-recovery.md | 8 + .../internal/subsystems/unified-resources.md | 8 + .../4766668c29936861.json | 62 ++++++ .../alertAssistantHandoffModel.test.ts | 31 +++ .../Alerts/alertAssistantHandoffModel.ts | 45 ++++- .../components/Workloads/GuestDrawer.test.tsx | 40 ++++ .../Workloads/GuestDrawerOverview.tsx | 4 +- .../Workloads/NodeDrawerOverview.tsx | 3 +- .../components/Workloads/guestDrawerModel.ts | 13 +- .../shared/DrawerAttentionSection.tsx | 9 +- .../SharedPrimitives.guardrails.test.ts | 15 ++ .../components/shared/TemperatureGauge.tsx | 10 +- .../features/alerts/__tests__/helpers.test.ts | 153 ++++++++++++++- .../alerts/metricAlertPresentation.ts | 157 +++++++++++++++ .../proxmox/ProxmoxMailGatewayDrawer.tsx | 3 +- .../features/proxmox/ProxmoxNodesTable.tsx | 23 +++ .../__tests__/ProxmoxNodesTable.test.tsx | 94 ++++++++- .../storageRowAlertPresentation.test.ts | 25 +++ .../storageRowAlertPresentation.ts | 12 ++ frontend-modern/src/types/api.ts | 26 +++ .../src/utils/__tests__/temperature.test.ts | 16 ++ frontend-modern/src/utils/temperature.ts | 22 ++- internal/alerts/active_cleanup.go | 2 +- internal/alerts/active_persistence.go | 5 + internal/alerts/alerts_test.go | 54 +++++ internal/alerts/canonical_metric.go | 22 ++- internal/alerts/metric_status.go | 79 ++++++++ internal/alerts/model.go | 20 ++ internal/alerts/tracking_cleanup.go | 4 +- internal/alerts/unified_eval_test.go | 185 ++++++++++++++++++ internal/models/metrics_types_test.go | 18 ++ internal/models/models.go | 52 +++++ .../monitoring/monitor_alert_handling_test.go | 36 ++++ internal/monitoring/monitor_alert_sync.go | 6 +- 40 files changed, 1297 insertions(+), 26 deletions(-) create mode 100644 frontend-modern/browser-verification/4766668c29936861.json create mode 100644 frontend-modern/src/features/alerts/metricAlertPresentation.ts create mode 100644 internal/alerts/metric_status.go diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index b51b2d57dc..408160343f 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -15,6 +15,14 @@ ## Purpose +### Alert projections carry a volatile metric status — issue #2068 + +`models.Alert` gains an optional `metricStatus` (`models.MetricAlertStatus`) +beside the alert-engine metadata. It is a live evaluation snapshot owned by +the alerts subsystem. Agent report admission, host continuity and +re-enrollment never read or write it, and it is not persisted with agent or +state records. + ### Source-record lookup preserves report admission Host and Docker report admission still performs canonical host-view matching diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index cb03c572e2..1f4f14d152 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -15,6 +15,23 @@ ## Purpose +### Open threshold alerts carry a volatile live status — issue #2068 + +`evaluateCanonicalMetricAlert` builds `MetricStatus` from the evaluator's own +inputs on every evaluation, including the hysteresis hold and the recovery +run, where `Value`, `Message` and `LastSeen` stay at the last breach and no +branch may notify. The phase comes from the trigger, the clear level handed to +the reducer (a value at or below it counts toward recovery) and the reducer's +recovery run; progress uses that run's monotonic elapsed time. The status is +stripped from the active-state checkpoint, the recovery mirror and the +degraded recovery projection, so a held alert adds no durable writes, and +after a restart it is absent until the next evaluation. Resolved records carry +no live status. Stale-alert cleanup and the retention TTLs use the latest +observation (`lastObservedAt`), so an alert held below its trigger for over a +day is not auto-resolved as unmonitored. Frontend surfaces format the status +through `features/alerts/metricAlertPresentation.ts` and never re-derive the +phase or recovery timing. + ### Retained guest observations are not alert evidence Guest memory marked `last-known`, `unavailable` or an unknown observation state diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index e410966406..6c38e23650 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -20,6 +20,19 @@ ## Purpose +### Open threshold alerts expose their live evaluation — issue #2068 + +`Alert.metricStatus` (`/api/alerts/active`, the websocket state and +`/api/state`) is optional and additive. It carries `phase` (`breaching`, +`latched` or `recovering`), `value` (the value compared with the rule, the +average for a rolling-average rule) with `rawValue` and +`evaluationWindowSeconds` when windowed, `unit`, `observedAt`, `trigger`, +`recovery` (the alert clears at or below it), `recoveryDelaySeconds`, and, +while recovering, `recoveryStartedAt` and `recoveryElapsedSeconds`. Legacy +`value`, `message` and `lastSeen` keep meaning the last reading that met the +trigger. The field is absent for non-threshold alerts and until the first +evaluation after a restart, and clients then fall back to `message`. + ### Organisation deletion retains data after incomplete monitoring shutdown Authenticated organisation-owner deletion waits for tenant-loop exit and sealed diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 308d1470e2..1d8ce207c1 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -22,6 +22,14 @@ All drawer History fallbacks must distinguish current observations from retained, unavailable or freshness-unknown guest readings, using original memory evidence and filesystem read reasons. +### Drawer attention detail line and temperature alert floor — issue #2068 + +`DrawerAttentionSection` items accept an optional `detail` (a muted second +line) and `title` (hover text). `TemperatureGauge` accepts `alertSeverity` +and `title`, and `getTemperatureTextClass` treats an open alert's severity as +a floor: a reading that has dipped under an open alert's trigger keeps the +alert's tone, while the reading still wins when it is worse. + ### Canonical drawer History preserves guest read provenance The shared resource drawer passes selected memory observation state/source/time diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index f02a9c2224..2becd58456 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -17,6 +17,14 @@ ## Purpose +### Active-alert state projection carries the live metric status — issue #2068 + +`activeAlertsSnapshot` copies the alert manager's volatile `MetricStatus` +into `models.Alert.metricStatus`, so the websocket and `/api/state` alert +lists carry the reading being evaluated now for a threshold alert held below +its trigger, while `value` and `message` keep the last breach. The projection +adds no copy of its own: `GetActiveAlerts` already returns deep clones. + ### Fresh broadcast projection owns one resource/target capture Canonical live, fallback and standalone-host continuity broadcast reads prefer diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 50344121e9..44f18cda41 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -17,6 +17,13 @@ Drawer observation labels derive locally from the selected snapshot without renewing timestamps, adding reads, remounting charts or changing stored metric coordinates. +### Guest drawer alert copy formats backend status only + +Guest drawer attention rows format `alert.metricStatus` through +`getGuestDrawerAlertAttention` and the shared `getAlertAttentionCopy`. No +threshold comparison, recovery timing or resource-metric lookup runs in the +drawer, and the host-capacity memory comparison stays a suffix on that copy. + ### Drawer provenance stays local to snapshot presentation Guest and canonical resource drawer memory presentation share the same pure diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index ebbc81311a..75882a9d08 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -21,6 +21,14 @@ ## Purpose +### Storage usage headline follows the alert phase — issue #2068 + +`describeStorageAlertHeadline` says "Over N% usage limit" only while a usage +alert is breaching. When `metricStatus.phase` is `latched` or `recovering`, +the row says usage is under the limit and names the clear level, so the +headline never contradicts the usage bar beside it. Fill-forecast headlines +are unchanged. + ### Guest-memory read state stays independent of backup disk state The existing guest drawer retains Core's selected memory provenance through diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index a22df85cf1..2bd755d13a 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -23,6 +23,14 @@ and sort the complete canonical change table while startup and ingestion wait. ## Purpose +### Proxmox node temperature cell follows its own open alert — issue #2068 + +`ProxmoxNodesTable` finds the node's own temperature alert through +`hostOverrideIdCandidates` and passes its severity (unless acknowledged) and +the shared live-status copy to `TemperatureGauge`. Other node alerts never +recolour the cell. The Mail Gateway drawer formats its attention rows through +`getAlertAttentionCopy`. + ### Canonical Discovery honours VM guest-read deferrals The canonical resource drawer uses the same fixed PVE VM lock, backup and diff --git a/frontend-modern/browser-verification/4766668c29936861.json b/frontend-modern/browser-verification/4766668c29936861.json new file mode 100644 index 0000000000..af31ef6f50 --- /dev/null +++ b/frontend-modern/browser-verification/4766668c29936861.json @@ -0,0 +1,62 @@ +{ + "version": 1, + "base_sha": "92d946d057ddeefcae4d247a87e88f2245abca4a", + "verified_at": "2026-10-06T12:36:53Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts", + "frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx", + "frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx", + "frontend-modern/src/components/Workloads/guestDrawerModel.ts", + "frontend-modern/src/components/shared/DrawerAttentionSection.tsx", + "frontend-modern/src/components/shared/TemperatureGauge.tsx", + "frontend-modern/src/features/alerts/metricAlertPresentation.ts", + "frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx", + "frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx", + "frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts", + "frontend-modern/src/types/api.ts", + "frontend-modern/src/utils/temperature.ts" + ], + "content_sha256": { + "frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts": "3c6f6f01df4a13b9fb3084b398be093473b63cdb71bbc01db1dbf41531e15edd", + "frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx": "667a9b9e6fd4385a23b950f18be16fb71cbd6eccc2a4aafb57d2c4532b3af6f1", + "frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx": "5adf11bfc948cae2cca66fde5e2132c66bbb01388efdbc23cbf022b449ae3b0c", + "frontend-modern/src/components/Workloads/guestDrawerModel.ts": "a922c4c26813a676be12e06376bdb40df2aca8bfe5fc850b76009a504dada663", + "frontend-modern/src/components/shared/DrawerAttentionSection.tsx": "2a4e15e781a6158e253b6b934198d7aed71275416eb12c07ade68d5fa80c0332", + "frontend-modern/src/components/shared/TemperatureGauge.tsx": "612172a594698216b8a5fe247e5fa7361946056e02eaba9e71cfe827d0dc5023", + "frontend-modern/src/features/alerts/metricAlertPresentation.ts": "2c875f102207c36d21d2868fa6bf05be7a5ae3e1cfb0351f54c2e89d448259bc", + "frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx": "c979737f3825f12da80b25f50083b15960f7217525905c01d27edc4b26e1fbc3", + "frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx": "b3ebf1259281b7d590c4cbd0eee9ed54ad060ed0e6fd1d80f740d44c95cd3107", + "frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts": "ad5c973c1fbe64a94ad98d8f89d2ac3e51dacad1b1d06af82f77d0d32462ea16", + "frontend-modern/src/types/api.ts": "0db85005fb024e72434e35fdb7eb9ce13d70361569563579365a877fac714f79", + "frontend-modern/src/utils/temperature.ts": "3381a7948f19ae52bf0c72d6fc011ded3691aeac9fdbc9d795dc298af68c4690" + }, + "routes": [ + "/proxmox" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 375, + "height": 812 + } + ], + "states": [ + "Private mock stack built from this change (sidecar backend on 7721, Vite on 5241); pve2's CPU temperature driven through 88, 78 and 72 degrees C by a local-only, uncommitted mock overlay", + "Breaching (88C, trigger 80/clear 75): node drawer Needs attention reads 'Temperature 88°C, above the 80°C alert level' with 'Clears once it drops to 75°C or lower and stays there for 5 minutes.'", + "Held (78C): drawer reads 'Temperature 78°C now, back under the 80°C alert level' / 'Stays open until it reaches 75°C or lower and stays there for 5 minutes.'; legacy alert message still 'Node temperature at 88.0°C' in the API; TEMP cell amber with the same explanation as hover text", + "Recovering (72C, below the 75C warning colour): drawer reads 'Temperature 72°C now, recovering' / 'Clears after 5 minutes at 75°C or lower, 1 minute so far.'; TEMP cell 72°C stays amber (text-amber-600) while 68°C on the next node is green", + "Resolved after 5 minutes at 72C: alert closed with 'Resolved: Node temperature at 72.0°C', TEMP cell green with no hover text, Needs attention section gone", + "Narrow 375px: drawer attention copy wraps inside the card, TEMP column hidden as before", + "Guest drawer, Mail Gateway drawer, storage headline and Assistant handoff copy covered by component and model tests, not exercised live" + ], + "interactions": [ + "Expanded the West Production B node row to open its inline drawer at desktop and phone widths", + "Read the TEMP cell colour class and title text through the DOM in each phase", + "Reloaded the page mid-recovery to confirm the cell and drawer render from fresh state", + "Resized between 1440x900 and 375x812" + ] +} diff --git a/frontend-modern/src/components/Alerts/__tests__/alertAssistantHandoffModel.test.ts b/frontend-modern/src/components/Alerts/__tests__/alertAssistantHandoffModel.test.ts index 9f23bdbe9a..1d9d579301 100644 --- a/frontend-modern/src/components/Alerts/__tests__/alertAssistantHandoffModel.test.ts +++ b/frontend-modern/src/components/Alerts/__tests__/alertAssistantHandoffModel.test.ts @@ -157,4 +157,35 @@ describe('alertAssistantHandoffModel', () => { const briefing = handoff.context.briefing as { detailLines: string[] }; expect(briefing.detailLines).toContain('Current value 92.5% · threshold 80.0%'); }); + + it('hands the assistant the live reading, not the breach an open alert still holds', () => { + const handoff = buildAlertAssistantHandoff({ + alert: makeAlert({ + type: 'temperature', + message: 'Node temperature at 80.0°C', + value: 80, + threshold: 80, + metricStatus: { + phase: 'recovering', + value: 72, + unit: '°C', + observedAt: '2026-05-07T10:04:50.000Z', + trigger: 80, + recovery: 75, + recoveryDelaySeconds: 300, + recoveryElapsedSeconds: 120, + }, + }), + now: new Date('2026-05-07T10:05:00.000Z'), + }); + + const briefing = handoff.context.briefing as { detailLines: string[] }; + expect(briefing.detailLines[0]).toContain('72.0°C'); + expect(briefing.detailLines.join('\n')).not.toContain('Node temperature at 80.0°C'); + expect(handoff.context.handoffContext).toContain('Current Value: 72.0°C'); + expect(handoff.context.handoffContext).toContain('Alert Phase: recovering'); + expect(handoff.context.handoffContext).toContain('Clears At: 75.0°C or lower for 300s'); + expect(handoff.context.handoffContext).toContain('Recovery Progress: 120s of 300s'); + expect(handoff.context.handoffContext).toContain('Last Reading At Or Above Threshold: 80.0°C'); + }); }); diff --git a/frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts b/frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts index 3a6a08b9c7..4ca70859cb 100644 --- a/frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts +++ b/frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts @@ -2,6 +2,7 @@ import type { Alert } from '@/types/api'; import type { AIChatContext } from '@/stores/aiChat'; import { DEFAULT_LOCALE, t, type SupportedLocale } from '@/i18n'; import { getCanonicalAlertId } from '@/features/alerts/identity'; +import { getMetricAlertPresentation } from '@/features/alerts/metricAlertPresentation'; import { formatAlertValue } from '@/utils/alertFormatters'; import { isMetricAlertType } from '@/utils/alerts'; import { isPulseSystemAlert } from '@/utils/alertScope'; @@ -35,7 +36,16 @@ export function buildAlertAssistantHandoff({ // to convey what's wrong. const systemScoped = isPulseSystemAlert(alert); const hasMetricValues = !systemScoped && isMetricAlertType(alert.type); - const currentValue = hasMetricValues ? formatAlertValue(alert.value, alert.type) : ''; + // An open threshold alert keeps value/message at its last breach while it + // holds; the live status is what the metric reads now. + const metricStatus = hasMetricValues ? alert.metricStatus : undefined; + const presentation = metricStatus ? getMetricAlertPresentation(alert, now.getTime()) : null; + const currentValue = hasMetricValues + ? formatAlertValue(metricStatus ? metricStatus.value : alert.value, alert.type) + : ''; + const briefingMessage = presentation + ? `${presentation.summary}. ${presentation.detail}` + : alert.message; const thresholdValue = hasMetricValues ? formatAlertValue(alert.threshold, alert.type) : ''; const nodeLabel = !systemScoped && alert.node ? alert.nodeDisplayName || alert.node : ''; const levelLabel = formatAlertLevel(alert.level); @@ -98,8 +108,8 @@ export function buildAlertAssistantHandoff({ }) : undefined, nodeLabel ? t('alerts.assistant.detail.node', { node: nodeLabel }) : undefined, - alert.message - ? t('alerts.assistant.detail.message', { message: alert.message }) + briefingMessage + ? t('alerts.assistant.detail.message', { message: briefingMessage }) : undefined, ].filter((line): line is string => Boolean(line)), actionLabel: t('alerts.assistant.action.investigate', { alertIdentifier }), @@ -149,6 +159,7 @@ function buildAlertAssistantModelContext({ systemScoped ? undefined : formatContextLine('Resource ID', alert.resourceId), formatContextLine('Current Value', currentValue), formatContextLine('Threshold', thresholdValue), + ...metricStatusContextLines(alert), formatContextLine('Duration', durationText), formatContextLine('Node', nodeLabel), formatContextLine('Message', alert.message), @@ -158,6 +169,34 @@ function buildAlertAssistantModelContext({ .join('\n'); } +const METRIC_PHASE_CONTEXT: Record = { + breaching: 'breaching (at or above the threshold)', + latched: 'holding (below the threshold, not yet down to the clear level)', + recovering: 'recovering (at or below the clear level, waiting out the recovery delay)', +}; + +function metricStatusContextLines(alert: Alert): Array { + const status = isMetricAlertType(alert.type) ? alert.metricStatus : undefined; + if (!status) return []; + const delay = status.recoveryDelaySeconds ?? 0; + return [ + formatContextLine('Alert Phase', METRIC_PHASE_CONTEXT[status.phase] ?? status.phase), + formatContextLine( + 'Clears At', + `${formatAlertValue(status.recovery, alert.type)} or lower${delay > 0 ? ` for ${delay}s` : ''}`, + ), + status.phase === 'recovering' && status.recoveryElapsedSeconds + ? formatContextLine('Recovery Progress', `${status.recoveryElapsedSeconds}s of ${delay}s`) + : undefined, + status.phase === 'breaching' + ? undefined + : formatContextLine( + 'Last Reading At Or Above Threshold', + formatAlertValue(alert.value, alert.type), + ), + ]; +} + function formatContextLine(label: string, value?: string | number | null): string | undefined { if (value === undefined || value === null) return undefined; const text = String(value).trim(); diff --git a/frontend-modern/src/components/Workloads/GuestDrawer.test.tsx b/frontend-modern/src/components/Workloads/GuestDrawer.test.tsx index 443cbbd5d5..0438d3f0a8 100644 --- a/frontend-modern/src/components/Workloads/GuestDrawer.test.tsx +++ b/frontend-modern/src/components/Workloads/GuestDrawer.test.tsx @@ -528,6 +528,46 @@ describe('GuestDrawer', () => { ).toBeInTheDocument(); }); + it('leads with the live reading when an open alert holds below its trigger', () => { + solidRender(() => ( + + )); + + expect(screen.getByText('Memory 87% now, back under the 90% alert level')).toBeInTheDocument(); + expect( + screen.getByText('Stays open until it reaches 85% or lower and stays there for 5 minutes.'), + ).toBeInTheDocument(); + expect(screen.queryByText('VM memory at 91.0%')).not.toBeInTheDocument(); + }); + it('keeps generic Assistant and context-copy actions out of the drawer header', () => { render(() => ); diff --git a/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx b/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx index 8ad1d0c370..de41cfe2e4 100644 --- a/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx +++ b/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx @@ -21,7 +21,7 @@ import { import { AvailabilityProbeSuggestionCard } from './AvailabilityProbeSuggestionCard'; import { buildWorkloadsDiskPresentation } from './diskListModel'; import { - getGuestDrawerAlertMessage, + getGuestDrawerAlertAttention, getGuestDrawerMemoryRows, getGuestDrawerMemoryReading, getGuestDrawerGuestReadPresentation, @@ -297,7 +297,7 @@ export function GuestDrawerOverview(props: GuestDrawerOverviewProps) { ({ id: alert.id, - message: getGuestDrawerAlertMessage(alert, { + ...getGuestDrawerAlertAttention(alert, { guest: props.guest, memoryDisplayBasis: props.memoryDisplayBasis, parentMemoryTotal: props.parentMemoryTotal, diff --git a/frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx b/frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx index 50957eac70..218cd127ba 100644 --- a/frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx +++ b/frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx @@ -9,6 +9,7 @@ import { } from '@/components/shared/DetailSectionTable'; import type { Alert, Disk, HostGPUSensor, Node, Temperature } from '@/types/api'; import { alertTypeDisplayLabel } from '@/features/alerts/helpers'; +import { getAlertAttentionCopy } from '@/features/alerts/metricAlertPresentation'; import { formatBytes, normalizeDiskArray } from '@/utils/format'; import type { MetricDisplayThresholds } from '@/utils/metricThresholds'; import { getProxmoxUpdateEvidencePresentation } from '@/utils/proxmoxUpdateEvidence'; @@ -370,7 +371,7 @@ export function NodeDrawerOverview(props: NodeDrawerOverviewProps) { ({ id: alert.id, - message: alert.message, + ...getAlertAttentionCopy(alert), subject: cleanText(alert.resourceName) || cleanText(alert.nodeDisplayName) || diff --git a/frontend-modern/src/components/Workloads/guestDrawerModel.ts b/frontend-modern/src/components/Workloads/guestDrawerModel.ts index f78290928c..27fa776ad3 100644 --- a/frontend-modern/src/components/Workloads/guestDrawerModel.ts +++ b/frontend-modern/src/components/Workloads/guestDrawerModel.ts @@ -7,6 +7,10 @@ import type { } from '@/api/charts'; import { formatHistoryChartTooltipValue } from '@/components/shared/historyChartModel'; +import { + getAlertAttentionCopy, + type AlertAttentionCopy, +} from '@/features/alerts/metricAlertPresentation'; import { formatBytes, formatPercent, getBackupInfo, type BackupThresholds } from '@/utils/format'; import { getWorkloadGuestDiskStatusMessage, @@ -82,12 +86,13 @@ export interface GuestDrawerBackupPresentation { export const isGuestDrawerVM = (guest: Guest): guest is VM => resolveWorkloadType(guest) === 'vm'; -export const getGuestDrawerAlertMessage = ( +export const getGuestDrawerAlertAttention = ( alert: Alert, context: Pick, -): string => { +): AlertAttentionCopy => { + const copy = getAlertAttentionCopy(alert); if (context.memoryDisplayBasis !== 'host' || alert.type.trim().toLowerCase() !== 'memory') { - return alert.message; + return copy; } const used = context.guest.memory?.used; @@ -104,7 +109,7 @@ export const getGuestDrawerAlertMessage = ( const comparison = hostShare ? `guest allocation · ${hostShare} of host capacity` : 'guest allocation · row uses host capacity'; - return `${alert.message} (${comparison})`; + return { ...copy, message: `${copy.message} (${comparison})` }; }; const getGuestDrawerDiskUsage = (guest: Guest): number | undefined => { diff --git a/frontend-modern/src/components/shared/DrawerAttentionSection.tsx b/frontend-modern/src/components/shared/DrawerAttentionSection.tsx index c0d8fd86d1..fc3d5c6b5f 100644 --- a/frontend-modern/src/components/shared/DrawerAttentionSection.tsx +++ b/frontend-modern/src/components/shared/DrawerAttentionSection.tsx @@ -10,6 +10,10 @@ import { Button } from './Button'; export interface DrawerAttentionItem { id: string; message: string; + /** Secondary line, e.g. what clears an alert whose reading has dropped. */ + detail?: string; + /** Hover text when it should say more than the message. */ + title?: string; subject?: string; metric?: string; severity?: string; @@ -73,10 +77,13 @@ export const DrawerAttentionSection: Component = (p

{item.message}

+ +

{item.detail}

+ { expect(drawerAttentionSectionSource).toContain('Show fewer alerts'); expect(drawerAttentionSectionSource).toContain('aria-expanded={expanded()}'); expect(drawerAttentionSectionSource).not.toContain('DetailSectionTable'); + // A threshold alert held under its trigger leads with the live reading and + // a second line saying what clears it, never only its stale breach text. + expect(drawerAttentionSectionSource).toContain('item.detail'); + expect(drawerAttentionSectionSource).toContain('item.title ?? item.message'); + for (const liveAlertConsumer of [ + 'src/components/Workloads/NodeDrawerOverview.tsx', + 'src/features/proxmox/ProxmoxMailGatewayDrawer.tsx', + ]) { + const source = readFrontendSource(liveAlertConsumer); + expect(source).toContain('...getAlertAttentionCopy(alert)'); + expect(source).not.toContain('message: alert.message'); + } + expect(readFrontendSource('src/components/Workloads/guestDrawerModel.ts')).toContain( + 'getAlertAttentionCopy(alert)', + ); const overviewConsumers = [ 'src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx', diff --git a/frontend-modern/src/components/shared/TemperatureGauge.tsx b/frontend-modern/src/components/shared/TemperatureGauge.tsx index 01e0c179bc..f5622ab88b 100644 --- a/frontend-modern/src/components/shared/TemperatureGauge.tsx +++ b/frontend-modern/src/components/shared/TemperatureGauge.tsx @@ -14,6 +14,9 @@ interface TemperatureGaugeProps { warning?: number; thresholds?: MetricDisplayThresholds | null; metric?: TemperatureDisplayMetric; + /** Severity of an open alert on this reading; keeps its tone while the alert holds. */ + alertSeverity?: 'warning' | 'critical' | null; + title?: string; class?: string; } @@ -30,11 +33,14 @@ export const TemperatureGauge: Component = (props) => { }); const textColorClass = createMemo(() => - getTemperatureTextClass(props.value, explicitThresholds(), props.metric), + getTemperatureTextClass(props.value, explicitThresholds(), props.metric, props.alertSeverity), ); return ( - + {formatTemperature(props.value)} ); diff --git a/frontend-modern/src/features/alerts/__tests__/helpers.test.ts b/frontend-modern/src/features/alerts/__tests__/helpers.test.ts index 160cc8da7a..676038b14f 100644 --- a/frontend-modern/src/features/alerts/__tests__/helpers.test.ts +++ b/frontend-modern/src/features/alerts/__tests__/helpers.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Alert, MetricAlertStatus } from '@/types/api'; import { clampMaxAlertsPerHour, fallbackMaxAlertsPerHour, @@ -12,6 +13,11 @@ import { alertTypeDisplayLabel, clampEscalationDelayMinutes, } from '@/features/alerts/helpers'; +import { + METRIC_ALERT_STATUS_STALE_MS, + getAlertAttentionCopy, + getMetricAlertPresentation, +} from '@/features/alerts/metricAlertPresentation'; describe('alerts helpers', () => { describe('clampEscalationDelayMinutes', () => { @@ -229,3 +235,148 @@ describe('alerts helpers', () => { }); }); }); + +const NOW = Date.parse('2026-10-06T10:52:00Z'); + +// minipc on 2026-10-06: the alert opened at 80°C, the node then read 71-76°C, +// and the drawer kept saying "Node temperature at 80.0°C". +const minipcAlert = (status?: Partial): Alert => ({ + id: 'homelab-minipc::metric-threshold:temperature', + type: 'temperature', + level: 'warning', + resourceId: 'homelab-minipc', + resourceName: 'minipc', + node: 'minipc', + instance: 'homelab', + message: 'Node temperature at 80.0°C', + value: 80, + threshold: 80, + startTime: '2026-10-06T07:57:44Z', + lastSeen: '2026-10-06T10:48:52Z', + acknowledged: false, + metricStatus: status + ? { + phase: 'latched', + value: 76, + unit: '°C', + observedAt: new Date(NOW - 10_000).toISOString(), + trigger: 80, + recovery: 75, + recoveryDelaySeconds: 300, + ...status, + } + : undefined, +}); + +describe('getMetricAlertPresentation', () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it('leads with the reading now and what clears an alert held under its trigger', () => { + const presentation = getMetricAlertPresentation( + minipcAlert({ phase: 'latched', value: 76 }), + NOW, + ); + expect(presentation).toMatchObject({ + phase: 'latched', + stale: false, + summary: 'Temperature 76°C now, back under the 80°C alert level', + detail: 'Stays open until it reaches 75°C or lower and stays there for 5 minutes.', + phaseLabel: 'Alert still open', + }); + expect(presentation?.lastBreach).toMatch(/^Last reading at or above 80°C: 80°C, /); + }); + + it('says a reading at the clear level is recovering, with progress once it has some', () => { + expect( + getMetricAlertPresentation( + minipcAlert({ phase: 'recovering', value: 72, recoveryElapsedSeconds: 0 }), + NOW, + ), + ).toMatchObject({ + summary: 'Temperature 72°C now, recovering', + detail: 'Clears after 5 minutes at 75°C or lower.', + phaseLabel: 'Recovering', + }); + expect( + getMetricAlertPresentation( + minipcAlert({ phase: 'recovering', value: 72, recoveryElapsedSeconds: 120 }), + NOW, + )?.detail, + ).toBe('Clears after 5 minutes at 75°C or lower, 2 minutes so far.'); + }); + + it('names the alert level while the reading is still over it', () => { + const presentation = getMetricAlertPresentation( + minipcAlert({ phase: 'breaching', value: 82 }), + NOW, + ); + expect(presentation).toMatchObject({ + summary: 'Temperature 82°C, above the 80°C alert level', + detail: 'Clears once it drops to 75°C or lower and stays there for 5 minutes.', + phaseLabel: 'Above 80°C', + }); + expect(presentation?.lastBreach).toBeUndefined(); + }); + + it('stops calling an old evaluation "now"', () => { + const presentation = getMetricAlertPresentation( + minipcAlert({ + phase: 'latched', + value: 76, + observedAt: new Date(NOW - METRIC_ALERT_STATUS_STALE_MS - 60_000).toISOString(), + }), + NOW, + ); + expect(presentation?.stale).toBe(true); + expect(presentation?.summary).toMatch(/^Last reading: Temperature 76°C, /); + expect(presentation?.summary).not.toContain('now'); + expect(presentation?.phaseLabel).toBe('No recent reading'); + }); + + it('shows the averaged value the rule follows beside the latest sample', () => { + const alert: Alert = { + ...minipcAlert(), + type: 'cpu', + metricStatus: { + phase: 'latched', + value: 84.2, + rawValue: 61, + evaluationWindowSeconds: 600, + unit: '%', + observedAt: new Date(NOW).toISOString(), + trigger: 90, + recovery: 80, + }, + }; + expect(getMetricAlertPresentation(alert, NOW)).toMatchObject({ + summary: 'CPU averaged 84% over 10 minutes, latest 61% now, back under the 90% alert level', + detail: 'Stays open until it reaches 80% or lower.', + }); + }); + + it('returns null without a live status so callers keep the alert message', () => { + expect(getMetricAlertPresentation(minipcAlert(), NOW)).toBeNull(); + expect(getAlertAttentionCopy(minipcAlert(), NOW)).toEqual({ + message: 'Node temperature at 80.0°C', + }); + }); +}); + +describe('getAlertAttentionCopy', () => { + it('replaces the stale breach message and keeps the breach in the hover text', () => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + const copy = getAlertAttentionCopy(minipcAlert({ phase: 'recovering', value: 72 }), NOW); + vi.useRealTimers(); + expect(copy.message).toBe('Temperature 72°C now, recovering'); + expect(copy.detail).toBe('Clears after 5 minutes at 75°C or lower.'); + expect(copy.title).toContain('Last reading at or above 80°C: 80°C'); + expect(copy.message).not.toContain('80'); + }); +}); diff --git a/frontend-modern/src/features/alerts/metricAlertPresentation.ts b/frontend-modern/src/features/alerts/metricAlertPresentation.ts new file mode 100644 index 0000000000..8d72d7c312 --- /dev/null +++ b/frontend-modern/src/features/alerts/metricAlertPresentation.ts @@ -0,0 +1,157 @@ +import type { Alert, MetricAlertPhase, MetricAlertStatus } from '@/types/api'; +import { formatRelativeTime } from '@/utils/format'; +import { formatTemperature } from '@/utils/temperature'; +import { alertTypeDisplayLabel } from './helpers'; + +/** + * A live status older than this is no longer "now": the resource stopped + * reporting, or Pulse stopped evaluating it. Generous enough for the slowest + * poll intervals. + */ +export const METRIC_ALERT_STATUS_STALE_MS = 10 * 60 * 1000; + +export interface MetricAlertPresentation { + phase: MetricAlertPhase; + /** The live status is too old to describe as the current reading. */ + stale: boolean; + /** Leads with the reading Pulse is evaluating now and where it stands. */ + summary: string; + /** What clears the alert from here. */ + detail: string; + /** Short state for compact cells and badges. */ + phaseLabel: string; + /** The last reading that met the trigger, once the live one differs. */ + lastBreach?: string; +} + +const formatMetricValue = (value: number, unit: string | undefined): string => { + if (unit === '°C') return formatTemperature(value); + if (unit === '%') { + return `${Math.abs(value) >= 10 ? Math.round(value) : Number(value.toFixed(1))}%`; + } + const rounded = Number(value.toFixed(1)); + return unit ? `${rounded} ${unit}` : String(rounded); +}; + +const formatSeconds = (seconds: number): string => { + if (seconds < 60) return `${seconds} ${seconds === 1 ? 'second' : 'seconds'}`; + if (seconds < 3600) { + const minutes = Math.round(seconds / 60); + return `${minutes} ${minutes === 1 ? 'minute' : 'minutes'}`; + } + const hours = Number((seconds / 3600).toFixed(1)); + return `${hours} ${hours === 1 ? 'hour' : 'hours'}`; +}; + +const describeReading = (label: string, status: MetricAlertStatus): string => { + const value = formatMetricValue(status.value, status.unit); + const window = status.evaluationWindowSeconds ?? 0; + if (window <= 0) return `${label} ${value}`; + const latest = + typeof status.rawValue === 'number' && Number.isFinite(status.rawValue) + ? `, latest ${formatMetricValue(status.rawValue, status.unit)}` + : ''; + return `${label} averaged ${value} over ${formatSeconds(window)}${latest}`; +}; + +const describeClearRule = (status: MetricAlertStatus, lead: string): string => { + const recovery = formatMetricValue(status.recovery, status.unit); + const delay = status.recoveryDelaySeconds ?? 0; + return delay > 0 + ? `${lead} ${recovery} or lower and stays there for ${formatSeconds(delay)}.` + : `${lead} ${recovery} or lower.`; +}; + +const isUsableStatus = (status: MetricAlertStatus | undefined): status is MetricAlertStatus => + !!status && + Number.isFinite(status.value) && + Number.isFinite(status.trigger) && + Number.isFinite(status.recovery) && + (status.phase === 'breaching' || status.phase === 'latched' || status.phase === 'recovering'); + +/** + * Describes an open threshold alert from the backend's live evaluation, so + * every surface says what the reading is now and why the alert is still open + * instead of repeating the last breach. Returns null when the alert carries + * no live status (non-threshold alerts, or the moments after a restart). + */ +export function getMetricAlertPresentation( + alert: Pick, + now: number = Date.now(), +): MetricAlertPresentation | null { + const status = alert.metricStatus; + if (!isUsableStatus(status)) return null; + + const label = alertTypeDisplayLabel(alert.type); + const reading = describeReading(label, status); + const trigger = formatMetricValue(status.trigger, status.unit); + const observedAt = Date.parse(status.observedAt); + const stale = !Number.isFinite(observedAt) || now - observedAt > METRIC_ALERT_STATUS_STALE_MS; + + let summary: string; + let detail: string; + let phaseLabel: string; + switch (status.phase) { + case 'breaching': + summary = `${reading}, above the ${trigger} alert level`; + detail = describeClearRule(status, 'Clears once it drops to'); + phaseLabel = `Above ${trigger}`; + break; + case 'latched': + summary = `${reading} now, back under the ${trigger} alert level`; + detail = describeClearRule(status, 'Stays open until it reaches'); + phaseLabel = 'Alert still open'; + break; + case 'recovering': { + const delay = status.recoveryDelaySeconds ?? 0; + const elapsed = status.recoveryElapsedSeconds ?? 0; + const recovery = formatMetricValue(status.recovery, status.unit); + summary = `${reading} now, recovering`; + detail = + delay > 0 + ? `Clears after ${formatSeconds(delay)} at ${recovery} or lower${ + elapsed > 0 ? `, ${formatSeconds(Math.min(elapsed, delay))} so far` : '' + }.` + : `Clears at ${recovery} or lower.`; + phaseLabel = 'Recovering'; + break; + } + } + + if (stale) { + summary = `Last reading: ${reading}${ + Number.isFinite(observedAt) ? `, ${formatRelativeTime(observedAt)}` : '' + }`; + phaseLabel = 'No recent reading'; + } + + let lastBreach: string | undefined; + if (status.phase !== 'breaching' && Number.isFinite(alert.value)) { + const when = alert.lastSeen ? `, ${formatRelativeTime(alert.lastSeen)}` : ''; + lastBreach = `Last reading at or above ${trigger}: ${formatMetricValue(alert.value, status.unit)}${when}`; + } + + return { phase: status.phase, stale, summary, detail, phaseLabel, lastBreach }; +} + +export interface AlertAttentionCopy { + message: string; + detail?: string; + title?: string; +} + +/** Copy for an alert row in a drawer's "Needs attention" list. */ +export function getAlertAttentionCopy( + alert: Pick, + now: number = Date.now(), +): AlertAttentionCopy { + const presentation = getMetricAlertPresentation(alert, now); + if (!presentation) return { message: alert.message }; + return { + message: presentation.summary, + detail: presentation.detail, + title: [presentation.summary, presentation.detail, presentation.lastBreach] + .filter(Boolean) + .join('\n'), + }; +} diff --git a/frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx b/frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx index bca5931740..6211981044 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx @@ -1,6 +1,7 @@ import { For, Show, createMemo, createResource, type Component } from 'solid-js'; import { DrawerAttentionSection } from '@/components/shared/DrawerAttentionSection'; import { alertTypeDisplayLabel } from '@/features/alerts/helpers'; +import { getAlertAttentionCopy } from '@/features/alerts/metricAlertPresentation'; import type { Alert } from '@/types/api'; import { Card } from '@/components/shared/Card'; import { DrawerSubjectHeading } from '@/components/shared/DrawerSubjectHeading'; @@ -351,7 +352,7 @@ export const ProxmoxMailGatewayDrawer: Component<{ ({ id: alert.id, - message: alert.message, + ...getAlertAttentionCopy(alert), subject: name(), metric: alertTypeDisplayLabel(alert.type), severity: alert.level, diff --git a/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx b/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx index 932f0767e3..191809f431 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx @@ -21,6 +21,7 @@ import { StackedDiskBar } from '@/components/Workloads/StackedDiskBar'; import { MetricMiniSparkline } from '@/components/Workloads/MetricMiniSparkline'; import { TemperatureGauge } from '@/components/shared/TemperatureGauge'; import { hostOverrideIdCandidates } from '@/features/alerts/alertOverridesModel'; +import { getMetricAlertPresentation } from '@/features/alerts/metricAlertPresentation'; import { useBreakpoint } from '@/hooks/useBreakpoint'; import { TableCell, TableRow } from '@/components/shared/Table'; import { SUMMARY_ROW_ACTION_BUTTON_FOCUS_CLASS } from '@/components/shared/summaryInteractionA11y'; @@ -451,6 +452,26 @@ export const ProxmoxNodesTable: Component<{ const alertResourceIds = () => hostOverrideIdCandidates(node); const temperatureThresholds = () => alertsActivation.getMetricThresholds('node', 'temperature', alertResourceIds()); + // The node's own temperature alert holds after the reading dips + // under its trigger; the cell keeps the alert's tone and says why + // instead of reading green beside a warning row. + const temperatureAlert = createMemo(() => + getAlertsForResource(alertResourceIds(), activeAlerts, alertsEnabled()).find( + (alert) => alert.type === 'temperature', + ), + ); + const temperatureAlertSeverity = () => { + const alert = temperatureAlert(); + if (!alert || alert.acknowledged) return null; + return alert.level === 'critical' ? 'critical' : 'warning'; + }; + const temperatureAlertTitle = () => { + const alert = temperatureAlert(); + const presentation = alert ? getMetricAlertPresentation(alert) : null; + return presentation + ? `${presentation.summary}. ${presentation.detail}` + : alert?.message; + }; const cpuThresholds = () => alertsActivation.getMetricThresholds('node', 'cpu', alertResourceIds()); const memoryThresholds = () => @@ -729,6 +750,8 @@ export const ProxmoxNodesTable: Component<{ diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx b/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx index e985df726d..fc4e8b9ae9 100644 --- a/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx +++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx @@ -46,8 +46,18 @@ vi.mock('@/components/Workloads/MetricMiniSparkline', () => ({ })); vi.mock('@/components/shared/TemperatureGauge', () => ({ - TemperatureGauge: (props: { value: number; thresholds?: unknown }) => { - temperatureGaugeMock({ value: props.value, thresholds: props.thresholds }); + TemperatureGauge: (props: { + value: number; + thresholds?: unknown; + alertSeverity?: string | null; + title?: string; + }) => { + temperatureGaugeMock({ + value: props.value, + thresholds: props.thresholds, + alertSeverity: props.alertSeverity, + title: props.title, + }); return
; }, })); @@ -358,6 +368,86 @@ describe('ProxmoxNodesTable', () => { ); }); + it('keeps an open temperature alert visible on a reading that dipped under its trigger', () => { + // minipc on 2026-10-06: the row warned "80.0°C" while the cell read a green 72°C. + activeAlertsMock.value = { + 'cpu-alert': { + id: 'cpu-alert', + resourceId: 'agent:pve-node-1', + type: 'cpu', + level: 'critical', + message: 'Node cpu at 97.0%', + value: 97, + acknowledged: false, + }, + 'temperature-alert': { + id: 'temperature-alert', + resourceId: 'agent:pve-node-1', + type: 'temperature', + level: 'warning', + message: 'Node temperature at 80.0°C', + value: 80, + acknowledged: false, + metricStatus: { + phase: 'recovering', + value: 72, + unit: '°C', + observedAt: new Date().toISOString(), + trigger: 80, + recovery: 75, + recoveryDelaySeconds: 300, + }, + }, + }; + + render(() => ( + } + emptyTitle="No Proxmox VE nodes" + emptyDescription="No nodes" + /> + )); + + // The node's own temperature alert sets the tone; its CPU alert does not. + expect(temperatureGaugeMock).toHaveBeenLastCalledWith( + expect.objectContaining({ + value: 72, + alertSeverity: 'warning', + title: 'Temperature 72°C now, recovering. Clears after 5 minutes at 75°C or lower.', + }), + ); + }); + + it('leaves the temperature tone to the reading once the alert is acknowledged', () => { + activeAlertsMock.value = { + 'temperature-alert': { + id: 'temperature-alert', + resourceId: 'agent:pve-node-1', + type: 'temperature', + level: 'warning', + message: 'Node temperature at 80.0°C', + value: 80, + acknowledged: true, + }, + }; + + render(() => ( + } + emptyTitle="No Proxmox VE nodes" + emptyDescription="No nodes" + /> + )); + + expect(temperatureGaugeMock).toHaveBeenLastCalledWith( + expect.objectContaining({ alertSeverity: null, title: 'Node temperature at 80.0°C' }), + ); + }); + it('keeps the Proxmox cache-inclusive memory comparison label on node rows', () => { render(() => ( { expect(result.dataAlertState).toBe('acknowledged'); }); + it('does not claim usage is over the limit while the alert holds below it', () => { + const holding = (phase: 'latched' | 'recovering') => + makeAlert({ + metricStatus: { + phase, + value: phase === 'latched' ? 83 : 79, + unit: '%', + observedAt: '2026-10-04T12:30:00Z', + trigger: 85, + recovery: 80, + recoveryDelaySeconds: 300, + }, + }); + expect(describeStorageAlertHeadline(holding('latched'))).toBe( + 'Under 85% limit, clears at 80% or lower', + ); + expect(describeStorageAlertHeadline(holding('latched'), { compact: true })).toBe('Clears ≤80%'); + expect(describeStorageAlertHeadline(holding('recovering'))).toBe( + 'Recovering, clears at 80% or lower', + ); + expect(describeStorageAlertHeadline(holding('recovering'), { compact: true })).toBe( + 'Recovering', + ); + }); + it('explains a highlighted row with a fill forecast, a crossed limit or the alert text', () => { const forecast = (days: number) => makeAlert({ threshold: 100, metadata: { forecastDaysToFull: days } }); diff --git a/frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts b/frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts index 57c6e34cff..30d7a3aa1b 100644 --- a/frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts +++ b/frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts @@ -78,6 +78,18 @@ export const describeStorageAlertHeadline = ( if (options.compact) return `Full in ~${rounded}d`; return `Full in ~${rounded} ${rounded === 1 ? 'day' : 'days'}`; } + const status = alert.type === 'usage' ? alert.metricStatus : undefined; + if (status && status.phase !== 'breaching') { + // Usage has dropped back under the limit but the alert holds until it + // reaches the clear level, so "Over" would contradict the bar beside it. + const recovery = `${Math.round(status.recovery)}%`; + if (status.phase === 'recovering') { + return options.compact ? 'Recovering' : `Recovering, clears at ${recovery} or lower`; + } + return options.compact + ? `Clears ≤${recovery}` + : `Under ${Math.round(status.trigger)}% limit, clears at ${recovery} or lower`; + } if (alert.type === 'usage' && alert.threshold > 0 && alert.threshold < 100) { return options.compact ? `Over ${alert.threshold}%` : `Over ${alert.threshold}% usage limit`; } diff --git a/frontend-modern/src/types/api.ts b/frontend-modern/src/types/api.ts index 5e7da6d6de..a455f1c4a2 100644 --- a/frontend-modern/src/types/api.ts +++ b/frontend-modern/src/types/api.ts @@ -1380,6 +1380,32 @@ export interface Alert { latestTransition?: LifecycleTransition; transitions?: LifecycleTransition[]; evidence?: EvidenceEnvelope[]; + /** + * Live evaluation behind an open threshold alert. `value`, `message` and + * `lastSeen` keep the last reading that met the trigger; this is the + * reading being evaluated now and why the alert has not cleared. + */ + metricStatus?: MetricAlertStatus; +} + +export type MetricAlertPhase = 'breaching' | 'latched' | 'recovering'; + +export interface MetricAlertStatus { + phase: MetricAlertPhase; + /** Value compared with the rule (the average for a rolling-average rule). */ + value: number; + /** Latest sample when the rule evaluates a rolling average. */ + rawValue?: number; + evaluationWindowSeconds?: number; + unit?: string; + observedAt: string; + /** Opens at or above this value. */ + trigger: number; + /** Clears after staying at or below this value for recoveryDelaySeconds. */ + recovery: number; + recoveryDelaySeconds?: number; + recoveryStartedAt?: string; + recoveryElapsedSeconds?: number; } export interface ResolvedAlert extends Alert { diff --git a/frontend-modern/src/utils/__tests__/temperature.test.ts b/frontend-modern/src/utils/__tests__/temperature.test.ts index ec4824f67f..5224735935 100644 --- a/frontend-modern/src/utils/__tests__/temperature.test.ts +++ b/frontend-modern/src/utils/__tests__/temperature.test.ts @@ -138,6 +138,22 @@ describe('temperature', () => { expect(getTemperatureTextClass(85, thresholds)).toContain('text-red-600'); }); + it("keeps an open alert's tone while the reading holds under its trigger", () => { + const thresholds = { warning: 75, critical: 80 }; + // minipc: alert open at 80°C, now reading 72°C on its way down. + expect(getTemperatureTextClass(72, thresholds)).toContain('text-green-600'); + expect(getTemperatureTextClass(72, thresholds, 'temperature', 'warning')).toContain( + 'text-amber-600', + ); + expect(getTemperatureTextClass(72, thresholds, 'temperature', 'critical')).toContain( + 'text-red-600', + ); + // The reading still wins when it is worse than the alert. + expect(getTemperatureTextClass(86, thresholds, 'temperature', 'warning')).toContain( + 'text-red-600', + ); + }); + it('uses disk temperature defaults for disk temperature displays', () => { expect(getTemperatureTextClass(49, undefined, 'diskTemperature')).toContain('text-green-600'); expect(getTemperatureTextClass(50, undefined, 'diskTemperature')).toContain('text-amber-600'); diff --git a/frontend-modern/src/utils/temperature.ts b/frontend-modern/src/utils/temperature.ts index f8b822504f..6b993af762 100644 --- a/frontend-modern/src/utils/temperature.ts +++ b/frontend-modern/src/utils/temperature.ts @@ -109,15 +109,31 @@ export type TemperatureDisplayMetric = Extract< 'temperature' | 'diskTemperature' >; +const TEMPERATURE_SEVERITY_RANK = { normal: 0, warning: 1, critical: 2 } as const; + +/** + * Colour for a temperature reading. An open alert on the reading sets a floor: + * the alert holds until the reading falls to its clear level, so a reading + * that has dipped under the trigger must not look healthy beside the alert. + */ export const getTemperatureTextClass = ( celsius: number | null | undefined, thresholds?: MetricDisplayThresholds | null, metric: TemperatureDisplayMetric = 'temperature', + openAlertSeverity?: 'warning' | 'critical' | null, ): string => { if (celsius === null || celsius === undefined || !Number.isFinite(celsius)) { return 'text-muted'; } - return TEMPERATURE_TEXT_CLASSES[ - getMetricSeverity(celsius, metric, thresholds ?? getDefaultDisplayMetricThresholds(metric)) - ]; + const readingSeverity = getMetricSeverity( + celsius, + metric, + thresholds ?? getDefaultDisplayMetricThresholds(metric), + ); + const severity = + openAlertSeverity && + TEMPERATURE_SEVERITY_RANK[openAlertSeverity] > TEMPERATURE_SEVERITY_RANK[readingSeverity] + ? openAlertSeverity + : readingSeverity; + return TEMPERATURE_TEXT_CLASSES[severity]; }; diff --git a/internal/alerts/active_cleanup.go b/internal/alerts/active_cleanup.go index 9d0a19af68..e8d24899db 100644 --- a/internal/alerts/active_cleanup.go +++ b/internal/alerts/active_cleanup.go @@ -19,7 +19,7 @@ func (m *Manager) Cleanup(maxAge time.Duration) { if alert == nil { return true } - lastSeen := alert.LastSeen + lastSeen := alert.lastObservedAt() if lastSeen.IsZero() { lastSeen = alert.StartTime } diff --git a/internal/alerts/active_persistence.go b/internal/alerts/active_persistence.go index d5c0c122a1..a1a094767c 100644 --- a/internal/alerts/active_persistence.go +++ b/internal/alerts/active_persistence.go @@ -180,6 +180,10 @@ func (m *Manager) snapshotActiveAlerts() []*Alert { clone := alert.Clone() backfillCanonicalIdentity(clone) clone.ID = exportedAlertID(clone, clone.ID) + // The live status changes on every poll while a threshold alert + // holds; keeping it out of the checkpoint keeps an unchanged alert + // set byte-identical, so the recovery mirror is not rewritten. + clone.MetricStatus = nil alerts = append(alerts, clone) } m.mu.RUnlock() @@ -308,6 +312,7 @@ func (m *Manager) setActiveRecoveryAlert(alert *Alert, fallback string) { clone := alert.Clone() backfillCanonicalIdentity(clone) clone.ID = exportedAlertID(clone, clone.ID) + clone.MetricStatus = nil storageKey := activeAlertStorageKey(clone, fallback) if storageKey == "" { return diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go index 68daa1b65c..f95e034cbf 100644 --- a/internal/alerts/alerts_test.go +++ b/internal/alerts/alerts_test.go @@ -21689,3 +21689,57 @@ func TestCleanupRetentionRequiresObservationInactivity(t *testing.T) { }) } } + +func TestMetricStatusStaysOutOfDurableAlertSnapshots(t *testing.T) { + m, clock := newMetricStatusTestManager(t) + fireNodeTemperatureAlert(t, m, clock, 85) + clock.advance(30 * time.Second) + checkMetricStatusNode(m, 78) + + live := m.GetActiveAlerts() + if len(live) != 1 || live[0].MetricStatus == nil || live[0].MetricStatus.Value != 78 { + t.Fatalf("live alerts do not carry the current reading: %+v", live) + } + for _, alert := range m.snapshotActiveAlerts() { + if alert.MetricStatus != nil { + t.Fatalf("durable checkpoint carries volatile status for %s: %+v", alert.ID, alert.MetricStatus) + } + } + for _, alert := range m.activeRecoverySnapshot() { + if alert.MetricStatus != nil { + t.Fatalf("recovery projection carries volatile status for %s: %+v", alert.ID, alert.MetricStatus) + } + } +} + +func TestStaleCleanupKeepsThresholdAlertStillBeingEvaluated(t *testing.T) { + m := newTestManager(t) + now := time.Now() + breachedLongAgo := now.Add(-StaleTrackingThreshold - time.Hour) + + held := &Alert{ + ID: "held::metric-threshold:usage", Type: "usage", ResourceID: "held", + StartTime: breachedLongAgo, LastSeen: breachedLongAgo, + MetricStatus: &models.MetricAlertStatus{Phase: models.MetricAlertPhaseLatched, ObservedAt: now}, + } + abandoned := &Alert{ + ID: "gone::metric-threshold:usage", Type: "usage", ResourceID: "gone", + StartTime: breachedLongAgo, LastSeen: breachedLongAgo, + MetricStatus: &models.MetricAlertStatus{Phase: models.MetricAlertPhaseLatched, ObservedAt: breachedLongAgo}, + } + m.mu.Lock() + m.activeAlerts[held.ID] = held + m.activeAlerts[abandoned.ID] = abandoned + m.mu.Unlock() + + m.cleanupStaleMaps() + + m.mu.RLock() + defer m.mu.RUnlock() + if _, ok := m.activeAlerts[held.ID]; !ok { + t.Fatal("cleanup resolved a held alert whose condition is still being evaluated") + } + if _, ok := m.activeAlerts[abandoned.ID]; ok { + t.Fatal("cleanup kept an alert nobody has evaluated for over a day") + } +} diff --git a/internal/alerts/canonical_metric.go b/internal/alerts/canonical_metric.go index fcfa75baa5..4d56b7ca48 100644 --- a/internal/alerts/canonical_metric.go +++ b/internal/alerts/canonical_metric.go @@ -261,6 +261,17 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec alertMetadata[k] = v } } + metricStatus := buildMetricAlertStatus(metricStatusInput{ + value: value, + triggered: triggered, + trigger: spec.MetricThreshold.Trigger, + clear: clearThreshold, + recoveryDelaySeconds: stability.RecoveryDelaySeconds, + unit: metricStatusUnit(metricType), + window: windowed, + incident: incident, + observedAt: observedAt, + }) if !exists { alert := &Alert{ @@ -278,6 +289,7 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec StartTime: alertStartTime, LastSeen: observedAt, Metadata: alertMetadata, + MetricStatus: metricStatus, } applyCanonicalIdentity(alert, spec.ID, string(spec.Kind)) @@ -331,12 +343,16 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec } if !triggered && primary == "" { - // Hysteresis latch: below trigger but above recovery — hold - // without refreshing, as the pre-cutover engine did. + // Hysteresis hold or recovery run: Value, Message and LastSeen + // keep the last breach, and nothing here may notify. Only the + // volatile live status moves, so surfaces can say what the + // reading is now and why the alert has not cleared. + existingAlert.MetricStatus = metricStatus return } oldLevel := existingAlert.Level + existingAlert.MetricStatus = metricStatus existingAlert.LastSeen = observedAt existingAlert.Value = value existingAlert.Threshold = spec.MetricThreshold.Trigger @@ -390,6 +406,8 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec existingAlert = existingAlert.Clone() existingAlert.Value = value existingAlert.LastSeen = observedAt + // A resolved record is history; the live status describes open alerts only. + existingAlert.MetricStatus = nil message, _ := metricAlertMessage(resourceType, metricType, value, opts) existingAlert.Message = "Resolved: " + message diff --git a/internal/alerts/metric_status.go b/internal/alerts/metric_status.go new file mode 100644 index 0000000000..7bc19204b1 --- /dev/null +++ b/internal/alerts/metric_status.go @@ -0,0 +1,79 @@ +package alerts + +import ( + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer" + "github.com/rcourtman/pulse-go-rewrite/internal/models" +) + +// metricStatusInput is what the canonical metric evaluator compared on one +// observation: the value after any rolling-average window, the exact rule +// handed to the reducer, and the incident the reducer holds afterwards. +type metricStatusInput struct { + value float64 + triggered bool + trigger float64 + clear float64 + recoveryDelaySeconds int + unit string + window metricWindowObservation + incident reducer.Incident + observedAt time.Time +} + +// metricStatusUnit is the unit of an evaluated metric value. CPU and memory +// share the "%" of usage even though their legacy messages omit a unit. +func metricStatusUnit(metricType string) string { + switch metricType { + case "temperature", "disk_temperature", "diskTemperature": + return "°C" + case "diskRead", "diskWrite", "networkIn", "networkOut": + return "MB/s" + default: + return "%" + } +} + +// buildMetricAlertStatus derives the live status of an open threshold alert +// from the evaluator's own inputs, so surfaces never re-derive the phase or +// the recovery timer from thresholds and a resource reading of their own. +func buildMetricAlertStatus(input metricStatusInput) *models.MetricAlertStatus { + // The reducer treats a non-positive clear as "clear at the trigger". + recovery := input.clear + if recovery <= 0 { + recovery = input.trigger + } + status := &models.MetricAlertStatus{ + Phase: models.MetricAlertPhaseBreaching, + Value: input.value, + Unit: input.unit, + ObservedAt: input.observedAt, + Trigger: input.trigger, + Recovery: recovery, + RecoveryDelaySeconds: input.recoveryDelaySeconds, + } + if input.window.WindowSeconds > 0 { + raw := input.window.CurrentValue + status.RawValue = &raw + status.EvaluationWindowSeconds = input.window.WindowSeconds + } + if input.triggered { + return status + } + if input.value > recovery || input.incident.RecoverySince.IsZero() { + status.Phase = models.MetricAlertPhaseLatched + return status + } + status.Phase = models.MetricAlertPhaseRecovering + startedAt := input.incident.RecoverySince + status.RecoveryStartedAt = &startedAt + elapsed := input.observedAt.Sub(input.incident.RecoverySince) + if input.incident.RecoveryTicksSupplied { + elapsed = input.incident.RecoveryElapsed + } + if elapsed > 0 { + status.RecoveryElapsedSeconds = int(elapsed / time.Second) + } + return status +} diff --git a/internal/alerts/model.go b/internal/alerts/model.go index 4b6324a2fd..66eecd2a29 100644 --- a/internal/alerts/model.go +++ b/internal/alerts/model.go @@ -4,6 +4,7 @@ import ( "strings" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" ) @@ -144,6 +145,23 @@ type Alert struct { LatestTransition *operationaltrust.LifecycleTransition `json:"latestTransition,omitempty"` Transitions []operationaltrust.LifecycleTransition `json:"transitions,omitempty"` Evidence []operationaltrust.EvidenceEnvelope `json:"evidence,omitempty"` + // MetricStatus is the live evaluation behind an open threshold alert. It + // is volatile and stripped from durable snapshots. + MetricStatus *models.MetricAlertStatus `json:"metricStatus,omitempty"` +} + +// lastObservedAt is when the alert's condition was last evaluated. A +// threshold alert held below its trigger keeps LastSeen at the last breach, +// so its live status carries the newer observation. +func (a *Alert) lastObservedAt() time.Time { + if a == nil { + return time.Time{} + } + observed := a.LastSeen + if a.MetricStatus != nil && a.MetricStatus.ObservedAt.After(observed) { + observed = a.MetricStatus.ObservedAt + } + return observed } // Clone returns a deep copy of the alert so it can be safely shared across goroutines. @@ -176,6 +194,8 @@ func (a *Alert) Clone() *Alert { clone.Metadata = cloneMetadata(a.Metadata) } + clone.MetricStatus = a.MetricStatus.Clone() + if a.OperationalRecord != nil { value := a.OperationalRecord.Clone() clone.OperationalRecord = &value diff --git a/internal/alerts/tracking_cleanup.go b/internal/alerts/tracking_cleanup.go index a328600064..9e5882c2a1 100644 --- a/internal/alerts/tracking_cleanup.go +++ b/internal/alerts/tracking_cleanup.go @@ -156,7 +156,9 @@ func (m *Manager) cleanupStaleMaps() { staleAlerts := make([]string, 0) for storageKey, alert := range m.activeAlerts { alertID := effectiveAlertID(alert, storageKey) - if alert != nil && now.Sub(alert.LastSeen) > staleThreshold { + // A threshold alert holding below its trigger is still observed + // even though LastSeen stays at its last breach. + if alert != nil && now.Sub(alert.lastObservedAt()) > staleThreshold { staleAlerts = append(staleAlerts, alertID) } } diff --git a/internal/alerts/unified_eval_test.go b/internal/alerts/unified_eval_test.go index de010d214c..72d803fc9b 100644 --- a/internal/alerts/unified_eval_test.go +++ b/internal/alerts/unified_eval_test.go @@ -2,9 +2,11 @@ package alerts import ( "sort" + "sync/atomic" "testing" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer" alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" @@ -1583,3 +1585,186 @@ func TestUnifiedMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) t.Fatalf("stabilityWindowSeconds = %v, want %d", got, defaultNoisyGaugeStabilitySeconds) } } + +// metricStatusClock drives a manager's wall and monotonic clocks together so +// recovery runs measure exactly the time the test advances. +type metricStatusClock struct { + now time.Time + tick time.Duration +} + +func (c *metricStatusClock) advance(d time.Duration) { + c.now = c.now.Add(d) + c.tick += d +} + +func newMetricStatusTestManager(t *testing.T) (*Manager, *metricStatusClock) { + t.Helper() + m := newTestManager(t) + // Recently-resolved retention prunes against the real clock, so start there. + clock := &metricStatusClock{now: time.Now().UTC()} + m.now = func() time.Time { return clock.now } + m.intentClock = func() time.Duration { return clock.tick } + m.mu.Lock() + m.config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 80, Clear: 75} + m.mu.Unlock() + return m, clock +} + +// fireNodeTemperatureAlert holds a breaching reading until the factory +// stability delay lets the alert open, as a real poll loop would. +func fireNodeTemperatureAlert(t *testing.T, m *Manager, clock *metricStatusClock, celsius float64) *Alert { + t.Helper() + for attempt := 0; attempt < 40; attempt++ { + checkMetricStatusNode(m, celsius) + if alert := activeNodeTemperatureAlert(t, m); alert != nil { + return alert + } + clock.advance(30 * time.Second) + } + t.Fatalf("%v°C never opened the temperature alert", celsius) + return nil +} + +func checkMetricStatusNode(m *Manager, celsius float64) { + m.CheckNode(models.Node{ + ID: "homelab-minipc", + Name: "minipc", + Instance: "homelab", + Status: "online", + CPU: 0.2, + Memory: models.Memory{Usage: 40}, + Disk: models.Disk{Usage: 40}, + Temperature: &models.Temperature{ + Available: true, + CPUPackage: celsius, + }, + }) +} + +func activeNodeTemperatureAlert(t *testing.T, m *Manager) *Alert { + t.Helper() + m.mu.RLock() + defer m.mu.RUnlock() + alert, ok := m.getActiveAlertNoLock(canonicalMetricStateID("homelab-minipc", "temperature")) + if !ok { + return nil + } + return alert.Clone() +} + +func TestOpenTemperatureAlertReportsLiveReadingWhileItHolds(t *testing.T) { + m, clock := newMetricStatusTestManager(t) + var notified atomic.Int32 + m.SetAlertCallback(func(*Alert) { notified.Add(1) }) + + fired := fireNodeTemperatureAlert(t, m, clock, 85) + status := fired.MetricStatus + if status == nil || status.Phase != models.MetricAlertPhaseBreaching || status.Value != 85 || + status.Trigger != 80 || status.Recovery != 75 || status.Unit != "°C" { + t.Fatalf("breaching status = %+v", status) + } + if status.RecoveryDelaySeconds <= 0 { + t.Fatalf("temperature alerts carry a recovery delay, got %+v", status) + } + delay := time.Duration(status.RecoveryDelaySeconds) * time.Second + firedMessage, firedLastSeen, notifications := fired.Message, fired.LastSeen, notified.Load() + + // Between the clear and trigger levels the alert holds. The legacy + // snapshot keeps the breach; only the live status moves. + clock.advance(30 * time.Second) + checkMetricStatusNode(m, 78) + held := activeNodeTemperatureAlert(t, m) + if held == nil { + t.Fatal("78°C cleared an alert that should hold until 75°C") + } + if held.Value != 85 || held.Message != firedMessage || !held.LastSeen.Equal(firedLastSeen) { + t.Fatalf("hold rewrote the breach snapshot: value=%v message=%q lastSeen=%v", held.Value, held.Message, held.LastSeen) + } + if got := held.MetricStatus; got == nil || got.Phase != models.MetricAlertPhaseLatched || got.Value != 78 || + !got.ObservedAt.Equal(clock.now) || got.RecoveryStartedAt != nil { + t.Fatalf("latched status = %+v", got) + } + + // At or below the recovery level the recovery run starts. + clock.advance(30 * time.Second) + recoveryStart := clock.now + checkMetricStatusNode(m, 72) + if got := activeNodeTemperatureAlert(t, m).MetricStatus; got == nil || + got.Phase != models.MetricAlertPhaseRecovering || got.Value != 72 || + got.RecoveryStartedAt == nil || !got.RecoveryStartedAt.Equal(recoveryStart) || got.RecoveryElapsedSeconds != 0 { + t.Fatalf("recovery start status = %+v", got) + } + clock.advance(delay / 2) + checkMetricStatusNode(m, 75) + if got := activeNodeTemperatureAlert(t, m).MetricStatus; got == nil || + got.Phase != models.MetricAlertPhaseRecovering || got.RecoveryElapsedSeconds != int(delay/2/time.Second) { + t.Fatalf("recovery progress status = %+v", got) + } + + // A reading back above the recovery level resets the run. + clock.advance(30 * time.Second) + checkMetricStatusNode(m, 77) + if got := activeNodeTemperatureAlert(t, m).MetricStatus; got == nil || + got.Phase != models.MetricAlertPhaseLatched || got.RecoveryStartedAt != nil || got.RecoveryElapsedSeconds != 0 { + t.Fatalf("status after recovery reset = %+v", got) + } + if got := notified.Load(); got != notifications { + t.Fatalf("holding and recovering sent %d notifications", got-notifications) + } + + // A full recovery delay at or below 75°C resolves the alert. + checkMetricStatusNode(m, 72) + clock.advance(delay) + checkMetricStatusNode(m, 72) + if alert := activeNodeTemperatureAlert(t, m); alert != nil { + t.Fatalf("alert still open after a full recovery delay: %+v", alert.MetricStatus) + } + m.resolvedMutex.RLock() + defer m.resolvedMutex.RUnlock() + resolved := 0 + for _, entry := range m.recentlyResolved { + if entry == nil || entry.Type != "temperature" { + continue + } + resolved++ + if entry.MetricStatus != nil { + t.Fatalf("resolved record kept the live status of an open alert: %+v", entry.MetricStatus) + } + } + if resolved != 1 { + t.Fatalf("resolved temperature records = %d, want 1", resolved) + } +} + +func TestBuildMetricAlertStatusCarriesWindowAndMonotonicRecovery(t *testing.T) { + observedAt := time.Date(2026, 10, 6, 10, 49, 0, 0, time.UTC) + status := buildMetricAlertStatus(metricStatusInput{ + value: 74, + trigger: 80, + clear: 75, + recoveryDelaySeconds: 300, + unit: "%", + window: metricWindowObservation{Value: 74, CurrentValue: 91, WindowSeconds: 600, Ready: true}, + incident: reducer.Incident{ + RecoverySince: observedAt.Add(-10 * time.Minute), + RecoveryElapsed: 90 * time.Second, + RecoveryTicksSupplied: true, + }, + observedAt: observedAt, + }) + if status.Phase != models.MetricAlertPhaseRecovering || status.Value != 74 || + status.RawValue == nil || *status.RawValue != 91 || status.EvaluationWindowSeconds != 600 { + t.Fatalf("windowed status = %+v", status) + } + // Elapsed time comes from the monotonic run, not wall-clock arithmetic. + if status.RecoveryElapsedSeconds != 90 { + t.Fatalf("recovery elapsed = %d, want 90", status.RecoveryElapsedSeconds) + } + + // The reducer clears at the trigger when no clear level is configured. + noClear := buildMetricAlertStatus(metricStatusInput{value: 85, triggered: true, trigger: 80, observedAt: observedAt}) + if noClear.Recovery != 80 || noClear.RawValue != nil { + t.Fatalf("status without clear level = %+v", noClear) + } +} diff --git a/internal/models/metrics_types_test.go b/internal/models/metrics_types_test.go index 86d4df09d0..8f5edabedc 100644 --- a/internal/models/metrics_types_test.go +++ b/internal/models/metrics_types_test.go @@ -451,3 +451,21 @@ func TestPhysicalDiskSnapshotPreservesIndependentPollingCadence(t *testing.T) { t.Fatal("snapshot mutation changed collector state") } } + +func TestMetricAlertStatusCloneIsDeep(t *testing.T) { + raw := 91.0 + startedAt := time.Date(2026, 10, 6, 10, 49, 0, 0, time.UTC) + status := &MetricAlertStatus{ + Phase: MetricAlertPhaseRecovering, Value: 74, RawValue: &raw, + RecoveryStartedAt: &startedAt, RecoveryElapsedSeconds: 90, + } + clone := status.Clone() + *clone.RawValue = 10 + *clone.RecoveryStartedAt = startedAt.Add(time.Hour) + if *status.RawValue != 91 || !status.RecoveryStartedAt.Equal(startedAt) { + t.Fatalf("clone shares pointers with the original: %+v", status) + } + if (*MetricAlertStatus)(nil).Clone() != nil { + t.Fatal("nil status must clone to nil") + } +} diff --git a/internal/models/models.go b/internal/models/models.go index 5f1f2af5b8..a08334e25a 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -110,6 +110,58 @@ type Alert struct { // Metadata carries alert-engine annotations (notably resourceType) so the // frontend can classify an alert without re-deriving resource identity. Metadata map[string]interface{} `json:"metadata,omitempty"` + // MetricStatus is the live evaluation behind an open threshold alert. + MetricStatus *MetricAlertStatus `json:"metricStatus,omitempty"` +} + +// Metric alert phases. A threshold alert stays open after its reading drops +// below the trigger: it holds until the reading reaches the recovery level, +// then must stay there for the recovery delay before it clears. +const ( + MetricAlertPhaseBreaching = "breaching" + MetricAlertPhaseLatched = "latched" + MetricAlertPhaseRecovering = "recovering" +) + +// MetricAlertStatus describes the reading Pulse is evaluating now for an open +// threshold alert and why the alert is still open. The alert's Value, Message +// and LastSeen keep describing the last reading that met the trigger, which +// can be minutes or days old while the alert holds, so surfaces lead with +// this instead. It is volatile: rebuilt on every evaluation, never persisted. +type MetricAlertStatus struct { + Phase string `json:"phase"` + // Value is the value compared with the rule. For a rolling-average rule + // that is the average; RawValue then carries the latest sample. + Value float64 `json:"value"` + RawValue *float64 `json:"rawValue,omitempty"` + EvaluationWindowSeconds int `json:"evaluationWindowSeconds,omitempty"` + Unit string `json:"unit,omitempty"` + ObservedAt time.Time `json:"observedAt"` + // Trigger opens the alert at or above this value; Recovery is the level + // the value must be at or below for RecoveryDelaySeconds to clear it. + Trigger float64 `json:"trigger"` + Recovery float64 `json:"recovery"` + RecoveryDelaySeconds int `json:"recoveryDelaySeconds,omitempty"` + // Recovery progress, present only in the recovering phase. + RecoveryStartedAt *time.Time `json:"recoveryStartedAt,omitempty"` + RecoveryElapsedSeconds int `json:"recoveryElapsedSeconds,omitempty"` +} + +// Clone returns a deep copy. +func (s *MetricAlertStatus) Clone() *MetricAlertStatus { + if s == nil { + return nil + } + clone := *s + if s.RawValue != nil { + raw := *s.RawValue + clone.RawValue = &raw + } + if s.RecoveryStartedAt != nil { + startedAt := *s.RecoveryStartedAt + clone.RecoveryStartedAt = &startedAt + } + return &clone } // ResolvedAlert represents a recently resolved alert diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go index 498bd35b8d..9d85fedba4 100644 --- a/internal/monitoring/monitor_alert_handling_test.go +++ b/internal/monitoring/monitor_alert_handling_test.go @@ -1466,3 +1466,39 @@ func TestMonitorDelayedPartialResolutionKeepsDestinationRecovery(t *testing.T) { t.Fatal("unannounced destination received a recovery") } } + +func TestSyncAlertsToStateCarriesLiveMetricStatusOfHeldAlert(t *testing.T) { + m := &Monitor{ + state: models.NewState(), + alertManager: alerts.NewManagerWithDataDir(t.TempDir()), + } + defer m.alertManager.Stop() + + cfg := m.alertManager.GetConfig() + cfg.TimeThresholds = map[string]int{"node": 0} + cfg.NodeDefaults.Temperature = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75} + m.alertManager.UpdateConfig(cfg) + + node := models.Node{ + ID: "homelab-minipc", Name: "minipc", Instance: "homelab", Status: "online", + Temperature: &models.Temperature{Available: true, CPUPackage: 85}, + } + m.alertManager.CheckNode(node) + node.Temperature = &models.Temperature{Available: true, CPUPackage: 78} + m.alertManager.CheckNode(node) + + m.syncAlertsToState() + + var held *models.Alert + for i, alert := range m.state.GetSnapshot().ActiveAlerts { + if alert.Type == "temperature" { + held = &m.state.GetSnapshot().ActiveAlerts[i] + } + } + require.NotNil(t, held, "expected the temperature alert to stay open at 78°C") + require.Equal(t, 85.0, held.Value, "the legacy value keeps the last breach") + require.NotNil(t, held.MetricStatus, "websocket state must carry the live reading") + require.Equal(t, models.MetricAlertPhaseLatched, held.MetricStatus.Phase) + require.Equal(t, 78.0, held.MetricStatus.Value) + require.Equal(t, 75.0, held.MetricStatus.Recovery) +} diff --git a/internal/monitoring/monitor_alert_sync.go b/internal/monitoring/monitor_alert_sync.go index 920381c2c2..9fc1154af9 100644 --- a/internal/monitoring/monitor_alert_sync.go +++ b/internal/monitoring/monitor_alert_sync.go @@ -71,8 +71,10 @@ func (m *Monitor) activeAlertsSnapshot() []models.Alert { Acknowledged: alert.Acknowledged, AckTime: alert.AckTime, AckUser: alert.AckUser, - // GetActiveAlerts returns deep clones, so the map is already private. - Metadata: alert.Metadata, + // GetActiveAlerts returns deep clones, so the map and the live + // metric status are already private. + Metadata: alert.Metadata, + MetricStatus: alert.MetricStatus, }) } return modelAlerts From eef9d900b05262f7f0993ee1212464137f126eed Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:25:00 +0100 Subject: [PATCH 02/53] Fit the temperature alert floor inside the discovery chunk The open-alert floor added to getTemperatureTextClass used a severity rank table. utils/temperature.ts ships in the shared discovery chunk, which already sat 0.01 kB under its 9.57 kB gzip budget, so the table pushed it to 9.60 kB and failed the Frontend bundle check. Below critical an open alert's severity is never lower than the reading's, so a single comparison gives the same colour without the table. The chunk builds at 9.55 kB and the colour tests are unchanged. --- .../68deecd720b4873b.json | 35 +++++++++++++++++++ frontend-modern/src/utils/temperature.ts | 12 +++---- 2 files changed, 39 insertions(+), 8 deletions(-) create mode 100644 frontend-modern/browser-verification/68deecd720b4873b.json diff --git a/frontend-modern/browser-verification/68deecd720b4873b.json b/frontend-modern/browser-verification/68deecd720b4873b.json new file mode 100644 index 0000000000..cdebbcffa6 --- /dev/null +++ b/frontend-modern/browser-verification/68deecd720b4873b.json @@ -0,0 +1,35 @@ +{ + "version": 1, + "base_sha": "08e785eb8d475a6906bc5135d6379c0ced78054b", + "verified_at": "2026-10-06T13:24:56Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/utils/temperature.ts" + ], + "content_sha256": { + "frontend-modern/src/utils/temperature.ts": "91d17ef07ce5cda8128f1b4bf65feecfd9101526c8ff7900ab9edc0431133838" + }, + "routes": [ + "/proxmox" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 375, + "height": 812 + } + ], + "states": [ + "Private mock stack built from this commit; pve2's CPU temperature pinned to 88 then 72 degrees C by a local-only, uncommitted mock overlay (trigger 80, clear 75)", + "Recovering at 72C with the alert still open: pve2 TEMP cell stays text-amber-600 with hover 'Temperature 72°C now, recovering. Clears after 5 minutes at 75°C or lower, 20 seconds so far.'", + "Nodes without an alert (51, 53, 42C) render text-green-600 with no hover text, unchanged", + "Narrow 375px: node table renders without horizontal overflow; TEMP column hidden as before" + ], + "interactions": [ + "Loaded /proxmox at 1440x900 and read each TEMP cell's colour class and title through the DOM", + "Resized to 375x812 and checked the node rows and page overflow" + ] +} diff --git a/frontend-modern/src/utils/temperature.ts b/frontend-modern/src/utils/temperature.ts index 6b993af762..15f608fbac 100644 --- a/frontend-modern/src/utils/temperature.ts +++ b/frontend-modern/src/utils/temperature.ts @@ -109,8 +109,6 @@ export type TemperatureDisplayMetric = Extract< 'temperature' | 'diskTemperature' >; -const TEMPERATURE_SEVERITY_RANK = { normal: 0, warning: 1, critical: 2 } as const; - /** * Colour for a temperature reading. An open alert on the reading sets a floor: * the alert holds until the reading falls to its clear level, so a reading @@ -130,10 +128,8 @@ export const getTemperatureTextClass = ( metric, thresholds ?? getDefaultDisplayMetricThresholds(metric), ); - const severity = - openAlertSeverity && - TEMPERATURE_SEVERITY_RANK[openAlertSeverity] > TEMPERATURE_SEVERITY_RANK[readingSeverity] - ? openAlertSeverity - : readingSeverity; - return TEMPERATURE_TEXT_CLASSES[severity]; + // Below critical, an open alert's severity is never lower than the reading's. + return TEMPERATURE_TEXT_CLASSES[ + readingSeverity === 'critical' || !openAlertSeverity ? readingSeverity : openAlertSeverity + ]; }; From b9cd50ca4e8c9d88b75dec40d9fc9055f2b84c21 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:42:50 +0100 Subject: [PATCH 03/53] Record the discovery chunk's measured size in its bundle budget The shared discovery chunk carries utils/temperature.ts. Growth from earlier changes had already taken it to the edge of its 9.57 kB gzip limit (about 9.56 kB on CI for main alone), so the roughly 20-byte open-alert floor in getTemperatureTextClass failed the Frontend check at 9.58 kB even after compacting it. Accept that intentional cost by recording the CI-measured 9,810 gzip bytes as this chunk's baseline. Every growth limit, every other chunk baseline, the totals and the startup-loading checks are unchanged. --- frontend-modern/.bundlesize.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend-modern/.bundlesize.json b/frontend-modern/.bundlesize.json index cea9cb20ad..d6c1a693f3 100644 --- a/frontend-modern/.bundlesize.json +++ b/frontend-modern/.bundlesize.json @@ -86,7 +86,7 @@ "baselineGzip": 8806 }, "discovery": { - "baselineGzip": 8522 + "baselineGzip": 9810 }, "useUnifiedResources": { "baselineGzip": 7595 From 11984fc63cee2769dd0f2bec0db093320560bd8f Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:43:55 +0100 Subject: [PATCH 04/53] Keep metric status tests clear of main's unified eval changes Main's platform-stamping change adds a strings import beside the sync/atomic import this branch added, so the two edits conflict in the import block. Count notifications through a buffered channel instead of an atomic, take main's version of the file, and keep this branch's tests where they were, so the merge with main is clean. --- internal/alerts/unified_eval_test.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/internal/alerts/unified_eval_test.go b/internal/alerts/unified_eval_test.go index 72d803fc9b..e5c732cee1 100644 --- a/internal/alerts/unified_eval_test.go +++ b/internal/alerts/unified_eval_test.go @@ -2,7 +2,7 @@ package alerts import ( "sort" - "sync/atomic" + "strings" "testing" "time" @@ -425,6 +425,15 @@ func TestCheckUnifiedResourceSupportsKubernetesTrueNASAndVMwareMetricTargets(t * if got := alert.Metadata["resourceType"]; got != tt.wantResourceType { t.Fatalf("resourceType metadata = %v, want %s", got, tt.wantResourceType) } + // resourceType is a display label, so the owning platform travels separately. + wantPlatform := map[string]string{ + "Kubernetes": "kubernetes", + "TrueNAS": "truenas", + "vSphere": "vmware-vsphere", + }[strings.Fields(tt.wantResourceType)[0]] + if got := alert.Metadata["platformType"]; got != wantPlatform { + t.Fatalf("platformType metadata = %v, want %s", got, wantPlatform) + } }) } } @@ -1655,8 +1664,9 @@ func activeNodeTemperatureAlert(t *testing.T, m *Manager) *Alert { func TestOpenTemperatureAlertReportsLiveReadingWhileItHolds(t *testing.T) { m, clock := newMetricStatusTestManager(t) - var notified atomic.Int32 - m.SetAlertCallback(func(*Alert) { notified.Add(1) }) + // Buffered so a dispatch never blocks the evaluator; len() counts sends. + notified := make(chan struct{}, 64) + m.SetAlertCallback(func(*Alert) { notified <- struct{}{} }) fired := fireNodeTemperatureAlert(t, m, clock, 85) status := fired.MetricStatus @@ -1668,7 +1678,7 @@ func TestOpenTemperatureAlertReportsLiveReadingWhileItHolds(t *testing.T) { t.Fatalf("temperature alerts carry a recovery delay, got %+v", status) } delay := time.Duration(status.RecoveryDelaySeconds) * time.Second - firedMessage, firedLastSeen, notifications := fired.Message, fired.LastSeen, notified.Load() + firedMessage, firedLastSeen, notifications := fired.Message, fired.LastSeen, len(notified) // Between the clear and trigger levels the alert holds. The legacy // snapshot keeps the breach; only the live status moves. @@ -1709,7 +1719,7 @@ func TestOpenTemperatureAlertReportsLiveReadingWhileItHolds(t *testing.T) { got.Phase != models.MetricAlertPhaseLatched || got.RecoveryStartedAt != nil || got.RecoveryElapsedSeconds != 0 { t.Fatalf("status after recovery reset = %+v", got) } - if got := notified.Load(); got != notifications { + if got := len(notified); got != notifications { t.Fatalf("holding and recovering sent %d notifications", got-notifications) } From 60fb6e842b702c35c93ce7c5e8397487c15aab6a Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:37:40 +0100 Subject: [PATCH 05/53] Show real CPU low and record in the Proxmox node drawer The node drawer's CPU low and CPU record rows repeated the current reading for every Proxmox node without a host agent. The adapter never read proxmox.temperatureDetails, where the API publishes the poller's full reading including the low and record values, and its fallback to the scalar proxmox.temperature copied that one value into both fields. On the dev stack, delly and minipc showed CPU low = CPU record = the current temperature. nodeFromResource now takes the first source with a usable reading: temperatureDetails comes after the platform record and the agent facet, so agent-linked nodes render exactly as before. The bare-reading fallback leaves the low and record unset, so the drawer omits those rows instead of inventing them. The poller also reset those values to the current reading every poll; that is fixed separately in the previous-node projection. With both, the rows show the low and record the poller tracks. --- .../subsystems/performance-and-scalability.md | 5 ++ .../internal/subsystems/unified-resources.md | 9 +++ .../541999f4ef0c4619.json | 36 ++++++++++++ .../resourceStateAdapters.coverage2.test.ts | 4 +- .../__tests__/resourceStateAdapters.test.ts | 55 +++++++++++++++++++ .../src/utils/resourceStateAdapters.ts | 24 +++++--- 6 files changed, 123 insertions(+), 10 deletions(-) create mode 100644 frontend-modern/browser-verification/541999f4ef0c4619.json diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 50344121e9..9a0ad0a645 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -159,6 +159,11 @@ clearing, false/zero values, unchanged peer-site rows and full/fast equivalence. merges, row, Overview and History with synthetic inputs. This is no native QGA/thaw, installed recovery, fleet-performance or release-availability claim. +Choosing a node's temperature source in `resourceStateAdapters.ts` reads one +more field of the facet already in the canonical payload +(`proxmox.temperatureDetails`) when earlier sources have no usable reading. It +adds no resource read, merge pass, timer or subscription. + ### Unavailable filesystem values share the existing disk model The filesystem list and guest Overview use the same nullable measured-byte ratio diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 9a954da020..40bf3e9036 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -3182,6 +3182,15 @@ footer after the visible rows. The footer may dim the final row edge and expose the remaining-row count, but expanding or collapsing it only changes the client-side visible slice; it must not refetch, reorder, or redefine the canonical resource collection. +The shared `nodeFromResource` adapter takes a node's temperature from the first +source with a usable reading: a platform or Proxmox temperature record, the +agent facet, then `proxmox.temperatureDetails`. That last field is the Proxmox +poller's full reading for a node without a host agent, including the CPU low +and record values it tracks; `proxmox.temperature` is only the scalar maximum. +A bare scalar reading carries no history, so the adapter leaves the CPU low and +record unset rather than repeating the current value, and the node drawer omits +those rows. `resourceStateAdapters.test.ts` pins both with the API's payload +shape. The registry and presentation coalescer also own metric-source freshness. When two source facets contribute the same metric, source priority decides only if both sources have equivalent freshness. A stale source must not hold CPU, diff --git a/frontend-modern/browser-verification/541999f4ef0c4619.json b/frontend-modern/browser-verification/541999f4ef0c4619.json new file mode 100644 index 0000000000..c648f4ae89 --- /dev/null +++ b/frontend-modern/browser-verification/541999f4ef0c4619.json @@ -0,0 +1,36 @@ +{ + "version": 1, + "base_sha": "3b290640a51ef0f488f9cd06b4aa644adbdcc6b3", + "verified_at": "2026-10-06T15:25:00Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/utils/resourceStateAdapters.ts" + ], + "content_sha256": { + "frontend-modern/src/utils/resourceStateAdapters.ts": "5b170b1c0d735afc584b0784403cd3510a8ab12aeb3e387e9baec6b80fcb5853" + }, + "routes": [ + "/proxmox" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack built from this tree (backend :7681, Vite :5199); /api/resources confirmed the real node payload shape: proxmox.temperature is the scalar maximum and proxmox.temperatureDetails the full reading", + "Proxmox node Analytics A (pve31) as a PVE-only node (agent facet removed from REST and websocket payloads), main's adapter swapped in: drawer Thermals showed CPU low 82\u00b0C and CPU record 82\u00b0C, the current reading repeated as history", + "Same PVE-only node with this change and no low/record in temperatureDetails: CPU low and CPU record rows absent, Temperature and NVMe still shown", + "Same PVE-only node with this change and temperatureDetails carrying cpuMin 38.5 and cpuMaxRecord 91.2: drawer showed CPU low 39\u00b0C and CPU record 91\u00b0C at 1440x900 and 390x844", + "Agent-linked mock node (agent facet present) with this change, with and without an injected record: drawer identical to main (Temperature from the agent facet, no CPU low/record rows), table TEMP column unchanged at 82\u00b0C" + ], + "interactions": [ + "Opened the node drawer from the row's detail toggle (aria-controls proxmox-host-drawer-) on /proxmox at desktop and phone widths", + "Read the Thermals card rows and the row's TEMP cell, and took screenshots, for main's adapter and for this change, via Playwright with REST and websocket payload rewriting" + ] +} diff --git a/frontend-modern/src/utils/__tests__/resourceStateAdapters.coverage2.test.ts b/frontend-modern/src/utils/__tests__/resourceStateAdapters.coverage2.test.ts index 679c95f6a3..d78b7157dc 100644 --- a/frontend-modern/src/utils/__tests__/resourceStateAdapters.coverage2.test.ts +++ b/frontend-modern/src/utils/__tests__/resourceStateAdapters.coverage2.test.ts @@ -981,7 +981,9 @@ describe('buildTemperature (via nodeFromResource)', () => { ); expect(node?.temperature?.cpuPackage).toBe(42); expect(node?.temperature?.cpuMax).toBe(42); - expect(node?.temperature?.cpuMin).toBe(42); + // A bare reading has no low or record history to report. + expect(node?.temperature?.cpuMin).toBeUndefined(); + expect(node?.temperature?.cpuMaxRecord).toBeUndefined(); expect(node?.temperature?.available).toBe(true); expect(node?.temperature?.hasCPU).toBe(true); }); diff --git a/frontend-modern/src/utils/__tests__/resourceStateAdapters.test.ts b/frontend-modern/src/utils/__tests__/resourceStateAdapters.test.ts index bea2d130c6..dbb6aef82c 100644 --- a/frontend-modern/src/utils/__tests__/resourceStateAdapters.test.ts +++ b/frontend-modern/src/utils/__tests__/resourceStateAdapters.test.ts @@ -254,6 +254,61 @@ describe('resourceStateAdapters nodeFromResource', () => { ]); }); + it("reads a Proxmox node's CPU low and record from the poller's temperature details", () => { + // The unified resource API publishes the scalar maximum as + // proxmox.temperature and the full reading as proxmox.temperatureDetails. + const proxmox = { + nodeName: 'pve-node-2', + temperature: 58, + temperatureDetails: { + available: true, + hasCPU: true, + cpuPackage: 52, + cpuMax: 58, + cpuMin: 41, + cpuMaxRecord: 76, + minRecorded: '2026-10-06T08:00:00Z', + maxRecorded: '2026-10-06T12:00:00Z', + cores: [{ core: 0, temp: 58 }], + lastUpdate: '2026-10-06T15:00:00Z', + }, + }; + const node = nodeFromResource({ + ...createNodeResource({ proxmox }), + temperature: 58, + proxmox, + } as unknown as Resource); + + expect(node?.temperature).toMatchObject({ + available: true, + cpuPackage: 52, + cpuMax: 58, + cpuMin: 41, + cpuMaxRecord: 76, + lastUpdate: '2026-10-06T15:00:00Z', + }); + expect(node?.temperature?.cores).toEqual([{ core: 0, temp: 58 }]); + + // Without details there is no history: the current reading must not be + // reported as the node's low or record. + const bare = nodeFromResource({ + ...createNodeResource({ proxmox: { nodeName: 'pve-node-2', temperature: 58 } }), + temperature: 58, + } as unknown as Resource); + expect(bare?.temperature?.cpuPackage).toBe(58); + expect(bare?.temperature?.cpuMin).toBeUndefined(); + expect(bare?.temperature?.cpuMaxRecord).toBeUndefined(); + + // A node with a host agent keeps reading the agent facet first. + const agentLinked = nodeFromResource({ + ...createNodeResource({ proxmox, agent: { agentId: 'pve-agent', temperature: 61 } }), + temperature: 61, + proxmox, + } as unknown as Resource); + expect(agentLinked?.temperature?.cpuPackage).toBe(61); + expect(agentLinked?.temperature?.cpuMin).toBeUndefined(); + }); + it('preserves linked-agent GPU telemetry for the Proxmox node drawer', () => { const gpu = { id: '0', diff --git a/frontend-modern/src/utils/resourceStateAdapters.ts b/frontend-modern/src/utils/resourceStateAdapters.ts index fef157dbc0..f78e577757 100644 --- a/frontend-modern/src/utils/resourceStateAdapters.ts +++ b/frontend-modern/src/utils/resourceStateAdapters.ts @@ -1378,13 +1378,19 @@ const buildTemperature = ( nodeMeta?: Record, ): Temperature | undefined => { const platform = resourcePlatformData(resource); - const raw = - asRecord(platform?.temperature) || - asRecord(nodeMeta?.temperature) || - asRecord(platform?.agent) || - undefined; - - if (raw) { + // Take the first source with a usable reading. A Proxmox node without a host + // agent publishes its full reading, including the CPU low and record values + // the poller tracks, only as proxmox.temperatureDetails (proxmox.temperature + // is the scalar maximum), so it comes before the bare-reading fallback. + const candidates = [ + platform?.temperature, + nodeMeta?.temperature, + platform?.agent, + nodeMeta?.temperatureDetails, + ]; + for (const candidate of candidates) { + const raw = asRecord(candidate); + if (!raw) continue; const available = asBoolean(raw.available); const cpuPackage = asNumber(raw.cpuPackage) ?? asNumber(raw.temperature) ?? asNumber(raw.cpu); const lastUpdate = toISOTime(raw.lastUpdate, resource.lastSeen); @@ -1441,11 +1447,11 @@ const buildTemperature = ( if (typeof resource.temperature === 'number' && Number.isFinite(resource.temperature)) { const temp = resource.temperature; + // A bare reading carries no low or record history. Leave them unset so the + // drawer omits those rows instead of repeating the current value as a record. return { cpuPackage: temp, cpuMax: temp, - cpuMin: temp, - cpuMaxRecord: temp, available: true, hasCPU: true, lastUpdate: toISOTime(undefined, resource.lastSeen), From bbd44455c85b8216a198c4c12111da980a60d11f Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:41:25 +0100 Subject: [PATCH 06/53] Show retained disk temperatures as last known, not live Normalization keeps a disk's last known temperature when the current observation is not available: a disk in standby, or a linked host agent past its reporting lease. The Physical Disks table, the pool drawer's disk list and the disk drawer card still painted that value with the live threshold colour, so a silent agent's frozen 72C looked exactly like a disk running hot right now, and a sleeping disk looked freshly read. The drawer even printed "Temperature is temporarily unavailable" directly above a red Temperature card. One helper in diskPresentation.ts now decides: only an available temperature state, or a source that predates collection state, is a current reading. Anything else renders muted with no threshold colour, a dotted underline, the collection reason as its title and screen-reader "last known" text; the drawer card reads "Last known temperature". The pool drawer also formats through the shared temperature formatter, so it follows the Fahrenheit setting. --- .../subsystems/frontend-primitives.md | 9 +++ .../internal/subsystems/storage-recovery.md | 17 ++++++ .../d0dfa366febe8d30.json | 50 +++++++++++++++ .../src/components/Storage/DiskDetail.tsx | 4 +- .../src/components/Storage/DiskList.tsx | 40 ++++++++---- .../components/Storage/StoragePoolDetail.tsx | 20 ++++-- .../Storage/__tests__/DiskDetail.test.tsx | 4 ++ .../Storage/__tests__/DiskList.test.tsx | 53 ++++++++++++++++ .../__tests__/diskDetailPresentation.test.ts | 44 +++++++++++++ .../__tests__/diskPresentation.test.ts | 61 +++++++++++++++++++ .../storagePoolDetailPresentation.test.ts | 10 +++ .../storageBackups/diskDetailPresentation.ts | 22 ++++++- .../storageBackups/diskPresentation.ts | 45 ++++++++++++++ .../storagePoolDetailPresentation.ts | 10 +++ 14 files changed, 368 insertions(+), 21 deletions(-) create mode 100644 frontend-modern/browser-verification/d0dfa366febe8d30.json diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 0257eb60aa..f169522281 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -7758,6 +7758,15 @@ candidates. Domain ownership stays with Patrol intelligence, storage recovery, alerts, and unified resources; the primitive layer owns consistent rendering, accessibility, and handoff behavior only. +The same storage presenters own the last-known disk temperature treatment. A +temperature whose collection state is not `available` renders through +`PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS` (muted, dotted underline, help +cursor) with its reason as the title and screen-reader "last known" text, +never through the threshold colour classes, so a retained reading cannot look +hot or healthy. Table, drawer and pool surfaces share that one decision in +`frontend-modern/src/features/storageBackups/diskPresentation.ts` rather than +each re-reading `collection.temperature`. + The focused browser proofs are `frontend-modern/src/features/patrol/__tests__/patrolRunAcceptance.test.ts`, `frontend-modern/src/components/Storage/__tests__/DiskDetail.test.tsx`, diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 626b08f763..5862022a37 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -3784,6 +3784,23 @@ remain explicit. In particular, members behind a shared controller must not inherit aggregate I/O counters, and a disk drawer must hide live-I/O charts when per-member collection is unavailable or unsupported. +A retained disk temperature reads as last known, never as a live reading. The +Physical Disks table, the disk drawer and the pool drawer's linked-disk list +decide through `isPhysicalDiskTemperatureCurrent` and +`getPhysicalDiskTemperaturePresentation` in +`frontend-modern/src/features/storageBackups/diskPresentation.ts`: only an +`available` temperature state, or a source that predates collection state, is +current and takes the per-disk-type threshold colour. Any other state, such as +a disk in standby or a host agent past its reporting lease, renders the value +muted with no hot or warm colour, a dotted underline, a title carrying the +collection reason and screen-reader "last known" text; the drawer card is +labelled "Last known temperature" beneath the existing collection message. +This is presentation only. It does not change the stored value, history writes +or alert evaluation. Verification: `diskPresentation.test.ts`, +`diskDetailPresentation.test.ts`, `storagePoolDetailPresentation.test.ts`, +`DiskList.test.tsx` (a row moving between current, retained and legacy states) +and `DiskDetail.test.tsx`. + Direct SATA, SAS, and NVMe device fallback identities remain compatible. Controller-member fallback identities add controller/target scope only where needed to keep multiple members distinct; serial and WWN continue to take diff --git a/frontend-modern/browser-verification/d0dfa366febe8d30.json b/frontend-modern/browser-verification/d0dfa366febe8d30.json new file mode 100644 index 0000000000..f7f5062854 --- /dev/null +++ b/frontend-modern/browser-verification/d0dfa366febe8d30.json @@ -0,0 +1,50 @@ +{ + "version": 1, + "base_sha": "f4191494584f05c731f5c7a9aaa15c3b777a3f13", + "verified_at": "2026-10-06T16:39:54Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Storage/DiskDetail.tsx", + "frontend-modern/src/components/Storage/DiskList.tsx", + "frontend-modern/src/components/Storage/StoragePoolDetail.tsx", + "frontend-modern/src/features/storageBackups/diskDetailPresentation.ts", + "frontend-modern/src/features/storageBackups/diskPresentation.ts", + "frontend-modern/src/features/storageBackups/storagePoolDetailPresentation.ts" + ], + "content_sha256": { + "frontend-modern/src/components/Storage/DiskDetail.tsx": "85a1bb30e4914c98b65b820e649322de9bf5412f913d85a7638d3f8400e987e2", + "frontend-modern/src/components/Storage/DiskList.tsx": "6e4d7ce022fe2b1262581cf8482e1ad11bb287de7a3c255d639a6eeac5c5cc31", + "frontend-modern/src/components/Storage/StoragePoolDetail.tsx": "020f64a6044f17619f30e3dd79e49fb863dcc9775de6a65a47f97b8fcca02819", + "frontend-modern/src/features/storageBackups/diskDetailPresentation.ts": "7d01c07370a710227a39423670a5f1611bbfa64ba5488aa9a845fbdf2fe5176d", + "frontend-modern/src/features/storageBackups/diskPresentation.ts": "9ae039f89878e5274cbe6fdff436d3175a8656d717ffb51e72e5bb96d8f62d7c", + "frontend-modern/src/features/storageBackups/storagePoolDetailPresentation.ts": "f52b9a2ada8e10bc84fb29ac9bc389a0d255fa794d99ff4c237e604e4cd30130" + }, + "routes": [ + "/proxmox/storage?tab=disks&q=West%20Production", + "/proxmox/storage?q=analytics-c-service-pool" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack (50 nodes) whose generator was overlaid locally, never committed, to stamp real collector states on disk temperatures: a SATA disk on pve1 retaining 36C with unavailable 'disk is in standby', an NVMe disk on pve2 retaining 72C with unavailable 'host agent stopped reporting', a SATA disk on pve3 reading 72C live (available, host_agent), an NVMe disk on pve3 retaining 41C as missing with no reason, every other Proxmox disk available, TrueNAS disks with no collection state", + "Same-session control with origin/main's files swapped in under Vite: the pve2 silent-agent disk showed a red 72C identical to the live hot pve3 disk, the standby disk a green 36C, and its drawer printed 'Temperature is temporarily unavailable: host agent stopped reporting' directly above a red Temperature 72C card; the pool list showed the retained 71C in red", + "Physical Disks table after the change, light 1440 and dark 390: the three retained readings render muted (rgb(71,85,105) light, rgb(148,163,184) dark) with a dotted underline, titles 'Last known reading, not current: host agent stopped reporting', '... disk is in standby' and 'Last known reading, not current', and screen-reader text ', last known'; the live 72C stays red, 51C amber, the rest green; no clipping in the 41px phone Temp column", + "Disk drawer: silent-agent and standby disks show the existing collection message above a 'Last known temperature' card with a muted value; the live hot disk keeps 'Temperature 72C' in red with no message", + "Pool drawer for analytics-c-service-pool (pve33, ZFS devices sda2 and sdb2 linked to overlay disks): the retained 71C reads muted, dotted, with the reason title; the live 73C beside it stays red", + "Re-run on the rebased tree (origin/main f419149458, none of the changed files differ from the verified copies): identical results, no page errors at either width" + ], + "interactions": [ + "Loaded the Physical Disks tab filtered to West Production and read every Temp cell's text, computed colour, decoration, title and overflow", + "Opened and closed the inline disk drawer for the silent-agent, standby and live hot disks at both viewports and read the collection message and temperature card", + "Opened the analytics-c-service-pool row on the Pools tab and read the linked-disk temperatures in its Physical Disks card", + "Swapped origin/main's versions of the six runtime files in under the running Vite and repeated the same reads, then restored the change and confirmed the diff byte-identical" + ] +} diff --git a/frontend-modern/src/components/Storage/DiskDetail.tsx b/frontend-modern/src/components/Storage/DiskDetail.tsx index 3588836cc0..b90f772797 100644 --- a/frontend-modern/src/components/Storage/DiskDetail.tsx +++ b/frontend-modern/src/components/Storage/DiskDetail.tsx @@ -17,7 +17,7 @@ import { DISK_DETAIL_HEALTH_SUMMARY_CLASS, DISK_DETAIL_HISTORY_RANGE_OPTIONS, DISK_DETAIL_LIVE_CHARTS, - getDiskAttributeValueTextClass, + getDiskAttributeCardValueTextClass, getDiskDetailHistoryFallbackMessage, getDiskDetailLiveBadgeLabel, } from '@/features/storageBackups/diskDetailPresentation'; @@ -166,7 +166,7 @@ export const DiskDetail: Component = (props) => { )} diff --git a/frontend-modern/src/components/Storage/DiskList.tsx b/frontend-modern/src/components/Storage/DiskList.tsx index fdff3b3d36..4d463bb5c2 100644 --- a/frontend-modern/src/components/Storage/DiskList.tsx +++ b/frontend-modern/src/components/Storage/DiskList.tsx @@ -15,7 +15,7 @@ import { } from '@/components/shared/Table'; import { useAlertsActivation } from '@/stores/alertsActivation'; import { formatBytes } from '@/utils/format'; -import { formatTemperature, getTemperatureTextClass } from '@/utils/temperature'; +import { getTemperatureTextClass } from '@/utils/temperature'; import { PHYSICAL_DISK_CELL_DEVICE_CLASS, PHYSICAL_DISK_CELL_DISK_CLASS, @@ -63,6 +63,7 @@ import { PHYSICAL_DISK_NAME_WRAP_CLASS, PHYSICAL_DISK_SIZE_VALUE_CLASS, PHYSICAL_DISK_TEMPERATURE_CLASS, + PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS, PHYSICAL_DISK_VALUE_TEXT_CLASS, PHYSICAL_DISK_TABLE_BODY_CLASS, PHYSICAL_DISK_TABLE_CLASS, @@ -83,6 +84,7 @@ import { getPhysicalDiskParentLabel, getPhysicalDiskRoleLabel, getPhysicalDiskTableLayoutModeForContainer, + getPhysicalDiskTemperaturePresentation, isPhysicalDiskColumnVisible, type PhysicalDiskTableColumnId, } from '@/features/storageBackups/diskPresentation'; @@ -335,6 +337,9 @@ export const DiskList: Component = (props) => { const status = createMemo(() => getPhysicalDiskHealthStatus(data())); const hostLabel = createMemo(() => getPhysicalDiskHostLabel(data(), disk)); const healthSummary = createMemo(() => getPhysicalDiskHealthSummary(status())); + const temperature = createMemo(() => + getPhysicalDiskTemperaturePresentation(data()), + ); const healthCompactLabel = createMemo(() => getPhysicalDiskHealthCompactLabel(status().label), ); @@ -509,18 +514,31 @@ export const DiskList: Component = (props) => { data-storage-column="temp" > 0} + when={temperature()} fallback={—} > - - {formatTemperature(data().temperature)} - + {(reading) => ( + + {reading().label} + + , last known + + + )} diff --git a/frontend-modern/src/components/Storage/StoragePoolDetail.tsx b/frontend-modern/src/components/Storage/StoragePoolDetail.tsx index c2e7383750..c1b9521523 100644 --- a/frontend-modern/src/components/Storage/StoragePoolDetail.tsx +++ b/frontend-modern/src/components/Storage/StoragePoolDetail.tsx @@ -14,6 +14,7 @@ import { getLinkedDiskHealthDotVariant, getLinkedDiskTemperatureTextClass, } from '@/features/storageBackups/diskDetailPresentation'; +import { PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS } from '@/features/storageBackups/diskPresentation'; import { useAlertsActivation } from '@/stores/alertsActivation'; import { STORAGE_POOL_DETAIL_HISTORY_RANGE_OPTIONS, @@ -39,6 +40,7 @@ import { } from '@/features/storageBackups/detailPresentation'; import type { StorageRecord } from '@/features/storageBackups/models'; import type { Resource } from '@/types/resource'; +import { formatTemperature } from '@/utils/temperature'; import { StorageDetailKeyValueRow } from './StorageDetailKeyValueRow'; import { useStoragePoolDetailModel } from './useStoragePoolDetailModel'; @@ -279,12 +281,20 @@ export const StoragePoolDetail: Component = (props) => { 0}> - {disk.temperature}°C + {formatTemperature(disk.temperature)} + + , last known +
diff --git a/frontend-modern/src/components/Storage/__tests__/DiskDetail.test.tsx b/frontend-modern/src/components/Storage/__tests__/DiskDetail.test.tsx index e1630ffa95..6d921d6624 100644 --- a/frontend-modern/src/components/Storage/__tests__/DiskDetail.test.tsx +++ b/frontend-modern/src/components/Storage/__tests__/DiskDetail.test.tsx @@ -99,6 +99,10 @@ describe('DiskDetail', () => { expect( screen.getByText('Temperature is temporarily unavailable: collection deadline exceeded'), ).toBeInTheDocument(); + // The retained value sits under its own label with no health tone. + expect(screen.queryByText('Temperature')).not.toBeInTheDocument(); + expect(screen.getByText('Last known temperature')).toBeInTheDocument(); + expect(screen.getByText('42°C')).toHaveClass('text-muted'); expect( screen.getByText('Disk I/O is unsupported: per-member counters unavailable'), ).toBeInTheDocument(); diff --git a/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx b/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx index ae75eb0693..73b9cd330d 100644 --- a/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx +++ b/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx @@ -84,6 +84,59 @@ describe('DiskList', () => { cleanup(); }); + it('shows a retained temperature as last known, not as a live threshold reading', async () => { + const withTemperature = (collection: NonNullable['collection']) => + buildDisk('sda', 'tower', { diskType: 'sata', temperature: 72, collection }); + const [disks, setDisks] = createSignal([ + withTemperature({ temperature: { state: 'available', source: 'host_agent' } }), + ]); + const view = render(() => ( + {}} + /> + )); + const reading = () => + view.container.querySelector( + '[data-row-id="sda"] td[data-storage-column="temp"] [data-temperature-reading]', + )!; + + expect(reading()).toHaveAttribute('data-temperature-reading', 'current'); + expect(reading()).toHaveClass('text-red-600'); + expect(reading()).not.toHaveAttribute('title'); + expect(reading()).toHaveTextContent(/^72°C$/); + + // The same row, retained after its host agent stopped reporting. + setDisks([ + withTemperature({ + temperature: { + state: 'unavailable', + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }), + ]); + await waitFor(() => + expect(reading()).toHaveAttribute('data-temperature-reading', 'last-known'), + ); + expect(reading()).toHaveClass('text-muted'); + expect(reading().className).not.toMatch(/text-(red|amber|green)-/); + expect(reading()).toHaveAttribute( + 'title', + 'Last known reading, not current: host agent stopped reporting', + ); + expect(reading()).toHaveTextContent('72°C, last known'); + + // A source that predates collection state still reads as current. + setDisks([withTemperature(undefined)]); + await waitFor(() => expect(reading()).toHaveAttribute('data-temperature-reading', 'current')); + expect(reading()).toHaveClass('text-red-600'); + }); + it('refreshes keyed disk rows without losing the expanded detail or keyboard focus', async () => { const initial = buildDisk('sda', 'tower', { diskType: 'ssd', wearout: 96 }); const [disks, setDisks] = createSignal([initial]); diff --git a/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.test.ts b/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.test.ts index 89a8caacb3..392e467d21 100644 --- a/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.test.ts +++ b/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { DISK_DETAIL_HISTORY_RANGE_OPTIONS, DISK_DETAIL_LIVE_CHARTS, + getDiskAttributeCardValueTextClass, getDiskAttributeValueTextClass, getDiskDetailAttributeCards, getDiskDetailHealthPresentation, @@ -18,6 +19,49 @@ describe('diskDetailPresentation', () => { expect(getDiskAttributeValueTextClass(false)).toBe('text-red-600 dark:text-red-400'); }); + it('labels a retained temperature as last known without a health tone', () => { + const disk = { + node: 'tower', + instance: 'cluster-main', + devPath: '/dev/sda', + model: 'Archive HDD', + serial: 'SERIAL-1', + wwn: '', + size: 1, + health: 'PASSED', + wearout: -1, + type: 'sata', + temperature: 72, + rpm: 7200, + used: '', + riskReasons: [] as string[], + }; + const live = getDiskDetailAttributeCards( + { ...disk, collection: { temperature: { state: 'available', source: 'host_agent' } } }, + { warning: 50, critical: 55 }, + ); + expect(live).toEqual([{ label: 'Temperature', value: '72°C', ok: false }]); + expect(getDiskAttributeCardValueTextClass(live[0])).toBe('text-red-600 dark:text-red-400'); + + const retained = getDiskDetailAttributeCards( + { + ...disk, + collection: { + temperature: { + state: 'unavailable', + source: 'smartctl', + reason: 'disk is in standby', + }, + }, + }, + { warning: 50, critical: 55 }, + ); + expect(retained).toEqual([ + { label: 'Last known temperature', value: '72°C', ok: true, lastKnown: true }, + ]); + expect(getDiskAttributeCardValueTextClass(retained[0])).toBe('text-muted'); + }); + it('returns canonical linked-disk state presentation', () => { expect(getLinkedDiskHealthDotVariant(true)).toBe('warning'); expect(getLinkedDiskHealthDotVariant(false)).toBe('success'); diff --git a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts index 935bcce339..a18d7ccdfe 100644 --- a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts +++ b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts @@ -37,10 +37,12 @@ import { getPhysicalDiskSourceKey, getPhysicalDiskSourceBadgePresentation, getPhysicalDiskTableLayoutModeForContainer, + getPhysicalDiskTemperaturePresentation, hasUnraidPhysicalDiskFaultSignal, hasPhysicalDiskSmartWarning, isPhysicalDiskWearoutReported, isPhysicalDiskColumnVisible, + isPhysicalDiskTemperatureCurrent, isUnraidPhysicalDisk, matchesPhysicalDiskSearch, normalizePhysicalDiskFacetFilter, @@ -70,6 +72,65 @@ function makeDiskData( } describe('diskPresentation', () => { + it('reads a disk temperature as current only when its collection state says so', () => { + // Sources that predate collection state, and empty states, stay current. + expect(isPhysicalDiskTemperatureCurrent(undefined)).toBe(true); + expect(isPhysicalDiskTemperatureCurrent({})).toBe(true); + expect(isPhysicalDiskTemperatureCurrent({ temperature: { state: '' as never } })).toBe(true); + expect( + isPhysicalDiskTemperatureCurrent({ temperature: { state: 'available', source: 'smartctl' } }), + ).toBe(true); + for (const state of ['unavailable', 'unsupported', 'missing'] as const) { + expect(isPhysicalDiskTemperatureCurrent({ temperature: { state } })).toBe(false); + } + + expect( + getPhysicalDiskTemperaturePresentation( + makeDiskData({ + temperature: 72, + collection: { temperature: { state: 'available', source: 'host_agent' } }, + }), + ), + ).toEqual({ label: '72°C', current: true, title: undefined }); + expect( + getPhysicalDiskTemperaturePresentation( + makeDiskData({ + temperature: 72, + collection: { + temperature: { + state: 'unavailable', + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }, + }), + ), + ).toEqual({ + label: '72°C', + current: false, + title: 'Last known reading, not current: host agent stopped reporting', + }); + expect( + getPhysicalDiskTemperaturePresentation( + makeDiskData({ + temperature: 41, + collection: { temperature: { state: 'missing', source: 'smartctl', reason: ' ' } }, + }), + ), + ).toEqual({ label: '41°C', current: false, title: 'Last known reading, not current' }); + // No reading at all is not a last-known reading either. + for (const temperature of [0, -1, NaN]) { + expect( + getPhysicalDiskTemperaturePresentation( + makeDiskData({ + temperature, + collection: { temperature: { state: 'unavailable', reason: 'disk is in standby' } }, + }), + ), + ).toBeNull(); + } + }); + it('distinguishes unsupported, unavailable, and unexpectedly missing disk evidence', () => { expect( getPhysicalDiskFieldStatusMessage('Disk I/O', { diff --git a/frontend-modern/src/features/storageBackups/__tests__/storagePoolDetailPresentation.test.ts b/frontend-modern/src/features/storageBackups/__tests__/storagePoolDetailPresentation.test.ts index dc34f25a70..42a4b15b04 100644 --- a/frontend-modern/src/features/storageBackups/__tests__/storagePoolDetailPresentation.test.ts +++ b/frontend-modern/src/features/storageBackups/__tests__/storagePoolDetailPresentation.test.ts @@ -299,6 +299,13 @@ describe('storagePoolDetailPresentation', () => { sizeBytes: 6_000, temperature: 32, spunDown: true, + collection: { + temperature: { + state: 'unavailable', + source: 'unraid', + reason: 'disk is reported spun down', + }, + }, readCount: 10, writeCount: 20, errorCount: 16, @@ -309,10 +316,13 @@ describe('storagePoolDetailPresentation', () => { const linkedDisks = getStoragePoolLinkedDisks(record, disks); expect(linkedDisks.map((disk) => disk.role)).toEqual(['parity', 'data']); + expect(linkedDisks[0].temperatureLastKnownTitle).toBeUndefined(); expect(linkedDisks[1]).toEqual( expect.objectContaining({ devPath: '/dev/sdc', sizeLabel: '5.86 KB', + temperature: 32, + temperatureLastKnownTitle: 'Last known reading, not current: disk is reported spun down', spunDown: true, errorCount: 16, ioLabel: 'R 10 / W 20', diff --git a/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts b/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts index 810daab8d7..75f66c2cce 100644 --- a/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts +++ b/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts @@ -1,18 +1,22 @@ import { getPhysicalDiskHealthStatus, getPhysicalDiskHealthSummary, + getPhysicalDiskTemperaturePresentation, type PhysicalDiskPresentationData, } from '@/features/storageBackups/diskPresentation'; import type { HistoryTimeRange } from '@/api/charts'; import { formatPowerOnHours } from '@/utils/format'; import { getMetricSeverity, type MetricDisplayThresholds } from '@/utils/metricThresholds'; -import { formatTemperature } from '@/utils/temperature'; import type { StatusIndicatorVariant } from '@/utils/status'; export function getDiskAttributeValueTextClass(ok: boolean): string { return ok ? 'text-green-600 dark:text-green-400' : 'text-red-600 dark:text-red-400'; } +export function getDiskAttributeCardValueTextClass(card: DiskDetailAttributeCard): string { + return card.lastKnown ? 'text-muted' : getDiskAttributeValueTextClass(card.ok); +} + export function getLinkedDiskHealthDotVariant(hasIssue: boolean): StatusIndicatorVariant { return hasIssue ? 'warning' : 'success'; } @@ -38,6 +42,8 @@ export type DiskDetailAttributeCard = { label: string; value: string; ok: boolean; + /** A retained value that is not a current reading. */ + lastKnown?: boolean; }; export type DiskDetailChartOption = { @@ -126,12 +132,22 @@ export function getDiskDetailAttributeCards( }); } - if (Number.isFinite(disk.temperature) && disk.temperature > 0) { + const temperature = getPhysicalDiskTemperaturePresentation(disk); + if (temperature?.current) { cards.push({ label: 'Temperature', - value: formatTemperature(disk.temperature), + value: temperature.label, ok: getMetricSeverity(disk.temperature, 'diskTemperature', diskTempThresholds) !== 'critical', }); + } else if (temperature) { + // A retained reading is neither healthy nor hot now. The collection + // message above the cards says why it is not current. + cards.push({ + label: 'Last known temperature', + value: temperature.label, + ok: true, + lastKnown: true, + }); } if (attrs.powerCycles != null) { diff --git a/frontend-modern/src/features/storageBackups/diskPresentation.ts b/frontend-modern/src/features/storageBackups/diskPresentation.ts index a7e24152c2..2499be5969 100644 --- a/frontend-modern/src/features/storageBackups/diskPresentation.ts +++ b/frontend-modern/src/features/storageBackups/diskPresentation.ts @@ -14,6 +14,7 @@ import { getAllFilterOptionLabel } from '@/components/shared/filterOptionPresent import { getPhysicalDiskNodeIdentity } from '@/components/Storage/diskResourceUtils'; import { getInfrastructureSettingsLocationLabel } from '@/utils/infrastructureSettingsPresentation'; import { normalizeStorageSourceKey, storageSourceMatchesFilter } from '@/utils/storageSources'; +import { formatTemperature } from '@/utils/temperature'; import type { NormalizedHealth, StorageHealthFilter } from './models'; import { matchesStorageNodeTerms, parseStorageSearchQuery } from './storageSearchQuery'; @@ -253,6 +254,11 @@ export const PHYSICAL_DISK_HEALTH_WRAP_CLASS = // that would compete with it for the track. export const PHYSICAL_DISK_HEALTH_LABEL_CLASS = 'shrink-0 text-[11px] font-semibold'; export const PHYSICAL_DISK_TEMPERATURE_CLASS = 'text-[11px] font-medium'; +// A retained reading takes no threshold colour, so it cannot read as a disk +// running hot or cool right now. The dotted underline marks the value as one +// its title explains. +export const PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS = + 'text-muted underline decoration-dotted underline-offset-2 cursor-help'; export const PHYSICAL_DISK_SIZE_VALUE_CLASS = 'text-[11px] text-base-content'; const titleize = (value: string | undefined | null): string => @@ -467,6 +473,45 @@ export function getPhysicalDiskFieldStatusMessage( } } +// Normalization may keep the last known temperature when the current +// observation is not available, such as a disk in standby or a host agent that +// stopped reporting. The value is a current reading only when its collection +// state is available, or when the source predates collection state. +export function isPhysicalDiskTemperatureCurrent( + collection: PhysicalDiskCollectionStatus | null | undefined, +): boolean { + const state = collection?.temperature?.state; + return !state || state === 'available'; +} + +export function getPhysicalDiskLastKnownTemperatureTitle( + status: PhysicalDiskFieldStatus | null | undefined, +): string { + const reason = status?.reason?.trim(); + return reason ? `Last known reading, not current: ${reason}` : 'Last known reading, not current'; +} + +export interface PhysicalDiskTemperaturePresentation { + label: string; + current: boolean; + /** Set only for a last-known reading: why the value is not current. */ + title?: string; +} + +export function getPhysicalDiskTemperaturePresentation( + disk: Pick, +): PhysicalDiskTemperaturePresentation | null { + if (!Number.isFinite(disk.temperature) || disk.temperature <= 0) return null; + const current = isPhysicalDiskTemperatureCurrent(disk.collection); + return { + label: formatTemperature(disk.temperature), + current, + title: current + ? undefined + : getPhysicalDiskLastKnownTemperatureTitle(disk.collection?.temperature), + }; +} + export function getPhysicalDiskCollectionMessages(disk: PhysicalDiskPresentationData): string[] { const collection = disk.collection; if (!collection) return []; diff --git a/frontend-modern/src/features/storageBackups/storagePoolDetailPresentation.ts b/frontend-modern/src/features/storageBackups/storagePoolDetailPresentation.ts index 166571e68f..cca8d9a7f4 100644 --- a/frontend-modern/src/features/storageBackups/storagePoolDetailPresentation.ts +++ b/frontend-modern/src/features/storageBackups/storagePoolDetailPresentation.ts @@ -1,4 +1,5 @@ import type { HistoryTimeRange, ResourceType as HistoryChartResourceType } from '@/api/charts'; +import { getPhysicalDiskTemperaturePresentation } from '@/features/storageBackups/diskPresentation'; import type { StorageRecord } from '@/features/storageBackups/models'; import { getStorageRecordContent, @@ -27,6 +28,8 @@ export type StoragePoolDetailLinkedDisk = { sizeLabel: string; diskType: string; temperature: number; + /** Set only when the temperature is a retained, not current, reading. */ + temperatureLastKnownTitle?: string; hasIssue: boolean; spunDown: boolean; errorCount: number; @@ -305,6 +308,12 @@ const readDiskTemperature = (disk: Resource): number => { return typeof physicalDisk.temperature === 'number' ? physicalDisk.temperature : 0; }; +const readDiskTemperatureLastKnownTitle = (disk: Resource): string | undefined => + getPhysicalDiskTemperaturePresentation({ + temperature: readDiskTemperature(disk), + collection: readPhysicalDisk(disk).collection, + })?.title; + const readDiskType = (disk: Resource): string => { const physicalDisk = readPhysicalDisk(disk); return typeof physicalDisk.diskType === 'string' ? physicalDisk.diskType.trim() : ''; @@ -424,6 +433,7 @@ export function getStoragePoolLinkedDisks( sizeLabel: readDiskSizeLabel(disk), diskType: readDiskType(disk), temperature: readDiskTemperature(disk), + temperatureLastKnownTitle: readDiskTemperatureLastKnownTitle(disk), hasIssue: readDiskHasIssue(disk), spunDown: readDiskSpunDown(disk), errorCount: readDiskErrorCount(disk), From 267ffcc3afed2c062c40006a45feff860d226334 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:23:55 +0100 Subject: [PATCH 07/53] Keep node temperature selection when details lack a CPU reading The first version of this change walked every temperature source and took the first usable one, which changed two cases that worked before. Details holding only NVMe or GPU readings displaced a valid scalar CPU reading, so the node lost its CPU temperature. An unusable earlier record no longer fell back to the scalar; the walk went on to a later record that was marked unavailable and suppressed the display. The record selection is back to exactly what it was. Only when it yields nothing usable does the adapter use proxmox.temperatureDetails, and only if that holds a current CPU reading; otherwise it falls back to the scalar as before, still without inventing a low or record. The contract notes now say the adapter forwards the poller's low and record values and that keeping them across polls is the poller's job. --- .../subsystems/performance-and-scalability.md | 6 +- .../internal/subsystems/unified-resources.md | 22 +-- .../ae27640ea2ed301f.json | 37 +++++ .../__tests__/resourceStateAdapters.test.ts | 28 ++++ .../src/utils/resourceStateAdapters.ts | 145 ++++++++++-------- 5 files changed, 161 insertions(+), 77 deletions(-) create mode 100644 frontend-modern/browser-verification/ae27640ea2ed301f.json diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 9a0ad0a645..3b4a3751d0 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -161,8 +161,10 @@ QGA/thaw, installed recovery, fleet-performance or release-availability claim. Choosing a node's temperature source in `resourceStateAdapters.ts` reads one more field of the facet already in the canonical payload -(`proxmox.temperatureDetails`) when earlier sources have no usable reading. It -adds no resource read, merge pass, timer or subscription. +(`proxmox.temperatureDetails`), only when the existing record selection yields +no usable reading, and keeps it only when it holds a CPU reading. That is one +extra record mapping per such node per adapter call. It adds no resource read, +merge pass, timer or subscription. ### Unavailable filesystem values share the existing disk model diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 40bf3e9036..aa5a022d7e 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -3182,15 +3182,19 @@ footer after the visible rows. The footer may dim the final row edge and expose the remaining-row count, but expanding or collapsing it only changes the client-side visible slice; it must not refetch, reorder, or redefine the canonical resource collection. -The shared `nodeFromResource` adapter takes a node's temperature from the first -source with a usable reading: a platform or Proxmox temperature record, the -agent facet, then `proxmox.temperatureDetails`. That last field is the Proxmox -poller's full reading for a node without a host agent, including the CPU low -and record values it tracks; `proxmox.temperature` is only the scalar maximum. -A bare scalar reading carries no history, so the adapter leaves the CPU low and -record unset rather than repeating the current value, and the node drawer omits -those rows. `resourceStateAdapters.test.ts` pins both with the API's payload -shape. +The shared `nodeFromResource` adapter selects a node's temperature record as +before: the platform temperature record, else the Proxmox temperature record, +else the agent facet. When that selection yields no usable reading, it uses +`proxmox.temperatureDetails` if that holds a current CPU reading, ahead of the +bare scalar fallback. That field is +the Proxmox poller's full reading for a node without a host agent, including +the CPU low and record values the poller supplies; `proxmox.temperature` is +only the scalar maximum. The adapter forwards those values; keeping them across +polls is the poller's job (`monitor_previous_state.go` and the temperature carry +in monitoring). A bare scalar reading carries no history, so the adapter leaves +the CPU low and record unset rather than repeating the current value, and the +node drawer omits those rows. `resourceStateAdapters.test.ts` pins these cases +with the API's payload shape. The registry and presentation coalescer also own metric-source freshness. When two source facets contribute the same metric, source priority decides only if both sources have equivalent freshness. A stale source must not hold CPU, diff --git a/frontend-modern/browser-verification/ae27640ea2ed301f.json b/frontend-modern/browser-verification/ae27640ea2ed301f.json new file mode 100644 index 0000000000..8c18857784 --- /dev/null +++ b/frontend-modern/browser-verification/ae27640ea2ed301f.json @@ -0,0 +1,37 @@ +{ + "version": 1, + "base_sha": "60fb6e842b702c35c93ce7c5e8397487c15aab6a", + "verified_at": "2026-10-06T16:05:00Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/utils/resourceStateAdapters.ts" + ], + "content_sha256": { + "frontend-modern/src/utils/resourceStateAdapters.ts": "744fb071acc493f973c6064e7c7eaaac831c9cef2a88491a8b51e49a08f139be" + }, + "routes": [ + "/proxmox" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack built from this tree (backend :7681, Vite :5199) after restoring the original record selection and gating temperatureDetails on a current CPU reading", + "Proxmox node Analytics A (pve31) made PVE-only (agent facet removed from REST and websocket payloads), compared on identical data: main's adapter showed CPU low 63\u00b0C and CPU record 63\u00b0C (the current reading repeated); this change showed CPU low 39\u00b0C and CPU record 91\u00b0C from an injected temperatureDetails record (cpuMin 38.5, cpuMaxRecord 91.2) at 1440x900 and 390x844", + "Same PVE-only node without a record in temperatureDetails: CPU low and CPU record rows absent; NVMe row and table TEMP unchanged", + "Operator-context Temperature row: hidden on main and on this change at 63\u00b0C (it appears only in the warning band), so no difference", + "Agent-linked mock node with and without an injected record, desktop and phone width: no CPU low/record rows and Temperature from the agent facet, as on main", + "Unit tests cover the two cases this commit fixes: details with only NVMe do not displace a valid scalar CPU reading, and an unusable legacy record still falls back to the scalar" + ], + "interactions": [ + "Opened the node drawer from the row's detail toggle (aria-controls proxmox-host-drawer-) on /proxmox at desktop and phone widths", + "Read the Thermals and Operator context rows and the row's TEMP cell, and took screenshots, for main's adapter and for this change via Playwright with REST and websocket payload rewriting" + ] +} diff --git a/frontend-modern/src/utils/__tests__/resourceStateAdapters.test.ts b/frontend-modern/src/utils/__tests__/resourceStateAdapters.test.ts index dbb6aef82c..fa946df7c7 100644 --- a/frontend-modern/src/utils/__tests__/resourceStateAdapters.test.ts +++ b/frontend-modern/src/utils/__tests__/resourceStateAdapters.test.ts @@ -299,6 +299,34 @@ describe('resourceStateAdapters nodeFromResource', () => { expect(bare?.temperature?.cpuMin).toBeUndefined(); expect(bare?.temperature?.cpuMaxRecord).toBeUndefined(); + // Details without a CPU reading do not displace a valid scalar CPU reading. + const sensorsOnly = nodeFromResource({ + ...createNodeResource({ + proxmox: { + nodeName: 'pve-node-2', + temperatureDetails: { + available: true, + hasCPU: false, + nvme: [{ device: 'nvme0', temp: 44 }], + }, + }, + }), + temperature: 58, + } as unknown as Resource); + expect(sensorsOnly?.temperature?.cpuPackage).toBe(58); + expect(sensorsOnly?.temperature?.available).toBe(true); + + // An unusable earlier record still falls back to the scalar reading. + const legacyUnavailable = nodeFromResource({ + ...createNodeResource({ + temperature: { available: false }, + proxmox: { nodeName: 'pve-node-2', temperature: { available: false, cpuPackage: 58 } }, + }), + temperature: 58, + } as unknown as Resource); + expect(legacyUnavailable?.temperature?.cpuPackage).toBe(58); + expect(legacyUnavailable?.temperature?.available).toBe(true); + // A node with a host agent keeps reading the agent facet first. const agentLinked = nodeFromResource({ ...createNodeResource({ proxmox, agent: { agentId: 'pve-agent', temperature: 61 } }), diff --git a/frontend-modern/src/utils/resourceStateAdapters.ts b/frontend-modern/src/utils/resourceStateAdapters.ts index f78e577757..72efec2fb5 100644 --- a/frontend-modern/src/utils/resourceStateAdapters.ts +++ b/frontend-modern/src/utils/resourceStateAdapters.ts @@ -1373,77 +1373,90 @@ const buildDisk = (metric: Resource['disk'], fallback?: Record) }; }; +const temperatureFromRecord = ( + raw: Record, + resource: Resource, +): Temperature | undefined => { + const available = asBoolean(raw.available); + const cpuPackage = asNumber(raw.cpuPackage) ?? asNumber(raw.temperature) ?? asNumber(raw.cpu); + const lastUpdate = toISOTime(raw.lastUpdate, resource.lastSeen); + if (available || typeof cpuPackage === 'number') { + return { + cpuPackage, + cpuMax: asNumber(raw.cpuMax), + cpuMin: asNumber(raw.cpuMin), + cpuMaxRecord: asNumber(raw.cpuMaxRecord), + minRecorded: asString(raw.minRecorded), + maxRecorded: asString(raw.maxRecorded), + cores: asArray(raw.cores) + .map((entry) => { + const rec = asRecord(entry); + if (!rec) return null; + const core = asNumber(rec.core); + const temp = asNumber(rec.temp); + if (typeof core !== 'number' || typeof temp !== 'number') return null; + return { core, temp }; + }) + .filter((entry): entry is NonNullable => Boolean(entry)), + gpu: asArray(raw.gpu) + .map((entry) => { + const rec = asRecord(entry); + if (!rec) return null; + const device = asString(rec.device); + if (!device) return null; + return { + device, + edge: asNumber(rec.edge), + junction: asNumber(rec.junction), + mem: asNumber(rec.mem), + }; + }) + .filter((entry): entry is NonNullable => Boolean(entry)), + nvme: asArray(raw.nvme) + .map((entry) => { + const rec = asRecord(entry); + if (!rec) return null; + const device = asString(rec.device); + const temp = asNumber(rec.temp); + if (!device || typeof temp !== 'number') return null; + return { device, temp }; + }) + .filter((entry): entry is NonNullable => Boolean(entry)), + available: available ?? true, + hasCPU: asBoolean(raw.hasCPU) ?? (typeof cpuPackage === 'number' ? true : undefined), + hasGPU: asBoolean(raw.hasGPU), + hasNVMe: asBoolean(raw.hasNVMe), + lastUpdate, + }; + } + return undefined; +}; + +const hasCPUTemperatureReading = (temperature: Temperature): boolean => + (temperature.cpuPackage ?? 0) > 0 || + (temperature.cpuMax ?? 0) > 0 || + (temperature.cores ?? []).some((core) => core.temp > 0); + const buildTemperature = ( resource: Resource, nodeMeta?: Record, ): Temperature | undefined => { const platform = resourcePlatformData(resource); - // Take the first source with a usable reading. A Proxmox node without a host - // agent publishes its full reading, including the CPU low and record values - // the poller tracks, only as proxmox.temperatureDetails (proxmox.temperature - // is the scalar maximum), so it comes before the bare-reading fallback. - const candidates = [ - platform?.temperature, - nodeMeta?.temperature, - platform?.agent, - nodeMeta?.temperatureDetails, - ]; - for (const candidate of candidates) { - const raw = asRecord(candidate); - if (!raw) continue; - const available = asBoolean(raw.available); - const cpuPackage = asNumber(raw.cpuPackage) ?? asNumber(raw.temperature) ?? asNumber(raw.cpu); - const lastUpdate = toISOTime(raw.lastUpdate, resource.lastSeen); - if (available || typeof cpuPackage === 'number') { - return { - cpuPackage, - cpuMax: asNumber(raw.cpuMax), - cpuMin: asNumber(raw.cpuMin), - cpuMaxRecord: asNumber(raw.cpuMaxRecord), - minRecorded: asString(raw.minRecorded), - maxRecorded: asString(raw.maxRecorded), - cores: asArray(raw.cores) - .map((entry) => { - const rec = asRecord(entry); - if (!rec) return null; - const core = asNumber(rec.core); - const temp = asNumber(rec.temp); - if (typeof core !== 'number' || typeof temp !== 'number') return null; - return { core, temp }; - }) - .filter((entry): entry is NonNullable => Boolean(entry)), - gpu: asArray(raw.gpu) - .map((entry) => { - const rec = asRecord(entry); - if (!rec) return null; - const device = asString(rec.device); - if (!device) return null; - return { - device, - edge: asNumber(rec.edge), - junction: asNumber(rec.junction), - mem: asNumber(rec.mem), - }; - }) - .filter((entry): entry is NonNullable => Boolean(entry)), - nvme: asArray(raw.nvme) - .map((entry) => { - const rec = asRecord(entry); - if (!rec) return null; - const device = asString(rec.device); - const temp = asNumber(rec.temp); - if (!device || typeof temp !== 'number') return null; - return { device, temp }; - }) - .filter((entry): entry is NonNullable => Boolean(entry)), - available: available ?? true, - hasCPU: asBoolean(raw.hasCPU) ?? (typeof cpuPackage === 'number' ? true : undefined), - hasGPU: asBoolean(raw.hasGPU), - hasNVMe: asBoolean(raw.hasNVMe), - lastUpdate, - }; - } - } + const raw = + asRecord(platform?.temperature) || + asRecord(nodeMeta?.temperature) || + asRecord(platform?.agent) || + undefined; + const fromRecord = raw ? temperatureFromRecord(raw, resource) : undefined; + if (fromRecord) return fromRecord; + + // A Proxmox node without a host agent publishes its full reading, including + // the CPU low and record values the poller tracks, only as + // proxmox.temperatureDetails (proxmox.temperature is the scalar maximum). Use + // it ahead of the bare-reading fallback when it holds a current CPU reading. + const details = asRecord(nodeMeta?.temperatureDetails); + const fromDetails = details ? temperatureFromRecord(details, resource) : undefined; + if (fromDetails?.available && hasCPUTemperatureReading(fromDetails)) return fromDetails; if (typeof resource.temperature === 'number' && Number.isFinite(resource.temperature)) { const temp = resource.temperature; From 3ca8b9647ab992101adf9153b71b5ef1757ac82f Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:03:55 +0100 Subject: [PATCH 08/53] Move the live metric status projection test clear of main Main appended TestSilentLinkedAgentStopsRefreshingNodeTemperatureAlert at the end of monitor_alert_handling_test.go, where this branch had appended its own projection test, so the two edits conflict. Move the test unchanged in substance to monitor_alert_intent_test.go, which main has not touched, so the merge with main is clean. --- .../monitoring/monitor_alert_handling_test.go | 36 --------------- .../monitoring/monitor_alert_intent_test.go | 44 +++++++++++++++++++ 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go index 9d85fedba4..498bd35b8d 100644 --- a/internal/monitoring/monitor_alert_handling_test.go +++ b/internal/monitoring/monitor_alert_handling_test.go @@ -1466,39 +1466,3 @@ func TestMonitorDelayedPartialResolutionKeepsDestinationRecovery(t *testing.T) { t.Fatal("unannounced destination received a recovery") } } - -func TestSyncAlertsToStateCarriesLiveMetricStatusOfHeldAlert(t *testing.T) { - m := &Monitor{ - state: models.NewState(), - alertManager: alerts.NewManagerWithDataDir(t.TempDir()), - } - defer m.alertManager.Stop() - - cfg := m.alertManager.GetConfig() - cfg.TimeThresholds = map[string]int{"node": 0} - cfg.NodeDefaults.Temperature = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75} - m.alertManager.UpdateConfig(cfg) - - node := models.Node{ - ID: "homelab-minipc", Name: "minipc", Instance: "homelab", Status: "online", - Temperature: &models.Temperature{Available: true, CPUPackage: 85}, - } - m.alertManager.CheckNode(node) - node.Temperature = &models.Temperature{Available: true, CPUPackage: 78} - m.alertManager.CheckNode(node) - - m.syncAlertsToState() - - var held *models.Alert - for i, alert := range m.state.GetSnapshot().ActiveAlerts { - if alert.Type == "temperature" { - held = &m.state.GetSnapshot().ActiveAlerts[i] - } - } - require.NotNil(t, held, "expected the temperature alert to stay open at 78°C") - require.Equal(t, 85.0, held.Value, "the legacy value keeps the last breach") - require.NotNil(t, held.MetricStatus, "websocket state must carry the live reading") - require.Equal(t, models.MetricAlertPhaseLatched, held.MetricStatus.Phase) - require.Equal(t, 78.0, held.MetricStatus.Value) - require.Equal(t, 75.0, held.MetricStatus.Recovery) -} diff --git a/internal/monitoring/monitor_alert_intent_test.go b/internal/monitoring/monitor_alert_intent_test.go index 6832ec92ce..1b3c0485e2 100644 --- a/internal/monitoring/monitor_alert_intent_test.go +++ b/internal/monitoring/monitor_alert_intent_test.go @@ -258,3 +258,47 @@ func TestResolveBackupIntentContextAcceptsSynthesizedJobGuestTask(t *testing.T) t.Fatalf("guest whose job section already finished should not carry intent: %+v", got) } } + +func TestSyncAlertsToStateCarriesLiveMetricStatusOfHeldAlert(t *testing.T) { + m := &Monitor{ + state: models.NewState(), + alertManager: alerts.NewManagerWithDataDir(t.TempDir()), + } + defer m.alertManager.Stop() + + cfg := m.alertManager.GetConfig() + cfg.TimeThresholds = map[string]int{"node": 0} + cfg.NodeDefaults.Temperature = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75} + m.alertManager.UpdateConfig(cfg) + + node := models.Node{ + ID: "homelab-minipc", Name: "minipc", Instance: "homelab", Status: "online", + Temperature: &models.Temperature{Available: true, CPUPackage: 85}, + } + m.alertManager.CheckNode(node) + node.Temperature = &models.Temperature{Available: true, CPUPackage: 78} + m.alertManager.CheckNode(node) + + m.syncAlertsToState() + + var held *models.Alert + snapshot := m.state.GetSnapshot() + for i := range snapshot.ActiveAlerts { + if snapshot.ActiveAlerts[i].Type == "temperature" { + held = &snapshot.ActiveAlerts[i] + } + } + if held == nil { + t.Fatal("expected the temperature alert to stay open at 78°C") + } + if held.Value != 85 { + t.Fatalf("legacy value = %v, want the last breach 85", held.Value) + } + status := held.MetricStatus + if status == nil { + t.Fatal("websocket state must carry the live reading") + } + if status.Phase != models.MetricAlertPhaseLatched || status.Value != 78 || status.Recovery != 75 { + t.Fatalf("live status = %+v, want latched at 78 clearing at 75", status) + } +} From 0c130a4855e4467c5056292abab1d9103f0b7c01 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:46:16 +0100 Subject: [PATCH 09/53] Key previous Proxmox nodes by source ID so poll carry-overs match The poller reads the previous poll's nodes back from the unified read state, but previousNodeFromView returned the registry's unified resource ID and no temperature. Every lookup keyed on the Proxmox source ID missed: nodeLastOnline in the offline grace policy, the temperature carry, the CPU low/record tracking (reset to the current reading every poll) and the node memory carry. When a whole instance poll failed, markPVEInstanceNodesUnreachable wrote the projected nodes back, so each failed poll re-ingested the node under a new unified ID. Cluster members also lost the cluster identity their canonical ID derives from, so they changed ID for the length of an outage. The projection now returns the source ID, the stored temperature details, the cluster name and the linked agent ID, and previousNodesForInstance restores the config-derived identity stamps (provider-scoped flag, native aliases, TLS fingerprint) the way pollPVENode writes them. Reviving those paths needed four bounds: - The node memory carry uses the poller's own validated snapshot rather than the read state, which holds the agent's reading on a node merged with a host agent, and keeps used + cache + free equal to the total. - A carried temperature keeps a lapsed host agent's lease. A reading stamped no later than the agent's last report is not carried, so the five-minute carry window cannot hold a silent agent's reading in alerts and history past its lease, which #2582 guarantees. - A carried reading keeps the node's CPU and GPU temperatures only. Its SMART and NVMe rows also stamp the node's physical disks, so a disk that went to standby would be stamped and recorded with its pre-sleep value. - Outage preservation and authoritative membership reconciliation keep a node's temperature only under those rules and drop it when the node goes offline. TestPollPVEInstanceMarksStaleNodesOfflineWhenGetNodesReturnsEmpty relied on the dead nodeLastOnline lookup and now ages that sighting too. The silent-agent test from #2582 now rebuilds the store as production does and feeds the polled node back, which is what exercises the carry. --- .../v6/internal/subsystems/monitoring.md | 34 +- internal/monitoring/host_agent_temps.go | 101 +++-- .../memory_trust_characterization_test.go | 6 +- .../monitoring/monitor_additional_test.go | 427 ++++++++++++++++++ .../monitoring/monitor_alert_handling_test.go | 28 +- .../monitoring/monitor_extra_coverage_test.go | 22 +- internal/monitoring/monitor_memory_test.go | 5 + internal/monitoring/monitor_polling_node.go | 32 +- .../monitor_polling_node_helpers.go | 42 +- internal/monitoring/monitor_previous_state.go | 33 +- internal/monitoring/monitor_pve.go | 31 +- internal/monitoring/monitor_pve_cluster.go | 2 + .../monitoring/node_memory_sources_test.go | 1 - 13 files changed, 671 insertions(+), 93 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 40873c0536..ba87140d89 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -3772,7 +3772,39 @@ cluster sensor cache, which keeps its own recency check. When every source returns nothing, `internal/monitoring/monitor_polling_node_helpers.go` may carry a previous reading, with its original `LastUpdate`, only inside the carry window (twice the PVE polling interval, never under five minutes); an older reading is -dropped rather than re-presented as current. +dropped rather than re-presented as current. A carried reading also keeps a +lapsed host agent's lease. Agent readings are stamped with the agent's report +time, so `carriedTemperatureOutlivesAgentLease` drops a carried reading stamped +no later than the last report of the node's matched agent (by link, then +hostname, whether or not that report had sensors) once that agent has lapsed. +Readings record no source, so the bound is conservative: an SSH, cluster-cache +or agent-plus-SSH merged reading (stamped with the agent's time) of that age is +dropped too, being already older than the lease, while a reading stamped after +that report keeps the carry window. A carried reading (`carriedNodeTemperature`) +keeps only the node's CPU and GPU temperatures: its SMART and NVMe rows would +also stamp the node's physical disks, whose freshness and standby state a +carried copy cannot follow (a disk that spun down would be stamped and recorded +with its pre-sleep value), so they are left out, and nothing is carried when +only they remain. +The poller reads its previous poll's nodes back from the unified read state +through `internal/monitoring/monitor_previous_state.go`. `nodeLastOnline` and +the temperature carry with its CPU low/record tracking match them by node ID, +which is the Proxmox source ID, so the projection must return +`NodeView.SourceID()`, never the unified resource ID, and carry the stored +`TemperatureDetails`. Nodes preserved through an instance outage +(`preserveOrExpireNodes`) or authoritative membership reconciliation are written +back into state from that projection, so it also restores the identity inputs +the registry derives a node's canonical ID and cross-view merges from: cluster +name, provider-scoped flag, native aliases, TLS fingerprint and linked agent ID, +the config-derived ones stamped the way `pollPVENode` stamps them. With the +unified ID a node re-keys on every failed poll, and without its cluster +identity a cluster member changes canonical ID for the length of an outage. +A preserved node keeps its +temperature only under the same carry rules and loses it when it goes +offline. Node memory carry-over after a failed status read uses the poller's own +validated `NodeMemorySnapshot`, not the read state, which holds the agent's +reading on a node merged with a host agent, and keeps used, cache and free +summing to the total. Legacy SSH temperature collection must also use the Pulse sensor-wrapper contract before falling back to raw lm-sensors output. `internal/monitoring/temperature.go` must request `/usr/local/sbin/pulse-sensors` when it exists, parse the wrapper diff --git a/internal/monitoring/host_agent_temps.go b/internal/monitoring/host_agent_temps.go index 661caeaf6e..8b3fc4d3e0 100644 --- a/internal/monitoring/host_agent_temps.go +++ b/internal/monitoring/host_agent_temps.go @@ -86,44 +86,7 @@ func (m *Monitor) getHostAgentTemperatureByID(nodeID, nodeName string) *models.T return nil } - hosts := readState.Hosts() - if len(hosts) == 0 { - // No host agents at all — check cluster sensor cache as fallback - return m.getClusterSensorTemperature(nodeName) - } - - var matchedHost *unifiedresources.HostView - - // First, try to find a host agent that is explicitly linked to this node - // via LinkedNodeID. This is the most reliable method and handles duplicate - // hostnames correctly. - if nodeID != "" { - for i := range hosts { - if hosts[i].LinkedNodeID() == nodeID { - matchedHost = hosts[i] - log.Debug(). - Str("nodeID", nodeID). - Str("hostAgentID", hosts[i].ID()). - Str("hostname", hosts[i].Hostname()). - Msg("Matched host agent to node via LinkedNodeID") - break - } - } - } - - // Fallback: match by hostname if no linked host was found - // This maintains backwards compatibility for setups where linking hasn't occurred yet - if matchedHost == nil { - nodeLower := strings.ToLower(strings.TrimSpace(nodeName)) - for i := range hosts { - hostnameLower := strings.ToLower(strings.TrimSpace(hosts[i].Hostname())) - if hostnameLower == nodeLower { - matchedHost = hosts[i] - break - } - } - } - + matchedHost := hostAgentForNode(readState.Hosts(), nodeID, nodeName) if matchedHost == nil { // No directly-linked host agent found — check cluster sensor cache return m.getClusterSensorTemperature(nodeName) @@ -153,6 +116,68 @@ func (m *Monitor) getHostAgentTemperatureByID(nodeID, nodeName string) *models.T return convertUnifiedHostSensorsToTemperature(sensors, agentStatus.LastSeen) } +// carriedTemperatureOutlivesAgentLease reports whether a temperature a node +// carries over from an earlier poll may have come from the node's host agent +// after that agent stopped reporting. Agent readings are stamped with the +// agent's report time and only count inside its reporting lease, so a reading +// stamped no later than a lapsed agent's last report is treated as the agent's +// and is not carried. Readings carry no source, so an SSH or cluster-cache +// reading of that age is dropped too; it is already older than the lease, and +// the bound errs toward a gap rather than a stale value. A reading stamped +// after that report keeps the ordinary carry window. +func (m *Monitor) carriedTemperatureOutlivesAgentLease(nodeID, nodeName string, temp *models.Temperature, now time.Time) bool { + if temp == nil { + return false + } + readState := m.GetUnifiedReadStateOrSnapshot() + if readState == nil { + return false + } + matchedHost := hostAgentForNode(readState.Hosts(), nodeID, nodeName) + if matchedHost == nil { + return false + } + agentStatus, ok := matchedHost.SourceStatus(unifiedresources.SourceAgent) + if !ok || agentStatus.LastSeen.IsZero() || + hostAgentReportCurrent(agentStatus.LastSeen, matchedHost.IntervalSeconds(), now) { + return false + } + return !temp.LastUpdate.After(agentStatus.LastSeen) +} + +// hostAgentForNode returns the host agent for a Proxmox node: the agent linked +// to the node ID, else one whose hostname matches. This correctly handles +// clusters where multiple nodes may have the same hostname (e.g., "px1" on +// different IPs). +func hostAgentForNode(hosts []*unifiedresources.HostView, nodeID, nodeName string) *unifiedresources.HostView { + // First, try to find a host agent that is explicitly linked to this node + // via LinkedNodeID. This is the most reliable method and handles duplicate + // hostnames correctly. + if nodeID != "" { + for i := range hosts { + if hosts[i].LinkedNodeID() == nodeID { + log.Debug(). + Str("nodeID", nodeID). + Str("hostAgentID", hosts[i].ID()). + Str("hostname", hosts[i].Hostname()). + Msg("Matched host agent to node via LinkedNodeID") + return hosts[i] + } + } + } + + // Fallback: match by hostname if no linked host was found + // This maintains backwards compatibility for setups where linking hasn't occurred yet + nodeLower := strings.ToLower(strings.TrimSpace(nodeName)) + for i := range hosts { + hostnameLower := strings.ToLower(strings.TrimSpace(hosts[i].Hostname())) + if hostnameLower == nodeLower { + return hosts[i] + } + } + return nil +} + func convertUnifiedHostSensorsToTemperature(sensors *unifiedresources.HostSensorMeta, lastSeen time.Time) *models.Temperature { if sensors == nil { return nil diff --git a/internal/monitoring/memory_trust_characterization_test.go b/internal/monitoring/memory_trust_characterization_test.go index 60c583c3a1..366164e81d 100644 --- a/internal/monitoring/memory_trust_characterization_test.go +++ b/internal/monitoring/memory_trust_characterization_test.go @@ -203,7 +203,7 @@ func TestPollPVENodeMemoryTrustCharacterization(t *testing.T) { MaxCPU: 8, } - modelNode, _, _, err := mon.pollPVENode(context.Background(), "test", &mon.config.PVEInstances[0], client, node, "healthy", nil, nil) + modelNode, _, _, err := mon.pollPVENode(context.Background(), "test", &mon.config.PVEInstances[0], client, node, "healthy", nil) if err != nil { t.Fatalf("pollPVENode() error = %v", err) } @@ -258,7 +258,7 @@ func TestPollPVENodePreservesPreviousSnapshotDuringTransientFallback(t *testing. MaxCPU: 8, } - first, _, _, err := mon.pollPVENode(context.Background(), "test", &mon.config.PVEInstances[0], client, node, "healthy", nil, nil) + first, _, _, err := mon.pollPVENode(context.Background(), "test", &mon.config.PVEInstances[0], client, node, "healthy", nil) if err != nil { t.Fatalf("first pollPVENode() error = %v", err) } @@ -274,7 +274,6 @@ func TestPollPVENodePreservesPreviousSnapshotDuringTransientFallback(t *testing. client, node, "healthy", - map[string]models.Memory{first.ID: first.Memory}, nil, ) if err != nil { @@ -302,7 +301,6 @@ func TestPollPVENodePreservesPreviousSnapshotDuringTransientFallback(t *testing. client, node, "healthy", - map[string]models.Memory{second.ID: second.Memory}, nil, ) if err != nil { diff --git a/internal/monitoring/monitor_additional_test.go b/internal/monitoring/monitor_additional_test.go index bc771e5d42..6a9b566dae 100644 --- a/internal/monitoring/monitor_additional_test.go +++ b/internal/monitoring/monitor_additional_test.go @@ -1158,3 +1158,430 @@ func TestTenantDeletionWaitsForSlowMonitoringLoop(t *testing.T) { } mtm.FinishTenantDeletion("deletion") } + +// The poller reads the previous poll's nodes back through the unified read +// state. Those lookups key on the Proxmox source ID, so the projection must +// return it (not the registry's unified resource ID) and carry the stored +// temperature; otherwise the node's CPU low/record values reset to the +// current reading every poll and a failed collection never carries over. +func TestPreviousNodeProjectionFeedsPollerTemperatureTracking(t *testing.T) { + instanceCfg := config.PVEInstance{Name: "pve1", Host: "https://10.0.0.1:8006"} + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m := &Monitor{ + config: &config.Config{TemperatureMonitoringEnabled: true, PVEInstances: []config.PVEInstance{instanceCfg}}, + state: models.NewState(), + resourceStore: adapter, + } + base := models.Node{ID: "pve1-node1", NodeIdentity: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", Type: "node"} + + // publish rebuilds the unified store from state the way the monitor does, + // alongside the host agents' latest reports. + var hosts []models.Host + publish := func() { + snapshot := m.state.GetSnapshot() + snapshot.Hosts = hosts + adapter.PopulateFromSnapshot(snapshot) + } + // poll delivers an agent report (none when agentReading is 0), runs one PVE + // poll's temperature step against the previous poll's nodes, and publishes + // the polled node. + poll := func(agentReading float64) *models.Temperature { + t.Helper() + hosts = nil + if agentReading > 0 { + hosts = []models.Host{{ + ID: "agent-1", Hostname: "node1", Status: "online", LinkedNodeID: base.ID, IntervalSeconds: 30, LastSeen: time.Now(), + Sensors: models.HostSensorSummary{TemperatureCelsius: map[string]float64{"cpu_package": agentReading}}, + }} + } + publish() + prevNodes := m.snapshotPrevNodes("pve1") + polled := base + polled.LastSeen = time.Now() + m.collectNodeTemperatureData(context.Background(), "pve1", &instanceCfg, proxmox.Node{Node: "node1"}, &polled, prevNodes, "online") + m.state.UpdateNodesForInstance("pve1", []models.Node{polled}) + publish() + return polled.Temperature + } + + if temp := poll(50); temp == nil || temp.CPUMin != 50 || temp.CPUMaxRecord != 50 { + t.Fatalf("first reading = %#v, want min and record initialised to 50", temp) + } + prevNodes := m.snapshotPrevNodes("pve1") + if len(prevNodes) != 1 || prevNodes[0].ID != base.ID || prevNodes[0].Temperature == nil { + t.Fatalf("previous nodes = %#v, want source ID %q with its temperature", prevNodes, base.ID) + } + + if temp := poll(62); temp == nil || temp.CPUPackage != 62 || temp.CPUMin != 50 || temp.CPUMaxRecord != 62 { + t.Fatalf("second reading = %#v, want current 62, low 50, record 62", temp) + } + third := poll(45) + if third == nil || third.CPUPackage != 45 || third.CPUMin != 45 || third.CPUMaxRecord != 62 { + t.Fatalf("third reading = %#v, want current 45, low 45, record 62", third) + } + + // No agent and no SSH collector: the last reading carries over with its + // original timestamp, inside the carry window. + carried := poll(0) + if carried == nil || carried.CPUPackage != 45 || !carried.LastUpdate.Equal(third.LastUpdate) { + t.Fatalf("carried reading = %#v, want the previous 45 with its original timestamp", carried) + } +} + +// When a whole Proxmox instance stops answering, its last-known nodes are +// written back into state through the offline grace policy. That writeback +// must keep each node the same resource: the poller's source ID and the +// config-derived identity the registry derives a cluster member's canonical ID +// from. Past grace the node goes offline and stops presenting a temperature. +func TestPVEOutageWritebackKeepsNodeIdentity(t *testing.T) { + for _, tc := range []struct { + name string + instances []config.PVEInstance + wantID string + providerScoped bool + }{ + { + name: "cluster member", + instances: []config.PVEInstance{{Name: "pve-a", Host: "https://10.0.0.1:8006", IsCluster: true, ClusterName: "lab"}}, + wantID: "lab-node1", + }, + { + name: "cluster name shared by two connections", + instances: []config.PVEInstance{ + {Name: "pve-a", Host: "https://10.0.0.1:8006", IsCluster: true, ClusterName: "lab"}, + {Name: "pve-b", Host: "https://10.0.1.1:8006", IsCluster: true, ClusterName: "lab"}, + }, + wantID: "pve-a-node1", + providerScoped: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m := &Monitor{ + config: &config.Config{PVEInstances: tc.instances}, + state: models.NewState(), + resourceStore: adapter, + nodeLastOnline: make(map[string]time.Time), + } + instanceCfg := m.getInstanceConfig("pve-a") + if got := m.pveNodeID("pve-a", instanceCfg, "node1"); got != tc.wantID { + t.Fatalf("pveNodeID = %q, want %q", got, tc.wantID) + } + now := time.Now() + polled := m.placeholderNodeForInstance("pve-a", instanceCfg, "node1") + polled.Status = "online" + polled.ConnectionHealth = "healthy" + polled.LastSeen = now + polled.Temperature = &models.Temperature{CPUPackage: 55, Available: true, HasCPU: true, LastUpdate: now} + if polled.ProviderScopedIdentity != tc.providerScoped { + t.Fatalf("ProviderScopedIdentity = %v, want %v", polled.ProviderScopedIdentity, tc.providerScoped) + } + m.nodeLastOnline[polled.ID] = now + m.state.UpdateNodesForInstance("pve-a", []models.Node{polled}) + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + unifiedID := adapter.Nodes()[0].ID() + + for cycle := 0; cycle < 3; cycle++ { + m.markPVEInstanceNodesUnreachable("pve-a") + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + stateNodes := m.state.GetSnapshot().Nodes + if len(stateNodes) != 1 || stateNodes[0].ID != tc.wantID || stateNodes[0].ClusterName != "lab" || + stateNodes[0].ProviderScopedIdentity != tc.providerScoped { + t.Fatalf("cycle %d: state nodes = %#v, want %q kept with its cluster identity", cycle, stateNodes, tc.wantID) + } + if stateNodes[0].Status != "online" || stateNodes[0].Temperature == nil { + t.Fatalf("cycle %d: node within grace = %#v, want online with its recent temperature", cycle, stateNodes[0]) + } + if nodes := adapter.Nodes(); len(nodes) != 1 || nodes[0].ID() != unifiedID { + t.Fatalf("cycle %d: unified nodes = %v, want the single resource %q", cycle, nodes, unifiedID) + } + } + + // The process last saw the node online beyond the grace period. + m.nodeLastOnline[polled.ID] = now.Add(-2 * nodeOfflineGracePeriod) + polled.LastSeen = now.Add(-2 * nodeOfflineGracePeriod) + m.state.UpdateNodesForInstance("pve-a", []models.Node{polled}) + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + m.markPVEInstanceNodesUnreachable("pve-a") + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + expired := m.state.GetSnapshot().Nodes + if len(expired) != 1 || expired[0].ID != tc.wantID || expired[0].Status != "offline" || expired[0].Temperature != nil { + t.Fatalf("expired nodes = %#v, want %q offline without a temperature", expired, tc.wantID) + } + if nodes := adapter.Nodes(); len(nodes) != 1 || nodes[0].ID() != unifiedID { + t.Fatalf("unified nodes after expiry = %v, want the single resource %q", nodes, unifiedID) + } + }) + } +} + +// Within grace a preserved node keeps its temperature only while the reading +// is inside the same carry window a failed collection gets. +func TestPreserveOrExpireNodesBoundsGraceTemperature(t *testing.T) { + m := &Monitor{config: &config.Config{}, state: models.NewState(), nodeLastOnline: make(map[string]time.Time)} + now := time.Now() + m.nodeLastOnline["pve-node1"] = now + stale := now.Add(-m.nodeTemperatureCarryWindow() - time.Minute) + preserved := m.preserveOrExpireNodes([]models.Node{{ + ID: "pve-node1", Name: "node1", Status: "online", LastSeen: now, + Temperature: &models.Temperature{CPUPackage: 70, Available: true, LastUpdate: stale}, + }}) + if len(preserved) != 1 || preserved[0].Status != "online" || preserved[0].Temperature != nil { + t.Fatalf("preserved = %#v, want online within grace without the stale temperature", preserved) + } +} + +// When detailed node status fails, the poller may carry its own last trusted +// memory reading for one cycle. On a node merged with a host agent, the read +// state holds the agent's memory, which is not the Proxmox value the snapshot +// vouches for, so the carry must use the snapshot and keep its used, cache +// and free split summing to the total. +func TestPollPVENodeCarriesOwnMemorySnapshot(t *testing.T) { + t.Setenv("PULSE_DATA_DIR", t.TempDir()) + const gib = uint64(1024 * 1024 * 1024) + + mon := newTestPVEMonitor("test") + defer mon.alertManager.Stop() + defer mon.notificationMgr.Stop() + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + mon.resourceStore = adapter + + client := &stubPVEClient{nodeStatus: &proxmox.NodeStatus{ + Memory: &proxmox.MemoryStatus{Total: 16 * gib, Used: 12 * gib, Free: 1 * gib, Available: 8 * gib}, + }} + node := proxmox.Node{Node: "node1", Status: "online", MaxMem: 16 * gib, Mem: 15 * gib, MaxCPU: 8} + + first, _, _, err := mon.pollPVENode(context.Background(), "test", &mon.config.PVEInstances[0], client, node, "healthy", nil) + if err != nil { + t.Fatalf("first pollPVENode() error = %v", err) + } + if !first.Memory.HasKnownUsage() || first.Memory.Used != int64(8*gib) || first.Memory.Cache == 0 { + t.Fatalf("first.Memory = %+v, want 8 GiB used with reclaimable cache split out", first.Memory) + } + mon.state.UpdateNodesForInstance("test", []models.Node{first}) + snapshot := mon.state.GetSnapshot() + snapshot.Hosts = []models.Host{{ + ID: "agent-1", Hostname: "node1", Status: "online", LinkedNodeID: first.ID, IntervalSeconds: 30, LastSeen: time.Now(), + Memory: models.Memory{Total: int64(16 * gib), Used: int64(3 * gib), Free: int64(13 * gib), Usage: 18.75}, + }} + adapter.PopulateFromSnapshot(snapshot) + + client.nodeStatus = nil + second, _, _, err := mon.pollPVENode(context.Background(), "test", &mon.config.PVEInstances[0], client, node, "healthy", mon.snapshotPrevNodes("test")) + if err != nil { + t.Fatalf("second pollPVENode() error = %v", err) + } + if second.Memory.Used != first.Memory.Used || second.Memory.Cache != first.Memory.Cache || second.Memory.Free != first.Memory.Free { + t.Fatalf("second.Memory = %+v, want the poller's own reading %+v", second.Memory, first.Memory) + } + if second.Memory.Used+second.Memory.Cache+second.Memory.Free != second.Memory.Total { + t.Fatalf("second.Memory = %+v, want used + cache + free = total", second.Memory) + } + if snap := mon.nodeSnapshots[makeNodeSnapshotKey("test", "node1")]; snap.MemorySource != "previous-snapshot" { + t.Fatalf("snapshot.MemorySource = %q, want previous-snapshot", snap.MemorySource) + } +} + +// Authoritative cluster membership writes previous nodes back too. Read from +// the unified store, an offline member keeps its source ID and agent link but +// stops presenting a temperature, and a member missing from /nodes keeps a +// reading only inside the carry window. +func TestPVEMembershipReconcileKeepsPreviousNodeIdentity(t *testing.T) { + cfg := &config.Config{PVEInstances: []config.PVEInstance{{ + Name: "cluster-api", Host: "https://pve-a:8006", IsCluster: true, ClusterName: "production", + }}} + m := newUnreachableTestMonitor(t, cfg) + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m.resourceStore = adapter + instanceCfg := &cfg.PVEInstances[0] + now := time.Now() + polled := func(name string, tempAge time.Duration) models.Node { + node := m.placeholderNodeForInstance("cluster-api", instanceCfg, name) + node.Status = "online" + node.ConnectionHealth = "healthy" + node.LastSeen = now + node.Temperature = &models.Temperature{CPUPackage: 60, Available: true, HasCPU: true, LastUpdate: now.Add(-tempAge)} + return node + } + offlineMember := polled("pve-a", 0) + offlineMember.LinkedAgentID = "agent-pve-a" + m.state.UpdateNodesForInstance("cluster-api", []models.Node{ + offlineMember, polled("pve-b", 0), polled("pve-c", m.nodeTemperatureCarryWindow()+time.Minute), + }) + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + + client := &membershipPVEClient{statuses: []proxmox.ClusterStatus{ + {Type: "cluster", Name: "production", Quorate: 1}, + {Type: "node", Name: "pve-a", Online: 0}, + {Type: "node", Name: "pve-b", Online: 1}, + {Type: "node", Name: "pve-c", Online: 1}, + }} + got := pveNodeByName(m.reconcilePVENodeInventory( + context.Background(), "cluster-api", instanceCfg, client, nil, m.snapshotPrevNodes("cluster-api"), + )) + + if node := got["pve-a"]; node.ID != "production-pve-a" || node.Status != "offline" || node.Temperature != nil || node.LinkedAgentID != "agent-pve-a" { + t.Fatalf("offline member = %#v, want production-pve-a offline with its agent link and no temperature", node) + } + if node := got["pve-b"]; node.ID != "production-pve-b" || node.Status != "unknown" || node.Temperature == nil { + t.Fatalf("unobserved member = %#v, want production-pve-b unknown with its recent temperature", node) + } + if node := got["pve-c"]; node.ID != "production-pve-c" || node.Status != "unknown" || node.Temperature != nil { + t.Fatalf("unobserved member = %#v, want production-pve-c unknown without its stale temperature", node) + } + + // Writing the reconciled inventory back keeps every member the same + // unified resource. + unifiedIDs := func() map[string]string { + ids := make(map[string]string) + for _, node := range adapter.Nodes() { + ids[node.NodeName()] = node.ID() + } + return ids + } + before := unifiedIDs() + m.state.UpdateNodesForInstance("cluster-api", sortedPVENodeInventory(got)) + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + if after := unifiedIDs(); len(after) != 3 || after["pve-a"] != before["pve-a"] || after["pve-b"] != before["pve-b"] || after["pve-c"] != before["pve-c"] { + t.Fatalf("unified IDs after writeback = %v, want %v", after, before) + } +} + +// A temperature carried over from an earlier poll keeps a lapsed host agent's +// lease: a reading the agent supplied (stamped no later than its last report) +// is dropped once the agent stops reporting, even when its latest report had +// no sensors, while a reading taken after that report keeps the carry window. +// The same rule bounds what an instance outage preserves within node grace. +func TestNodeTemperatureCarryRespectsAgentLease(t *testing.T) { + now := time.Now() + silentSince := now.Add(-hostAgentHealthWindow(30) - time.Minute) + instanceCfg := config.PVEInstance{Name: "pve1", Host: "https://10.0.0.1:8006"} + base := models.Node{ID: "pve1-node1", NodeIdentity: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", Type: "node", LastSeen: now} + + for _, tc := range []struct { + name string + readingAt time.Time + agentSensors bool + wantCarried bool + }{ + {name: "reading from the silent agent", readingAt: silentSince, agentSensors: true}, + {name: "silent agent whose last report had no sensors", readingAt: silentSince.Add(-30 * time.Second)}, + {name: "SSH reading taken after the agent went silent", readingAt: now.Add(-time.Minute), agentSensors: true, wantCarried: true}, + } { + t.Run(tc.name, func(t *testing.T) { + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m := &Monitor{ + config: &config.Config{TemperatureMonitoringEnabled: true, PVEInstances: []config.PVEInstance{instanceCfg}}, + state: models.NewState(), + resourceStore: adapter, + nodeLastOnline: map[string]time.Time{base.ID: now}, + } + agent := models.Host{ID: "agent-1", Hostname: "node1", Status: "offline", LinkedNodeID: base.ID, IntervalSeconds: 30, LastSeen: silentSince} + if tc.agentSensors { + agent.Sensors = models.HostSensorSummary{TemperatureCelsius: map[string]float64{"cpu_package": 92}} + } + published := base + published.Temperature = &models.Temperature{CPUPackage: 92, CPUMin: 92, CPUMaxRecord: 92, Available: true, HasCPU: true, LastUpdate: tc.readingAt} + m.state.UpdateNodesForInstance("pve1", []models.Node{published}) + snapshot := m.state.GetSnapshot() + snapshot.Hosts = []models.Host{agent} + adapter.PopulateFromSnapshot(snapshot) + + polled := base + m.collectNodeTemperatureData(context.Background(), "pve1", &instanceCfg, proxmox.Node{Node: "node1"}, &polled, m.snapshotPrevNodes("pve1"), "online") + if carried := polled.Temperature != nil; carried != tc.wantCarried { + t.Fatalf("failed collection carried = %v (%#v), want %v", carried, polled.Temperature, tc.wantCarried) + } + + preserved := m.preserveOrExpireNodes(m.snapshotPrevNodes("pve1")) + if len(preserved) != 1 || preserved[0].Status != "online" { + t.Fatalf("preserved = %#v, want the node online within grace", preserved) + } + if kept := preserved[0].Temperature != nil; kept != tc.wantCarried { + t.Fatalf("outage preservation kept temperature = %v, want %v", kept, tc.wantCarried) + } + }) + } +} + +// A carried node reading keeps the node's CPU temperature steady through a +// failed collection, but never its disk temperatures: those also stamp the +// node's physical disks, which have their own freshness and standby state. +// Here a current agent whose only sensor is a SMART disk reports that disk in +// standby, so collection returns nothing; the disk must not be re-stamped with +// its pre-sleep reading through the carry. +func TestCarriedNodeTemperatureLeavesDiskReadingsOut(t *testing.T) { + now := time.Now() + instanceCfg := config.PVEInstance{Name: "pve1", Host: "https://10.0.0.1:8006"} + base := models.Node{ID: "pve1-node1", NodeIdentity: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", Type: "node", LastSeen: now} + disk := models.PhysicalDisk{ID: "pve1-node1-sda", Node: "node1", Instance: "pve1", DevPath: "/dev/sda", Serial: "WD-STANDBY", Type: "hdd"} + + for _, tc := range []struct { + name string + previous models.Temperature + wantCPU bool + }{ + { + name: "CPU and disk reading", + previous: models.Temperature{ + Available: true, HasCPU: true, CPUPackage: 50, HasSMART: true, HasNVMe: true, + SMART: []models.DiskTemp{{Device: "/dev/sda", Serial: "WD-STANDBY", Temperature: 38}}, + NVMe: []models.NVMeTemp{{Device: "nvme0", Temp: 44}}, + }, + wantCPU: true, + }, + { + name: "disk-only reading", + previous: models.Temperature{ + Available: true, HasSMART: true, + SMART: []models.DiskTemp{{Device: "/dev/sda", Serial: "WD-STANDBY", Temperature: 38}}, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m := &Monitor{ + config: &config.Config{TemperatureMonitoringEnabled: true, PVEInstances: []config.PVEInstance{instanceCfg}}, + state: models.NewState(), + resourceStore: adapter, + nodeLastOnline: map[string]time.Time{base.ID: now}, + } + published := base + previous := tc.previous + previous.LastUpdate = now.Add(-time.Minute) + published.Temperature = &previous + m.state.UpdateNodesForInstance("pve1", []models.Node{published}) + snapshot := m.state.GetSnapshot() + snapshot.Hosts = []models.Host{{ + ID: "agent-1", Hostname: "node1", Status: "online", LinkedNodeID: base.ID, IntervalSeconds: 30, LastSeen: now, + Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{Device: "sda", Serial: "WD-STANDBY", Standby: true}}}, + }} + adapter.PopulateFromSnapshot(snapshot) + + polled := base + m.collectNodeTemperatureData(context.Background(), "pve1", &instanceCfg, proxmox.Node{Node: "node1"}, &polled, m.snapshotPrevNodes("pve1"), "online") + if tc.wantCPU { + if polled.Temperature == nil || polled.Temperature.CPUPackage != 50 || !polled.Temperature.LastUpdate.Equal(previous.LastUpdate) { + t.Fatalf("carried temperature = %#v, want CPU 50 with its original timestamp", polled.Temperature) + } + if len(polled.Temperature.SMART) != 0 || len(polled.Temperature.NVMe) != 0 || polled.Temperature.HasSMART || polled.Temperature.HasNVMe { + t.Fatalf("carried temperature kept disk readings: %#v", polled.Temperature) + } + } else if polled.Temperature != nil { + t.Fatalf("carried temperature = %#v, want none when only disk readings were left", polled.Temperature) + } + if stamped := mergeNVMeTempsIntoDisks([]models.PhysicalDisk{disk}, []models.Node{polled}); stamped[0].Temperature != 0 { + t.Fatalf("standby disk stamped with %d°C from a carried reading, want no reading", stamped[0].Temperature) + } + + // Outage preservation follows the same rule. + preserved := m.preserveOrExpireNodes(m.snapshotPrevNodes("pve1")) + if len(preserved) != 1 { + t.Fatalf("preserved = %#v, want one node", preserved) + } + if got := preserved[0].Temperature; (got != nil) != tc.wantCPU || (got != nil && (len(got.SMART) != 0 || len(got.NVMe) != 0)) { + t.Fatalf("preserved temperature = %#v, want CPU only (%v)", got, tc.wantCPU) + } + }) + } +} diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go index 8ecdeb2250..1e84d90190 100644 --- a/internal/monitoring/monitor_alert_handling_test.go +++ b/internal/monitoring/monitor_alert_handling_test.go @@ -1547,22 +1547,24 @@ func TestSilentLinkedAgentStopsRefreshingNodeTemperatureAlert(t *testing.T) { alertConfig.NodeDefaults.Temperature = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75} manager.UpdateConfig(alertConfig) - registry := unifiedresources.NewRegistry(nil) + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) monitor := &Monitor{ config: &config.Config{TemperatureMonitoringEnabled: true}, state: models.NewState(), - resourceStore: unifiedresources.NewMonitorAdapter(registry), + resourceStore: adapter, } node := models.Node{ ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", ConnectionHealth: "healthy", LinkedAgentID: "agent-1", } - // ingest records a PVE poll that just saw the node, plus the agent as of its - // last report. + // ingest rebuilds the store, as each monitor refresh does, from the last PVE + // poll's node (with the temperature it published, which the next poll reads + // back as its previous node) plus the agent as of its last report. + lastPolled := node ingest := func(agentLastReport time.Time, agentStatus string) { - polledNode := node + polledNode := lastPolled polledNode.LastSeen = time.Now() - registry.IngestSnapshot(models.StateSnapshot{ + adapter.PopulateFromSnapshot(models.StateSnapshot{ Nodes: []models.Node{polledNode}, Hosts: []models.Host{{ ID: "agent-1", Hostname: "node1", Status: agentStatus, LinkedNodeID: node.ID, @@ -1574,13 +1576,14 @@ func TestSilentLinkedAgentStopsRefreshingNodeTemperatureAlert(t *testing.T) { // poll runs the node temperature step of a PVE poll (no SSH collector) and // evaluates the node's alerts with the result. poll := func() *models.Temperature { - _, prevNodes := monitor.snapshotPrevNodes("pve1") + prevNodes := monitor.snapshotPrevNodes("pve1") polled := node monitor.collectNodeTemperatureData( context.Background(), "pve1", &config.PVEInstance{Name: "pve1"}, proxmox.Node{Node: node.Name}, &polled, prevNodes, "online", ) manager.CheckNode(polled) + lastPolled = polled return polled.Temperature } temperatureAlert := func() alerts.Alert { @@ -1605,8 +1608,15 @@ func TestSilentLinkedAgentStopsRefreshingNodeTemperatureAlert(t *testing.T) { opened := temperatureAlert() // The agent goes silent while PVE polling continues: the merged row stays - // fresh, but the agent's retained sensors no longer count as a reading. - ingest(time.Now().Add(-hostAgentHealthWindow(30)-time.Minute), "offline") + // fresh, but the agent's retained sensors no longer count as a reading, and + // the poller does not carry the agent's last reading past its lease. With no + // clock to advance, age the agent's last report and the reading it produced + // by the same amount. + silence := hostAgentHealthWindow(30) + time.Minute + agedReading := *lastPolled.Temperature + agedReading.LastUpdate = agedReading.LastUpdate.Add(-silence) + lastPolled.Temperature = &agedReading + ingest(agentReport.Add(-silence), "offline") time.Sleep(5 * time.Millisecond) require.Nil(t, poll(), "a silent agent's retained sensors must not be presented as a current reading") held := temperatureAlert() diff --git a/internal/monitoring/monitor_extra_coverage_test.go b/internal/monitoring/monitor_extra_coverage_test.go index 4060fc88cc..23caafd87e 100644 --- a/internal/monitoring/monitor_extra_coverage_test.go +++ b/internal/monitoring/monitor_extra_coverage_test.go @@ -2132,22 +2132,32 @@ func TestMonitor_PreviousNodesForInstance_Extra(t *testing.T) { registry := unifiedresources.NewRegistry(nil) registry.IngestSnapshot(models.StateSnapshot{ Nodes: []models.Node{ - {ID: "node-1", Instance: "pve1", Name: "a", DisplayName: "a", Memory: models.Memory{Total: 100}}, + { + ID: "node-1", Instance: "pve1", Name: "a", DisplayName: "a", ClusterName: "lab", + Memory: models.Memory{Total: 100, Used: 40, Free: 60, Usage: 40}, + Temperature: &models.Temperature{CPUPackage: 52, CPUMin: 41, CPUMaxRecord: 70, Available: true, HasCPU: true}, + }, {ID: "node-2", Instance: "pve2", Name: "b", DisplayName: "b", Memory: models.Memory{Total: 200}}, }, }) m.resourceStore = unifiedresources.NewMonitorAdapter(registry) - prevNodeMemory, prevNodes := m.previousNodesForInstance("pve1") + prevNodes := m.previousNodesForInstance("pve1") if len(prevNodes) != 1 || prevNodes[0].Instance != "pve1" || prevNodes[0].DisplayName != "a" { t.Fatalf("expected only pve1 nodes, got %#v", prevNodes) } - if mem, ok := prevNodeMemory[prevNodes[0].ID]; !ok || mem.Total != 100 { - t.Fatalf("expected node memory for node-1, got %#v", prevNodeMemory) + // The poller matches previous nodes by its own source ID, never the + // registry's unified resource ID. + if prevNodes[0].ID != "node-1" || prevNodes[0].NodeIdentity != "node-1" { + t.Fatalf("previous node ID = %q / identity %q, want source ID node-1", prevNodes[0].ID, prevNodes[0].NodeIdentity) } - if len(prevNodeMemory) != 1 { - t.Fatalf("expected node-2 to be filtered out, got %#v", prevNodeMemory) + if prevNodes[0].ClusterName != "lab" || prevNodes[0].Memory.Total != 100 || prevNodes[0].Memory.Used != 40 { + t.Fatalf("previous node lost cluster or memory: %#v", prevNodes[0]) + } + temp := prevNodes[0].Temperature + if temp == nil || temp.CPUPackage != 52 || temp.CPUMin != 41 || temp.CPUMaxRecord != 70 { + t.Fatalf("previous node temperature = %#v, want the stored reading with its min/max records", temp) } } diff --git a/internal/monitoring/monitor_memory_test.go b/internal/monitoring/monitor_memory_test.go index df2a659a1b..84042d56fd 100644 --- a/internal/monitoring/monitor_memory_test.go +++ b/internal/monitoring/monitor_memory_test.go @@ -366,6 +366,11 @@ func TestPollPVEInstanceMarksStaleNodesOfflineWhenGetNodesReturnsEmpty(t *testin staleNode := first.Nodes[0] staleNode.LastSeen = time.Now().Add(-nodeOfflineGracePeriod - 2*time.Second) mon.state.UpdateNodesForInstance("test", []models.Node{staleNode}) + // The grace policy also counts this process's own online sighting, keyed + // by the node's source ID, so age that too. + mon.mu.Lock() + mon.nodeLastOnline[staleNode.ID] = staleNode.LastSeen + mon.mu.Unlock() client.nodes = nil mon.pollPVEInstance(context.Background(), "test", client) diff --git a/internal/monitoring/monitor_polling_node.go b/internal/monitoring/monitor_polling_node.go index 035897cdc3..a2b301f63a 100644 --- a/internal/monitoring/monitor_polling_node.go +++ b/internal/monitoring/monitor_polling_node.go @@ -96,26 +96,31 @@ func collectPVENodeNetworkInterfaces( return mapPVENodeNetworkInterfaces(interfaces) } -func (m *Monitor) canCarryForwardNodeMemory(instance, node string, now time.Time) bool { +// carryForwardNodeMemory returns the node's last memory reading from this +// poller's own snapshot when that reading came from a trusted source recently +// enough to stand in for one failed status read. The unified read state is not +// used here: on a node merged with a host agent it holds the agent's reading, +// not the Proxmox value this snapshot vouches for. +func (m *Monitor) carryForwardNodeMemory(instance, node string, now time.Time) (models.Memory, bool) { if m == nil { - return false + return models.Memory{}, false } m.diagMu.RLock() snapshot, ok := m.nodeSnapshots[makeNodeSnapshotKey(instance, node)] m.diagMu.RUnlock() if !ok || !snapshot.Memory.HasKnownUsage() || snapshot.RetrievedAt.IsZero() { - return false + return models.Memory{}, false } age := now.Sub(snapshot.RetrievedAt) if age < 0 || age > nodeMemoryCarryForwardMaxAge { - return false + return models.Memory{}, false } switch CanonicalMemorySource(snapshot.MemorySource) { case "available-field", "derived-free-buffers-cached", "derived-total-minus-used", "rrd-memavailable", "rrd-memused": - return true + return snapshot.Memory, true default: - return false + return models.Memory{}, false } } @@ -126,7 +131,6 @@ func (m *Monitor) pollPVENode( client PVEClientInterface, node proxmox.Node, connectionHealthStr string, - prevNodeMemory map[string]models.Memory, prevInstanceNodes []models.Node, ) (models.Node, string, string, error) { nodeStart := time.Now() @@ -314,10 +318,8 @@ func (m *Monitor) pollPVENode( } // If we couldn't update memory metrics using detailed status, preserve previous accurate values if available - if !memoryUpdated && - effectiveStatus == "online" && - m.canCarryForwardNodeMemory(instanceName, node.Node, time.Now()) { - if prevMem, exists := prevNodeMemory[modelNode.ID]; exists && prevMem.HasKnownUsage() { + if !memoryUpdated && effectiveStatus == "online" { + if prevMem, ok := m.carryForwardNodeMemory(instanceName, node.Node, time.Now()); ok { total := int64(node.MaxMem) if total == 0 { total = prevMem.Total @@ -326,14 +328,14 @@ func (m *Monitor) pollPVENode( if total > 0 && used > total { used = total } - free := total - used - if free < 0 { - free = 0 - } + // Keep the snapshot's used | cache | free split summing to total. + cache := min(max(prevMem.Cache, 0), max(total-used, 0)) + free := max(total-used-cache, 0) preserved := prevMem preserved.Total = total preserved.Used = used + preserved.Cache = cache preserved.Free = free preserved.Usage = safePercentage(float64(used), float64(total)) diff --git a/internal/monitoring/monitor_polling_node_helpers.go b/internal/monitoring/monitor_polling_node_helpers.go index ced65961a2..e8ef1e849a 100644 --- a/internal/monitoring/monitor_polling_node_helpers.go +++ b/internal/monitoring/monitor_polling_node_helpers.go @@ -331,7 +331,15 @@ func (m *Monitor) collectNodeTemperatureData( } } - if prevTemp != nil && time.Since(prevTemp.LastUpdate) > m.nodeTemperatureCarryWindow() { + if m.carriedTemperatureOutlivesAgentLease(modelNode.ID, node.Node, prevTemp, time.Now()) { + // The reading came from a host agent that has stopped reporting. + // Its reporting lease already rode out transient gaps, so + // carrying it would outlive that lease. + log.Debug(). + Str("node", node.Node). + Time("lastUpdate", prevTemp.LastUpdate). + Msg("Dropped temperature data (host agent stopped reporting)") + } else if prevTemp != nil && time.Since(prevTemp.LastUpdate) > m.nodeTemperatureCarryWindow() { // Collection has stayed broken past the carry window. Re-presenting // the old reading would let alerts, history and the UI treat it as // a live measurement forever, so the node has no reading now. @@ -340,11 +348,9 @@ func (m *Monitor) collectNodeTemperatureData( Bool("isCluster", modelNode.IsClusterMember). Time("lastUpdate", prevTemp.LastUpdate). Msg("Dropped stale temperature data (collection has not returned a reading within the carry window)") - } else if prevTemp != nil { - // Clone the previous temperature to avoid modifying historical data - preserved := *prevTemp - preserved.LastUpdate = prevTemp.LastUpdate // Keep original update time to indicate staleness - modelNode.Temperature = &preserved + } else if preserved := carriedNodeTemperature(prevTemp); preserved != nil { + // The carried copy keeps its original LastUpdate to indicate staleness. + modelNode.Temperature = preserved log.Debug(). Str("node", node.Node). Bool("isCluster", modelNode.IsClusterMember). @@ -361,6 +367,30 @@ func (m *Monitor) collectNodeTemperatureData( } } +// carriedNodeTemperature returns the part of an earlier reading a node may +// keep presenting while collection fails: its CPU and GPU temperatures, with +// the original LastUpdate. Disk temperatures (SMART, NVMe) are left out. They +// also stamp the node's physical disks, which have their own freshness and +// standby state that a carried copy cannot follow: a disk that spun down would +// be stamped, and recorded, with its pre-sleep reading. Returns nil when the +// earlier reading held nothing else. +func carriedNodeTemperature(prev *models.Temperature) *models.Temperature { + if prev == nil || !prev.Available { + return nil + } + carried := *prev + carried.Cores = append([]models.CoreTemp(nil), prev.Cores...) + carried.GPU = append([]models.GPUTemp(nil), prev.GPU...) + carried.SMART = []models.DiskTemp{} + carried.HasSMART = false + carried.NVMe = []models.NVMeTemp{} + carried.HasNVMe = false + if !hasUsableTemperatureReading(&carried) { + return nil + } + return &carried +} + func (m *Monitor) applyNodePendingUpdates(ctx context.Context, instanceName string, client PVEClientInterface, node proxmox.Node, nodeID string, effectiveStatus string, modelNode *models.Node) { if modelNode == nil { return diff --git a/internal/monitoring/monitor_previous_state.go b/internal/monitoring/monitor_previous_state.go index bfd30496c9..79568436ef 100644 --- a/internal/monitoring/monitor_previous_state.go +++ b/internal/monitoring/monitor_previous_state.go @@ -3,6 +3,7 @@ package monitoring import ( "strings" + "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" ) @@ -89,24 +90,31 @@ func (m *Monitor) previousGuestContextForInstance(instanceName string) previousG return ctx } -func (m *Monitor) previousNodesForInstance(instanceName string) (map[string]models.Memory, []models.Node) { - prevNodeMemory := make(map[string]models.Memory) +func (m *Monitor) previousNodesForInstance(instanceName string) []models.Node { prevInstanceNodes := make([]models.Node, 0) readState := m.GetUnifiedReadStateOrSnapshot() if readState == nil { - return prevNodeMemory, prevInstanceNodes + return prevInstanceNodes } + // The registry keeps no copy of the identity stamps pollPVENode derives + // from configuration, so restore them the same way. Nodes preserved through + // an instance outage are written back to state, and without these the + // registry would derive a different canonical identity for them. + instanceCfg := m.getInstanceConfig(instanceName) + providerScoped := m.pveNodeUsesProviderScopedIdentity(instanceName, instanceCfg) for _, existingNode := range readState.Nodes() { if existingNode == nil || existingNode.Instance() != instanceName { continue } modelNode := previousNodeFromView(existingNode) - prevNodeMemory[modelNode.ID] = modelNode.Memory + modelNode.ProviderScopedIdentity = providerScoped + modelNode.NativeNameAliases = config.PVEClusterNodeNativeAliases(instanceCfg, modelNode.Name) + modelNode.TLSFingerprint = pveNodeTLSFingerprint(instanceCfg, modelNode.Name) prevInstanceNodes = append(prevInstanceNodes, modelNode) } - return prevNodeMemory, prevInstanceNodes + return prevInstanceNodes } func previousVMFromView(vm *unifiedresources.VMView) models.VM { @@ -199,20 +207,33 @@ func previousNodeFromView(node *unifiedresources.NodeView) models.Node { if node == nil { return models.Node{} } + // The poller keys nodes by their Proxmox source ID (nodeLastOnline, the + // temperature carry, state writes). The unified resource ID is a separate + // registry key, so returning it would match nothing, and writing it back + // during an outage would re-key the node on every failed poll. + id := node.SourceID() + if id == "" { + id = node.ID() + } return models.Node{ - ID: node.ID(), + ID: id, + NodeIdentity: id, Name: node.NodeName(), DisplayName: node.Name(), Instance: node.Instance(), Host: node.HostURL(), Status: string(node.Status()), + Type: "node", Uptime: node.Uptime(), IsClusterMember: node.IsClusterMember(), + ClusterName: node.ClusterName(), LastSeen: node.LastSeen(), LoadAverage: node.LoadAverage(), PVEVersion: node.PVEVersion(), KernelVersion: node.KernelVersion(), NetworkInterfaces: hostNetworkInterfacesFromReadStateView(node.NetworkInterfaces()), + Temperature: node.TemperatureDetails(), + LinkedAgentID: node.LinkedAgentID(), Memory: models.Memory{ Used: node.MemoryUsed(), Total: node.MemoryTotal(), diff --git a/internal/monitoring/monitor_pve.go b/internal/monitoring/monitor_pve.go index 1394b9a20d..7f5405f896 100644 --- a/internal/monitoring/monitor_pve.go +++ b/internal/monitoring/monitor_pve.go @@ -587,7 +587,7 @@ func (m *Monitor) updatePVEConnectionHealth(ctx context.Context, instanceName st return connectionHealthStr } -func (m *Monitor) snapshotPrevNodes(instanceName string) (map[string]models.Memory, []models.Node) { +func (m *Monitor) snapshotPrevNodes(instanceName string) []models.Node { return m.previousNodesForInstance(instanceName) } @@ -598,7 +598,6 @@ func (m *Monitor) pollPVENodesParallel( client PVEClientInterface, nodes []proxmox.Node, connectionHealthStr string, - prevNodeMemory map[string]models.Memory, prevInstanceNodes []models.Node, debugEnabled bool, ) ([]models.Node, map[string]string, map[string]string) { @@ -627,7 +626,7 @@ func (m *Monitor) pollPVENodesParallel( go func(node proxmox.Node) { defer wg.Done() - modelNode, effectiveStatus, diskSource, _ := m.pollPVENode(ctx, instanceName, instanceCfg, client, node, connectionHealthStr, prevNodeMemory, prevInstanceNodes) + modelNode, effectiveStatus, diskSource, _ := m.pollPVENode(ctx, instanceName, instanceCfg, client, node, connectionHealthStr, prevInstanceNodes) resultChan <- nodePollResult{ node: modelNode, @@ -655,7 +654,7 @@ func (m *Monitor) pollPVENodesParallel( // leaves the last successful snapshot in state forever, so a shut-down host // keeps showing its final online status (#1441). func (m *Monitor) markPVEInstanceNodesUnreachable(instanceName string) { - _, prevInstanceNodes := m.snapshotPrevNodes(instanceName) + prevInstanceNodes := m.snapshotPrevNodes(instanceName) if len(prevInstanceNodes) == 0 { // Never polled successfully this process lifetime (e.g. Pulse // started while the host was already down). Synthesize offline @@ -762,6 +761,7 @@ func (m *Monitor) preserveOrExpireNodes(prevInstanceNodes []models.Node) []model if nodeCopy.ConnectionHealth == "" || strings.EqualFold(nodeCopy.ConnectionHealth, "error") { nodeCopy.ConnectionHealth = "degraded" } + m.boundCarriedNodeTemperature(&nodeCopy, now) preserved = append(preserved, nodeCopy) continue } @@ -770,11 +770,28 @@ func (m *Monitor) preserveOrExpireNodes(prevInstanceNodes []models.Node) []model nodeCopy.ConnectionHealth = "error" nodeCopy.Uptime = 0 nodeCopy.CPU = 0 + nodeCopy.Temperature = nil preserved = append(preserved, nodeCopy) } return preserved } +// boundCarriedNodeTemperature applies the rules a failed live collection's +// carry follows to a temperature a node keeps from an earlier poll: the carry +// window, a lapsed host agent's lease for a reading that agent supplied, and +// no disk temperatures (carriedNodeTemperature). +func (m *Monitor) boundCarriedNodeTemperature(node *models.Node, now time.Time) { + if node.Temperature == nil { + return + } + if now.Sub(node.Temperature.LastUpdate) > m.nodeTemperatureCarryWindow() || + m.carriedTemperatureOutlivesAgentLease(node.ID, node.Name, node.Temperature, now) { + node.Temperature = nil + return + } + node.Temperature = carriedNodeTemperature(node.Temperature) +} + func (m *Monitor) seedNodeDisplayNames(modelNodes []models.Node) { for i := range modelNodes { if modelNodes[i].DisplayName != "" { @@ -1466,8 +1483,9 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie // Check if client is a ClusterClient to determine health status connectionHealthStr := m.updatePVEConnectionHealth(ctx, instanceName, client) - // Capture previous memory metrics so we can preserve them if detailed status fails - prevNodeMemory, prevInstanceNodes := m.snapshotPrevNodes(instanceName) + // Capture the previous poll's nodes for network, temperature and + // inventory continuity when this poll cannot read them. + prevInstanceNodes := m.snapshotPrevNodes(instanceName) // Convert to models modelNodes, nodeEffectiveStatus, nodeDiskSources := m.pollPVENodesParallel( @@ -1477,7 +1495,6 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie client, nodes, connectionHealthStr, - prevNodeMemory, prevInstanceNodes, debugEnabled, ) diff --git a/internal/monitoring/monitor_pve_cluster.go b/internal/monitoring/monitor_pve_cluster.go index 083948619f..d03383f925 100644 --- a/internal/monitoring/monitor_pve_cluster.go +++ b/internal/monitoring/monitor_pve_cluster.go @@ -431,11 +431,13 @@ func (m *Monitor) reconcilePVENodeInventory( node.ConnectionHealth = "error" node.CPU = 0 node.Uptime = 0 + node.Temperature = nil } else if _, observed := currentByName[key]; !observed { node.Status = "unknown" node.ConnectionHealth = "degraded" node.CPU = 0 node.Uptime = 0 + m.boundCarriedNodeTemperature(&node, time.Now()) } reconciled[key] = node } diff --git a/internal/monitoring/node_memory_sources_test.go b/internal/monitoring/node_memory_sources_test.go index 70687c67d0..5e93730088 100644 --- a/internal/monitoring/node_memory_sources_test.go +++ b/internal/monitoring/node_memory_sources_test.go @@ -242,7 +242,6 @@ func TestPollPVENodePrefersLinkedHostDiskOverRootFS(t *testing.T) { }, "healthy", nil, - nil, ) if err != nil { t.Fatalf("pollPVENode() error = %v", err) From 112ce030d7ae4287a35e32d7bf9e722b971afdb1 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:55:10 +0100 Subject: [PATCH 10/53] Stop reports and resource history calling a moved alert recovered A node alert that moves to its linked Pulse agent closes with a moved_to_agent resolution (#2580), and #2591 taught Patrol, alert history and the assistant to say so. Two readers still showed the close as a recovery. Performance reports dropped the resolution when copying recently resolved alerts into reporting.AlertInfo. A node report generated in the five-minute window after a handover read HEALTHY, "All systems operating normally", counted the alert as triggered and resolved, and told the AI narrator resolved: true, although nothing had seen the condition clear. Report rows now carry the resolution and the alert's resource. pkg/reporting treats an alert that closed without recovering as still live unless the report also shows its successor's reading for that metric (an open alert, or a recovery since the handover), so a handover never reads as healthy and one condition is counted once; fleet reports match handovers across all their resources. The PDF card, Quick Stats, active list and heuristic narrative say where the alert went and to check the agent, the alerts table shows "Moved" with the engine's summary, fleets count it as active and name the agent, and the narrator gets resolved: false, the summary and whether the successor's alert is listed, with prompt rules for both. Resource change readers labelled every alert_resolved change "Alert resolved" in green and prefixed the headline with it, so a moved close read "Alert resolved: Alert moved to pve1 (Host Agent). This is not a recovery...". getResourceChangePresentation now reads the alert_resolution metadata: such a change is "Alert moved" in the neutral blue of the history badge, and its summary is the whole headline in the drawer, ResourceChangeSummary and Patrol's assessment handoff. Reports requested as agent, which is how the Reports page addresses every Proxmox node, still carry no alert rows at all, and node alert history is stored under legacy node IDs that the drawer never queries; both are separate fixes. --- .../v6/internal/subsystems/agent-lifecycle.md | 5 + .../v6/internal/subsystems/ai-runtime.md | 11 +- .../v6/internal/subsystems/api-contracts.md | 33 +++ .../subsystems/frontend-primitives.md | 6 + .../subsystems/patrol-intelligence.md | 6 + .../internal/subsystems/storage-recovery.md | 4 + .../internal/subsystems/unified-resources.md | 20 +- .../3cc753f21d394de0.json | 45 +++ .../Infrastructure/ResourceChangeSummary.tsx | 4 +- .../ResourceDetailDrawerOverviewTab.tsx | 4 +- .../ResourceDetailDrawer.history.test.tsx | 60 ++++ .../patrolInvestigationContextModel.test.ts | 27 ++ .../patrol/patrolInvestigationContextModel.ts | 8 + .../resourceChangePresentation.test.ts | 55 ++++ .../src/utils/resourceChangePresentation.ts | 62 ++++ internal/ai/report_narrator.go | 24 +- internal/ai/report_narrator_test.go | 42 +++ internal/api/contract_test.go | 61 ++++ internal/api/metrics_reporting_handlers.go | 91 +++--- pkg/reporting/alert_state.go | 229 +++++++++++++++ pkg/reporting/alert_state_test.go | 273 ++++++++++++++++++ pkg/reporting/fleet_narrative.go | 30 +- pkg/reporting/narrative.go | 77 ++--- pkg/reporting/pdf.go | 165 ++++++----- pkg/reporting/reporting.go | 22 ++ 25 files changed, 1159 insertions(+), 205 deletions(-) create mode 100644 frontend-modern/browser-verification/3cc753f21d394de0.json create mode 100644 pkg/reporting/alert_state.go create mode 100644 pkg/reporting/alert_state_test.go diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index ec3fc5d064..b88a424f2d 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -2727,6 +2727,11 @@ agent inventory, registration state, or command-channel readiness. reports, but workspace logo material remains API/security/reporting ownership and must not become agent credential, install-token, or fleet lifecycle state. + Report alert rows built there carry an alert's handover resolution, so a + node alert that moved to its linked Pulse agent reads as moved, not + recovered. That is report presentation only: the report never decides + agent linkage, ownership of a metric, or agent freshness, and it adds no + lifecycle state. The same isolation rule applies to Patrol investigation-record propagation through shared AI intelligence handlers and `internal/api/router.go`: lifecycle surfaces may observe the resulting resource context, but they must diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 56b2a47c66..d233ab4bf9 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -4651,9 +4651,14 @@ metadata when it projects from the canonical timeline). `TestService_buildAlertContext_HandoverIsNotARecovery`, `TestExecuteListResolvedAlertsReportsHandover` and `TestIncidentStore_HandoverCloseIsNotARecovery` pin it. Performance reports -(`pkg/reporting`, the report narrator) do not carry the resolution yet: a -report generated within the five-minute recently-resolved window after a -handover still lists that alert as resolved. The projection also collapses open/resolved churn: +carry the resolution too: the report narrator sends such an alert with +`resolved: false` and the engine's summary as `resolution`, plus +`successor_alert_listed` when the same report lists the alert the resource +it moved to raised for that metric. Its system prompt forbids counting the +alert as resolved or calling the resource healthy on its account, and tells +the model to describe a covered condition once, from the successor's alert +(`TestBuildReportNarratorPayload_HandoverIsNotARecovery`). The +deterministic verdict lives in `pkg/reporting` (see api-contracts). The projection also collapses open/resolved churn: when a record's timeline holds at least four open-to-resolved or resolved-to-open transitions inside the last 24 hours, the item carries a `flapping` summary (transition count, window, first and latest transition) and diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 8454240e42..b8d575681b 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -9598,6 +9598,39 @@ narrator must fail closed: nil provider, parse failure, timeout, or empty response causes the engine to fall back to the heuristic narrative without surfacing the AI failure to the caller, so reporting is never blocked by AI availability. +Report alerts keep the alert engine's resolution. The node, VM and +container enrichers in `internal/api/metrics_reporting_handlers.go` build +every row through `reportActiveAlertInfo` and `reportResolvedAlertInfo`, +which carry the alert's `ResourceID` and copy `models.AlertResolution` into +`reporting.AlertInfo.Resolution`, so a node alert that moved to its Pulse +agent inside the five-minute recently-resolved window is never reported as +a recovery (`TestContract_ReportingAlertsCarryHandoverResolution`). In +`pkg/reporting`, `AlertInfo.Recovered()` is true only for a close without a +resolution, and `unresolvedAlerts` treats an alert that closed without +recovering as still live unless the report also shows its successor's own +reading for that metric: an alert of the same type on the successor or one +of its children that is open, or recovered at or after the handover (an +earlier successor recovery does not count). That alert then carries the +condition, so it is counted once. Fleet reports match each handover against +every resource's alerts (`unresolvedAlertsAmong` over +`MultiReportData.fleetAlerts`). The single-resource PDF health card, Quick +Stats and active list and the heuristic narrator's verdict, observations and +recommendations read that rule: a lone handover renders WARNING (or CRITICAL +at its level) with `Alert moved to - check the agent for the current +reading`, and the alerts table shows `Moved` in neutral text with the +engine's summary as a note. Fleet reports count it as an active alert in the +aggregate, health card, table and resource blocks, and the fleet heuristic +recommends checking the agent it moved to. "Triggered and resolved" counts +recoveries only, and the report narrator's payload marks a covered handover +with `successor_alert_listed` so the model describes the condition once. +Reports requested as `agent`, which is how the Reports picker addresses every +Proxmox node and standalone agent in v6, carry no alert rows yet: +`enrichReportRequest` enriches node, VM and container reports only, so an +agent report's deterministic verdict ignores its machine's alerts (HEALTHY, +or NO DATA without metrics). The rule above reaches those reports once they +attach the machine's alerts. The fleet heuristic offers its all-clear +pattern and "No fleet-wide action required" only while no alert, moved +ones included, is active. Multi-resource fleet reports (`engine.GenerateMulti`) now also carry an optional fleet-level narrative through a distinct `pkg/reporting.FleetNarrator` interface, kept separate from the diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 710f4ef07e..fc27a49e3a 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -7120,6 +7120,12 @@ row carries the account in the badge title and the phone card renders it under the message, while `getAlertHistoryStatusPresentation` and `getAlertResolutionDetail` in `utils/alertIncidentPresentation.ts` own the wording. +Resource-change readers follow the same split: `getResourceChangePresentation` +and `formatResourceChangeHeadline` in `utils/resourceChangePresentation.ts` +own the `Alert moved` label, its neutral-blue tone and the summary headline +for an `alert_resolved` change with `alert_resolution` metadata, and the +Patrol assessment handoff consumes that headline instead of composing its own +kind prefix. `frontend-modern/src/features/alerts/useAlertHistoryState.ts` re-exposes the `getResource` resolver it is already given, and `frontend-modern/src/features/alerts/AlertResourceIncidentsPanel.tsx` reads it diff --git a/docs/release-control/v6/internal/subsystems/patrol-intelligence.md b/docs/release-control/v6/internal/subsystems/patrol-intelligence.md index 251bfc8252..46765dbb24 100644 --- a/docs/release-control/v6/internal/subsystems/patrol-intelligence.md +++ b/docs/release-control/v6/internal/subsystems/patrol-intelligence.md @@ -1388,6 +1388,12 @@ condition went, and that it is not a recovery) sits directly under the summary rather than only inside Evidence and history (`getAttentionHandoverClose` in `patrolHomePresentation.ts`). The timeline names that cause `Ownership transferred`, never `Recovery evidence`. +The assessment handoff tells the same close the same way: a recent +`alert_resolved` change carrying `alert_resolution` metadata enters the +handoff context and briefing evidence as the alert engine's summary through +the shared `formatResourceChangeHeadline`, never as `Alert resolved: ...` +(`patrolInvestigationContextModel.ts`, pinned by +`patrolInvestigationContextModel.test.ts`). The active Patrol queue now uses compact severity-accented rows for prioritisation and a single focused review panel for the selected issue. The diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 626b08f763..e90edfe932 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -1470,6 +1470,10 @@ recovery scope, or a storage/recovery-owned secret source. generated report output when a separate reporting surface exposes it, but workspace logo settings are not backup artifacts, recovery-point metadata, restore evidence, or storage-provider credentials. + The alert rows that request assembly builds now keep an alert's + handover resolution (moved to a Pulse agent, not recovered). That changes + how a report states alert health only; it opens no storage, backup or + recovery path, and report backups still come from the recovery store. Update-plan readiness payloads and apply-route readiness enforcement are adjacent shared API context only. Storage and recovery surfaces may observe the resulting update state if a future settings flow links to recovery diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 7fe6d54e2f..829435c249 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -4228,10 +4228,22 @@ An `alert_resolved` change for a close that was not a recovery carries the alert engine's summary as its reason and the reason code as `alert_resolution` metadata (`AlertTimelineChange.ResolutionReason` and `ResolutionSummary`), so incident projection titles the close with where the alert went instead of -"Alert resolved: ". The resource drawer still labels the change -by kind (`Alert resolved`) and prefixes its headline with that label; only the -reason after it says the alert moved. Ordinary recoveries omit the key, and -every other lifecycle kind ignores the fields. +"Alert resolved: ". Frontend readers label the change through +`getResourceChangePresentation(change)` rather than by kind alone: with the +metadata set, the drawer's change history titles the entry `Alert moved` and +the `ResourceChangeSummary` badge reads `Alert moved` in the neutral blue of +the alert history's "moved to agent" badge (`Alert closed` for a reason code +the build does not know), and +`formatResourceChangeHeadline` uses the summary as the whole headline instead +of prefixing `Alert resolved:`; Patrol's assessment context reuses that +headline. A node's own drawer does not list the move yet: node and guest +alert changes are still recorded under their legacy alert resource IDs +(`MonitorAdapter.RecordChange` maps only Docker IDs), and Proxmox platform +rows open the drawer in the `table-row` presentation, which fetches no +remote history. The facet badges +still count these changes under `alert_resolved`, because the counts carry no +metadata. Ordinary recoveries omit the key, and every other lifecycle kind +ignores the fields. Alert-scoped incident memory may still project those events for one investigation thread, but the durable source of truth for resource-affecting alert lifecycle and diff --git a/frontend-modern/browser-verification/3cc753f21d394de0.json b/frontend-modern/browser-verification/3cc753f21d394de0.json new file mode 100644 index 0000000000..2c2808e61e --- /dev/null +++ b/frontend-modern/browser-verification/3cc753f21d394de0.json @@ -0,0 +1,45 @@ +{ + "version": 1, + "base_sha": "d26a4837315b40b47a73dbf1ff1cb4c8d6075225", + "verified_at": "2026-10-06T15:52:00Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Infrastructure/ResourceChangeSummary.tsx", + "frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx", + "frontend-modern/src/features/patrol/patrolInvestigationContextModel.ts", + "frontend-modern/src/utils/resourceChangePresentation.ts" + ], + "content_sha256": { + "frontend-modern/src/components/Infrastructure/ResourceChangeSummary.tsx": "a7d8728f72de5b9acd73832064f037af97978b78859cb2c69a01e25f0750e6d5", + "frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx": "f0f868d614ef70165b77e80f220cc5927936bdfcfa4eff3a8f18b728acaf5b52", + "frontend-modern/src/features/patrol/patrolInvestigationContextModel.ts": "e172bfead859ece6f7f9998dcd765c6f348bbfc25214089d6ba1a150f30d6fec", + "frontend-modern/src/utils/resourceChangePresentation.ts": "5a8d67301975d89545c866116e1e0e7e633881b3b3ce4da37b825a46cf65265d" + }, + "routes": [ + "/proxmox/overview?agent=Production+West-pve1", + "/standalone/availability?view=fleet" + ], + "viewports": [ + { + "width": 1280, + "height": 900 + }, + { + "width": 375, + "height": 812 + } + ], + "states": [ + "Private mock stack (backend :7741 from this branch plus a local-only go build -overlay hook, Vite :5241 from this clone): the real alerts manager raised pve1 node memory and CPU alerts, the hook closed the memory alert through clearAlertWithResolution with a moved_to_agent resolution (successor pve1 (Host Agent)), and threshold restore recovered CPU normally; the monitor recorded both alert_resolved changes, the moved one with the engine summary as reason and alert_resolution metadata", + "Proxmox overview, West Production A inline drawer (table-row presentation): Overview and History tabs render, no change history section, no facets request; table-row drawers skip remote history by design and node alert changes are stored under the legacy node ID mock-cluster-1-pve1, so the move cannot appear in a node drawer today", + "Availability checks Fleet view, Rack UPS network card full drawer, with the two producer-made alert_resolved rows (plus their alert_fired rows) copied from mock-cluster-1-pve1 to this endpoint's canonical ID in unified_resources.db as a fixture: at 1280 the change history lists 'Alert moved' over 'Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading.' and the ordinary close still reads 'Alert resolved' over 'Alert resolved: Node cpu 5-minute average at 33.7% (current 33.7%)'", + "Same drawer at 375: both entries wrap inside the card, the moved entry is titled Alert moved, no horizontal page scroll; facet chips still read 'Alert resolved 2' because kind counts carry no metadata (documented in unified-resources)", + "Console: only the known private-port websocket rejection (ws://127.0.0.1:5241/ws, ALLOWED_ORIGINS lists 5173/7655), no errors from the drawer" + ], + "interactions": [ + "Expanded West Production A on /proxmox/overview, opened its History tab and checked the DOM and network log for a change history section and a facets request", + "Switched Availability checks to the Fleet view through View preferences and opened the Rack UPS network card tile", + "Scrolled the change history to the Alert moved entry at 1280x900 and read every entry title and reason", + "Reloaded at 375x812, reopened the tile, read the entries and compared document width with the viewport" + ] +} diff --git a/frontend-modern/src/components/Infrastructure/ResourceChangeSummary.tsx b/frontend-modern/src/components/Infrastructure/ResourceChangeSummary.tsx index 05518bf5e9..a05fd6374e 100644 --- a/frontend-modern/src/components/Infrastructure/ResourceChangeSummary.tsx +++ b/frontend-modern/src/components/Infrastructure/ResourceChangeSummary.tsx @@ -2,7 +2,7 @@ import { type Component, For, Show, createMemo } from 'solid-js'; import type { ResourceChange } from '@/types/resource'; import { formatRelativeTime } from '@/utils/format'; import { - getResourceChangeKindPresentation, + getResourceChangePresentation, getResourceChangeSourceAdapterPresentation, getResourceChangeSourceTypePresentation, formatResourceChangeHeadline, @@ -83,7 +83,7 @@ export const ResourceChangeSummary: Component = (pro const shouldShowReason = reason.length > 0 && reason !== headline; const resourceHref = buildResourceHref(change.resourceId); const relatedResources = (change.relatedResources ?? []).slice(0, 3); - const kindPresentation = getResourceChangeKindPresentation(change.kind); + const kindPresentation = getResourceChangePresentation(change); const sourceTypePresentation = getResourceChangeSourceTypePresentation( change.sourceType, ); diff --git a/frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx b/frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx index dc927cdc7f..189a4f6942 100644 --- a/frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx +++ b/frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx @@ -48,6 +48,7 @@ import { RESOURCE_CHANGE_SOURCE_ADAPTER_ORDER, RESOURCE_CHANGE_SOURCE_TYPE_ORDER, getResourceChangeKindPresentation, + getResourceChangePresentation, getResourceChangeSourceAdapterPresentation, getResourceChangeSourceTypePresentation, } from '@/utils/resourceChangePresentation'; @@ -592,7 +593,8 @@ export const ResourceDetailDrawerOverviewTab: Component {(change) => { - const kindPresentation = getResourceChangeKindPresentation(change.kind); + // A close that was not a recovery reads as a move, not "Alert resolved". + const kindPresentation = getResourceChangePresentation(change); const sourceTypePresentation = getResourceChangeSourceTypePresentation( change.sourceType, ); diff --git a/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailDrawer.history.test.tsx b/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailDrawer.history.test.tsx index 800c3f0b7f..f5e53ea7de 100644 --- a/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailDrawer.history.test.tsx +++ b/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailDrawer.history.test.tsx @@ -916,6 +916,66 @@ describe('ResourceDetailDrawer change history section', () => { expect(panel.queryByText('Runs on')).toBeNull(); }); + it('labels an alert that moved to its Pulse agent as a move, not a recovery', async () => { + const summary = + 'Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading.'; + facetBundleMock.getFacetBundle.mockResolvedValueOnce({ + recentChanges: [ + { + id: 'change-moved', + observedAt: '2026-10-06T15:00:00Z', + occurredAt: '2026-10-06T15:00:00Z', + resourceId: 'node:pve1', + kind: 'alert_resolved', + sourceType: 'heuristic', + confidence: 'high', + reason: summary, + metadata: { + alert_identifier: 'pve1-memory', + alert_type: 'memory', + alert_resolution: 'moved_to_agent', + }, + }, + { + id: 'change-recovered', + observedAt: '2026-10-06T14:00:00Z', + resourceId: 'node:pve1', + kind: 'alert_resolved', + sourceType: 'heuristic', + confidence: 'high', + reason: 'Alert resolved: Node CPU at 91%', + metadata: { alert_identifier: 'pve1-cpu', alert_type: 'cpu' }, + }, + ], + counts: { recentChanges: 2, recentChangeKinds: { alert_resolved: 2 } }, + }); + + render(() => ( + + )); + + await screen.findByText('Change history'); + const panel = within(screen.getByTestId('resource-change-history-section')); + const movedReason = await panel.findByText(summary); + const movedEntry = movedReason.parentElement as HTMLElement; + expect(within(movedEntry).getByText('Alert moved')).toBeInTheDocument(); + expect(within(movedEntry).queryByText('Alert resolved')).toBeNull(); + + // An ordinary recovery keeps its kind label. + const recoveredEntry = panel.getByText('Alert resolved: Node CPU at 91%') + .parentElement as HTMLElement; + expect(within(recoveredEntry).getByText('Alert resolved')).toBeInTheDocument(); + }); + it('surfaces resource-scoped action history from the canonical action audit API', async () => { actionAuditMock.listActionAudits.mockResolvedValueOnce({ available: true, diff --git a/frontend-modern/src/features/patrol/__tests__/patrolInvestigationContextModel.test.ts b/frontend-modern/src/features/patrol/__tests__/patrolInvestigationContextModel.test.ts index 9e11a59853..3b9656c15b 100644 --- a/frontend-modern/src/features/patrol/__tests__/patrolInvestigationContextModel.test.ts +++ b/frontend-modern/src/features/patrol/__tests__/patrolInvestigationContextModel.test.ts @@ -165,6 +165,33 @@ describe('patrolInvestigationContextModel', () => { expect(handoff.context.handoffContext).not.toContain('online to online'); }); + it("tells a moved alert close in the alert engine's words, not as a recovery", () => { + const summary = + 'Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading.'; + const handoff = buildPatrolAssessmentAssistantHandoff({ + assessment: { title: 'Issues detected' }, + supportingEvidence: { + recentChanges: [ + { + id: 'change-moved', + observedAt: '2026-10-06T15:00:00Z', + resourceId: 'node:pve1', + kind: 'alert_resolved', + sourceType: 'heuristic', + confidence: 'high', + reason: summary, + metadata: { alert_resolution: 'moved_to_agent', alert_type: 'memory' }, + }, + ], + }, + activeFindings: [], + }); + + expect(handoff.context.handoffContext).toContain(summary); + expect(handoff.context.handoffContext).not.toContain('Alert resolved'); + expect(handoff.context.briefing?.evidence?.join('\n')).toContain(summary); + }); + it('includes bounded related-resource context in assessment handoff evidence', () => { const longRelatedResource = 'storage-pool-with-a-very-long-description-that-keeps-going-beyond-the-handoff-limit-for-operators'; diff --git a/frontend-modern/src/features/patrol/patrolInvestigationContextModel.ts b/frontend-modern/src/features/patrol/patrolInvestigationContextModel.ts index 1d1a648191..2cf8f21eaa 100644 --- a/frontend-modern/src/features/patrol/patrolInvestigationContextModel.ts +++ b/frontend-modern/src/features/patrol/patrolInvestigationContextModel.ts @@ -15,7 +15,9 @@ import type { } from '@/stores/aiChat'; import type { ResourceChange } from '@/types/resource'; import { + formatResourceChangeHeadline, formatResourceChangeKind, + getResourceChangeAlertResolution, sortResourceChangesByObservedAt, } from '@/utils/resourceChangePresentation'; import { @@ -1736,6 +1738,12 @@ function formatAssessmentCorrelationContextLine( } function formatAssessmentRecentChangeSummary(change: ResourceChange): string { + // An alert that closed without recovering (moved to its Pulse agent) is + // told in the alert engine's words, never as "Alert resolved". + if (getResourceChangeAlertResolution(change)) { + return truncateContextText(formatResourceChangeHeadline(change), 160); + } + const kind = formatResourceChangeKind(change.kind); if (isCommandBearingResourceChange(change)) { return `${kind}: execution event recorded`; diff --git a/frontend-modern/src/utils/__tests__/resourceChangePresentation.test.ts b/frontend-modern/src/utils/__tests__/resourceChangePresentation.test.ts index c6539e2bbf..60c2bb9be0 100644 --- a/frontend-modern/src/utils/__tests__/resourceChangePresentation.test.ts +++ b/frontend-modern/src/utils/__tests__/resourceChangePresentation.test.ts @@ -3,7 +3,9 @@ import { describe, expect, it } from 'vitest'; import { formatResourceChangeHeadline, formatResourceChangeKind, + getResourceChangeAlertResolution, getResourceChangeKindPresentation, + getResourceChangePresentation, getResourceChangeSourceAdapterPresentation, getResourceChangeSourceTypePresentation, sortResourceChangesByObservedAt, @@ -90,6 +92,59 @@ describe('resourceChangePresentation utils', () => { }); }); + it('presents an alert close that was not a recovery as a move', () => { + const summary = + 'Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading.'; + const moved = { + id: 'change-moved', + resourceId: 'node:pve1', + kind: 'alert_resolved' as const, + observedAt: '2026-10-06T15:00:00Z', + sourceType: 'heuristic' as const, + confidence: 'high' as const, + reason: summary, + metadata: { alert_resolution: 'moved_to_agent', alert_type: 'memory' }, + }; + + expect(getResourceChangeAlertResolution(moved)).toBe('moved_to_agent'); + expect(getResourceChangePresentation(moved)).toEqual({ + label: 'Alert moved', + plural: 'Alerts moved', + className: 'bg-blue-100 text-blue-700 dark:bg-blue-900/25 dark:text-blue-300', + }); + // The engine's account is the whole headline, never "Alert resolved: ...". + expect(formatResourceChangeHeadline(moved)).toBe(summary); + expect(formatResourceChangeHeadline({ ...moved, reason: undefined })).toBe( + 'Alert moved: node:pve1', + ); + + // A reason code this build does not know still never reads as a recovery. + expect( + getResourceChangePresentation({ + kind: 'alert_resolved', + metadata: { alert_resolution: 'some_future_reason' }, + }).label, + ).toBe('Alert closed'); + + // Ordinary recoveries, and other kinds, keep their kind presentation. + const recovered = { + ...moved, + metadata: { alert_type: 'memory' }, + reason: 'Node memory at 95%', + }; + expect(getResourceChangeAlertResolution(recovered)).toBeUndefined(); + expect(getResourceChangePresentation(recovered)).toEqual( + getResourceChangeKindPresentation('alert_resolved'), + ); + expect(formatResourceChangeHeadline(recovered)).toBe('Alert resolved: Node memory at 95%'); + expect( + getResourceChangePresentation({ + kind: 'alert_fired', + metadata: { alert_resolution: 'moved_to_agent' }, + }).label, + ).toBe('Alert fired'); + }); + it('formats shared confidence percentages', () => { expect(formatConfidencePercentage(0.5)).toBe('50%'); expect(formatConfidencePercentage(0.875)).toBe('88%'); diff --git a/frontend-modern/src/utils/resourceChangePresentation.ts b/frontend-modern/src/utils/resourceChangePresentation.ts index 8e5dd04848..bcb612cba1 100644 --- a/frontend-modern/src/utils/resourceChangePresentation.ts +++ b/frontend-modern/src/utils/resourceChangePresentation.ts @@ -210,6 +210,62 @@ export function getResourceChangeKindPresentation( ); } +// Metadata key carrying the reason code of an alert_resolved change whose +// close was not a recovery (unifiedresources.MetadataAlertResolution). +// Ordinary recoveries omit it. +const ALERT_RESOLUTION_METADATA_KEY = 'alert_resolution'; + +// A close that was not a recovery is neither a live problem nor a healthy +// outcome, so it reads in the same neutral blue as the alert history's +// "moved to agent" badge. +const ALERT_CLOSED_WITHOUT_RECOVERY_CLASS = + 'bg-blue-100 text-blue-700 dark:bg-blue-900/25 dark:text-blue-300'; + +const ALERT_CLOSED_WITHOUT_RECOVERY_PRESENTATIONS: Record = + { + moved_to_agent: { + label: 'Alert moved', + plural: 'Alerts moved', + className: ALERT_CLOSED_WITHOUT_RECOVERY_CLASS, + }, + }; + +const ALERT_CLOSED_WITHOUT_RECOVERY_FALLBACK: ResourceChangeLabelPresentation = { + label: 'Alert closed', + plural: 'Alerts closed', + className: ALERT_CLOSED_WITHOUT_RECOVERY_CLASS, +}; + +/** + * The reason code of an alert_resolved change that closed without recovering, + * such as "moved_to_agent" when a node alert moved to its Pulse agent. + * Undefined for an ordinary recovery and every other kind. + */ +export function getResourceChangeAlertResolution( + change: Pick, +): string | undefined { + if (change.kind !== 'alert_resolved') return undefined; + const reason = change.metadata?.[ALERT_RESOLUTION_METADATA_KEY]; + return typeof reason === 'string' && reason.trim() !== '' ? reason.trim() : undefined; +} + +/** + * Presentation for one change: its kind's, except that an alert close that + * was not a recovery reads as a move instead of a green "Alert resolved". + */ +export function getResourceChangePresentation( + change: Pick, +): ResourceChangeLabelPresentation { + const resolution = getResourceChangeAlertResolution(change); + if (resolution) { + return ( + ALERT_CLOSED_WITHOUT_RECOVERY_PRESENTATIONS[resolution] ?? + ALERT_CLOSED_WITHOUT_RECOVERY_FALLBACK + ); + } + return getResourceChangeKindPresentation(change.kind); +} + export function getResourceChangeSourceTypePresentation( sourceType: ResourceChangeSourceType | string, ): ResourceChangeLabelPresentation { @@ -266,6 +322,12 @@ export function formatResourceChangeKind(kind: ResourceChange['kind']): string { } export function formatResourceChangeHeadline(change: ResourceChange): string { + // The alert engine's account of a close that was not a recovery already + // says where the alert went, so it is the headline on its own. + if (getResourceChangeAlertResolution(change)) { + const reason = change.reason?.trim(); + return reason || `${getResourceChangePresentation(change).label}: ${change.resourceId}`; + } if (change.kind === 'state_transition' && change.from && change.to) { return `${formatResourceChangeKind(change.kind)}: ${change.from} → ${change.to}`; } diff --git a/internal/ai/report_narrator.go b/internal/ai/report_narrator.go index 8ea890cbc0..7160e880bd 100644 --- a/internal/ai/report_narrator.go +++ b/internal/ai/report_narrator.go @@ -38,6 +38,7 @@ You MUST: - Keep observations to short, concrete sentences. Avoid hedging adverbs ("perhaps", "seems"). State the fact and its implication. - When prior-period data is supplied, write a period_comparison paragraph describing the most material deltas (resource trends, new or resolved alerts, new findings). When no prior data is supplied, leave period_comparison empty. - Do NOT invent recommendations for problems that aren't in the data. If everything is healthy, say so plainly. +- An alert with a "resolution" closed without recovering, for example because the Pulse agent now monitoring the machine took its metric over. It is not a recovery: never count it as resolved or call the resource healthy because of it, and say where it went using the resolution text. When it also has "successor_alert_listed": true, the alerts list holds the alert the resource it moved to raised for that metric; describe the condition once, from that alert. DETECTION BOUNDARY — this is critical: Pulse Patrol is the canonical detection layer for this product. The patrol_findings array, when populated, contains issues Patrol has already classified with its own severity. Your job is to SUMMARIZE those findings, not to function as a parallel detector competing with them. @@ -118,7 +119,14 @@ type reportNarratorAlert struct { Message string `json:"message"` Value float64 `json:"value,omitempty"` Threshold float64 `json:"threshold,omitempty"` - Resolved bool `json:"resolved"` + // Resolved is true only for a recovery. An alert that closed without + // recovering, such as a node alert that moved to its Pulse agent, carries + // the alert engine's account in Resolution instead, and + // SuccessorAlertListed when this list also holds the alert the resource + // it moved to raised for the same metric. + Resolved bool `json:"resolved"` + Resolution string `json:"resolution,omitempty"` + SuccessorAlertListed bool `json:"successor_alert_listed,omitempty"` } type reportNarratorStorage struct { @@ -318,12 +326,14 @@ func buildReportNarratorPayload(in reporting.NarrativeInput) reportNarratorPaylo for _, alert := range in.Alerts { payload.Alerts = append(payload.Alerts, reportNarratorAlert{ - Type: alert.Type, - Level: alert.Level, - Message: alert.Message, - Value: alert.Value, - Threshold: alert.Threshold, - Resolved: alert.ResolvedTime != nil, + Type: alert.Type, + Level: alert.Level, + Message: alert.Message, + Value: alert.Value, + Threshold: alert.Threshold, + Resolved: alert.Recovered(), + Resolution: alert.ResolutionSummary(), + SuccessorAlertListed: alert.SuccessorAlertListed(in.Alerts), }) } diff --git a/internal/ai/report_narrator_test.go b/internal/ai/report_narrator_test.go index cd233f386b..fc56b4354a 100644 --- a/internal/ai/report_narrator_test.go +++ b/internal/ai/report_narrator_test.go @@ -272,3 +272,45 @@ func TestBuildReportNarratorPayload_PeriodFormatting(t *testing.T) { t.Errorf("Period.Hours = %d, want 24", payload.Period.Hours) } } + +// A node alert that moved to its Pulse agent closed without recovering, so the +// model must not see it as resolved: it gets the engine's account instead. +func TestBuildReportNarratorPayload_HandoverIsNotARecovery(t *testing.T) { + resolvedAt := time.Date(2026, 10, 6, 15, 0, 0, 0, time.UTC) + summary := "Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading." + payload := buildReportNarratorPayload(reporting.NarrativeInput{ + Alerts: []reporting.AlertInfo{ + { + Type: "memory", + Level: "warning", + Message: "Node memory at 95%", + ResolvedTime: &resolvedAt, + Resolution: &reporting.AlertResolution{ + Reason: reporting.AlertResolutionMovedToAgent, + SuccessorResourceID: "agent:host-1", + SuccessorName: "pve1 (Host Agent)", + Summary: summary, + }, + }, + {Type: "cpu", Level: "warning", Message: "Node CPU at 91%", ResolvedTime: &resolvedAt}, + {Type: "memory", Level: "warning", Message: "Agent memory at 94%", ResourceID: "agent:host-1"}, + }, + }) + if len(payload.Alerts) != 3 { + t.Fatalf("alerts = %+v", payload.Alerts) + } + moved, recovered, successor := payload.Alerts[0], payload.Alerts[1], payload.Alerts[2] + if moved.Resolved || moved.Resolution != summary || !moved.SuccessorAlertListed { + t.Fatalf("moved alert must carry its resolution, point at the listed agent alert and not read as resolved, got %+v", moved) + } + if successor.Resolved || successor.Resolution != "" || successor.SuccessorAlertListed { + t.Fatalf("the agent's open alert is an ordinary open alert, got %+v", successor) + } + if !recovered.Resolved || recovered.Resolution != "" { + t.Fatalf("an ordinary recovery stays resolved, got %+v", recovered) + } + if !strings.Contains(reportNarratorSystemPrompt, `An alert with a "resolution" closed without recovering`) || + !strings.Contains(reportNarratorSystemPrompt, `"successor_alert_listed": true`) { + t.Fatal("system prompt must tell the model a resolution is not a recovery and how to count it once") + } +} diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go index 32bfb19a12..6284d3e0a6 100644 --- a/internal/api/contract_test.go +++ b/internal/api/contract_test.go @@ -5272,6 +5272,67 @@ func TestContract_ReportingRequestCarriesEntitledReportBranding(t *testing.T) { } } +// A node alert that moved to its linked Pulse agent closed without +// recovering. The report request must keep that resolution and each alert's +// resource, so the report neither calls the node healthy nor counts the +// condition twice beside the agent's own alert. +func TestContract_ReportingAlertsCarryHandoverResolution(t *testing.T) { + now := time.Now() + summary := "Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading." + snapshot := reportingEnrichmentSnapshot{ + Nodes: []models.Node{{ID: "pve1-node", Name: "pve1"}}, + ActiveAlerts: []models.Alert{{ + ID: "agent:host-1-memory", + Type: "memory", + Level: "warning", + ResourceID: "agent:host-1", + Node: "pve1", + Message: "Agent memory at 94%", + StartTime: now.Add(-time.Minute), + }}, + RecentlyResolved: []models.ResolvedAlert{{ + Alert: models.Alert{ + ID: "pve1-node-memory", + Type: "memory", + Level: "warning", + ResourceID: "pve1-node", + Node: "pve1", + Message: "Node memory at 95%", + StartTime: now.Add(-time.Hour), + Resolution: &models.AlertResolution{ + Reason: "moved_to_agent", + SuccessorResourceID: "agent:host-1", + SuccessorName: "pve1 (Host Agent)", + Summary: summary, + }, + }, + ResolvedTime: now.Add(-2 * time.Minute), + }}, + } + req := reporting.MetricReportRequest{ResourceType: "node", ResourceID: "pve1-node"} + NewReportingHandlers(nil, nil).enrichNodeReport(&req, snapshot, now.Add(-24*time.Hour), now) + + if len(req.Alerts) != 2 { + t.Fatalf("expected the agent alert and the moved node alert, got %+v", req.Alerts) + } + active, moved := req.Alerts[0], req.Alerts[1] + if active.ResourceID != "agent:host-1" || active.ResolvedTime != nil || active.Resolution != nil { + t.Fatalf("active alert row = %+v", active) + } + if moved.ResourceID != "pve1-node" || moved.ResolvedTime == nil || moved.Recovered() { + t.Fatalf("moved alert must stay resolved-but-not-recovered, got %+v", moved) + } + want := reporting.AlertResolution{ + Reason: reporting.AlertResolutionMovedToAgent, + SuccessorResourceID: "agent:host-1", + SuccessorName: "pve1 (Host Agent)", + Summary: summary, + } + if moved.Resolution == nil || *moved.Resolution != want { + t.Fatalf("resolution = %+v, want %+v", moved.Resolution, want) + } +} + func TestContract_ReportBrandingSettingsRejectWorkspaceLogoPath(t *testing.T) { err := validateReportBrandingSettings(map[string]interface{}{ "displayName": "Client One", diff --git a/internal/api/metrics_reporting_handlers.go b/internal/api/metrics_reporting_handlers.go index ebf7c91682..ba2cbee35a 100644 --- a/internal/api/metrics_reporting_handlers.go +++ b/internal/api/metrics_reporting_handlers.go @@ -750,6 +750,37 @@ func reportSubjectIDMatches(req *reporting.MetricReportRequest, candidateID stri return req.MetricsResourceID != "" && candidateID == req.MetricsResourceID } +// reportActiveAlertInfo is the report row for an open alert. +func reportActiveAlertInfo(alert models.Alert) reporting.AlertInfo { + return reporting.AlertInfo{ + Type: alert.Type, + Level: alert.Level, + Message: alert.Message, + Value: alert.Value, + Threshold: alert.Threshold, + StartTime: alert.StartTime, + ResourceID: alert.ResourceID, + } +} + +// reportResolvedAlertInfo is the report row for a recently resolved alert. It +// keeps the alert's resolution, so a node alert that moved to its Pulse agent +// is never reported as a recovery. +func reportResolvedAlertInfo(resolved models.ResolvedAlert) reporting.AlertInfo { + info := reportActiveAlertInfo(resolved.Alert) + resolvedTime := resolved.ResolvedTime + info.ResolvedTime = &resolvedTime + if resolution := resolved.Resolution; resolution != nil { + info.Resolution = &reporting.AlertResolution{ + Reason: resolution.Reason, + SuccessorResourceID: resolution.SuccessorResourceID, + SuccessorName: resolution.SuccessorName, + Summary: resolution.Summary, + } + } + return info +} + // enrichNodeReport adds node-specific data to the report request func (h *ReportingHandlers) enrichNodeReport(req *reporting.MetricReportRequest, snapshot reportingEnrichmentSnapshot, start, end time.Time) { // Find the node @@ -791,29 +822,13 @@ func (h *ReportingHandlers) enrichNodeReport(req *reporting.MetricReportRequest, // Find alerts for this node for _, alert := range snapshot.ActiveAlerts { if reportSubjectIDMatches(req, alert.ResourceID) || alert.Node == node.Name { - req.Alerts = append(req.Alerts, reporting.AlertInfo{ - Type: alert.Type, - Level: alert.Level, - Message: alert.Message, - Value: alert.Value, - Threshold: alert.Threshold, - StartTime: alert.StartTime, - }) + req.Alerts = append(req.Alerts, reportActiveAlertInfo(alert)) } } for _, resolved := range snapshot.RecentlyResolved { if (reportSubjectIDMatches(req, resolved.ResourceID) || resolved.Node == node.Name) && resolved.ResolvedTime.After(start) && resolved.ResolvedTime.Before(end) { - resolvedTime := resolved.ResolvedTime - req.Alerts = append(req.Alerts, reporting.AlertInfo{ - Type: resolved.Type, - Level: resolved.Level, - Message: resolved.Message, - Value: resolved.Value, - Threshold: resolved.Threshold, - StartTime: resolved.StartTime, - ResolvedTime: &resolvedTime, - }) + req.Alerts = append(req.Alerts, reportResolvedAlertInfo(resolved)) } } @@ -922,29 +937,13 @@ func (h *ReportingHandlers) enrichVMReport(ctx context.Context, orgID string, re // Find alerts for this VM for _, alert := range snapshot.ActiveAlerts { if reportSubjectIDMatches(req, alert.ResourceID) { - req.Alerts = append(req.Alerts, reporting.AlertInfo{ - Type: alert.Type, - Level: alert.Level, - Message: alert.Message, - Value: alert.Value, - Threshold: alert.Threshold, - StartTime: alert.StartTime, - }) + req.Alerts = append(req.Alerts, reportActiveAlertInfo(alert)) } } for _, resolved := range snapshot.RecentlyResolved { if reportSubjectIDMatches(req, resolved.ResourceID) && resolved.ResolvedTime.After(start) && resolved.ResolvedTime.Before(end) { - resolvedTime := resolved.ResolvedTime - req.Alerts = append(req.Alerts, reporting.AlertInfo{ - Type: resolved.Type, - Level: resolved.Level, - Message: resolved.Message, - Value: resolved.Value, - Threshold: resolved.Threshold, - StartTime: resolved.StartTime, - ResolvedTime: &resolvedTime, - }) + req.Alerts = append(req.Alerts, reportResolvedAlertInfo(resolved)) } } @@ -1201,29 +1200,13 @@ func (h *ReportingHandlers) enrichContainerReport(ctx context.Context, orgID str // Find alerts for this container for _, alert := range snapshot.ActiveAlerts { if reportSubjectIDMatches(req, alert.ResourceID) { - req.Alerts = append(req.Alerts, reporting.AlertInfo{ - Type: alert.Type, - Level: alert.Level, - Message: alert.Message, - Value: alert.Value, - Threshold: alert.Threshold, - StartTime: alert.StartTime, - }) + req.Alerts = append(req.Alerts, reportActiveAlertInfo(alert)) } } for _, resolved := range snapshot.RecentlyResolved { if reportSubjectIDMatches(req, resolved.ResourceID) && resolved.ResolvedTime.After(start) && resolved.ResolvedTime.Before(end) { - resolvedTime := resolved.ResolvedTime - req.Alerts = append(req.Alerts, reporting.AlertInfo{ - Type: resolved.Type, - Level: resolved.Level, - Message: resolved.Message, - Value: resolved.Value, - Threshold: resolved.Threshold, - StartTime: resolved.StartTime, - ResolvedTime: &resolvedTime, - }) + req.Alerts = append(req.Alerts, reportResolvedAlertInfo(resolved)) } } diff --git a/pkg/reporting/alert_state.go b/pkg/reporting/alert_state.go new file mode 100644 index 0000000000..98cf4b1e8a --- /dev/null +++ b/pkg/reporting/alert_state.go @@ -0,0 +1,229 @@ +package reporting + +import ( + "fmt" + "strings" +) + +// AlertResolutionMovedToAgent is the reason code of an alert that closed +// because a linked Pulse agent took its metric over +// (alerts.AlertResolutionMovedToAgent). +const AlertResolutionMovedToAgent = "moved_to_agent" + +// Recovered reports whether the alert closed because its condition cleared. +// An open alert, or one that closed without recovering, is not recovered. +func (a AlertInfo) Recovered() bool { + return a.ResolvedTime != nil && a.Resolution == nil +} + +// ClosedWithoutRecovery reports whether the alert closed while its condition +// may still hold, such as a node alert handed to its Pulse agent. +func (a AlertInfo) ClosedWithoutRecovery() bool { + return a.ResolvedTime != nil && a.Resolution != nil +} + +// ResolutionOutcome is the short form of a close that was not a recovery, +// such as "moved to pve1 (Host Agent)". It is empty for an open alert and for +// an ordinary recovery. +func (a AlertInfo) ResolutionOutcome() string { + if !a.ClosedWithoutRecovery() { + return "" + } + if a.Resolution.Reason == AlertResolutionMovedToAgent { + return "moved to " + a.successorLabel() + } + return "closed without recovering" +} + +// ResolutionSummary is the alert engine's account of a close that was not a +// recovery, falling back to the short form when the engine supplied none. It +// is empty for an open alert and for an ordinary recovery. +func (a AlertInfo) ResolutionSummary() string { + if !a.ClosedWithoutRecovery() { + return "" + } + if summary := strings.TrimSpace(a.Resolution.Summary); summary != "" { + return summary + } + return fmt.Sprintf("Alert %s. This is not a recovery.", a.ResolutionOutcome()) +} + +func (a AlertInfo) successorLabel() string { + if a.Resolution != nil { + if name := strings.TrimSpace(a.Resolution.SuccessorName); name != "" { + return name + } + } + return "its Pulse agent" +} + +// unresolvedAlerts returns the alerts a report must treat as live: every open +// alert, and every alert that closed without recovering unless the report +// also shows its successor's own reading for that metric. That alert carries +// the condition's current state, so one condition is never counted twice and +// a handover never reads as a recovery. +func unresolvedAlerts(alerts []AlertInfo) []AlertInfo { + return unresolvedAlertsAmong(alerts, alerts) +} + +// unresolvedAlertsAmong is unresolvedAlerts with successors looked up in +// pool, so a fleet report matches a handover against every resource's alerts. +func unresolvedAlertsAmong(alerts, pool []AlertInfo) []AlertInfo { + out := make([]AlertInfo, 0, len(alerts)) + for _, alert := range alerts { + switch { + case alert.ResolvedTime == nil: + out = append(out, alert) + case alert.ClosedWithoutRecovery() && successorAlertFor(alert, pool) == nil: + out = append(out, alert) + } + } + return out +} + +// successorAlertFor returns the alert in pool that carries a moved alert's +// condition: one of the same type on its successor or one of the successor's +// children (an agent's filesystem alerts sit under "/disk:"), +// still open or recovered at or after the handover. A successor recovery from +// before the handover says nothing about the reading since, so it does not +// count. +func successorAlertFor(moved AlertInfo, pool []AlertInfo) *AlertInfo { + if !moved.ClosedWithoutRecovery() { + return nil + } + successor := strings.TrimSpace(moved.Resolution.SuccessorResourceID) + if successor == "" { + return nil + } + for i := range pool { + candidate := &pool[i] + if candidate.Type != moved.Type || candidate.ClosedWithoutRecovery() { + continue + } + if candidate.ResolvedTime != nil && candidate.ResolvedTime.Before(*moved.ResolvedTime) { + continue + } + resourceID := strings.TrimSpace(candidate.ResourceID) + if resourceID == successor || strings.HasPrefix(resourceID, successor+"/") { + return candidate + } + } + return nil +} + +// SuccessorAlertListed reports whether alerts shows the reading of the +// resource this alert moved to: an alert of the same type on the successor, +// open or recovered since the handover. The report then counts the condition +// once, through that alert. It is false for an open alert or a recovery. +func (a AlertInfo) SuccessorAlertListed(alerts []AlertInfo) bool { + return successorAlertFor(a, alerts) != nil +} + +// fleetAlerts is every alert listed across a fleet report's resources. +func (d *MultiReportData) fleetAlerts() []AlertInfo { + if d == nil { + return nil + } + var pool []AlertInfo + for _, rd := range d.Resources { + if rd != nil { + pool = append(pool, rd.Alerts...) + } + } + return pool +} + +// alertHealth is the health verdict a report's alerts support. The PDF health +// card and the heuristic narrator both read it, so they never disagree. +type alertHealth struct { + Status string // HEALTHY, WARNING or CRITICAL + Message string + Critical int + Warning int + // Unresolved is every alert the report treats as live (unresolvedAlerts). + Unresolved []AlertInfo +} + +func assessAlertHealth(alerts []AlertInfo) alertHealth { + health := alertHealth{Unresolved: unresolvedAlerts(alerts)} + open := 0 + var moved []AlertInfo + for _, alert := range health.Unresolved { + if alert.Level == "critical" { + health.Critical++ + } else { + health.Warning++ + } + if alert.ClosedWithoutRecovery() { + moved = append(moved, alert) + } else { + open++ + } + } + + switch { + case health.Critical > 0: + health.Status = "CRITICAL" + case health.Warning > 0: + health.Status = "WARNING" + default: + health.Status = "HEALTHY" + } + + switch { + case open == 0 && len(moved) > 0: + health.Message = movedAlertsHealthMessage(moved) + case health.Critical == 1: + health.Message = "1 critical issue requires immediate attention" + case health.Critical > 1: + health.Message = fmt.Sprintf("%d critical issues require immediate attention", health.Critical) + case health.Warning == 1: + health.Message = "1 warning detected - review recommended" + case health.Warning > 1: + health.Message = fmt.Sprintf("%d warnings detected - review recommended", health.Warning) + default: + health.Message = "All systems operating normally" + } + return health +} + +// movedAlertsHealthMessage says where alerts that closed without recovering +// went, so the report points at the resource that now holds the reading +// instead of calling this one healthy. +func movedAlertsHealthMessage(moved []AlertInfo) string { + first := moved[0] + for _, alert := range moved[1:] { + if alert.ResolutionOutcome() != first.ResolutionOutcome() { + return fmt.Sprintf("%d alerts closed without recovering - review recommended", len(moved)) + } + } + if first.Resolution.Reason != AlertResolutionMovedToAgent { + if len(moved) == 1 { + return "1 alert closed without recovering - review recommended" + } + return fmt.Sprintf("%d alerts closed without recovering - review recommended", len(moved)) + } + if len(moved) == 1 { + return fmt.Sprintf("Alert %s - check the agent for the current reading", first.ResolutionOutcome()) + } + return fmt.Sprintf("%d alerts %s - check the agent for the current reading", len(moved), first.ResolutionOutcome()) +} + +// movedAlertSuccessors names, once each and in report order, the Pulse agents +// that unresolved alerts moved to. +func movedAlertSuccessors(unresolved []AlertInfo) []string { + var successors []string + seen := make(map[string]struct{}) + for _, alert := range unresolved { + if !alert.ClosedWithoutRecovery() || alert.Resolution.Reason != AlertResolutionMovedToAgent { + continue + } + successor := alert.successorLabel() + if _, ok := seen[successor]; ok { + continue + } + seen[successor] = struct{}{} + successors = append(successors, successor) + } + return successors +} diff --git a/pkg/reporting/alert_state_test.go b/pkg/reporting/alert_state_test.go new file mode 100644 index 0000000000..175f019b78 --- /dev/null +++ b/pkg/reporting/alert_state_test.go @@ -0,0 +1,273 @@ +package reporting + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/go-pdf/fpdf" +) + +const movedAlertSummary = "Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading." + +// movedNodeAlert is a node memory alert that closed because the linked Pulse +// agent took the metric over, as GetRecentlyResolved reports it. +func movedNodeAlert(level string) AlertInfo { + resolved := time.Now().Add(-2 * time.Minute) + return AlertInfo{ + Type: "memory", + Level: level, + Message: "Node memory at 95%", + StartTime: time.Now().Add(-time.Hour), + ResolvedTime: &resolved, + ResourceID: "pve1-node", + Resolution: &AlertResolution{ + Reason: AlertResolutionMovedToAgent, + SuccessorResourceID: "agent:host-1", + SuccessorName: "pve1 (Host Agent)", + Summary: movedAlertSummary, + }, + } +} + +// A node alert that moved to its Pulse agent within the recently-resolved +// window must not turn the report HEALTHY: nothing saw the condition clear. +func TestHeuristicNarrator_MovedAlertIsNotARecovery(t *testing.T) { + out, err := HeuristicNarrator{}.Narrate(context.Background(), NarrativeInput{ + MetricStats: map[string]MetricStats{"memory": {Avg: 60, Max: 96}}, + Alerts: []AlertInfo{movedNodeAlert("warning")}, + }) + if err != nil { + t.Fatalf("Narrate: %v", err) + } + if out.HealthStatus != "WARNING" { + t.Fatalf("HealthStatus = %q, want WARNING", out.HealthStatus) + } + if want := "Alert moved to pve1 (Host Agent) - check the agent for the current reading"; out.HealthMessage != want { + t.Fatalf("HealthMessage = %q, want %q", out.HealthMessage, want) + } + + var movedBullet *NarrativeBullet + for i, bullet := range out.Observations { + if strings.Contains(bullet.Text, "triggered and resolved") { + t.Fatalf("moved alert counted as triggered and resolved: %+v", out.Observations) + } + if bullet.Text == movedAlertSummary { + movedBullet = &out.Observations[i] + } + } + if movedBullet == nil || movedBullet.Severity != NarrativeSeverityInfo { + t.Fatalf("expected an observation recording the handover, got %+v", out.Observations) + } + + recs := strings.Join(out.Recommendations, "\n") + if !strings.Contains(recs, "Check the current reading on pve1 (Host Agent)") { + t.Fatalf("expected a recommendation to check the agent, got %q", recs) + } + if strings.Contains(recs, "No immediate action required") { + t.Fatalf("a moved alert is not a reason to stand down, got %q", recs) + } + + critical := assessAlertHealth([]AlertInfo{movedNodeAlert("critical")}) + if critical.Status != "CRITICAL" || critical.Critical != 1 { + t.Fatalf("a moved critical alert keeps its level, got %+v", critical) + } + + // An ordinary recovery is still a recovery. + recovered := movedNodeAlert("warning") + recovered.Resolution = nil + plain, _ := HeuristicNarrator{}.Narrate(context.Background(), NarrativeInput{ + MetricStats: map[string]MetricStats{"memory": {Avg: 60, Max: 70}}, + Alerts: []AlertInfo{recovered}, + }) + if plain.HealthStatus != "HEALTHY" || plain.HealthMessage != "All systems operating normally" { + t.Fatalf("recovered alert should leave the report healthy, got %q / %q", plain.HealthStatus, plain.HealthMessage) + } +} + +// When the report also lists the alert the successor raised for the same +// metric, that alert carries the condition: the moved alert must not count +// it a second time. +func TestUnresolvedAlerts_SuccessorAlertCarriesTheCondition(t *testing.T) { + agentMemory := AlertInfo{Type: "memory", Level: "warning", Message: "Agent memory at 94%", ResourceID: "agent:host-1"} + health := assessAlertHealth([]AlertInfo{movedNodeAlert("warning"), agentMemory}) + if health.Warning != 1 || len(health.Unresolved) != 1 || health.Unresolved[0] != agentMemory { + t.Fatalf("expected only the agent's own alert to count, got %+v", health) + } + if health.Message != "1 warning detected - review recommended" { + t.Fatalf("HealthMessage = %q", health.Message) + } + out, _ := HeuristicNarrator{}.Narrate(context.Background(), NarrativeInput{ + MetricStats: map[string]MetricStats{"memory": {Avg: 60, Max: 96}}, + Alerts: []AlertInfo{movedNodeAlert("warning"), agentMemory}, + }) + if strings.Contains(strings.Join(out.Recommendations, "\n"), "Check the current reading") { + t.Fatalf("covered handover should not ask to check the agent: %+v", out.Recommendations) + } + + // The agent's filesystem alerts sit under the agent's resource ID. + movedDisk := movedNodeAlert("warning") + movedDisk.Type = "disk" + agentRoot := AlertInfo{Type: "disk", Level: "warning", ResourceID: "agent:host-1/disk:root"} + if got := unresolvedAlerts([]AlertInfo{movedDisk, agentRoot}); len(got) != 1 || got[0] != agentRoot { + t.Fatalf("expected the agent filesystem alert to cover the moved disk alert, got %+v", got) + } + + // A different metric, or an agent whose ID merely shares a prefix, does + // not carry the moved alert's condition. + agentCPU := AlertInfo{Type: "cpu", Level: "warning", ResourceID: "agent:host-1"} + otherAgent := AlertInfo{Type: "memory", Level: "warning", ResourceID: "agent:host-10"} + if got := unresolvedAlerts([]AlertInfo{movedNodeAlert("warning"), agentCPU, otherAgent}); len(got) != 3 { + t.Fatalf("expected the moved alert to stay unresolved beside unrelated alerts, got %+v", got) + } +} + +func TestExecutiveSummary_MovedAlertIsNotHealthy(t *testing.T) { + data := &ReportData{ + Title: "pve1", + ResourceType: "node", + ResourceID: "pve1-node", + Start: time.Now().Add(-time.Hour), + End: time.Now(), + GeneratedAt: time.Now(), + Summary: MetricSummary{ByMetric: map[string]MetricStats{ + "cpu": {Avg: 5, Max: 12, Count: 60}, + "memory": {Avg: 60, Max: 96, Count: 60}, + }}, + TotalPoints: 120, + Alerts: []AlertInfo{movedNodeAlert("warning")}, + } + // fpdf escapes parentheses inside PDF string literals. + text := strings.NewReplacer(`\(`, "(", `\)`, ")").Replace(renderExecutiveSummaryText(t, data)) + for _, want := range []string{ + "WARNING", + "Alert moved to pve1 (Host Agent) - check the agent for the current reading", + "Node memory at 95% (moved to pve1 (Host Agent))", + "Moved", + "Moved: " + movedAlertSummary, + } { + if !strings.Contains(text, want) { + t.Errorf("expected %q in executive summary, got:\n%s", want, text) + } + } + if strings.Contains(text, "HEALTHY") || strings.Contains(text, "All systems operating normally") { + t.Errorf("a handover must not render the healthy card, got:\n%s", text) + } + + // The health card does not wrap, so the handover message must fit it. + pdf := fpdf.New("P", "mm", "A4", "") + pdf.SetFont("Arial", "", 11) + pageWidth, _ := pdf.GetPageSize() + if width := pdf.GetStringWidth(assessAlertHealth(data.Alerts).Message); width > pageWidth-40 { + t.Fatalf("handover health message is %.1fmm wide, card is %.1fmm", width, pageWidth-40) + } +} + +// A successor's reading only counts from the handover on: an agent recovery +// from before the node alert moved says nothing about the condition since. +func TestUnresolvedAlerts_SuccessorReadingMustFollowTheHandover(t *testing.T) { + moved := movedNodeAlert("warning") + before := moved.ResolvedTime.Add(-time.Minute) + after := moved.ResolvedTime.Add(time.Minute) + agentRecoveredBefore := AlertInfo{Type: "memory", Level: "warning", ResourceID: "agent:host-1", ResolvedTime: &before} + agentRecoveredAfter := AlertInfo{Type: "memory", Level: "warning", ResourceID: "agent:host-1", ResolvedTime: &after} + + stale := assessAlertHealth([]AlertInfo{agentRecoveredBefore, moved}) + if stale.Status != "WARNING" || len(stale.Unresolved) != 1 || stale.Unresolved[0] != moved { + t.Fatalf("an earlier agent recovery must not cover a later handover, got %+v", stale) + } + if moved.SuccessorAlertListed([]AlertInfo{agentRecoveredBefore, moved}) { + t.Fatal("SuccessorAlertListed must ignore a recovery from before the handover") + } + + cleared := assessAlertHealth([]AlertInfo{agentRecoveredAfter, moved}) + if cleared.Status != "HEALTHY" || len(cleared.Unresolved) != 0 { + t.Fatalf("an agent recovery after the handover shows the condition cleared, got %+v", cleared) + } + if !moved.SuccessorAlertListed([]AlertInfo{agentRecoveredAfter, moved}) { + t.Fatal("SuccessorAlertListed must accept a recovery after the handover") + } +} + +func TestBuildFleetNarrativeInput_MovedAlertStaysActive(t *testing.T) { + now := time.Now() + movedElsewhere := movedNodeAlert("critical") + movedElsewhere.Resolution = &AlertResolution{ + Reason: AlertResolutionMovedToAgent, + SuccessorResourceID: "agent:host-3", + SuccessorName: "pve3 (Host Agent)", + Summary: "Alert moved to pve3 (Host Agent). This is not a recovery: check the agent for the current reading.", + } + multi := &MultiReportData{ + Title: "Fleet", + Start: now.Add(-time.Hour), + End: now, + Resources: []*ReportData{ + { + ResourceID: "pve1-node", + ResourceType: "node", + Resource: &ResourceInfo{Name: "pve1", Status: "online"}, + Alerts: []AlertInfo{movedNodeAlert("warning")}, + }, + { + // The agent pve1's alert moved to is its own fleet entry. + ResourceID: "agent:host-1", + ResourceType: "agent", + Resource: &ResourceInfo{Name: "pve1 (Host Agent)", Status: "online"}, + Alerts: []AlertInfo{{Type: "memory", Level: "warning", ResourceID: "agent:host-1"}}, + }, + { + ResourceID: "pve3-node", + ResourceType: "node", + Resource: &ResourceInfo{Name: "pve3", Status: "online"}, + Alerts: []AlertInfo{movedElsewhere}, + }, + }, + } + in := buildFleetNarrativeInput(multi) + if in.Aggregate.TotalResolvedAlerts != 0 { + t.Fatalf("moved alerts are not resolved, got %d", in.Aggregate.TotalResolvedAlerts) + } + // pve1's handover is carried by the agent's own alert in another fleet + // entry; pve3's has no successor reading, so it stays live at its level. + if in.Aggregate.TotalActiveAlerts != 2 || in.Aggregate.TotalCriticalAlerts != 1 { + t.Fatalf("expected the agent alert and pve3's moved alert active, got active=%d critical=%d", + in.Aggregate.TotalActiveAlerts, in.Aggregate.TotalCriticalAlerts) + } + if got := in.Resources[0]; got.ActiveAlerts != 0 || len(got.MovedTo) != 0 { + t.Fatalf("pve1's covered handover must not count again, got %+v", got) + } + if got := in.Resources[2]; got.ActiveAlerts != 1 || len(got.MovedTo) != 1 || got.MovedTo[0] != "pve3 (Host Agent)" { + t.Fatalf("pve3 should keep its moved alert and name the agent, got %+v", got) + } + + out, _ := HeuristicFleetNarrator{}.NarrateFleet(context.Background(), in) + if out.HealthStatus != "CRITICAL" { + t.Fatalf("fleet HealthStatus = %q, want CRITICAL", out.HealthStatus) + } + recs := strings.Join(out.Recommendations, "\n") + if !strings.Contains(recs, "Check the current reading on pve3 (Host Agent)") { + t.Fatalf("expected a recommendation naming pve3's agent, got %q", recs) + } + if strings.Contains(recs, "pve1 (Host Agent)") || strings.Contains(recs, "No fleet-wide action required") { + t.Fatalf("unexpected recommendation, got %q", recs) + } + + // A covered handover leaves only the agent's open warning: still no + // all-clear in the patterns or the recommendations. + multi.Resources = multi.Resources[:2] + covered, _ := HeuristicFleetNarrator{}.NarrateFleet(context.Background(), buildFleetNarrativeInput(multi)) + if covered.HealthStatus != "WARNING" { + t.Fatalf("fleet HealthStatus = %q, want WARNING", covered.HealthStatus) + } + for _, pattern := range covered.Patterns { + if strings.Contains(pattern.Text, "nominal thresholds") { + t.Fatalf("an open alert rules out the all-clear pattern, got %+v", covered.Patterns) + } + } + if recs := strings.Join(covered.Recommendations, "\n"); strings.Contains(recs, "No fleet-wide action required") || + !strings.Contains(recs, "Review the active alerts") { + t.Fatalf("an open alert needs a review recommendation, got %q", recs) + } +} diff --git a/pkg/reporting/fleet_narrative.go b/pkg/reporting/fleet_narrative.go index 478eeb02b0..8adef1b4c5 100644 --- a/pkg/reporting/fleet_narrative.go +++ b/pkg/reporting/fleet_narrative.go @@ -38,6 +38,9 @@ type FleetResourceSummary struct { UnhealthyDisks int StoragePoolsHigh int Findings int + // MovedTo names the Pulse agents this resource's unresolved alerts + // moved to without recovering; those alerts also count as active. + MovedTo []string } // FleetAggregate is the fleet-wide rollup. Means are means-of-means: @@ -313,7 +316,9 @@ func fleetHeuristicPatterns(in FleetNarrativeInput) []NarrativeBullet { Severity: NarrativeSeverityInfo, }) } - if len(out) == 0 { + // Open alerts, including ones that moved to an agent without + // recovering, rule out the all-clear even when no metric pattern fired. + if len(out) == 0 && in.Aggregate.TotalActiveAlerts == 0 { out = append(out, NarrativeBullet{ Text: fmt.Sprintf("Fleet of %d resources operating within nominal thresholds", in.Aggregate.ResourceCount), Severity: NarrativeSeverityOK, @@ -373,6 +378,19 @@ func fleetHeuristicRecommendations(in FleetNarrativeInput) []string { if disksHigh > 0 { recs = append(recs, "Replace failing or end-of-life disks before they cause outage") } + seenSuccessors := make(map[string]struct{}) + for _, r := range in.Resources { + for _, successor := range r.MovedTo { + if _, ok := seenSuccessors[successor]; ok { + continue + } + seenSuccessors[successor] = struct{}{} + recs = append(recs, fmt.Sprintf("Check the current reading on %s - an alert moved there without recovering", successor)) + } + } + if len(recs) == 0 && a.TotalActiveAlerts > 0 { + recs = append(recs, "Review the active alerts on the resources listed above") + } if len(recs) == 0 { recs = append(recs, "No fleet-wide action required - continue routine monitoring") } @@ -402,6 +420,7 @@ func buildFleetNarrativeInput(data *MultiReportData) FleetNarrativeInput { var sumCPU, sumMem, sumDisk float64 var countCPU, countMem, countDisk int + fleetAlerts := data.fleetAlerts() for _, rd := range data.Resources { if rd == nil { @@ -450,11 +469,16 @@ func buildFleetNarrativeInput(data *MultiReportData) FleetNarrativeInput { } for _, alert := range rd.Alerts { - if alert.ResolvedTime != nil { + if alert.Recovered() { summary.ResolvedAlerts++ in.Aggregate.TotalResolvedAlerts++ - continue } + } + // An alert that moved without recovering stays active until the + // fleet report lists its successor's own alert. + unresolved := unresolvedAlertsAmong(rd.Alerts, fleetAlerts) + summary.MovedTo = movedAlertSuccessors(unresolved) + for _, alert := range unresolved { summary.ActiveAlerts++ in.Aggregate.TotalActiveAlerts++ if alert.Level == "critical" { diff --git a/pkg/reporting/narrative.go b/pkg/reporting/narrative.go index 370f13085b..5473e425cd 100644 --- a/pkg/reporting/narrative.go +++ b/pkg/reporting/narrative.go @@ -145,53 +145,11 @@ func (HeuristicNarrator) Narrate(_ context.Context, in NarrativeInput) (Narrativ } func heuristicHealthStatus(in NarrativeInput) string { - criticalAlerts := 0 - warningAlerts := 0 - for _, alert := range in.Alerts { - if alert.ResolvedTime != nil { - continue - } - if alert.Level == "critical" { - criticalAlerts++ - } else { - warningAlerts++ - } - } - switch { - case criticalAlerts > 0: - return "CRITICAL" - case warningAlerts > 0: - return "WARNING" - default: - return "HEALTHY" - } + return assessAlertHealth(in.Alerts).Status } func heuristicHealthMessage(in NarrativeInput) string { - criticalAlerts := 0 - warningAlerts := 0 - for _, alert := range in.Alerts { - if alert.ResolvedTime != nil { - continue - } - if alert.Level == "critical" { - criticalAlerts++ - } else { - warningAlerts++ - } - } - switch { - case criticalAlerts == 1: - return "1 critical issue requires immediate attention" - case criticalAlerts > 1: - return fmt.Sprintf("%d critical issues require immediate attention", criticalAlerts) - case warningAlerts == 1: - return "1 warning detected - review recommended" - case warningAlerts > 1: - return fmt.Sprintf("%d warnings detected - review recommended", warningAlerts) - default: - return "All systems operating normally" - } + return assessAlertHealth(in.Alerts).Message } // heuristicObservations mirrors the original generateObservations rules. @@ -273,7 +231,7 @@ func heuristicObservations(in NarrativeInput) []NarrativeBullet { resolved := 0 for _, alert := range in.Alerts { - if alert.ResolvedTime != nil { + if alert.Recovered() { resolved++ } } @@ -283,6 +241,17 @@ func heuristicObservations(in NarrativeInput) []NarrativeBullet { Severity: NarrativeSeverityInfo, }) } + // A close that was not a recovery is recorded as what happened. The + // health verdict, the active list and the recommendations carry the + // warning while the successor's reading is not in this report. + for _, alert := range in.Alerts { + if alert.ClosedWithoutRecovery() { + obs = append(obs, NarrativeBullet{ + Text: alert.ResolutionSummary(), + Severity: NarrativeSeverityInfo, + }) + } + } for _, disk := range in.Disks { if disk.WearLevel > 0 && disk.WearLevel <= 10 { @@ -314,19 +283,8 @@ func heuristicObservations(in NarrativeInput) []NarrativeBullet { // heuristicRecommendations mirrors the original generateRecommendations rules. func heuristicRecommendations(in NarrativeInput) []string { - criticalAlerts := 0 - warningAlerts := 0 - for _, alert := range in.Alerts { - if alert.ResolvedTime != nil { - continue - } - if alert.Level == "critical" { - criticalAlerts++ - } else { - warningAlerts++ - } - } - _ = warningAlerts // retained for future shaping; matches original signature + health := assessAlertHealth(in.Alerts) + criticalAlerts := health.Critical var recs []string @@ -344,6 +302,9 @@ func heuristicRecommendations(in NarrativeInput) []string { if criticalAlerts > 0 { recs = append(recs, "Investigate and resolve critical alerts immediately") } + for _, successor := range movedAlertSuccessors(health.Unresolved) { + recs = append(recs, fmt.Sprintf("Check the current reading on %s - an alert moved there without recovering", successor)) + } if stats, ok := in.MetricStats["memory"]; ok { if stats.Avg > 85 { diff --git a/pkg/reporting/pdf.go b/pkg/reporting/pdf.go index b1db00fdc1..00b8cb51a7 100644 --- a/pkg/reporting/pdf.go +++ b/pkg/reporting/pdf.go @@ -464,41 +464,23 @@ func (g *PDFGenerator) writeCoverPage(pdf *fpdf.Fpdf, data *ReportData) { func (g *PDFGenerator) writeExecutiveSummary(pdf *fpdf.Fpdf, data *ReportData) { pageWidth, _ := pdf.GetPageSize() - // Determine overall health status - healthStatus := "HEALTHY" + // Determine overall health status. An alert that closed without + // recovering (moved to its Pulse agent) still counts until the report + // lists the successor's own alert for that metric. + health := assessAlertHealth(data.Alerts) + healthStatus := health.Status healthColor := colorAccent // Green - healthMessage := "All systems operating normally" + healthMessage := health.Message - activeAlerts := 0 - criticalAlerts := 0 - warningAlerts := 0 - for _, alert := range data.Alerts { - if alert.ResolvedTime == nil { - activeAlerts++ - if alert.Level == "critical" { - criticalAlerts++ - } else { - warningAlerts++ - } - } - } + unresolved := health.Unresolved + activeAlerts := len(unresolved) + criticalAlerts := health.Critical + warningAlerts := health.Warning if criticalAlerts > 0 { - healthStatus = "CRITICAL" healthColor = colorDanger - if criticalAlerts == 1 { - healthMessage = "1 critical issue requires immediate attention" - } else { - healthMessage = fmt.Sprintf("%d critical issues require immediate attention", criticalAlerts) - } } else if warningAlerts > 0 { - healthStatus = "WARNING" healthColor = colorWarning - if warningAlerts == 1 { - healthMessage = "1 warning detected - review recommended" - } else { - healthMessage = fmt.Sprintf("%d warnings detected - review recommended", warningAlerts) - } } else if data.TotalPoints == 0 && len(data.Summary.ByMetric) == 0 { // No metrics arrived for the requested window. Reporting // HEALTHY would green-light an empty report and mislead the @@ -633,8 +615,8 @@ func (g *PDFGenerator) writeExecutiveSummary(pdf *fpdf.Fpdf, data *ReportData) { pdf.SetFont("Arial", "", 9) alertCount := 0 - for _, alert := range data.Alerts { - if alert.ResolvedTime == nil && alertCount < 5 { + for _, alert := range unresolved { + if alertCount < 5 { if alert.Level == "critical" { pdf.SetTextColor(colorDanger[0], colorDanger[1], colorDanger[2]) pdf.CellFormat(8, 5, "!", "", 0, "C", false, 0, "") @@ -643,11 +625,7 @@ func (g *PDFGenerator) writeExecutiveSummary(pdf *fpdf.Fpdf, data *ReportData) { pdf.CellFormat(8, 5, "!", "", 0, "C", false, 0, "") } pdf.SetTextColor(colorTextDark[0], colorTextDark[1], colorTextDark[2]) - msg := alert.Message - if len(msg) > 70 { - msg = msg[:67] + "..." - } - pdf.CellFormat(0, 5, msg, "", 1, "L", false, 0, "") + pdf.CellFormat(0, 5, unresolvedAlertLine(alert, 70), "", 1, "L", false, 0, "") alertCount++ } } @@ -1509,8 +1487,12 @@ func (g *PDFGenerator) writeAlertsSection(pdf *fpdf.Fpdf, data *ReportData) { // Started pdf.CellFormat(colWidths[3], 6, alert.StartTime.Format("Jan 02 15:04"), "1", 0, "C", fill, 0, "") - // Resolved - if alert.ResolvedTime != nil { + // Resolved. A close that was not a recovery reads "Moved" in + // neutral text, never as a green resolve time. + if alert.ClosedWithoutRecovery() { + pdf.SetTextColor(colorTextMuted[0], colorTextMuted[1], colorTextMuted[2]) + pdf.CellFormat(colWidths[4], 6, closedWithoutRecoveryLabel(alert), "1", 0, "C", fill, 0, "") + } else if alert.ResolvedTime != nil { pdf.SetTextColor(colorAccent[0], colorAccent[1], colorAccent[2]) pdf.CellFormat(colWidths[4], 6, alert.ResolvedTime.Format("Jan 02 15:04"), "1", 0, "C", fill, 0, "") } else { @@ -1523,9 +1505,62 @@ func (g *PDFGenerator) writeAlertsSection(pdf *fpdf.Fpdf, data *ReportData) { fill = !fill } + // Say once what each "Moved" row means, in the alert engine's words. + var notes []string + seenNotes := make(map[string]struct{}) + for _, alert := range data.Alerts { + summary := alert.ResolutionSummary() + if summary == "" { + continue + } + note := closedWithoutRecoveryLabel(alert) + ": " + summary + if _, ok := seenNotes[note]; ok { + continue + } + seenNotes[note] = struct{}{} + notes = append(notes, note) + } + if len(notes) > 0 { + pdf.Ln(2) + pdf.SetFont("Arial", "I", 8) + pdf.SetTextColor(colorTextMuted[0], colorTextMuted[1], colorTextMuted[2]) + for _, note := range notes { + pdf.MultiCell(0, 4, note, "", "L", false) + } + pdf.SetTextColor(colorTextDark[0], colorTextDark[1], colorTextDark[2]) + } + pdf.Ln(10) } +// closedWithoutRecoveryLabel is the alerts table's Resolved cell for a close +// that was not a recovery. +func closedWithoutRecoveryLabel(alert AlertInfo) string { + if alert.Resolution != nil && alert.Resolution.Reason == AlertResolutionMovedToAgent { + return "Moved" + } + return "Closed" +} + +// unresolvedAlertLine is one alert in an "Active Alerts" list, cut to max +// characters. An alert that closed without recovering keeps its outcome, +// such as "(moved to pve1 (Host Agent))", and gives up message text instead. +func unresolvedAlertLine(alert AlertInfo, max int) string { + suffix := "" + if outcome := alert.ResolutionOutcome(); outcome != "" { + suffix = " (" + outcome + ")" + } + room := max - len(suffix) + if room < 10 { + room = 10 + } + msg := alert.Message + if len(msg) > room { + msg = msg[:room-3] + "..." + } + return msg + suffix +} + // writeStorageSection writes storage pools table func (g *PDFGenerator) writeStorageSection(pdf *fpdf.Fpdf, data *ReportData) { if len(data.Storage) == 0 { @@ -1831,8 +1866,10 @@ func (g *PDFGenerator) GenerateMulti(data *MultiReportData) ([]byte, error) { _, pageHeight := pdf.GetPageSize() bottomLimit := pageHeight - 30 pageOpen := false + fleetAlerts := data.fleetAlerts() for _, rd := range data.Resources { - startNewPage := !pageOpen || pdf.GetY()+condensedResourceBlockHeight(rd) > bottomLimit + unresolved := unresolvedAlertsAmong(rd.Alerts, fleetAlerts) + startNewPage := !pageOpen || pdf.GetY()+condensedResourceBlockHeight(rd, unresolved) > bottomLimit if startNewPage { pdf.AddPage() g.addMultiPageHeader(pdf, data, "Resource Details") @@ -1843,7 +1880,7 @@ func (g *PDFGenerator) GenerateMulti(data *MultiReportData) ([]byte, error) { pdf.Line(20, pdf.GetY(), 190, pdf.GetY()) pdf.Ln(5) } - g.writeCondensedResourcePage(pdf, rd) + g.writeCondensedResourcePage(pdf, rd, unresolved) } // Add page numbers to all pages except cover @@ -2043,15 +2080,14 @@ func (g *PDFGenerator) writeFleetSummary(pdf *fpdf.Fpdf, data *MultiReportData) totalActive := 0 totalCritical := 0 totalWarning := 0 + fleetAlerts := data.fleetAlerts() for _, rd := range data.Resources { - for _, alert := range rd.Alerts { - if alert.ResolvedTime == nil { - totalActive++ - if alert.Level == "critical" { - totalCritical++ - } else { - totalWarning++ - } + for _, alert := range unresolvedAlertsAmong(rd.Alerts, fleetAlerts) { + totalActive++ + if alert.Level == "critical" { + totalCritical++ + } else { + totalWarning++ } } } @@ -2215,12 +2251,7 @@ func (g *PDFGenerator) writeFleetSummary(pdf *fpdf.Fpdf, data *MultiReportData) pdf.CellFormat(colWidths[6], 6, fmt.Sprintf("%.1f%%", avgDisk), "1", 0, "C", fill, 0, "") // Alerts count - alertCount := 0 - for _, alert := range rd.Alerts { - if alert.ResolvedTime == nil { - alertCount++ - } - } + alertCount := len(unresolvedAlertsAmong(rd.Alerts, fleetAlerts)) pdf.SetTextColor(getAlertCountColor(alertCount)[0], getAlertCountColor(alertCount)[1], getAlertCountColor(alertCount)[2]) pdf.CellFormat(colWidths[7], 6, fmt.Sprintf("%d", alertCount), "1", 0, "C", fill, 0, "") @@ -2395,17 +2426,12 @@ func writeFleetNarrativeSection(pdf *fpdf.Fpdf, fn *FleetNarrative) { // resource block needs so the fleet layout can decide whether it fits on // the current page. Estimates err slightly high; a block that still // overflows is carried across pages by fpdf's auto page break. -func condensedResourceBlockHeight(rd *ReportData) float64 { +func condensedResourceBlockHeight(rd *ReportData, unresolved []AlertInfo) float64 { height := 11.0 + 6.0 + 27.0 + 8.0 // name header + availability line + stats bar + padding if len(rd.Metrics["cpu"]) >= 2 || len(rd.Metrics["memory"]) >= 2 { height += 58 // chart title + canvas + legend } - active := 0 - for _, alert := range rd.Alerts { - if alert.ResolvedTime == nil { - active++ - } - } + active := len(unresolved) if active > 0 { shown := active if shown > 3 { @@ -2422,7 +2448,9 @@ func condensedResourceBlockHeight(rd *ReportData) float64 { return height } -func (g *PDFGenerator) writeCondensedResourcePage(pdf *fpdf.Fpdf, rd *ReportData) { +// writeCondensedResourcePage writes one resource's block of a fleet report. +// unresolved is the resource's live alerts matched against the whole fleet. +func (g *PDFGenerator) writeCondensedResourcePage(pdf *fpdf.Fpdf, rd *ReportData, unresolved []AlertInfo) { // Resource header resourceName := rd.ResourceID if rd.Resource != nil && rd.Resource.Name != "" { @@ -2586,13 +2614,8 @@ func (g *PDFGenerator) writeCondensedResourcePage(pdf *fpdf.Fpdf, rd *ReportData } } - // Active alerts (up to 3) - activeAlerts := make([]AlertInfo, 0) - for _, alert := range rd.Alerts { - if alert.ResolvedTime == nil { - activeAlerts = append(activeAlerts, alert) - } - } + // Active alerts (up to 3), including any that moved without recovering + activeAlerts := unresolved if len(activeAlerts) > 0 { pdf.Ln(3) @@ -2615,11 +2638,7 @@ func (g *PDFGenerator) writeCondensedResourcePage(pdf *fpdf.Fpdf, rd *ReportData } pdf.CellFormat(6, 5, "!", "", 0, "C", false, 0, "") pdf.SetTextColor(colorTextDark[0], colorTextDark[1], colorTextDark[2]) - msg := alert.Message - if len(msg) > 80 { - msg = msg[:77] + "..." - } - pdf.CellFormat(0, 5, msg, "", 1, "L", false, 0, "") + pdf.CellFormat(0, 5, unresolvedAlertLine(alert, 80), "", 1, "L", false, 0, "") } if len(activeAlerts) > 3 { pdf.SetTextColor(colorTextMuted[0], colorTextMuted[1], colorTextMuted[2]) diff --git a/pkg/reporting/reporting.go b/pkg/reporting/reporting.go index cc424e6bc8..b7b83c4644 100644 --- a/pkg/reporting/reporting.go +++ b/pkg/reporting/reporting.go @@ -115,6 +115,28 @@ type AlertInfo struct { StartTime time.Time ResolvedTime *time.Time // nil if still active Acknowledged bool + // ResourceID is the resource the alert was raised on, so an alert that + // moved can find the alert its successor raised for the same metric. + ResourceID string + // Resolution is set only on a resolved alert whose close was not a + // recovery, such as a node alert handed to its Pulse agent. A report + // never counts such an alert as resolved. + Resolution *AlertResolution +} + +// AlertResolution says why an alert closed without its condition clearing. +// It mirrors models.AlertResolution so this package stays free of internal +// dependencies. +type AlertResolution struct { + // Reason is the alert engine's reason code, such as + // AlertResolutionMovedToAgent. + Reason string + SuccessorResourceID string + SuccessorName string + // Summary is the alert engine's one-line account, such as "Alert moved + // to pve1 (Host Agent). This is not a recovery: check the agent for the + // current reading." + Summary string } // BackupInfo contains backup information for VMs/containers From 474dbc4f09974154211ec75c2d4114cfb647bf97 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:05:15 +0100 Subject: [PATCH 11/53] Record Proxmox and agent alert history on the canonical resource Alert lifecycle changes carry the alert's source-native resource ID, and MonitorAdapter.RecordChange only mapped full Docker container IDs. Proxmox node, guest and storage alerts and host-agent alerts were journaled under IDs such as "instance-node", "instance:node:vmid" or "agent:", while resource facets, resource intelligence and Assistant resource context read history by canonical ID, so those surfaces never saw the alerts. Resolve such references to the canonical resource and record a history binding, so the reference keeps reaching the same history after a guest migration or inventory removal. Only durable identities bind: the canonical ID, a retired era, a source ID, a node-scoped guest reference or an agent host ID. Names and hostnames never bind, so a VM named "pulse-system" cannot capture system alerts, and a reference two resources answer to binds to neither. Docker keeps its full-ID rule. Rows already journaled under a source reference join canonical history without being rewritten: each registry generation binds the unbound alert references it can resolve, those found at startup and those written before inventory named the resource, retrying each for 30 minutes. --- .../v6/internal/subsystems/monitoring.md | 9 + .../internal/subsystems/unified-resources.md | 23 +++ .../monitoring/monitor_alert_handling_test.go | 58 ++++++ internal/unifiedresources/history_identity.go | 188 ++++++++++++++++++ .../unifiedresources/history_identity_test.go | 160 +++++++++++++++ internal/unifiedresources/monitor_adapter.go | 38 +++- 6 files changed, 469 insertions(+), 7 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 6522d39af6..5e550c5cb0 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -1043,6 +1043,15 @@ join another container's history. The real alert-manager callback path is covered by `TestDockerAlertTimelineUsesCanonicalHistoryIdentity` in `internal/monitoring/monitor_alert_handling_test.go`. +Proxmox node, guest and storage alert lifecycle events carry source-native IDs +and pass through the same writer, which resolves them to the canonical +resource. Resource facets, resource intelligence and Assistant resource context +therefore include their alert history, including events emitted after the +resource left inventory. Event IDs still hash the alert's source reference, so +replay stays idempotent. The real alert-manager callback path is covered by +`TestProxmoxAlertTimelineUsesCanonicalHistoryIdentity` in +`internal/monitoring/monitor_alert_handling_test.go`. + TrueNAS native alert projection preserves the trimmed, uppercase provider level in ResourceIncident.NativeSeverity. INFO and NOTICE retain the same canonical monitor risk; consumers must not lose their distinct actionability when projecting provider evidence. Native CRITICAL, ALERT, and EMERGENCY all project to canonical critical severity; EMERGENCY must not be discarded as unknown or make a still-active condition appear recovered. WARNING remains warning, and INFO and NOTICE remain informational at this projection boundary. Verification: `TestIncidentProjectionPreservesNativeSeverity` in `internal/truenas/provider_pool_health_contract_test.go` covers all seven native levels and case/whitespace normalization. `TestTrueNASNativeSeverityDispatch` in `internal/alerts/truenas_native_dispatch_test.go` verifies downstream INFO suppression, NOTICE preservation, notification severity, duplicate-poll retention, and confirmed recovery callback identity. The TrueNAS lifecycle tests in `internal/alerts/unified_incidents_test.go` require repeated EMERGENCY evidence to interrupt recovery confirmation. These are fixture-based projection and manager checks, not appliance ingestion or external notification-provider receipt proof. diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 5c1dfdb1bb..8b80254b40 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -5838,6 +5838,29 @@ action requests, approvals, links or exclusions. Separate monitor, API and Assistant store handles must see current persisted aliases. Missing identity storage is an error, not evidence of empty history. Retention removes an alias only after neither identity has retained journal records. +Other alert source references resolve at `MonitorAdapter.RecordChange` +through `ResourceRegistry.resolveHistoryReference`, which accepts only durable +identities: the canonical ID, a retired era, a source ID (Proxmox node, guest +and storage IDs), a node-scoped Proxmox guest reference after migration, or +`agent:`. Names and hostnames never bind history, unlike the wider +`ResolveReferenceID` used by alert policy, so a resource named like a system +reference cannot capture its events. A reference two resources answer to binds +to neither: its event keeps its own reference and creates or changes no +binding, though an earlier binding still covers that reference on read. A +reference no resource +answers to follows its retained binding after inventory removal, or keeps its +own history. Docker references never take this path. Alert journal references +without a binding (read once per process, plus any written before inventory +named their resource) are retried on each published registry generation for +`legacyHistoryBindWindow`. Bound rows keep their recorded resource ID, so a +change list may show the source reference on rows written before the binding. +Only the full-presentation `ResourceDetailDrawer` reads this history in the UI +(facets and resource intelligence); the `table-row` drawer used by Proxmox node +and Agents rows, and the guest and storage drawers, do not request it. +Assistant and Patrol resource contexts read it from the store. Proof: +`TestHistoryIdentityMonitorAdapterResolvesProxmoxAlertReferences` and +`TestHistoryIdentityBindsLegacyAlertRowsFromRegistryGenerations` in +`internal/unifiedresources/history_identity_test.go`. That same shared timeline vocabulary now includes the `activity` change kind for provider-read breadcrumbs such as VMware tasks and events, plus the `vmware_adapter` source-adapter token for canonical provenance drill-down. diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go index 8ecdeb2250..d45c8f1937 100644 --- a/internal/monitoring/monitor_alert_handling_test.go +++ b/internal/monitoring/monitor_alert_handling_test.go @@ -92,6 +92,64 @@ func TestDockerAlertTimelineUsesCanonicalHistoryIdentity(t *testing.T) { t.Logf("DOCKER_HISTORY_LIFECYCLE %s", encoded) } +// Proxmox node and guest alerts carry source-native IDs. Their lifecycle must +// reach the canonical resource history that facets, the drawer and the +// assistant read, including events emitted after the resource left inventory. +func TestProxmoxAlertTimelineUsesCanonicalHistoryIdentity(t *testing.T) { + store, err := unifiedresources.NewSQLiteResourceStore(t.TempDir(), "default") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + manager := alerts.NewManagerWithDataDir(t.TempDir()) + t.Cleanup(manager.Stop) + config := manager.GetConfig() + config.Enabled = true + config.ActivationState = alerts.ActivationPending + config.TimeThresholds = map[string]int{"node": 0, "guest": 0} + config.SuppressionWindow = 0 + manager.UpdateConfig(config) + now := time.Now() + node := models.Node{ID: "lab-pve1", Name: "pve1", Instance: "lab", Host: "https://pve1.lab:8006", Status: "online", CPU: 0.99, LastSeen: now} + vm := models.VM{ID: "lab:pve1:101", VMID: 101, Name: "web", Node: "pve1", Instance: "lab", Status: "stopped", Type: "qemu", LastSeen: now} + registry := unifiedresources.NewRegistry(store) + registry.IngestSnapshot(models.StateSnapshot{Nodes: []models.Node{node}, VMs: []models.VM{vm}}) + monitor := &Monitor{alertManager: manager, resourceStore: unifiedresources.NewMonitorAdapter(registry)} + manager.SubscribeLifecycleCallback(monitor.handleAlertLifecycleEvent) + manager.CheckNode(node) + manager.CheckGuest(vm, vm.Instance) + manager.CheckGuest(vm, vm.Instance) + + canonical := map[string]string{} + for _, resource := range registry.List() { + canonical[resource.Name] = resource.ID + } + require.NotEqual(t, node.ID, canonical["pve1"]) + require.NotEqual(t, vm.ID, canonical["web"]) + filters := unifiedresources.ResourceChangeFilters{Kinds: []unifiedresources.ChangeKind{unifiedresources.ChangeAlertFired, unifiedresources.ChangeAlertResolved}} + for name, sourceID := range map[string]string{"pve1": node.ID, "web": vm.ID} { + changes, err := store.GetRecentChangesFiltered(canonical[name], time.Time{}, 10, filters) + require.NoError(t, err) + require.Len(t, changes, 1, name) + require.Equal(t, unifiedresources.ChangeAlertFired, changes[0].Kind) + require.Equal(t, canonical[name], changes[0].ResourceID) + kinds, err := store.CountRecentChangesByKind(canonical[name], time.Time{}) + require.NoError(t, err) + require.Equal(t, 1, kinds[unifiedresources.ChangeAlertFired], name) + legacy, err := store.GetRecentChangesFiltered(sourceID, time.Time{}, 10, filters) + require.NoError(t, err) + require.Equal(t, changes, legacy, "the source reference reads the same history") + } + + // The node recovers after it left inventory; its retained binding holds. + monitor.resourceStore = unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store)) + node.CPU = 0.05 + manager.CheckNode(node) + changes, err := store.GetRecentChangesFiltered(canonical["pve1"], time.Time{}, 10, filters) + require.NoError(t, err) + require.Len(t, changes, 2) + require.Equal(t, unifiedresources.ChangeAlertResolved, changes[0].Kind) + require.Equal(t, canonical["pve1"], changes[0].ResourceID) +} + func TestMonitor_HandleAlertFired_Extra(t *testing.T) { // 1. Alert is nil m1 := &Monitor{} diff --git a/internal/unifiedresources/history_identity.go b/internal/unifiedresources/history_identity.go index e71c71a2c4..9dc69d886e 100644 --- a/internal/unifiedresources/history_identity.go +++ b/internal/unifiedresources/history_identity.go @@ -4,7 +4,10 @@ import ( "database/sql" "encoding/hex" "fmt" + "log" "strings" + "sync" + "time" ) // legacyDockerHistoryIdentity accepts the source identifier emitted by Docker @@ -29,6 +32,61 @@ func legacyDockerHistoryIdentity(ref string) (sourceID, canonicalID string, ok b return sourceID, SourceSpecificID(ResourceTypeAppContainer, SourceDocker, sourceID), true } +// isDockerHistoryReference marks Docker alert source references, which only a +// complete container ID binds (legacyDockerHistoryIdentity). A container name +// or shortened ID must never join history through general reference matching. +func isDockerHistoryReference(ref string) bool { + return strings.HasPrefix(strings.TrimSpace(ref), "docker:") +} + +// resolveHistoryReference resolves a non-Docker alert reference to the +// resource whose history it joins. Only durable identities qualify: the +// canonical ID, a retired era, a source ID (Proxmox node, guest and storage +// IDs), a node-scoped Proxmox guest reference, or an agent alert reference +// ("agent:"). Names and hostnames never bind history, so a resource +// named like a system reference cannot capture its events. claimed reports +// that inventory answers to the reference, possibly ambiguously; an ambiguous +// reference resolves to nothing and must not follow a retained binding. +func (rr *ResourceRegistry) resolveHistoryReference(ref string) (resourceID string, claimed bool) { + ref = CanonicalResourceID(ref) + if rr == nil || ref == "" || isDockerHistoryReference(ref) { + return "", false + } + rr.mu.RLock() + defer rr.mu.RUnlock() + if rr.resources[ref] != nil { + return ref, true + } + if id := rr.supersededResourceIDLocked(ref); rr.resources[id] != nil { + return id, true + } + matches := make(map[string]struct{}, 1) + for _, mapping := range rr.bySource { + if id := mapping[ref]; rr.resources[id] != nil { + matches[id] = struct{}{} + } + } + if hostID, ok := strings.CutPrefix(ref, "agent:"); ok && hostID != "" && !strings.Contains(hostID, "/") { + if id := rr.bySource[SourceAgent][hostID]; rr.resources[id] != nil { + matches[id] = struct{}{} + } + } + if instance, _, vmid, ok := ParseProxmoxGuestSourceID(ref); ok { + for _, resourceType := range []ResourceType{ResourceTypeVM, ResourceTypeSystemContainer} { + if id := ProxmoxGuestCanonicalID(resourceType, instance, vmid); rr.resources[id] != nil { + matches[id] = struct{}{} + } + } + } + if len(matches) != 1 { + return "", len(matches) > 1 + } + for id := range matches { + resourceID = id + } + return resourceID, true +} + // resourceHistoryIdentityWriter binds a source reference to the canonical // resource of one event. It affects history lookup only, never operator state, // action requests, approvals or execution identities. @@ -90,6 +148,136 @@ func (m *MemoryStore) RecordChangeWithSourceIdentity(change ResourceChange, sour return m.recordChangeLocked(change) } +// legacyHistoryBindWindow bounds how long an unresolved alert journal +// reference is retried. Inventory normally names a resource within a few poll +// cycles; a reference still unresolved after the window names a removed +// resource or a sub-resource and keeps its own history. +const legacyHistoryBindWindow = 30 * time.Minute + +// legacyHistoryBinder exposes alert journal references that no history binding +// covers. Rows recorded before their reference resolved join canonical history +// through a binding; the rows themselves are never rewritten. +type legacyHistoryBinder interface { + unboundHistoryReferences() ([]string, error) + bindHistorySourceIdentities(bindings map[string]string) error +} + +// legacyHistoryBackfill holds unbound references with their retry deadlines: +// those already journaled when the process started, and those RecordChange +// wrote before inventory could name their resource. +type legacyHistoryBackfill struct { + mu sync.Mutex + loaded bool + pending map[string]time.Time +} + +func (b *legacyHistoryBackfill) addLocked(ref string, now time.Time) { + if b.pending == nil { + b.pending = make(map[string]time.Time) + } + if _, ok := b.pending[ref]; !ok { + b.pending[ref] = now.Add(legacyHistoryBindWindow) + } +} + +func (b *legacyHistoryBackfill) add(ref string, now time.Time) { + b.mu.Lock() + defer b.mu.Unlock() + b.addLocked(ref, now) +} + +// unboundHistoryReferences lists alert journal references without a binding. +// Docker references are excluded: the store migration binds full container +// IDs, and names or shortened IDs must never bind. +func (s *SQLiteResourceStore) unboundHistoryReferences() ([]string, error) { + rows, err := s.db.Query(`SELECT DISTINCT canonical_id FROM resource_changes + WHERE kind GLOB 'alert_*' AND canonical_id NOT GLOB 'docker:*' + AND canonical_id NOT IN (SELECT source_id FROM resource_history_aliases) + ORDER BY canonical_id`) + if err != nil { + return nil, fmt.Errorf("read unbound resource history references: %w", err) + } + defer rows.Close() + var refs []string + for rows.Next() { + var ref string + if err := rows.Scan(&ref); err != nil { + return nil, err + } + refs = append(refs, ref) + } + return refs, rows.Err() +} + +func (s *SQLiteResourceStore) bindHistorySourceIdentities(bindings map[string]string) error { + if len(bindings) == 0 { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return fmt.Errorf("begin resource history identities: %w", err) + } + defer func() { _ = tx.Rollback() }() + for sourceID, canonicalID := range bindings { + if _, err := tx.Exec(`INSERT INTO resource_history_aliases (source_id, canonical_id) VALUES (?, ?) + ON CONFLICT(source_id) DO UPDATE SET canonical_id = excluded.canonical_id`, sourceID, canonicalID); err != nil { + return fmt.Errorf("bind resource history identity: %w", err) + } + } + return tx.Commit() +} + +// bindLegacyHistory binds the pending references a published registry +// generation resolves, with the same rules as RecordChange. The journal is +// scanned once per process; later references arrive from RecordChange. +func (a *MonitorAdapter) bindLegacyHistory(registry *ResourceRegistry, now time.Time) { + binder, ok := registry.store.(legacyHistoryBinder) + if !ok { + return + } + backfill := &a.legacyHistory + backfill.mu.Lock() + defer backfill.mu.Unlock() + if !backfill.loaded { + refs, err := binder.unboundHistoryReferences() + if err != nil { + log.Printf("unifiedresources: %v", err) + return + } + backfill.loaded = true + for _, ref := range refs { + backfill.addLocked(ref, now) + } + } + if len(backfill.pending) == 0 { + return + } + bindings := make(map[string]string) + var settled []string + for ref, deadline := range backfill.pending { + if resolvedID, _ := registry.resolveHistoryReference(ref); resolvedID != "" { + if resolvedID != ref { + bindings[ref] = resolvedID + } + settled = append(settled, ref) + } else if !now.Before(deadline) { + delete(backfill.pending, ref) + } + } + if err := binder.bindHistorySourceIdentities(bindings); err != nil { + log.Printf("unifiedresources: %v", err) + return + } + for _, ref := range settled { + delete(backfill.pending, ref) + } + if len(bindings) > 0 { + log.Printf("unifiedresources: bound %d alert history references to canonical resources", len(bindings)) + } +} + // migrateResourceHistoryAliases adds an index for exact legacy Docker event // identities, including resources already removed from live inventory. The // event rows and all authority-bearing tables remain unchanged. diff --git a/internal/unifiedresources/history_identity_test.go b/internal/unifiedresources/history_identity_test.go index 21c4512dbf..935e260fb4 100644 --- a/internal/unifiedresources/history_identity_test.go +++ b/internal/unifiedresources/history_identity_test.go @@ -173,3 +173,163 @@ func TestHistoryIdentityRetentionAndUnavailableLookup(t *testing.T) { _, err = store.GetRecentChanges(canonical, time.Time{}, 10) require.Error(t, err) } + +func proxmoxHistoryIdentitySnapshot(now time.Time) models.StateSnapshot { + return models.StateSnapshot{ + Nodes: []models.Node{{ID: "lab-pve1", Name: "pve1", Instance: "lab", Host: "https://pve1.lab:8006", Status: "online", LinkedAgentID: "host-pve1", LastSeen: now}}, + Hosts: []models.Host{{ID: "host-pve1", Hostname: "pve1", LinkedNodeID: "lab-pve1", MachineID: "0123456789abcdef", Status: "online", LastSeen: now}}, + VMs: []models.VM{{ID: "lab:pve1:101", VMID: 101, Name: "web", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now}}, + Containers: []models.Container{{ID: "lab-pve1-102", VMID: 102, Name: "dns", Node: "pve1", Instance: "lab", Status: "running", Type: "lxc", LastSeen: now}}, + } +} + +func historyIdentityResourceID(t *testing.T, registry *ResourceRegistry, resourceType ResourceType, name string) string { + t.Helper() + for _, resource := range registry.List() { + if resource.Type == resourceType && resource.Name == name { + return resource.ID + } + } + t.Fatalf("no %s resource named %q", resourceType, name) + return "" +} + +// Proxmox alerts carry the source-native node and guest IDs. Their lifecycle +// must land in the history the drawer, facets and assistant read by canonical ID. +func TestHistoryIdentityMonitorAdapterResolvesProxmoxAlertReferences(t *testing.T) { + store, err := NewSQLiteResourceStore(t.TempDir(), "default") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + now := time.Now().UTC().Truncate(time.Second) + snapshot := proxmoxHistoryIdentitySnapshot(now) + // A guest named like the system alert reference must not capture it. + snapshot.VMs = append(snapshot.VMs, models.VM{ID: "lab:pve1:120", VMID: 120, Name: "pulse-system", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now}) + registry := NewRegistry(store) + registry.IngestSnapshot(snapshot) + adapter := NewMonitorAdapter(registry) + nodeID := historyIdentityResourceID(t, registry, ResourceTypeAgent, "pve1") + vmID := historyIdentityResourceID(t, registry, ResourceTypeVM, "web") + ctID := historyIdentityResourceID(t, registry, ResourceTypeSystemContainer, "dns") + + // "lab:pve2:101" names a node other than the guest's current one, as an + // alert raised before a live migration does; it still names the guest. + refs := []string{"lab-pve1", "agent:host-pve1", "lab:pve2:101", "lab-pve1-102", "pulse-system", "web"} + for i, ref := range refs { + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "fired-" + ref, ResourceID: ref, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Duration(i) * time.Second)})) + } + for canonicalID, want := range map[string][]string{nodeID: {"agent:host-pve1", "lab-pve1"}, vmID: {"lab:pve2:101"}, ctID: {"lab-pve1-102"}} { + got, err := store.GetRecentChanges(canonicalID, time.Time{}, 10) + require.NoError(t, err) + require.Len(t, got, len(want), canonicalID) + for i, ref := range want { + require.Equal(t, "fired-"+ref, got[i].ID) + require.Equal(t, canonicalID, got[i].ResourceID) + bound, found, err := store.ResolveHistorySourceIdentity(ref) + require.NoError(t, err) + require.True(t, found, ref) + require.Equal(t, canonicalID, bound) + } + kinds, err := store.CountRecentChangesByKind(canonicalID, time.Time{}) + require.NoError(t, err) + require.Equal(t, len(want), kinds[ChangeAlertFired], canonicalID) + } + // Names are display identities: they keep their own history and no binding. + for _, ref := range []string{"pulse-system", "web"} { + got, err := store.GetRecentChanges(ref, time.Time{}, 10) + require.NoError(t, err) + require.Len(t, got, 1, ref) + _, found, err := store.ResolveHistorySourceIdentity(ref) + require.NoError(t, err) + require.False(t, found, ref) + } + got, err := store.GetRecentChanges(historyIdentityResourceID(t, registry, ResourceTypeVM, "pulse-system"), time.Time{}, 10) + require.NoError(t, err) + require.Empty(t, got) + + // Recovery after the node left inventory still reaches the same history. + removed := NewMonitorAdapter(NewRegistry(store)) + require.NoError(t, removed.RecordChange(ResourceChange{ID: "resolved-lab-pve1", ResourceID: "lab-pve1", Kind: ChangeAlertResolved, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Minute)})) + got, err = store.GetRecentChanges(nodeID, time.Time{}, 10) + require.NoError(t, err) + require.Len(t, got, 3) + require.Equal(t, "resolved-lab-pve1", got[0].ID) + require.Equal(t, nodeID, got[0].ResourceID) + + // When two resources answer to the reference, neither the registry nor the + // retained binding may choose: the event keeps its own reference. + registry.mu.Lock() + registry.bySource[SourcePBS] = map[string]string{"lab-pve1": vmID} + registry.mu.Unlock() + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "ambiguous-lab-pve1", ResourceID: "lab-pve1", Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(2 * time.Minute)})) + var recordedAs string + require.NoError(t, store.db.QueryRow(`SELECT canonical_id FROM resource_changes WHERE id = 'ambiguous-lab-pve1'`).Scan(&recordedAs)) + require.Equal(t, "lab-pve1", recordedAs) + bound, _, err := store.ResolveHistorySourceIdentity("lab-pve1") + require.NoError(t, err) + require.Equal(t, nodeID, bound, "an ambiguous reference never rebinds") + got, err = store.GetRecentChanges(vmID, time.Time{}, 10) + require.NoError(t, err) + require.Len(t, got, 1) +} + +// Alert rows recorded under source-native references before they resolved join +// canonical history once a registry generation can name their resource: rows +// journaled before this process started, and rows written before inventory +// knew the resource. The rows themselves are never rewritten. +func TestHistoryIdentityBindsLegacyAlertRowsFromRegistryGenerations(t *testing.T) { + store, err := NewSQLiteResourceStore(t.TempDir(), "default") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + now := time.Now().UTC().Truncate(time.Second) + for i, ref := range []string{"lab-pve1", "lab-pve1-102", "lab:pve1:103", "lab:pve1:999", "docker:tower/worker"} { + require.NoError(t, store.RecordChange(ResourceChange{ID: "legacy-" + ref, ResourceID: ref, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Duration(i) * time.Second)})) + } + adapter := NewMonitorAdapter(NewRegistry(store)) + snapshot := proxmoxHistoryIdentitySnapshot(now) + adapter.PopulateFromSnapshot(snapshot) + registry := adapter.currentRegistry() + for ref, canonicalID := range map[string]string{ + "lab-pve1": historyIdentityResourceID(t, registry, ResourceTypeAgent, "pve1"), + "lab-pve1-102": historyIdentityResourceID(t, registry, ResourceTypeSystemContainer, "dns"), + } { + got, err := store.GetRecentChangesFiltered(canonicalID, time.Time{}, 10, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertFired}}) + require.NoError(t, err) + require.Len(t, got, 1, ref) + require.Equal(t, "legacy-"+ref, got[0].ID) + require.Equal(t, ref, got[0].ResourceID) + } + + // An alert raised after the journal scan, before inventory names its guest. + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "early-lab:pve1:104", ResourceID: "lab:pve1:104", Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Minute)})) + for _, ref := range []string{"lab:pve1:103", "lab:pve1:104"} { + _, found, err := store.ResolveHistorySourceIdentity(ref) + require.NoError(t, err) + require.False(t, found, ref) + } + // Both guests join inventory in a later generation and are bound then. + snapshot.VMs = append(snapshot.VMs, + models.VM{ID: "lab:pve1:103", VMID: 103, Name: "late", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now}, + models.VM{ID: "lab:pve1:104", VMID: 104, Name: "early", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now}) + adapter.PopulateFromSnapshot(snapshot) + for name, id := range map[string]string{"late": "legacy-lab:pve1:103", "early": "early-lab:pve1:104"} { + got, err := store.GetRecentChangesFiltered(historyIdentityResourceID(t, adapter.currentRegistry(), ResourceTypeVM, name), time.Time{}, 10, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertFired}}) + require.NoError(t, err) + require.Len(t, got, 1, name) + require.Equal(t, id, got[0].ID) + } + + // Unknown guests and Docker names stay unbound, and retries end at the deadline. + for _, ref := range []string{"lab:pve1:999", "docker:tower/worker"} { + _, found, err := store.ResolveHistorySourceIdentity(ref) + require.NoError(t, err) + require.False(t, found, ref) + } + require.Len(t, adapter.legacyHistory.pending, 1) + require.Contains(t, adapter.legacyHistory.pending, "lab:pve1:999") + adapter.legacyHistory.pending["lab:pve1:999"] = time.Now().Add(-time.Second) + adapter.PopulateFromSnapshot(snapshot) + require.Empty(t, adapter.legacyHistory.pending) + refs, err := store.unboundHistoryReferences() + require.NoError(t, err) + require.Equal(t, []string{"lab:pve1:999"}, refs) +} diff --git a/internal/unifiedresources/monitor_adapter.go b/internal/unifiedresources/monitor_adapter.go index 8bac41ceaa..94795237b4 100644 --- a/internal/unifiedresources/monitor_adapter.go +++ b/internal/unifiedresources/monitor_adapter.go @@ -26,6 +26,7 @@ type MonitorAdapter struct { activeAlerts []models.Alert lastRebuiltAt time.Time staleThresholds map[DataSource]time.Duration + legacyHistory legacyHistoryBackfill overlays overlayReadStateCache } @@ -195,13 +196,17 @@ func (a *MonitorAdapter) currentRegistry() *ResourceRegistry { } // RecordChange forwards canonical resource-history events into the underlying -// resource store when monitoring has a durable store attached. +// resource store when monitoring has a durable store attached. Alert lifecycle +// events carry source-native references (Proxmox node and guest IDs, agent +// IDs); they resolve here so history reads by canonical ID include them. func (a *MonitorAdapter) RecordChange(change ResourceChange) error { registry := a.currentRegistry() if registry == nil || registry.store == nil { return nil } sourceRef := change.ResourceID + history, hasHistory := registry.store.(resourceHistoryIdentityWriter) + unbound := false if sourceID, derivedID, ok := legacyDockerHistoryIdentity(sourceRef); ok { // Use exact source identity when inventory is present, including any // canonical identity merge. Full Docker IDs remain derivable after removal. @@ -212,7 +217,7 @@ func (a *MonitorAdapter) RecordChange(change ResourceChange) error { change.ResourceID = resolvedID } else { change.ResourceID = derivedID - if history, ok := registry.store.(resourceHistoryIdentityWriter); ok { + if hasHistory { id, found, err := history.ResolveHistorySourceIdentity(sourceRef) if err != nil { return err @@ -222,13 +227,31 @@ func (a *MonitorAdapter) RecordChange(change ResourceChange) error { } } } - } - if sourceRef != change.ResourceID { - if writer, ok := registry.store.(resourceHistoryIdentityWriter); ok { - return writer.RecordChangeWithSourceIdentity(change, sourceRef) + } else if resolvedID, claimed := registry.resolveHistoryReference(sourceRef); resolvedID != "" { + change.ResourceID = resolvedID + } else if hasHistory && !claimed && !isDockerHistoryReference(sourceRef) { + // No resource answers to it: keep the binding an earlier event recorded. + id, found, err := history.ResolveHistorySourceIdentity(CanonicalResourceID(sourceRef)) + if err != nil { + return err + } + if found { + change.ResourceID = id + } else { + unbound = true } } - return registry.store.RecordChange(change) + if hasHistory && CanonicalResourceID(sourceRef) != CanonicalResourceID(change.ResourceID) { + return history.RecordChangeWithSourceIdentity(change, sourceRef) + } + if err := registry.store.RecordChange(change); err != nil { + return err + } + // Inventory may name the resource in a later generation; bind the row then. + if _, ok := registry.store.(legacyHistoryBinder); unbound && ok { + a.legacyHistory.add(CanonicalResourceID(sourceRef), time.Now()) + } + return nil } // GetRecentChanges forwards canonical resource-history reads into the @@ -437,6 +460,7 @@ func (a *MonitorAdapter) replaceRegistryLocked(snapshot models.StateSnapshot, re a.activeAlerts = append([]models.Alert(nil), snapshot.ActiveAlerts...) a.lastRebuiltAt = rebuiltAt a.mu.Unlock() + a.bindLegacyHistory(rebuilt, time.Now()) } // ShouldSkipAPIPolling returns true when agent coverage indicates API From ba26b3581e2b8f4b81197a47927eef6eda402ab2 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:15:12 +0100 Subject: [PATCH 12/53] Settle slow polled reads and keep replication links honest A review of #2578 found three gaps it left. The shared poll in createNonSuspendingQuery replaced any read still in flight on every tick. The API answers 429 with Retry-After: 60 and apiFetch waits it out, so under rate limiting a polled query slower than its 30-second interval never settled. Replication reads also ignored the abort signal, so each replaced read kept running. A poll now skips its tick while a read an earlier poll started is still in flight; a refresh, source change or remount revalidation stays replaceable at the next tick as before. A polled read in flight for 150 seconds is presumed stalled and replaced, which also clears any spinner it held. Replication reads forward the signal to apiFetch. A failed first replication read made the job count zero, so a direct /proxmox/replication link fell to Overview without its error or Retry. Replication presence is now unknown until a read succeeds, and a failed poll after that keeps the last answer instead of reopening the route. The shared clock returned its last tick, so a cell mounted between ticks measured from up to 30 seconds ago and the desktop backup age could disagree with its band. Every read now returns the wall clock and the tick only schedules re-reads, which retires the backup Math.max workaround. The #2578 age tests now jump the clock and fire one tick instead of looping hundreds of re-renders, which timed out under load. --- .../subsystems/frontend-primitives.md | 22 ++- .../internal/subsystems/storage-recovery.md | 20 ++- .../internal/subsystems/unified-resources.md | 11 +- .../0a3f56b9810b4db5.json | 52 ++++++ ...ResourceDetailSummary.relativeAge.test.tsx | 30 +++- ...netesControllersTable.relativeAge.test.tsx | 4 +- .../features/proxmox/ProxmoxPageSurface.tsx | 16 +- .../proxmox/ProxmoxReplicationTable.tsx | 13 +- .../ProxmoxPageSurface.contract.test.tsx | 151 +++++++++++++++++- .../ProxmoxReplicationTable.test.tsx | 25 +++ .../proxmoxBackupsTableShared.test.tsx | 21 +++ .../proxmox/proxmoxBackupsTableShared.tsx | 7 +- .../createNonSuspendingQuery.test.tsx | 106 ++++++++++++ .../src/hooks/createNonSuspendingQuery.ts | 33 +++- .../utils/__tests__/relativeTimeClock.test.ts | 16 ++ .../src/utils/relativeTimeClock.ts | 19 ++- 16 files changed, 502 insertions(+), 44 deletions(-) create mode 100644 frontend-modern/browser-verification/0a3f56b9810b4db5.json diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 710f4ef07e..2bcd810d94 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -4649,6 +4649,21 @@ Each query run receives an `AbortSignal`; changing the source, resetting the query, or unmounting its owner must abort the superseded browser request before starting replacement work. Consumers must forward that signal through their API/cache layer when the transport supports cancellation. +A poll (`pollMs`) does not replace a read that an earlier poll started while +that read is still in flight; it skips the tick. The API answers 429 with +`Retry-After: 60` and `apiFetch` honours up to two minutes, so a polled read +slower than the poll interval would otherwise be discarded on every tick and +never settle. Any other read (a refresh, a source change, a remount +revalidation) stays replaceable at the next tick, so a slow or hung refresh is +replaced at most once before polling protects its replacement. A polled read +still in flight at the first poll tick at or after `STALLED_QUERY_READ_MS` (150 +seconds) is presumed stalled: that tick replaces it and settles it as a failed +read, clearing `loading`, setting `resolvedOnce` and publishing a timed-out +`error`, so no consumer is left waiting on a read that never returns. Each +consumer renders that failure as it renders any other (the Proxmox Replication +table shows its error and Retry; workload sparklines keep retained series). A +transport that always takes longer than that deadline plus one poll interval +never publishes. The settings reporting shell now also owns a deliberate split between historical performance reports and current-state VM inventory export. `frontend-modern/src/components/Settings/ReportingPanel.tsx`, @@ -5210,9 +5225,10 @@ cells: a value formatted through `formatPlatformTableRelativeTimeValue` or and tooltips, an open drawer's summary rows and annotations) passes `now` from the owning component's `useRelativeTimeNow`, and a countdown or freshness band derived from the same time (replication Next sync, backup age bands) reads it -too. The clock can trail the wall clock by up to one tick, so a check that -treats a future time as invalid (the backup age band) measures from the later -of the two. +too. Every read of the clock returns the wall clock; the 30-second tick only +tells readers to re-read, so a cell that mounts between ticks never measures +from a stale time and a timestamp from the last few seconds never reads as a +future time. Read-only metadata badges follow the same primitive-owned shell rule. `frontend-modern/src/components/shared/MetadataBadge.tsx` owns filled and outlined appearances, compact sizing, shape, typed tone vocabulary, fit diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 626b08f763..6d712201d4 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -986,15 +986,21 @@ composition, and visible source/state chips must render through `MetadataBadge` instead of restoring local rounded-sm xs badge spans. Backup ages read the frontend-primitives shared relative-time clock: `ProxmoxBackupAgeText` measures the full and compact age text and its -current, aging, or stale band from `useRelativeTimeNow`, never earlier than the -wall clock, so a backup row that stays mounted for days moves from current to -aging without new inventory, and a backup that finished inside the clock's last -tick never bands as a future, unknown age. `ProxmoxPageSurface` reads +current, aging, or stale band from `useRelativeTimeNow`, so a backup row that +stays mounted for days moves from current to aging without new inventory, and +because every clock read returns the wall clock, a backup that finished since +the last tick never bands as a future, unknown age. `ProxmoxPageSurface` reads replication jobs through `createNonSuspendingQuery` with a 30-second background poll, because the Replication table's Last sync and Next sync move on the same -clock: a failed read reports the error but keeps the last jobs and the tab, and -a direct replication link holds until the first read resolves instead of -falling back to Overview. The poll re-reads the existing read-only +clock. Each read forwards the query's abort signal, so a replaced or unmounted +read is cancelled. Whether replication exists is unknown until a read +succeeds, counted from the last reset (an organization switch): a failed read +before then holds a direct replication link on the error and its Retry instead +of falling back to Overview, and once a read has succeeded a failed poll +reports the error but keeps that answer (the jobs and the tab, or the Overview +fallback for an empty answer) rather than reopening the route. An access +denial (401 or 403) withdraws the jobs and the tab, and a direct link holds on +the denial so the operator sees why. The poll re-reads the existing read-only `/api/replication/jobs` snapshot and opens no new storage or recovery path. Proxmox backup-health and recoverable-artifact summary rows follow the same platform-table density contract as every other provider surface. Storage and diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 5c1dfdb1bb..d8f30a735c 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -1274,9 +1274,14 @@ tooltip, and the Proxmox replication Last sync and Next sync in the row, phone projection and disclosure. A silent agent or a stalled pvesr scheduler is the case whose age matters most, so a replication Next sync turns overdue on the clock without new data. Replication jobs bypass the unified-resource stream, -so `ProxmoxPageSurface` re-reads them in the background every 30 seconds; -moving ages over a snapshot read once at mount would age a job that keeps -syncing and count it overdue. +so `ProxmoxPageSurface` polls them in the background every 30 seconds, +skipping a tick while the previous poll's read is still in flight unless that +read has stalled past `STALLED_QUERY_READ_MS`; moving ages over a snapshot read +once at mount would age a job that keeps syncing and count it overdue. +`fetchReplicationJobs` takes the query's abort signal and passes it to +`apiFetch`, so a replaced or unmounted replication read stops instead of +running on in the background, and it throws `apiErrorFromResponse` errors that +keep the HTTP status, so a 401 or 403 withdraws the jobs as an access failure. Kubernetes name columns hold the chevron, status dot and name inside one track, so their md widths leave the name room for a typical node or service name at a 768px viewport: Nodes 20 percent (with Capacity at 16 so its diff --git a/frontend-modern/browser-verification/0a3f56b9810b4db5.json b/frontend-modern/browser-verification/0a3f56b9810b4db5.json new file mode 100644 index 0000000000..70d8f76336 --- /dev/null +++ b/frontend-modern/browser-verification/0a3f56b9810b4db5.json @@ -0,0 +1,52 @@ +{ + "version": 1, + "base_sha": "9aaac4d3e502c01913fb3637d0997b0ce03a8b7b", + "verified_at": "2026-10-06T19:14:07Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx", + "frontend-modern/src/features/proxmox/ProxmoxReplicationTable.tsx", + "frontend-modern/src/features/proxmox/proxmoxBackupsTableShared.tsx", + "frontend-modern/src/hooks/createNonSuspendingQuery.ts", + "frontend-modern/src/utils/relativeTimeClock.ts" + ], + "content_sha256": { + "frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx": "306a0f3a95585efd4d980549f33a41538a00975f62aff39873a75145e4d9bf5f", + "frontend-modern/src/features/proxmox/ProxmoxReplicationTable.tsx": "e56e51e972457dbfe34f5695b69423f1ac916df3213a61d028e05cbcecfd12bd", + "frontend-modern/src/features/proxmox/proxmoxBackupsTableShared.tsx": "33fd51603010d13040faa6fe6b3690d3e4bbe1c941c32ceb1aa0c972ec47370c", + "frontend-modern/src/hooks/createNonSuspendingQuery.ts": "6780323e7d9055b0839da57b6204fd68bfce9696e622362ea34a02d0907554bd", + "frontend-modern/src/utils/relativeTimeClock.ts": "0ff094e2dc98bf07b3792af478bfa90d3156143ddc5aef0e233de80868ec4dba" + }, + "routes": [ + "/proxmox/replication", + "/proxmox/backups", + "/docker/overview", + "/standalone/machines" + ], + "viewports": [ + { + "width": 1600, + "height": 1000 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack built from this tree on origin/main 9aaac4d3e5; Playwright drove it with /api/replication/jobs answers shaped per scenario through page.route", + "Direct /proxmox/replication link whose first read returns 500, at 1600px and 390px: stays on the route with \"Could not load replication jobs\" and Refresh (main's code fell back to Overview with no error); Refresh loads all 8 jobs", + "Jobs loaded, then a poll read answers 403: the Replication tab is withdrawn and the link shows the server's \"Insufficient permissions to read replication jobs\" with Refresh (main kept the tab advertised and a generic message)", + "Empty first answer, then a failed poll, then an empty answer: the link stays on Overview throughout and the route never reopens", + "page.clock installed, a polled replication read held open: the next two 30s poll ticks start no new request and abort none; releasing it lets it finish and the following tick starts the next read (main started reads 3 and 4 over the held read and never cancelled it)", + "Leaving the Proxmox page for Docker while a replication read is held aborts it (net::ERR_ABORTED); main let it run", + "Regression: with /ws frames held and keep-alive pongs, Machines Seen cells move from \"just now\" to \"10m ago\" over ten one-minute clock steps on one socket", + "Proxmox Backups at 1600px and 390px after the clock change retired the band's Math.max: 36 age cells each, none Unknown, every band colour matching its Current/Aging/Stale title, no page errors", + "Same-session control: the five runtime files swapped back to main under the running Vite reproduced every failure above, then restored and compared byte for byte" + ], + "interactions": [ + "Clicked Refresh on the replication error state at both widths", + "Clicked the Docker nav link while a replication read was in flight", + "page.clock.runFor poll steps with data-free pong frames sent before each step" + ] +} diff --git a/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.relativeAge.test.tsx b/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.relativeAge.test.tsx index bf17540823..786914a867 100644 --- a/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.relativeAge.test.tsx +++ b/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.relativeAge.test.tsx @@ -4,6 +4,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { formatPlatformTableDateTimeValue } from '@/features/platformPage/sharedPlatformPage'; import type { Resource } from '@/types/resource'; +import { RELATIVE_TIME_TICK_MS } from '@/utils/relativeTimeClock'; import { InlineResourceSummaryTables } from '../ResourceDetailSummary'; import { useResourceDetailDrawerDerivedState } from '../useResourceDetailDrawerDerivedState'; import type { UseResourceDetailDrawerStateResult } from '../useResourceDetailDrawerState'; @@ -47,6 +48,12 @@ afterEach(() => { vi.useRealTimers(); }); +// Time passes with no new data; the next clock tick re-reads the ages. +const hoursLater = (hours: number) => { + vi.setSystemTime(Date.now() + hours * 60 * 60 * 1000); + vi.advanceTimersByTime(RELATIVE_TIME_TICK_MS); +}; + describe('resource drawer summary relative ages', () => { it('keeps an open Kubernetes controller summary age moving without new data', () => { vi.useFakeTimers({ now: new Date('2026-05-24T13:31:00Z') }); @@ -73,7 +80,7 @@ describe('resource drawer summary relative ages', () => { within(section()).getByText(`${absoluteTime('2026-05-24T13:00:00Z')} (31m ago)`), ).toBeInTheDocument(); - vi.advanceTimersByTime(2 * 60 * 60 * 1000); + hoursLater(2); // The absolute time is unchanged; only the age after it moves. expect( @@ -95,6 +102,7 @@ describe('resource drawer summary relative ages', () => { containerState: 'running', createdAt: '2026-05-24T11:31:00Z', startedAt: '2026-05-24T13:21:00Z', + finishedAt: '2026-05-24T12:31:00Z', }, })} drawer={drawerState} @@ -108,14 +116,16 @@ describe('resource drawer summary relative ages', () => { .closest('tr'); expect(row('Created')).toHaveTextContent('2 hours ago'); expect(row('Started')).toHaveTextContent('10 mins ago'); + expect(row('Finished')).toHaveTextContent('1 hour ago'); - vi.advanceTimersByTime(3 * 60 * 60 * 1000); + hoursLater(3); expect(row('Created')).toHaveTextContent('5 hours ago'); expect(row('Started')).toHaveTextContent('3 hours ago'); + expect(row('Finished')).toHaveTextContent('4 hours ago'); }); - it('keeps an open drawer Last seen moving for a resource that stops reporting', () => { + it('keeps an open drawer Last seen and platform freshness ages moving without new data', () => { vi.useFakeTimers({ now: new Date('2026-05-24T13:31:00Z') }); const { state, dispose } = createRoot((dispose) => ({ @@ -124,7 +134,11 @@ describe('resource drawer summary relative ages', () => { id: 'node-1', type: 'agent', lastSeen: Date.parse('2026-05-24T13:30:00Z'), - }), + platformData: { + docker: { updatesLastCheckedAt: '2026-05-24T13:01:00Z' }, + pmg: { lastUpdated: '2026-05-24T13:21:00Z' }, + }, + } as Partial & Pick), debugEnabled: () => false, discoveryFeatureEnabled: () => false, resourceIntelligence: () => null, @@ -133,11 +147,15 @@ describe('resource drawer summary relative ages', () => { })); expect(state.lastSeen()).toBe('1 min ago'); + expect(state.dockerUpdatesCheckedRelative()).toBe('30 mins ago'); + expect(state.pmgUpdatedRelative()).toBe('10 mins ago'); - // The resource stays silent: its lastSeen never changes again. - vi.advanceTimersByTime(2 * 60 * 60 * 1000); + // The resource stays silent: none of its timestamps change again. + hoursLater(2); expect(state.lastSeen()).toBe('2 hours ago'); + expect(state.dockerUpdatesCheckedRelative()).toBe('2 hours ago'); + expect(state.pmgUpdatedRelative()).toBe('2 hours ago'); dispose(); }); }); diff --git a/frontend-modern/src/features/kubernetes/__tests__/KubernetesControllersTable.relativeAge.test.tsx b/frontend-modern/src/features/kubernetes/__tests__/KubernetesControllersTable.relativeAge.test.tsx index 7b547bf664..4f740ca6e7 100644 --- a/frontend-modern/src/features/kubernetes/__tests__/KubernetesControllersTable.relativeAge.test.tsx +++ b/frontend-modern/src/features/kubernetes/__tests__/KubernetesControllersTable.relativeAge.test.tsx @@ -17,6 +17,7 @@ vi.mock('@/api/resources', () => ({ })); import type { Resource } from '@/types/resource'; +import { RELATIVE_TIME_TICK_MS } from '@/utils/relativeTimeClock'; import { KubernetesControllersTable } from '../KubernetesControllersTable'; const makeResource = ({ @@ -85,7 +86,8 @@ describe('KubernetesControllersTable relative ages', () => { // No data changes: the timestamps stay put and only the clock moves. A // detail computed once at render would still read "31m ago" here. - vi.advanceTimersByTime(3 * 60 * 60 * 1000); + vi.setSystemTime(Date.parse('2026-05-24T16:31:00Z')); + vi.advanceTimersByTime(RELATIVE_TIME_TICK_MS); expect(screen.getByText('Completed 3h ago')).toBeInTheDocument(); expect(screen.getByText('Last success 3h ago')).toBeInTheDocument(); diff --git a/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx b/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx index 118fd48185..0cf4d4c4ba 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx @@ -52,6 +52,7 @@ import { ProxmoxMailGatewayTable } from './ProxmoxMailGatewayTable'; import { ProxmoxNodesTable } from './ProxmoxNodesTable'; import { ProxmoxReplicationTable, fetchReplicationJobs } from './ProxmoxReplicationTable'; import { createNonSuspendingQuery } from '@/hooks/createNonSuspendingQuery'; +import { getAPIReadAccessErrorMessage } from '@/utils/apiAccessError'; import { useUnifiedResources } from '@/hooks/useUnifiedResources'; import type { ReplicationJob } from '@/types/api'; import type { Resource } from '@/types/resource'; @@ -127,11 +128,22 @@ export function ProxmoxPageSurface() { // last jobs, so the tab does not vanish under the user. const replicationJobs = createNonSuspendingQuery({ source: () => 'proxmox-pve', - fetcher: () => fetchReplicationJobs(), + fetcher: (_source, signal) => fetchReplicationJobs(signal), initialValue: NO_REPLICATION_JOBS, pollMs: REPLICATION_JOBS_POLL_MS, }); const replicationJobCount = createMemo(() => replicationJobs.value().length); + // Replication presence is known once a read has succeeded. A failed read + // before that says nothing, so a direct link holds on its error and Retry + // instead of falling to Overview; a failed poll after that keeps the last + // answer rather than reopening a route an empty read already closed. An + // access denial withdraws that answer, so the link holds on the denial. + const replicationJobsConfirmed = createMemo((confirmed) => { + if (!replicationJobs.resolvedOnce()) return false; + const error = replicationJobs.error(); + if (getAPIReadAccessErrorMessage(error)) return false; + return confirmed || !error; + }, false); const visibleTabs = createMemo(() => { // An unknown snapshot is not evidence that every optional integration is // present. Never use estate-wide aggregations here: unrelated VMware VMs @@ -149,7 +161,7 @@ export function ProxmoxPageSurface() { // Do not discard a direct link while counts are still unknown: its own // resource query must be allowed to hydrate before deciding it is absent. const countsUnknown = - !tabEvidence.facets?.() || (requested === 'replication' && !replicationJobs.resolvedOnce()); + !tabEvidence.facets?.() || (requested === 'replication' && !replicationJobsConfirmed()); return countsUnknown || visibleTabIds().has(requested) ? requested : 'overview'; }); const shouldHydrateTab = (tab: ProxmoxPageTabId) => activeTab() === tab; diff --git a/frontend-modern/src/features/proxmox/ProxmoxReplicationTable.tsx b/frontend-modern/src/features/proxmox/ProxmoxReplicationTable.tsx index 918f4d8486..5a7e714cf4 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxReplicationTable.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxReplicationTable.tsx @@ -5,7 +5,7 @@ import { InlineDetailTableRow } from '@/components/shared/InlineDetailTableRow'; import { StatusDot } from '@/components/shared/StatusDot'; import type { StatusIndicatorVariant } from '@/utils/status'; import { TableCell, TableHead, TableRow } from '@/components/shared/Table'; -import { apiFetch } from '@/utils/apiClient'; +import { apiErrorFromResponse, apiFetch } from '@/utils/apiClient'; import { useRelativeTimeNow } from '@/utils/relativeTimeClock'; import { PlatformTableToolbar, @@ -264,10 +264,15 @@ export const compactReplicationNextSyncText = (text: string, tone: NextSyncTone) return text.replace(/^in /, ''); }; -export async function fetchReplicationJobs(): Promise { - const response = await apiFetch('/api/replication/jobs?platform=proxmox-pve'); +export async function fetchReplicationJobs(signal?: AbortSignal): Promise { + const response = await apiFetch('/api/replication/jobs?platform=proxmox-pve', { signal }); if (!response.ok) { - throw new Error(`Failed to load replication jobs (${response.status})`); + // A structured error keeps the HTTP status, so a 401/403 withdraws the + // jobs as an access failure instead of being retained as an outage. + throw await apiErrorFromResponse( + response, + `Failed to load replication jobs (${response.status})`, + ); } const payload = (await response.json()) as ReplicationJobsResponse; return Array.isArray(payload?.data) ? payload.data : []; diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx b/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx index 30583297ab..0979b3a487 100644 --- a/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx +++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx @@ -1,4 +1,4 @@ -import { cleanup, render, screen, waitFor } from '@solidjs/testing-library'; +import { cleanup, fireEvent, render, screen, waitFor } from '@solidjs/testing-library'; import { Route, Router } from '@solidjs/router'; import { createSignal } from 'solid-js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; @@ -21,7 +21,9 @@ const mockWorkloadSearch = vi.hoisted(() => vi.fn(() => '')); const mockSelectedNode = vi.hoisted(() => vi.fn<() => string | null>(() => null)); const mockHandleNodeSelect = vi.hoisted(() => vi.fn()); const mockWorkloadsOptions = vi.hoisted(() => vi.fn()); -const mockFetchReplicationJobs = vi.hoisted(() => vi.fn(() => Promise.resolve([] as unknown[]))); +const mockFetchReplicationJobs = vi.hoisted(() => + vi.fn((_signal?: AbortSignal) => Promise.resolve([] as unknown[])), +); const makeResource = (resource: Partial & Pick): Resource => ({ @@ -153,14 +155,18 @@ vi.mock('../ProxmoxNodesTable', () => ({ })); vi.mock('../ProxmoxReplicationTable', () => ({ - ProxmoxReplicationTable: (props: { jobs?: unknown[]; error?: unknown }) => ( + ProxmoxReplicationTable: (props: { jobs?: unknown[]; error?: unknown; onRetry: () => void }) => (
+ > + +
), - fetchReplicationJobs: () => mockFetchReplicationJobs(), + fetchReplicationJobs: (signal?: AbortSignal) => mockFetchReplicationJobs(signal), })); const renderSurface = () => @@ -698,6 +704,141 @@ describe('ProxmoxPageSurface contract', () => { } }); + it('holds a direct replication link on a failed first read and recovers on retry', async () => { + mockPathname.mockReturnValue('/proxmox/replication'); + const node = makeResource({ id: 'node:pve1', type: 'agent', name: 'pve1' }); + mockUseUnifiedResources.mockReturnValue({ + resources: () => [node], + facets: () => ({ incidentCount: 0, byType: {} }), + loading: () => false, + error: () => null, + refetch: vi.fn(async () => []), + }); + mockFetchReplicationJobs.mockReset(); + mockFetchReplicationJobs.mockRejectedValueOnce(new Error('replication read failed')); + mockFetchReplicationJobs.mockResolvedValueOnce([{ id: 'job-1' }]); + + try { + renderSurface(); + const tabs = screen.getByTestId('platform-section-tabs'); + // A failed read says nothing about whether replication exists, so the + // link keeps its error and Retry instead of falling back to Overview. + await waitFor(() => + expect(screen.getByTestId('replication-table')).toHaveAttribute('data-error', 'yes'), + ); + expect(tabs).toHaveAttribute('data-active', 'replication'); + expect(tabs).toHaveAttribute('data-tabs', 'overview'); + + fireEvent.click(screen.getByRole('button', { name: 'Retry replication' })); + await waitFor(() => + expect(screen.getByTestId('replication-table')).toHaveAttribute('data-jobs', '1'), + ); + expect(screen.getByTestId('replication-table')).toHaveAttribute('data-error', 'no'); + expect(tabs).toHaveAttribute('data-tabs', 'overview,replication'); + expect(tabs).toHaveAttribute('data-active', 'replication'); + } finally { + mockFetchReplicationJobs.mockReset(); + mockFetchReplicationJobs.mockImplementation(() => Promise.resolve([])); + } + }); + + it('keeps an empty replication answer through a failed poll instead of reopening the route', async () => { + vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] }); + mockPathname.mockReturnValue('/proxmox/replication'); + const node = makeResource({ id: 'node:pve1', type: 'agent', name: 'pve1' }); + mockUseUnifiedResources.mockReturnValue({ + resources: () => [node], + facets: () => ({ incidentCount: 0, byType: {} }), + loading: () => false, + error: () => null, + refetch: vi.fn(async () => []), + }); + mockFetchReplicationJobs.mockReset(); + mockFetchReplicationJobs.mockResolvedValueOnce([]); + mockFetchReplicationJobs.mockRejectedValueOnce(new Error('replication read failed')); + mockFetchReplicationJobs.mockResolvedValueOnce([]); + + try { + renderSurface(); + const tabs = screen.getByTestId('platform-section-tabs'); + // An empty answer means no replication: the link falls back to Overview. + await waitFor(() => expect(tabs).toHaveAttribute('data-active', 'overview')); + + vi.advanceTimersByTime(30_000); + await waitFor(() => expect(mockFetchReplicationJobs).toHaveBeenCalledTimes(2)); + await Promise.resolve(); + expect(tabs).toHaveAttribute('data-active', 'overview'); + expect(screen.queryByTestId('replication-table')).not.toBeInTheDocument(); + + vi.advanceTimersByTime(30_000); + await waitFor(() => expect(mockFetchReplicationJobs).toHaveBeenCalledTimes(3)); + expect(tabs).toHaveAttribute('data-active', 'overview'); + } finally { + vi.useRealTimers(); + mockFetchReplicationJobs.mockReset(); + mockFetchReplicationJobs.mockImplementation(() => Promise.resolve([])); + } + }); + + it('holds a direct replication link on an access denial that withdraws loaded jobs', async () => { + vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] }); + mockPathname.mockReturnValue('/proxmox/replication'); + const node = makeResource({ id: 'node:pve1', type: 'agent', name: 'pve1' }); + mockUseUnifiedResources.mockReturnValue({ + resources: () => [node], + facets: () => ({ incidentCount: 0, byType: {} }), + loading: () => false, + error: () => null, + refetch: vi.fn(async () => []), + }); + mockFetchReplicationJobs.mockReset(); + mockFetchReplicationJobs.mockResolvedValueOnce([{ id: 'job-1' }]); + mockFetchReplicationJobs.mockRejectedValueOnce( + Object.assign(new Error('Insufficient permissions'), { status: 403 }), + ); + + try { + renderSurface(); + const tabs = screen.getByTestId('platform-section-tabs'); + await waitFor(() => + expect(screen.getByTestId('replication-table')).toHaveAttribute('data-jobs', '1'), + ); + + vi.advanceTimersByTime(30_000); + await waitFor(() => + expect(screen.getByTestId('replication-table')).toHaveAttribute('data-error', 'yes'), + ); + // The denied jobs are withdrawn, so the tab goes; the link shows why. + expect(tabs).toHaveAttribute('data-tabs', 'overview'); + expect(tabs).toHaveAttribute('data-active', 'replication'); + } finally { + vi.useRealTimers(); + mockFetchReplicationJobs.mockReset(); + mockFetchReplicationJobs.mockImplementation(() => Promise.resolve([])); + } + }); + + it('cancels an in-flight replication read when the page unmounts', async () => { + mockPathname.mockReturnValue('/proxmox/replication'); + setResources([makeResource({ id: 'node:pve1', type: 'agent', name: 'pve1' })]); + mockFetchReplicationJobs.mockReset(); + mockFetchReplicationJobs.mockReturnValueOnce(new Promise(() => undefined)); + + try { + const view = renderSurface(); + expect(mockFetchReplicationJobs).toHaveBeenCalledTimes(1); + const signal = mockFetchReplicationJobs.mock.calls[0][0]; + expect(signal).toBeInstanceOf(AbortSignal); + expect(signal?.aborted).toBe(false); + + view.unmount(); + expect(signal?.aborted).toBe(true); + } finally { + mockFetchReplicationJobs.mockReset(); + mockFetchReplicationJobs.mockImplementation(() => Promise.resolve([])); + } + }); + it('does not surface stale-agent notices for development builds without an agent target', () => { mockVersionInfo.mockReturnValue({ version: '6.0.0-rc.6+git.172.g2c360f779.dirty', diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxReplicationTable.test.tsx b/frontend-modern/src/features/proxmox/__tests__/ProxmoxReplicationTable.test.tsx index 8e9f45cdec..938f07c21f 100644 --- a/frontend-modern/src/features/proxmox/__tests__/ProxmoxReplicationTable.test.tsx +++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxReplicationTable.test.tsx @@ -7,6 +7,7 @@ import { REPLICATION_MOBILE_COLUMNS, REPLICATION_MOBILE_COLUMN_WIDTHS, compactReplicationNextSyncText, + fetchReplicationJobs, formatMobileReplicationGuestLabel, } from '../ProxmoxReplicationTable'; import replicationTableSource from '../ProxmoxReplicationTable.tsx?raw'; @@ -134,6 +135,30 @@ describe('ProxmoxReplicationTable', () => { }); }); +describe('fetchReplicationJobs', () => { + it('passes the caller abort signal to the network request', async () => { + const fetchSpy = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(new Response(JSON.stringify({ data: [], meta: { total: 0 } }))); + const controller = new AbortController(); + + await expect(fetchReplicationJobs(controller.signal)).resolves.toEqual([]); + + const init = fetchSpy.mock.calls.find(([url]) => + String(url).includes('/api/replication/jobs'), + )?.[1]; + expect(init?.signal).toBe(controller.signal); + }); + + it('keeps the HTTP status on a failed read so access denials are recognised', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response(JSON.stringify({ message: 'Insufficient permissions' }), { status: 403 }), + ); + + await expect(fetchReplicationJobs()).rejects.toMatchObject({ status: 403 }); + }); +}); + describe('ProxmoxReplicationTable relative times', () => { const start = Date.parse('2026-10-04T12:00:00Z'); // Last synced 5 minutes before the table opens, next sync due 10 minutes after. diff --git a/frontend-modern/src/features/proxmox/__tests__/proxmoxBackupsTableShared.test.tsx b/frontend-modern/src/features/proxmox/__tests__/proxmoxBackupsTableShared.test.tsx index 2d6ec3ca25..ab23ae24f3 100644 --- a/frontend-modern/src/features/proxmox/__tests__/proxmoxBackupsTableShared.test.tsx +++ b/frontend-modern/src/features/proxmox/__tests__/proxmoxBackupsTableShared.test.tsx @@ -242,6 +242,27 @@ describe('backup age cell on the shared clock', () => { ); }); + it('keeps the full age text and its band in step between clock ticks', () => { + vi.useFakeTimers({ now: new Date('2026-10-04T12:00:00Z') }); + // Another mounted cell starts the shared clock. + render(() => ); + + // Before the next tick, a backup crosses seven days old as its row mounts. + vi.setSystemTime(new Date('2026-10-11T12:00:10Z')); + const createdAt = '2026-10-04T12:00:00Z'; + render(() => ( + + + + )); + + const cell = within(screen.getByTestId('boundary')).getByTitle(/backup age/); + expect(cell).toHaveTextContent('7d ago'); + expect(cell).toHaveClass('text-amber-600'); + }); + it('does not band a backup that finished after the last clock tick as unknown', () => { vi.useFakeTimers({ now: new Date('2026-10-04T12:00:00Z') }); // Another mounted cell starts the shared clock at 12:00:00. diff --git a/frontend-modern/src/features/proxmox/proxmoxBackupsTableShared.tsx b/frontend-modern/src/features/proxmox/proxmoxBackupsTableShared.tsx index 4828f3f754..099548660f 100644 --- a/frontend-modern/src/features/proxmox/proxmoxBackupsTableShared.tsx +++ b/frontend-modern/src/features/proxmox/proxmoxBackupsTableShared.tsx @@ -233,11 +233,8 @@ export function formatCompactBackupAge(createdAt: string, now?: number): string export function ProxmoxBackupAgeText(props: { artifact: RecoverableArtifact; compact?: boolean }) { // Backup rows stay mounted while an artifact's creation time never changes, - // so the age and its freshness band both read the shared clock. The clock - // can trail the wall clock by up to one tick, and a backup that finished - // inside that window must not band as a future (unknown) age. - const tickNow = useRelativeTimeNow(); - const now = () => Math.max(tickNow(), Date.now()); + // so the age and its freshness band both read the shared clock. + const now = useRelativeTimeNow(); const band = () => getRecoveryAgeBand(props.artifact.createdMs, now()); const title = () => { if (band() === 'unknown') return recoveryAgeTitleByBand.unknown; diff --git a/frontend-modern/src/hooks/__tests__/createNonSuspendingQuery.test.tsx b/frontend-modern/src/hooks/__tests__/createNonSuspendingQuery.test.tsx index ce8b4af4b4..95d0cc4c82 100644 --- a/frontend-modern/src/hooks/__tests__/createNonSuspendingQuery.test.tsx +++ b/frontend-modern/src/hooks/__tests__/createNonSuspendingQuery.test.tsx @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { cleanup, fireEvent, render, screen, waitFor } from '@solidjs/testing-library'; import { createSignal } from 'solid-js'; import { + STALLED_QUERY_READ_MS, createNonSuspendingQuery, getCreateNonSuspendingQueryCacheDiagnosticsForTest, resetCreateNonSuspendingQueryCacheForTest, @@ -312,6 +313,111 @@ it.each(['success', 'failure'] as const)( }, ); +it('lets a background read slower than the poll interval settle', async () => { + vi.useFakeTimers(); + // The second read takes 40s, for example while apiFetch waits out a 429 + // Retry-After, and the query polls every 30s. + const fetcher = vi + .fn() + .mockResolvedValueOnce('first') + .mockImplementationOnce( + () => new Promise((resolve) => setTimeout(() => resolve('second'), 40_000)), + ) + .mockResolvedValue('third'); + const Probe = () => { + const query = createNonSuspendingQuery({ + source: () => 'replication', + fetcher, + initialValue: '', + pollMs: 30_000, + }); + return {query.value()}; + }; + render(() => ); + await vi.advanceTimersByTimeAsync(0); + expect(screen.getByTestId('query')).toHaveTextContent('first'); + + // The 30s poll starts the slow read; the 60s poll must not discard it. + await vi.advanceTimersByTimeAsync(60_000); + expect(fetcher).toHaveBeenCalledTimes(2); + expect(fetcher.mock.calls[1][1].aborted).toBe(false); + + await vi.advanceTimersByTimeAsync(10_000); + expect(screen.getByTestId('query')).toHaveTextContent('second'); + + // Polling resumes on the next tick once nothing is in flight. + await vi.advanceTimersByTimeAsync(20_000); + expect(fetcher).toHaveBeenCalledTimes(3); + expect(screen.getByTestId('query')).toHaveTextContent('third'); +}); + +it('drops a stalled read loading state even when its replacement also hangs', async () => { + vi.useFakeTimers(); + const hang = () => new Promise(() => undefined); + const fetcher = vi.fn().mockResolvedValueOnce('first').mockImplementation(hang); + const Probe = () => { + const query = createNonSuspendingQuery({ + source: () => 'pbs-host', + fetcher, + initialValue: '', + pollMs: 30_000, + }); + return ( + <> + + {`${query.value()}|loading:${query.loading()}`} + + ); + }; + render(() => ); + await vi.advanceTimersByTimeAsync(0); + fireEvent.click(screen.getByRole('button', { name: 'Refresh' })); + expect(screen.getByTestId('query')).toHaveTextContent('first|loading:true'); + + // The refresh never returns, so the next poll replaces it; that read never + // returns either, and later polls leave it alone until it has stalled. + await vi.advanceTimersByTimeAsync(30_000); + expect(fetcher).toHaveBeenCalledTimes(3); + expect(fetcher.mock.calls[1][1].aborted).toBe(true); + await vi.advanceTimersByTimeAsync(STALLED_QUERY_READ_MS - 30_000); + expect(fetcher).toHaveBeenCalledTimes(3); + expect(screen.getByTestId('query')).toHaveTextContent('first|loading:true'); + + await vi.advanceTimersByTimeAsync(30_000); + expect(fetcher).toHaveBeenCalledTimes(4); + expect(fetcher.mock.calls[2][1].aborted).toBe(true); + expect(screen.getByTestId('query')).toHaveTextContent('first|loading:false'); +}); + +it('reports a never-loaded query as failed once its polled read stalls', async () => { + vi.useFakeTimers(); + // Every read hangs, from the first one on. + const fetcher = vi.fn().mockImplementation(() => new Promise(() => undefined)); + const Probe = () => { + const query = createNonSuspendingQuery({ + source: () => 'replication', + fetcher, + initialValue: '', + pollMs: 30_000, + }); + return ( + {`resolved:${query.resolvedOnce()}|error:${ + (query.error() as Error | null)?.message ?? '' + }|loading:${query.loading()}`} + ); + }; + render(() => ); + await vi.advanceTimersByTimeAsync(30_000); + expect(fetcher).toHaveBeenCalledTimes(2); + expect(screen.getByTestId('query')).toHaveTextContent('resolved:false|error:|loading:true'); + + await vi.advanceTimersByTimeAsync(STALLED_QUERY_READ_MS); + expect(fetcher).toHaveBeenCalledTimes(3); + expect(screen.getByTestId('query')).toHaveTextContent( + 'resolved:true|error:The request timed out.|loading:false', + ); +}); + const settle = async () => { await Promise.resolve(); await Promise.resolve(); diff --git a/frontend-modern/src/hooks/createNonSuspendingQuery.ts b/frontend-modern/src/hooks/createNonSuspendingQuery.ts index 920e0de316..0ef3a1a43c 100644 --- a/frontend-modern/src/hooks/createNonSuspendingQuery.ts +++ b/frontend-modern/src/hooks/createNonSuspendingQuery.ts @@ -26,6 +26,15 @@ interface QueryRunOptions { */ const RETAINED_QUERY_CACHE_MAX_ENTRIES = 64; const RETAINED_QUERY_CACHE_MAX_AGE_MS = 5 * 60_000; +// A poll does not replace a read an earlier poll started while it is still in +// flight: a read slower than the poll interval (the API answers 429 with +// Retry-After: 60, and apiFetch honours up to two minutes) would otherwise be +// discarded on every tick and never settle. Other reads stay replaceable at +// the next tick, so a slow refresh is replaced at most once. Past this age a +// polled read is presumed stalled: the next poll replaces it and reports it as +// a failed read, so a consumer that never loaded shows its error and Retry +// rather than spinning while every read hangs. +export const STALLED_QUERY_READ_MS = 150_000; type RetainedQueryCacheEntry = { cachedAt: number; @@ -179,6 +188,8 @@ export function createNonSuspendingQuery(options: CreateNonSuspendingQuery let latestRequestId = 0; let activeAbortController: AbortController | null = null; + let activeRequestStartedAt = 0; + let activeRequestIsPoll = false; const cancelActiveRequest = () => { activeAbortController?.abort(); @@ -209,10 +220,12 @@ export function createNonSuspendingQuery(options: CreateNonSuspendingQuery }); onCleanup(unsubscribeLiveQueryOrgSwitch); - const run = async (key: K, runOptions: QueryRunOptions = {}): Promise => { + const run = async (key: K, runOptions: QueryRunOptions = {}, poll = false): Promise => { cancelActiveRequest(); const abortController = new AbortController(); activeAbortController = abortController; + activeRequestStartedAt = Date.now(); + activeRequestIsPoll = poll; const requestId = ++latestRequestId; const retainedCacheKey = getRetainedCacheKey(key); const cacheGeneration = retainedQueryCacheGeneration; @@ -296,7 +309,23 @@ export function createNonSuspendingQuery(options: CreateNonSuspendingQuery if (typeof options.pollMs === 'number' && options.pollMs > 0) { const interval = setInterval(() => { - void refetch({ background: true }); + if (activeAbortController !== null && activeRequestIsPoll) { + if (Date.now() - activeRequestStartedAt < STALLED_QUERY_READ_MS) return; + // The stalled read is abandoned and settles as a failure: its + // replacement runs in the background and may hang too, so nothing + // else would clear a spinner or a never-loaded state. + batch(() => { + setLoading(false); + setError(new Error('The request timed out.')); + setResolvedOnce(true); + }); + } + const key = options.source(); + if (key === null) { + reset(); + return; + } + void run(key, { background: true }, true); }, options.pollMs); onCleanup(() => clearInterval(interval)); } diff --git a/frontend-modern/src/utils/__tests__/relativeTimeClock.test.ts b/frontend-modern/src/utils/__tests__/relativeTimeClock.test.ts index 5a92bbd0a3..ae65a87f5c 100644 --- a/frontend-modern/src/utils/__tests__/relativeTimeClock.test.ts +++ b/frontend-modern/src/utils/__tests__/relativeTimeClock.test.ts @@ -21,6 +21,22 @@ describe('useRelativeTimeNow', () => { expect(vi.getTimerCount()).toBe(0); }); + it('reads the wall clock between ticks, so a late reader never trails it', () => { + vi.useFakeTimers({ now: new Date('2026-10-06T10:00:00Z') }); + + const first = createRoot((dispose) => ({ now: useRelativeTimeNow(), dispose })); + // A second cell mounts 20s after the clock started, before the next tick. + vi.setSystemTime(new Date('2026-10-06T10:00:20Z')); + const second = createRoot((dispose) => ({ now: useRelativeTimeNow(), dispose })); + + expect(second.now()).toBe(Date.parse('2026-10-06T10:00:20Z')); + expect(first.now()).toBe(second.now()); + + first.dispose(); + second.dispose(); + expect(vi.getTimerCount()).toBe(0); + }); + it('reads the time directly outside a reactive owner', () => { vi.useFakeTimers({ now: new Date('2026-10-06T10:00:00Z') }); const now = useRelativeTimeNow(); diff --git a/frontend-modern/src/utils/relativeTimeClock.ts b/frontend-modern/src/utils/relativeTimeClock.ts index 43cd334e7e..0957a98ca1 100644 --- a/frontend-modern/src/utils/relativeTimeClock.ts +++ b/frontend-modern/src/utils/relativeTimeClock.ts @@ -7,19 +7,26 @@ import { createSignal, getOwner, onCleanup } from 'solid-js'; // when nothing does. export const RELATIVE_TIME_TICK_MS = 30_000; -const [now, setNow] = createSignal(Date.now()); +// The tick only tells readers to re-read; every read returns the wall clock. +// A cell that mounts between ticks would otherwise measure from up to one tick +// ago, so a timestamp from the last few seconds could read as in the future +// and two cells formatted from the same time could disagree. +const [tick, setTick] = createSignal(0); +const now = (): number => { + tick(); + return Date.now(); +}; let readers = 0; let timer: ReturnType | undefined; -// Inside a component or reactive root, the returned accessor is the shared -// ticking signal. Outside one there is nothing to stop a timer, so it reads the -// time directly. +// Inside a component or reactive root, the returned accessor tracks the shared +// tick. Outside one there is nothing to stop a timer, so it reads the time +// directly. export function useRelativeTimeNow(): () => number { if (!getOwner()) return () => Date.now(); readers += 1; if (timer === undefined) { - setNow(Date.now()); - timer = setInterval(() => setNow(Date.now()), RELATIVE_TIME_TICK_MS); + timer = setInterval(() => setTick((count) => count + 1), RELATIVE_TIME_TICK_MS); } onCleanup(() => { readers -= 1; From 95b29286dd2ee211241c5819eabd35365124cee8 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:09:04 +0100 Subject: [PATCH 13/53] Stop serving retained disk temperatures as current readings Normalization may keep a disk temperature it did not collect (a disk in standby, a host agent past its reporting lease) as long as its collection state says so. Two consumers still presented that value as current. The metrics-history handler answered an empty disk range with the disk's temperature as a "live" point at now, regardless of its state. Since the history writers stopped recording retained temperatures, a short-range chart for a silent agent's disk or a sleeping disk showed the retained value as a fresh reading. Mock mode also seeded synthetic series from it and padded sparse stored series out to now. All of these now require a temperature collected now (diskinventory.TemperatureCollected, the shared form of the rule the history writers apply). Unraid array-inventory rows carry no per-field provenance, so a temperature taken from one looked collected after the agent went silent. The adapter now derives its state from the host lease that ExpireHostTelemetry already records, and from the disk's spin state: unavailable once the host is offline or the disk is spun down. A SMART row that borrows the inventory reading takes that state, including available. The native row records only the withdrawing states, because the registry merge picks temperature and state separately and an available claim from it would mask the SMART row's own state. A withdrawn inventory temperature no longer counts as heat in disk risk. --- .../v6/internal/subsystems/agent-lifecycle.md | 5 + .../v6/internal/subsystems/api-contracts.md | 11 ++ .../v6/internal/subsystems/monitoring.md | 6 + .../internal/subsystems/security-privacy.md | 8 ++ .../internal/subsystems/storage-recovery.md | 9 ++ .../internal/subsystems/unified-resources.md | 14 ++ internal/api/contract_test.go | 124 ++++++++++++++++++ internal/api/router.go | 14 +- internal/models/models.go | 6 +- .../physical_disk_roundtrip_test.go | 85 ++++++++++++ .../unifiedresources/adapter_coverage_test.go | 117 +++++++++++++++++ internal/unifiedresources/adapters.go | 55 +++++++- pkg/diskinventory/status.go | 17 +++ pkg/diskinventory/status_test.go | 22 ++++ 14 files changed, 483 insertions(+), 10 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 6813b04986..4d1c57bcac 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -734,6 +734,11 @@ agent's SMART temperature and I/O counters as last-known values but marks them report replaces them. Expiry is compare-and-set on the report time the offline sweep judged stale, so a report admitted between that judgement and the expiry is never expired. +Unraid array-inventory temperatures follow the same lease: while the host is +offline the unified-resources adapter reports them `unavailable` with the same +reason (`models.HostAgentStoppedReportingReason`) and leaves them out of the +disk's risk, and the metrics-history API does not return any such retained +disk temperature as a live point. An enabled availability target assigned to a host agent creates an agent-lifecycle lease for that exact target/agent pairing. First assignment diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 8454240e42..80f345f31f 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -6308,6 +6308,17 @@ windows. `/api/metrics-store/history` must accept `resourceType=disk`, keep `MetricsTarget.ResourceID` that unified resources already expose, instead of leaving storage drawers or other callers to fork a disk-local history route or invent an alternate disk identity. +When a disk range has no samples, the `smart_temp` live fallback (single-metric +and all-metrics) is served only for a temperature that is collected +(`diskinventory.TemperatureCollected`). A temperature that normalization kept +under a non-available collection state, such as a disk in standby or a host +agent past its reporting lease, is omitted, so the range stays empty instead of +returning that value at `now` with `source: "live"`. Mock mode derives a +synthetic disk temperature series from the disk's reading, or pads a sparse +stored one out to `now`, only from a temperature collected now; without one the +stored samples are returned as they are. An empty mock range can still fall +through to the monitor's generic demo chart series for that disk ID, which is +not derived from the disk's reading. That same metrics-history contract also owns Kubernetes pod identity normalization. `/api/metrics-store/history` must accept legacy bare pod IDs such as `cluster-1:pod:pod-1`, canonicalize them onto the unified pod metrics diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 6522d39af6..94971c3220 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -3778,6 +3778,12 @@ predates collection state), so a retained reading has to keep its non-available state to stay out of history; a path that relabels a carried reading as available, such as a node-temperature carry stamped `proxmox_node_smart`, is not covered by this rule. +`diskinventory.TemperatureCollected` is the shared form of that test for +consumers outside the history writers: the metrics-history API serves a disk +`smart_temp` live point, and mock mode derives a synthetic disk temperature +series from the disk's reading or pads a stored one, only from a collected +temperature, so a retained standby or silent-agent reading never reappears as a +point at the current time. Unified-resource physical-disk round trips must retain named `StorageGroup` membership rather than degrading it to the generic `Used` filesystem label. diff --git a/docs/release-control/v6/internal/subsystems/security-privacy.md b/docs/release-control/v6/internal/subsystems/security-privacy.md index a2dc2b0c6e..3bd46e4a70 100644 --- a/docs/release-control/v6/internal/subsystems/security-privacy.md +++ b/docs/release-control/v6/internal/subsystems/security-privacy.md @@ -3065,3 +3065,11 @@ paused-delivery banner strings in English, German and Spanish. This is alert presentation copy only: no API token name, scope preset, privacy disclosure, telemetry statement or security term changes, and no request, storage or authorization path moves. + +### Retained disk temperatures are not served as live history + +`/api/metrics-store/history` now omits a disk's `smart_temp` live point, and +the mock-mode synthetic series derived from that reading, when the temperature +was not collected by the current observation. This only withholds a value the +caller was already authorized to read; the route, its authentication and scope +checks, tenant resolution and response shape are unchanged. diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 626b08f763..f3dc92fb2b 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -6739,3 +6739,12 @@ blank header is removed on the next save. Export and import pass through the same normalization. No new file, retention window, backup, migration or recovery authority is introduced; `TestWebhookMaskedValuesResolvePerKey` in `internal/api/alerting/notifications_test.go` covers the write path. + +### Retained disk temperatures open no storage or recovery path + +The metrics-history handler in `internal/api/router.go` serves a disk +temperature live point only when `diskinventory.TemperatureCollected` holds, +and the Unraid physical-disk adapter marks an inventory temperature kept past +the host agent's lease as no longer collected. Both change presentation of +values Pulse already holds: no metrics write, retention window, backup, +migration or recovery authority is added or moved. diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 5c1dfdb1bb..f90e5ef236 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -3576,6 +3576,20 @@ state the Proxmox row still carries from the same source past its reporting lease must not stay "collected" through the Proxmox row's copy of the agent's own state, including when its host is down and no disk poll refreshes that copy. +Unraid array-inventory rows carry no per-field provenance, so the adapter +derives the state of a temperature taken from one: `unavailable` from `unraid` +with "host agent stopped reporting" once `State.ExpireHostTelemetry` has marked +the host offline, or with "disk is reported spun down" for a spun-down disk, +and otherwise `available`. The value is kept as last-known context; it is +collected again only when a reporting host sends a positive reading for a disk +that is no longer spun down. A SMART row that falls back to +the inventory reading because it has none of its own takes that state with it. +The Unraid-native disk row records only the withdrawing states, never +`available`: the registry chooses a merged disk's temperature and its state +separately, and an `available` claim from the provenance-less row would +outrank the SMART row's explicit state for the same disk when the SMART +reading is the one shown. A withdrawn inventory temperature also stops +counting as heat in the disk's risk assessment. Cross-source correlation compares normalized serial and WWN values across fields without truncation, allowing a PVE bare-hex array-volume serial to join diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go index 32bfb19a12..b2cd38ac74 100644 --- a/internal/api/contract_test.go +++ b/internal/api/contract_test.go @@ -65,6 +65,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/pkg/audit" authpkg "github.com/rcourtman/pulse-go-rewrite/pkg/auth" "github.com/rcourtman/pulse-go-rewrite/pkg/cloudauth" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" "github.com/rcourtman/pulse-go-rewrite/pkg/edition" pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing" licensetestsupport "github.com/rcourtman/pulse-go-rewrite/pkg/licensing/testsupport" @@ -12546,6 +12547,129 @@ func TestContract_MetricsHistoryAgentTemperatureLiveFallback(t *testing.T) { } } +// Normalization may keep a disk temperature it did not collect this time: a +// host agent past its reporting lease, or a disk in standby. The history +// endpoint must not return that retained value as a live point at "now", and +// mock mode must not grow a synthetic series that ends on it. A collected +// temperature still answers an empty range with a live point. +func TestContract_MetricsHistoryDiskLivePointOnlyForCollectedTemperature(t *testing.T) { + state := models.NewState() + lastReport := time.Now().Add(-time.Hour) + state.UpsertHost(models.Host{ + ID: "agent-silent", Hostname: "silent", Status: "online", IntervalSeconds: 30, LastSeen: lastReport, + Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{ + Device: "/dev/sdb", Serial: "WD-SILENT-LIVE", Type: "sata", Health: "PASSED", Temperature: 41, + Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}, + }}}, + }) + if _, changed := state.ExpireHostTelemetry("agent-silent", lastReport); !changed { + t.Fatal("the silent agent's lease expiry did not mark its SMART readings") + } + registry := unifiedresources.NewRegistry(nil) + registry.IngestSnapshot(state.GetSnapshot()) + var resources []unifiedresources.Resource + for _, resource := range unifiedresources.NewMonitorAdapter(registry).GetAll() { + if resource.Type == unifiedresources.ResourceTypePhysicalDisk { + resources = append(resources, resource) + } + } + if len(resources) != 1 || resources[0].PhysicalDisk.Temperature != 41 { + t.Fatalf("silent agent disk resources = %+v, want one disk retaining 41", resources) + } + staticDisk := func(serial string, temperature int, status diskinventory.FieldStatus) unifiedresources.Resource { + return unifiedresources.Resource{ + ID: "physical-disk:" + serial, + Type: unifiedresources.ResourceTypePhysicalDisk, + PhysicalDisk: &unifiedresources.PhysicalDiskMeta{ + Serial: serial, + Temperature: temperature, + Collection: &diskinventory.CollectionStatus{Temperature: status}, + }, + MetricsTarget: &unifiedresources.MetricsTarget{ResourceType: "disk", ResourceID: serial}, + } + } + resources = append(resources, + staticDisk("WD-STANDBY-LIVE", 38, diskinventory.Unavailable("smartctl", "disk is in standby")), + staticDisk("WD-AWAKE-LIVE", 36, diskinventory.Available("smartctl")), + staticDisk("WD-ASLEEP-LIVE", 0, diskinventory.Unavailable("smartctl", "disk is in standby")), + ) + + store, err := metrics.NewStore(metrics.DefaultConfig(t.TempDir())) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + monitor := &monitoring.Monitor{} + monitor.SetResourceStore(staticPhysicalDiskResourceStore{resources: resources}) + setUnexportedField(t, monitor, "metricsHistory", monitoring.NewMetricsHistory(10, time.Hour)) + setUnexportedField(t, monitor, "metricsStore", store) + router := &Router{monitor: monitor} + + history := func(query string) (metricsHistoryResponse, map[string]json.RawMessage) { + t.Helper() + rec := httptest.NewRecorder() + router.handleMetricsHistory(rec, httptest.NewRequest(http.MethodGet, "/api/metrics-store/history?resourceType=disk&"+query, nil)) + if rec.Code != http.StatusOK { + t.Fatalf("%s: status = %d, body=%s", query, rec.Code, rec.Body.String()) + } + var single metricsHistoryResponse + var all struct { + Metrics map[string]json.RawMessage `json:"metrics"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &single); err != nil { + t.Fatalf("%s: decode: %v", query, err) + } + if err := json.Unmarshal(rec.Body.Bytes(), &all); err != nil { + t.Fatalf("%s: decode metrics: %v", query, err) + } + return single, all.Metrics + } + + for _, serial := range []string{"WD-SILENT-LIVE", "WD-STANDBY-LIVE"} { + resp, _ := history("resourceId=" + serial + "&metric=smart_temp&range=5m") + if resp.Source == "live" || len(resp.Points) != 0 { + t.Fatalf("%s: retained temperature served as current: source=%q points=%+v", serial, resp.Source, resp.Points) + } + if _, all := history("resourceId=" + serial + "&range=5m"); all["smart_temp"] != nil { + t.Fatalf("%s: retained temperature served in the all-metrics response: %s", serial, all["smart_temp"]) + } + } + resp, _ := history("resourceId=WD-AWAKE-LIVE&metric=smart_temp&range=5m") + if resp.Source != "live" || len(resp.Points) != 1 || resp.Points[0].Value != 36 { + t.Fatalf("collected temperature not served as a live point: source=%q points=%+v", resp.Source, resp.Points) + } + + // Mock mode pads a sparse stored series out to now from the disk's current + // temperature. A retained temperature is not current, so the stored + // samples stay as they are. + // Without a current reading (retained, absent, or no such disk) there is + // nothing to pad with, so the last stored sample is not extended either. + setMockModeForTest(t, true) + now := time.Now() + for _, serial := range []string{"WD-STANDBY-LIVE", "WD-ASLEEP-LIVE", "WD-REMOVED-LIVE"} { + store.Write("disk", serial, "smart_temp", 38, now.Add(-50*time.Minute)) + store.Write("disk", serial, "smart_temp", 38, now.Add(-40*time.Minute)) + } + store.Flush() + for _, serial := range []string{"WD-STANDBY-LIVE", "WD-ASLEEP-LIVE", "WD-REMOVED-LIVE"} { + resp, _ = history("resourceId=" + serial + "&metric=smart_temp&range=1h") + if resp.Source != "store" || len(resp.Points) != 2 { + t.Fatalf("%s: mock mode padded stored temperature history to now without a current reading: source=%q points=%d", serial, resp.Source, len(resp.Points)) + } + } + // With nothing stored, mock mode must not seed its synthetic series from a + // retained temperature either. (The monitor's generic demo series for the + // disk ID is not derived from the disk's reading.) + resp, _ = history("resourceId=WD-SILENT-LIVE&metric=smart_temp&range=1h") + if resp.Source == "mock_synthetic" { + t.Fatalf("mock mode synthesized history from a silent agent's retained temperature: points=%+v", resp.Points) + } + resp, _ = history("resourceId=WD-AWAKE-LIVE&metric=smart_temp&range=1h") + if resp.Source != "mock_synthetic" || len(resp.Points) < 2 || resp.Points[len(resp.Points)-1].Value != 36 { + t.Fatalf("mock mode stopped synthesizing a collected temperature: source=%q points=%+v", resp.Source, resp.Points) + } +} + func TestContract_MetricsHistoryCanonicalizesLegacyKubernetesPodIDs(t *testing.T) { mh := monitoring.NewMetricsHistory(1000, time.Hour) now := time.Now().UTC().Truncate(time.Second) diff --git a/internal/api/router.go b/internal/api/router.go index ceec1ce918..6f77a32878 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -6723,7 +6723,9 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request) return points } pd := disk.PhysicalDisk - if pd.Temperature > 0 { + // A retained temperature (standby, a silent host agent) is not a + // reading taken now. + if diskinventory.TemperatureCollected(pd.Temperature, pd.Collection) { points["smart_temp"] = monitoring.MetricPoint{Timestamp: now, Value: float64(pd.Temperature)} } if pd.SMART != nil { @@ -6802,7 +6804,8 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request) if mock.IsMockEnabled() && runtimeResourceType == "disk" { current := 0.0 - if disk := findDisk(resourceID); disk != nil && disk.PhysicalDisk != nil && metricType == "smart_temp" { + if disk := findDisk(resourceID); disk != nil && disk.PhysicalDisk != nil && metricType == "smart_temp" && + diskinventory.TemperatureCollected(disk.PhysicalDisk.Temperature, disk.PhysicalDisk.Collection) { current = float64(disk.PhysicalDisk.Temperature) } if current > 0 || metricType == "disk" || metricType == "diskread" || metricType == "diskwrite" { @@ -7189,7 +7192,12 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request) if len(points) > 0 && len(points) < targetPoints { current := points[len(points)-1].Value if metricType == "smart_temp" { - if disk := findDisk(resourceID); disk != nil && disk.PhysicalDisk != nil && disk.PhysicalDisk.Temperature > 0 { + // Pad a temperature series to now only with a reading + // collected now. Without one (retained, absent, or no + // such disk) the stored samples stay as they are. + current = 0 + if disk := findDisk(resourceID); disk != nil && disk.PhysicalDisk != nil && + diskinventory.TemperatureCollected(disk.PhysicalDisk.Temperature, disk.PhysicalDisk.Collection) { current = float64(disk.PhysicalDisk.Temperature) } } diff --git a/internal/models/models.go b/internal/models/models.go index 30aaa665e6..38ae9c18f3 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -5695,7 +5695,9 @@ func (s *State) ExpireHostTelemetry(hostID string, lastSeen time.Time) (Host, bo return Host{}, false } -const hostAgentStoppedReportingReason = "host agent stopped reporting" +// HostAgentStoppedReportingReason is the collection-state reason recorded on +// readings a host agent supplied once its reporting lease has expired. +const HostAgentStoppedReportingReason = "host agent stopped reporting" // expireHostSMARTReadings marks the temperature and I/O counters of one SMART // row as no longer collected, keeping their values as last-known evidence. @@ -5725,7 +5727,7 @@ func expireCollectedReading(status *diskinventory.FieldStatus, hasValue bool) bo default: return false } - *status = diskinventory.Unavailable(status.Source, hostAgentStoppedReportingReason) + *status = diskinventory.Unavailable(status.Source, HostAgentStoppedReportingReason) return true } diff --git a/internal/monitoring/physical_disk_roundtrip_test.go b/internal/monitoring/physical_disk_roundtrip_test.go index 74bca727c3..0c07448d64 100644 --- a/internal/monitoring/physical_disk_roundtrip_test.go +++ b/internal/monitoring/physical_disk_roundtrip_test.go @@ -633,3 +633,88 @@ func TestStandbyDiskRetainedTemperatureIsNotRecordedAsHistory(t *testing.T) { } } } + +// An Unraid agent reports array disk temperatures in its Unraid inventory, +// which carries no per-field provenance, and a SMART row without a temperature +// falls back to that inventory reading. Once the agent's reporting lease +// expires both rows stay as last-known context, so the canonical disks must say +// the retained temperature is no longer collected. The next report collects +// it again. +func TestSilentUnraidAgentDiskTemperatureIsRetainedButNotCollected(t *testing.T) { + now := time.Now() + state := models.NewState() + report := func(seen time.Time, temperature int) { + state.UpsertHost(models.Host{ + ID: "agent-tower", Hostname: "tower", Status: "online", IntervalSeconds: 30, LastSeen: seen, + Unraid: &models.HostUnraidStorage{ArrayStarted: true, Disks: []models.HostUnraidDisk{ + {Name: "disk1", Device: "sdc", Role: "data", Status: "online", Serial: "UNRAID-ONLY1", Temperature: temperature}, + {Name: "disk2", Device: "sdd", Role: "data", Status: "online", Serial: "UNRAID-SMART2", Temperature: temperature + 2}, + {Name: "disk3", Device: "sde", Role: "data", Status: "online", Serial: "UNRAID-SAS3", Temperature: temperature + 4}, + }}, + // The SMART rows for disk2 and disk3 carry no temperature of their + // own; disk3's smartctl could not read one at all. + Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{ + {Device: "/dev/sdd", Serial: "UNRAID-SMART2", Type: "sata", Health: "PASSED"}, + {Device: "/dev/sde", Serial: "UNRAID-SAS3", Type: "sas", Health: "PASSED", Collection: &diskinventory.CollectionStatus{ + Temperature: diskinventory.Unsupported("smartctl", "no temperature attribute"), + }}, + }}, + }) + } + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m := &Monitor{state: state, resourceStore: adapter, startTime: now.Add(-time.Hour)} + temperatures := func() map[string]diskinventory.FieldStatus { + t.Helper() + adapter.PopulateFromSnapshot(state.GetSnapshot()) + got := make(map[string]diskinventory.FieldStatus) + for _, view := range adapter.PhysicalDisks() { + status := diskinventory.FieldStatus{} + if collection := view.Collection(); collection != nil { + status = collection.Temperature + } + if _, seen := got[view.Serial()]; seen { + t.Fatalf("disk %s surfaced twice", view.Serial()) + } + got[view.Serial()] = status + wantTemperature := map[string]int{"UNRAID-ONLY1": 37, "UNRAID-SMART2": 39, "UNRAID-SAS3": 41}[view.Serial()] + if view.Temperature() != wantTemperature { + t.Fatalf("disk %s temperature = %d, want %d", view.Serial(), view.Temperature(), wantTemperature) + } + } + if len(got) != 3 { + t.Fatalf("canonical disks = %v, want the three Unraid disks", got) + } + return got + } + + // A reporting agent's readings are collected. The inventory-only disk + // keeps no state of its own (it counts as collected); the SMART rows that + // borrow the inventory reading take its provenance. + collected := map[string]diskinventory.FieldStatus{ + "UNRAID-ONLY1": {}, + "UNRAID-SMART2": diskinventory.Available("unraid"), + "UNRAID-SAS3": diskinventory.Available("unraid"), + } + report(now, 37) + for serial, status := range temperatures() { + if status != collected[serial] { + t.Fatalf("reporting agent: disk %s temperature status = %+v, want %+v", serial, status, collected[serial]) + } + } + + state.TouchHost("agent-tower", now.Add(-hostAgentHealthWindow(30)-time.Minute)) + m.evaluateHostAgents(now) + for serial, status := range temperatures() { + if status != diskinventory.Unavailable("unraid", "host agent stopped reporting") { + t.Fatalf("silent agent: disk %s retained temperature presented as collected: %+v", serial, status) + } + } + + report(time.Now(), 37) + m.evaluateHostAgents(time.Now()) + for serial, status := range temperatures() { + if status != collected[serial] { + t.Fatalf("resumed agent: disk %s temperature status = %+v, want collected again (%+v)", serial, status, collected[serial]) + } + } +} diff --git a/internal/unifiedresources/adapter_coverage_test.go b/internal/unifiedresources/adapter_coverage_test.go index 12d961af3e..154ca7e4aa 100644 --- a/internal/unifiedresources/adapter_coverage_test.go +++ b/internal/unifiedresources/adapter_coverage_test.go @@ -7,6 +7,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) func strPointer(v string) *string { @@ -1045,3 +1046,119 @@ func TestMetricPercent(t *testing.T) { }) } } + +// Unraid inventory temperatures have no agent provenance, so the adapter +// derives their state from the host's reporting lease and the disk's spin +// state. A native row records only a state that withdraws its reading, a SMART +// row that borrows the inventory reading takes the inventory's state, and a +// SMART row with its own reading keeps the agent's state. A withdrawn reading +// never counts as heat in the disk's risk. +func TestUnraidDiskTemperatureFollowsTheHostLease(t *testing.T) { + smartWithout := models.HostDiskSMART{Device: "/dev/sdc", Serial: "UNRAID-1", Collection: &diskinventory.CollectionStatus{ + Serial: diskinventory.Available("smartctl"), + Temperature: diskinventory.Unsupported("smartctl", "no temperature attribute"), + }} + smartWith := models.HostDiskSMART{Device: "/dev/sdc", Serial: "UNRAID-1", Temperature: 40, Collection: &diskinventory.CollectionStatus{ + Temperature: diskinventory.Unavailable("smartctl", models.HostAgentStoppedReportingReason), + }} + stopped := diskinventory.Unavailable("unraid", models.HostAgentStoppedReportingReason) + spunDown := diskinventory.Unavailable("unraid", "disk is reported spun down") + hotRisk := func(risk *PhysicalDiskRisk) bool { + if risk == nil { + return false + } + for _, reason := range risk.Reasons { + if reason.Code == "temperature_high" { + return true + } + } + return false + } + for _, tc := range []struct { + name string + status string + spunDown bool + native diskinventory.FieldStatus + borrowed diskinventory.FieldStatus + }{ + {"reporting", "online", false, diskinventory.FieldStatus{}, diskinventory.Available("unraid")}, + {"silent agent", "offline", false, stopped, stopped}, + {"spun down", "online", true, spunDown, spunDown}, + } { + inventory := models.HostUnraidDisk{Name: "disk1", Device: "sdc", Role: "data", Status: "online", Serial: "UNRAID-1", + Temperature: 72, SpunDown: tc.spunDown} + host := models.Host{ID: "agent-tower", Hostname: "tower", Status: tc.status, + Unraid: &models.HostUnraidStorage{Disks: []models.HostUnraidDisk{inventory}}} + collected := tc.borrowed.State == diskinventory.FieldAvailable + + native, _ := resourceFromHostUnraidPhysicalDisk(host, inventory) + got := native.PhysicalDisk + var nativeStatus diskinventory.FieldStatus + if got.Collection != nil { + nativeStatus = got.Collection.Temperature + } + if got.Temperature != 72 || nativeStatus != tc.native { + t.Fatalf("%s: Unraid row temperature=%d state=%+v, want 72 with %+v", tc.name, got.Temperature, nativeStatus, tc.native) + } + if hotRisk(got.Risk) != collected { + t.Fatalf("%s: Unraid row temperature risk = %v, want %v: %+v", tc.name, hotRisk(got.Risk), collected, got.Risk) + } + + borrowed, _ := resourceFromHostSMARTDisk(host, smartWithout) + got = borrowed.PhysicalDisk + if got.Temperature != 72 || got.Collection.Temperature != tc.borrowed || + got.Collection.Serial != diskinventory.Available("smartctl") { + t.Fatalf("%s: SMART row borrowing the inventory reading: temperature=%d collection=%+v", tc.name, got.Temperature, got.Collection) + } + if hotRisk(got.Risk) != collected { + t.Fatalf("%s: borrowed temperature risk = %v, want %v: %+v", tc.name, hotRisk(got.Risk), collected, got.Risk) + } + if smartWithout.Collection.Temperature.State != diskinventory.FieldUnsupported { + t.Fatal("the adapter mutated the SMART row's own collection state") + } + + own, _ := resourceFromHostSMARTDisk(host, smartWith) + if got := own.PhysicalDisk; got.Temperature != 40 || got.Collection.Temperature != smartWith.Collection.Temperature { + t.Fatalf("%s: SMART row with its own reading lost the agent's state: temperature=%d collection=%+v", tc.name, got.Temperature, got.Collection) + } + } + + cold := models.HostUnraidDisk{Name: "disk2", Device: "sdd", Serial: "UNRAID-2", SpunDown: true} + native, _ := resourceFromHostUnraidPhysicalDisk(models.Host{ID: "agent-tower", Status: "online"}, cold) + if native.PhysicalDisk.Collection != nil { + t.Fatalf("an Unraid row without a temperature claimed temperature state: %+v", native.PhysicalDisk.Collection) + } +} + +// The registry merges an Unraid inventory row and the agent's SMART row for the +// same disk, choosing the temperature and the collection state separately. A +// reporting host's inventory row must not lend the merged disk an "available" +// state when the reading it shows is the SMART row's, and that row says it was +// not collected. +func TestUnraidInventoryRowDoesNotMaskTheSMARTRowState(t *testing.T) { + state := models.NewState() + state.UpsertHost(models.Host{ + ID: "agent-tower", Hostname: "tower", Status: "online", + Unraid: &models.HostUnraidStorage{Disks: []models.HostUnraidDisk{ + {Name: "disk1", Device: "sdc", Role: "data", Status: "online", Serial: "UNRAID-MERGE1", Temperature: 37}, + }}, + Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{ + Device: "/dev/sdc", Serial: "UNRAID-MERGE1", Temperature: 40, + Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}, + }}}, + }) + registry := NewRegistry(nil) + registry.IngestSnapshot(state.GetSnapshot()) + var disks []*PhysicalDiskMeta + for _, resource := range registry.List() { + if resource.Type == ResourceTypePhysicalDisk { + disks = append(disks, resource.PhysicalDisk) + } + } + if len(disks) != 1 { + t.Fatalf("physical disks = %d, want the inventory and SMART rows merged into one", len(disks)) + } + if got := disks[0]; got.Temperature != 40 || diskinventory.TemperatureCollected(got.Temperature, got.Collection) { + t.Fatalf("merged disk presents the SMART row's uncollected reading as collected: temperature=%d collection=%+v", got.Temperature, got.Collection) + } +} diff --git a/internal/unifiedresources/adapters.go b/internal/unifiedresources/adapters.go index 67cfe887ab..5196aab935 100644 --- a/internal/unifiedresources/adapters.go +++ b/internal/unifiedresources/adapters.go @@ -804,8 +804,16 @@ func resourceFromHostUnraidPhysicalDisk(host models.Host, disk models.HostUnraid model := strings.TrimSpace(disk.Model) name := firstNonEmpty(model, disk.Name, disk.Device, host.Hostname) health := unraidPhysicalDiskHealth(disk) - assessment := assessUnraidPhysicalDisk(disk) + assessment := assessUnraidPhysicalDisk(host, disk) sizeBytes := unraidDiskSizeBytes(host, disk) + // The row records only a state that withdraws its reading. A positive + // claim from this provenance-less row would outrank another source's + // explicit state for the same disk when the registry merges them, even + // where that source's temperature is the one shown. + var collection *diskinventory.CollectionStatus + if status := unraidDiskTemperatureStatus(host, disk); status.State == diskinventory.FieldUnavailable { + collection = &diskinventory.CollectionStatus{Temperature: status} + } resource := Resource{ Type: ResourceTypePhysicalDisk, Name: name, @@ -829,6 +837,7 @@ func resourceFromHostUnraidPhysicalDisk(host models.Host, disk models.HostUnraid ReadCount: disk.ReadCount, WriteCount: disk.WriteCount, ErrorCount: disk.ErrorCount, + Collection: collection, Risk: physicalDiskRiskFromAssessment(assessment), }, } @@ -841,6 +850,29 @@ func resourceFromHostUnraidPhysicalDisk(host models.Host, disk models.HostUnraid return resource, identity } +// unraidTemperatureSource is the provenance of a temperature read from an +// Unraid agent's array inventory; the agent's standby SMART rows use it too. +const unraidTemperatureSource = "unraid" + +// unraidDiskTemperatureStatus returns the collection state of the temperature +// an Unraid inventory row carries. The inventory has no per-field provenance: +// each accepted report replaces it, and once the host's reporting lease +// expires State.ExpireHostTelemetry keeps it as last-known context and marks +// the host offline. A temperature kept past that point, or one a spun-down +// disk still carries, is not collected. +func unraidDiskTemperatureStatus(host models.Host, disk models.HostUnraidDisk) diskinventory.FieldStatus { + switch { + case disk.Temperature <= 0: + return diskinventory.FieldStatus{} + case strings.EqualFold(strings.TrimSpace(host.Status), "offline"): + return diskinventory.Unavailable(unraidTemperatureSource, models.HostAgentStoppedReportingReason) + case disk.SpunDown: + return diskinventory.Unavailable(unraidTemperatureSource, "disk is reported spun down") + default: + return diskinventory.Available(unraidTemperatureSource) + } +} + func resourceFromHostSMARTDisk(host models.Host, disk models.HostDiskSMART) (Resource, ResourceIdentity) { name := strings.TrimSpace(disk.Model) if name == "" { @@ -880,6 +912,7 @@ func resourceFromHostSMARTDisk(host models.Host, disk models.HostDiskSMART) (Res serial := strings.TrimSpace(disk.Serial) diskType := strings.TrimSpace(disk.Type) temperature := disk.Temperature + collection := diskinventory.CloneStatus(disk.Collection) health := strings.TrimSpace(disk.Health) if unraidDisk != nil { if model == "" { @@ -896,6 +929,13 @@ func resourceFromHostSMARTDisk(host models.Host, disk models.HostDiskSMART) (Res } if temperature <= 0 { temperature = unraidDisk.Temperature + // The reading is now the Unraid inventory's, and so is its state. + if status := unraidDiskTemperatureStatus(host, *unraidDisk); status.State != "" { + if collection == nil { + collection = &diskinventory.CollectionStatus{} + } + collection.Temperature = status + } } if health == "" || strings.EqualFold(health, "UNKNOWN") { health = unraidPhysicalDiskHealth(*unraidDisk) @@ -903,7 +943,7 @@ func resourceFromHostSMARTDisk(host models.Host, disk models.HostDiskSMART) (Res } assessment := storagehealth.AssessHostSMARTDisk(disk) if unraidDisk != nil { - assessment = storagehealth.SummarizeAssessments(assessment, assessUnraidPhysicalDisk(*unraidDisk)) + assessment = storagehealth.SummarizeAssessments(assessment, assessUnraidPhysicalDisk(host, *unraidDisk)) } wearout := physicalDiskWearoutFromSMARTAttributes(disk.Attributes) @@ -939,7 +979,7 @@ func resourceFromHostSMARTDisk(host models.Host, disk models.HostDiskSMART) (Res WriteCount: unraidDiskCounter(unraidDisk, "write"), ErrorCount: unraidDiskCounter(unraidDisk, "error"), IO: physicalDiskIOToMeta(disk.IO), - Collection: diskinventory.CloneStatus(disk.Collection), + Collection: collection, SMART: convertSMARTAttributes(disk.Attributes), Risk: physicalDiskRiskFromAssessment(assessment), }, @@ -1288,11 +1328,16 @@ func unraidPhysicalDiskHealth(disk models.HostUnraidDisk) string { } } -func assessUnraidPhysicalDisk(disk models.HostUnraidDisk) storagehealth.Assessment { +func assessUnraidPhysicalDisk(host models.Host, disk models.HostUnraidDisk) storagehealth.Assessment { + // A retained inventory temperature is not current evidence of heat. + temperature := 0 + if status := unraidDiskTemperatureStatus(host, disk); status.State == diskinventory.FieldAvailable { + temperature = disk.Temperature + } assessment := storagehealth.AssessSample(storagehealth.Sample{ Model: disk.Model, Health: unraidPhysicalDiskHealth(disk), - Temperature: disk.Temperature, + Temperature: temperature, Wearout: -1, }) addReason := func(code string, severity storagehealth.RiskLevel, summary string) { diff --git a/pkg/diskinventory/status.go b/pkg/diskinventory/status.go index 7041ab9883..f4db88f0c6 100644 --- a/pkg/diskinventory/status.go +++ b/pkg/diskinventory/status.go @@ -71,6 +71,23 @@ func CloneStatus(status *CollectionStatus) *CollectionStatus { return &clone } +// TemperatureCollected reports whether a disk temperature was collected by the +// observation that carries it. Normalization may keep a last-known temperature +// it did not collect (a disk in standby, a host agent past its reporting +// lease) under a non-available state; that value must not be recorded or +// presented as a current reading. A temperature without collection state +// predates this contract and counts as collected. +func TemperatureCollected(temperature int, status *CollectionStatus) bool { + if temperature <= 0 { + return false + } + if status == nil { + return true + } + state := status.Temperature.State + return state == "" || state == FieldAvailable +} + // MergeStatus keeps an available observation over a weaker state while still // allowing a current available observation to replace older provenance. func MergeStatus(existing, incoming *CollectionStatus) *CollectionStatus { diff --git a/pkg/diskinventory/status_test.go b/pkg/diskinventory/status_test.go index 238e4b7097..b418eac9da 100644 --- a/pkg/diskinventory/status_test.go +++ b/pkg/diskinventory/status_test.go @@ -65,3 +65,25 @@ func TestMergeReportedStatusLetsTheSourceWithdrawItsOwnAvailability(t *testing.T t.Fatal("MergeReportedStatus mutated its input") } } + +func TestTemperatureCollectedOnlyForAReadingTakenByThisObservation(t *testing.T) { + for _, tc := range []struct { + name string + temperature int + status *CollectionStatus + want bool + }{ + {"no reading", 0, &CollectionStatus{Temperature: Available("smartctl")}, false}, + {"collected", 41, &CollectionStatus{Temperature: Available("smartctl")}, true}, + {"predates collection state", 41, nil, true}, + {"no temperature state", 41, &CollectionStatus{Serial: Available("smartctl")}, true}, + {"retained in standby", 41, &CollectionStatus{Temperature: Unavailable("smartctl", "disk is in standby")}, false}, + {"retained past the agent lease", 41, &CollectionStatus{Temperature: Unavailable("unraid", "host agent stopped reporting")}, false}, + {"unsupported", 41, &CollectionStatus{Temperature: Unsupported("proxmox_disks", "not exposed")}, false}, + {"missing", 41, &CollectionStatus{Temperature: Missing("smartctl", "attribute absent")}, false}, + } { + if got := TemperatureCollected(tc.temperature, tc.status); got != tc.want { + t.Errorf("%s: TemperatureCollected = %v, want %v", tc.name, got, tc.want) + } + } +} From 8b61559f5aa062f1249b2cc5f603fb8996b8f197 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:16:56 +0100 Subject: [PATCH 14/53] Move the metric status declarations clear of main's alert resolution Main added Alert.resolution and its AlertResolution type at the end of the Alert struct in models.go and of the Alert interface in types/api.ts, exactly where this branch added metricStatus and its types, so the two edits conflict. Move this branch's field up beside metadata and its types below ResolvedAlert. Nothing else changes: the built frontend is byte-identical, and the Go types are only reordered. Contract-Neutral: declaration order only; no field, type or behaviour change --- .../57680ea92a178c44.json | 34 +++++++++++++++++++ frontend-modern/src/types/api.ts | 16 ++++----- internal/models/models.go | 16 ++++----- 3 files changed, 50 insertions(+), 16 deletions(-) create mode 100644 frontend-modern/browser-verification/57680ea92a178c44.json diff --git a/frontend-modern/browser-verification/57680ea92a178c44.json b/frontend-modern/browser-verification/57680ea92a178c44.json new file mode 100644 index 0000000000..30970a532a --- /dev/null +++ b/frontend-modern/browser-verification/57680ea92a178c44.json @@ -0,0 +1,34 @@ +{ + "version": 1, + "base_sha": "50b3fd822697e60a9675374e7b17de306da4f625", + "verified_at": "2026-10-06T19:11:04Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/types/api.ts" + ], + "content_sha256": { + "frontend-modern/src/types/api.ts": "e91be59f68e507aa1403e564ff1f9d25e2701cf68173911cf660030e6463323a" + }, + "routes": [ + "/proxmox" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 375, + "height": 812 + } + ], + "states": [ + "Type-only change: MetricAlertStatus and the Alert.metricStatus field moved to different positions in types/api.ts so they no longer abut main's Alert.resolution insertion", + "vite build of the tree before and after this change produces byte-identical output (diff -rq over every emitted asset), so no rendered surface can differ", + "The rendered surfaces themselves were verified live earlier on this branch (receipts 4766668c29936861 and 68deecd720b4873b): /proxmox node drawer and TEMP cell through breaching, held, recovering and resolved at 1440x900 and 375x812" + ], + "interactions": [ + "Built frontend-modern with vite before and after the move and compared every output file", + "Relies on the earlier live passes for drawer expansion, TEMP cell hover text and narrow layout; no runtime bytes changed since" + ] +} diff --git a/frontend-modern/src/types/api.ts b/frontend-modern/src/types/api.ts index a455f1c4a2..2ac721a40f 100644 --- a/frontend-modern/src/types/api.ts +++ b/frontend-modern/src/types/api.ts @@ -1376,16 +1376,20 @@ export interface Alert { ackUser?: string; correlation?: AlertCorrelation; metadata?: Record; - operationalRecord?: OperationalRecord; - latestTransition?: LifecycleTransition; - transitions?: LifecycleTransition[]; - evidence?: EvidenceEnvelope[]; /** * Live evaluation behind an open threshold alert. `value`, `message` and * `lastSeen` keep the last reading that met the trigger; this is the * reading being evaluated now and why the alert has not cleared. */ metricStatus?: MetricAlertStatus; + operationalRecord?: OperationalRecord; + latestTransition?: LifecycleTransition; + transitions?: LifecycleTransition[]; + evidence?: EvidenceEnvelope[]; +} + +export interface ResolvedAlert extends Alert { + resolvedTime: string; } export type MetricAlertPhase = 'breaching' | 'latched' | 'recovering'; @@ -1408,10 +1412,6 @@ export interface MetricAlertStatus { recoveryElapsedSeconds?: number; } -export interface ResolvedAlert extends Alert { - resolvedTime: string; -} - // One entry from the append-only alert event log // (GET /api/alerts/events): a lifecycle transition or a notification // decision, including suppressions with the mechanism that held them. diff --git a/internal/models/models.go b/internal/models/models.go index a08334e25a..bf5567c7de 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -107,11 +107,17 @@ type Alert struct { Acknowledged bool `json:"acknowledged"` AckTime *time.Time `json:"ackTime,omitempty"` AckUser string `json:"ackUser,omitempty"` + // MetricStatus is the live evaluation behind an open threshold alert. + MetricStatus *MetricAlertStatus `json:"metricStatus,omitempty"` // Metadata carries alert-engine annotations (notably resourceType) so the // frontend can classify an alert without re-deriving resource identity. Metadata map[string]interface{} `json:"metadata,omitempty"` - // MetricStatus is the live evaluation behind an open threshold alert. - MetricStatus *MetricAlertStatus `json:"metricStatus,omitempty"` +} + +// ResolvedAlert represents a recently resolved alert +type ResolvedAlert struct { + Alert + ResolvedTime time.Time `json:"resolvedTime"` } // Metric alert phases. A threshold alert stays open after its reading drops @@ -164,12 +170,6 @@ func (s *MetricAlertStatus) Clone() *MetricAlertStatus { return &clone } -// ResolvedAlert represents a recently resolved alert -type ResolvedAlert struct { - Alert - ResolvedTime time.Time `json:"resolvedTime"` -} - // Node represents a Proxmox VE node type Node struct { ID string `json:"id"` From b09aad56d6630df363f2975f43bcc3946785400f Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:25:24 +0100 Subject: [PATCH 15/53] Point the disk detail boundary guard at the card tone helper The disk drawer now picks each attribute card's value tone through getDiskAttributeCardValueTextClass, which mutes a last-known temperature and otherwise delegates to getDiskAttributeValueTextClass. The resource-type boundary guard still required the old helper name in DiskDetail.tsx, so it failed on the previous commit. It now pins the shared card helper, keeping the intent that the drawer takes its tone from the shared presenter rather than a local class map. --- .../src/utils/__tests__/frontendResourceTypeBoundaries.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend-modern/src/utils/__tests__/frontendResourceTypeBoundaries.test.ts b/frontend-modern/src/utils/__tests__/frontendResourceTypeBoundaries.test.ts index dd056e05e3..399a9c8e07 100644 --- a/frontend-modern/src/utils/__tests__/frontendResourceTypeBoundaries.test.ts +++ b/frontend-modern/src/utils/__tests__/frontendResourceTypeBoundaries.test.ts @@ -1303,7 +1303,7 @@ describe('frontend resource type boundaries', () => { expect(storagePoolDetailSource).not.toContain("'text-yellow-500'"); expect(storagePoolDetailSource).not.toContain("'text-yellow-600 dark:text-yellow-400 italic'"); expect(storagePoolDetailSource).not.toContain("'text-red-600 dark:text-red-400 font-medium'"); - expect(diskDetailSource).toContain('getDiskAttributeValueTextClass'); + expect(diskDetailSource).toContain('getDiskAttributeCardValueTextClass'); expect(diskDetailSource).toContain('attributeCards()'); expect(diskDetailSource).toContain('historyCharts()'); expect(diskDetailSource).toContain('getDiskDetailHistoryFallbackMessage'); From b248a9a3679b4c0a1f0e01ea02e5b51155c45cf7 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:31:29 +0100 Subject: [PATCH 16/53] Record the browser pass for the merged alert types Merging main combined this branch's metricStatus declarations with main's alert resolution declarations in types/api.ts, so the file now matches neither parent and needs its own receipt. It emits no runtime code: the build with main's version and with the merged one is byte-identical, and the type-check and alert suites pass. --- .../17aa647ac2cd3920.json | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 frontend-modern/browser-verification/17aa647ac2cd3920.json diff --git a/frontend-modern/browser-verification/17aa647ac2cd3920.json b/frontend-modern/browser-verification/17aa647ac2cd3920.json new file mode 100644 index 0000000000..0938ce7330 --- /dev/null +++ b/frontend-modern/browser-verification/17aa647ac2cd3920.json @@ -0,0 +1,35 @@ +{ + "version": 1, + "base_sha": "e25472feab5b9bcc8acdfea3f946c675475fe7cd", + "verified_at": "2026-10-06T19:31:23Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/types/api.ts" + ], + "content_sha256": { + "frontend-modern/src/types/api.ts": "f050360724c96235f134ed79aa96436a8c414a7d1f96722bb008efa141b2e0fb" + }, + "routes": [ + "/proxmox" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 375, + "height": 812 + } + ], + "states": [ + "Merged types/api.ts is the union of this branch's Alert.metricStatus/MetricAlertStatus declarations and main's Alert.resolution/AlertResolution declarations; it emits no runtime code", + "vite build with main's types/api.ts and with the merged file produce byte-identical output (diff -rq over every emitted asset)", + "tsc --noEmit passes on the merged tree; alert presentation, guardrail, Proxmox nodes table, guest drawer, assistant handoff, storage headline and temperature suites pass (7 files, 300 tests)", + "Rendered surfaces were verified live earlier on this branch (receipts 4766668c29936861 and 68deecd720b4873b); no runtime bytes changed since" + ], + "interactions": [ + "Built frontend-modern with vite using main's and the merged types/api.ts and compared every output file", + "Ran tsc and the alert-related vitest suites on the merged tree" + ] +} From 28d5f4425712cd4778f78e7c62ed245c64a72047 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:12:57 +0100 Subject: [PATCH 17/53] Judge disk risk only on a temperature collected now Since #2593, a disk whose host agent stops reporting keeps its last SMART temperature as a last-known value marked unavailable, and a disk in standby keeps its pre-sleep reading the same way. The shared disk risk assessment still fed that value into its temperature rule, so a silent agent's frozen 72C made the disk read "Replace Now: Disk temperature is 72C" with a warning status, while the temperature cell beside it now says the reading is not current. storagehealth.CollectedTemperature passes a temperature through only when its collection state is available, or when the source predates collection state. AssessPhysicalDisk, AssessHostSMARTDisk and the registry's recompute from merged metadata use it, so a retained reading stays on the resource as history but adds no temperature_high reason and leaves the disk online. Health, wearout and SMART counters are unchanged: they are durable evidence and keep judging as before. --- .../v6/internal/subsystems/monitoring.md | 11 ++ .../internal/subsystems/unified-resources.md | 8 + internal/storagehealth/risk.go | 23 ++- internal/storagehealth/risk_test.go | 57 +++++++ .../unifiedresources/physical_disk_risk.go | 2 +- internal/unifiedresources/registry_test.go | 153 ++++++++++++++++++ 6 files changed, 251 insertions(+), 3 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 6522d39af6..4804448c5e 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -4327,6 +4327,17 @@ emit structured storage topology such as Unraid per-disk state, the shared assessment layer must derive canonical risk and alert severity from that richer disk topology instead of letting coarser aggregate counters override it and flap the operator-facing storage alert surface. +That shared assessment judges only a disk temperature that was collected now. +`storagehealth.CollectedTemperature` passes a reading through when its +`collection.temperature` state is `available`, or when the source predates +collection state, and otherwise drops it before `AssessPhysicalDisk` and +`AssessHostSMARTDisk` build their sample. A last known value retained for a +disk in standby or a host agent past its reporting lease therefore raises no +`temperature_high` reason and cannot turn the disk's verdict or status into +a warning, while the value itself stays on the resource as last-known +history. Proof: `TestDiskAssessmentsIgnoreRetainedTemperature` and +`TestCollectedTemperatureKeepsOnlyCurrentReadings` in +`internal/storagehealth/risk_test.go`. That same monitoring-owned storage polling boundary also owns cluster-shared Proxmox storage status coherence. `internal/monitoring/monitor_polling_storage.go` must merge shared storage observations across nodes into one cluster-scoped diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 5c1dfdb1bb..cffb750018 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -4123,6 +4123,14 @@ incidents during cross-source merges. A provider alert such as TrueNAS `truenas_smart` is not presentation-only context; it must become a canonical `physicalDisk.risk.reasons` entry so hybrid agent/API disk resources keep one shared disk-health truth after deduplication. +That shared risk contract judges only a collected temperature. Both the +adapter assessment and the registry's recompute from merged metadata +(`physicalDiskAssessmentFromMeta`) pass the temperature through +`storagehealth.CollectedTemperature`, so a retained `unavailable`, +`unsupported` or `missing` temperature stays visible as last-known evidence +but adds no `temperature_high` reason and leaves the disk `online` rather +than `warning`. Proof: `TestPhysicalDiskRiskIgnoresRetainedTemperature` in +`internal/unifiedresources/registry_test.go`. That same canonical disk contract now also owns recent aggregate temperature history. When a provider such as TrueNAS can supply `disk.temperature_agg` min/avg/max readings, it must project those onto diff --git a/internal/storagehealth/risk.go b/internal/storagehealth/risk.go index 1c618b8bc4..de165bbb5b 100644 --- a/internal/storagehealth/risk.go +++ b/internal/storagehealth/risk.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) type RiskLevel string @@ -76,11 +77,29 @@ type Sample struct { UnsafeShutdowns int64 } +// CollectedTemperature returns a disk temperature only when its collection +// state marks it as observed now. Normalization may keep the last known value +// when the current observation is not available, such as a disk in standby or +// a host agent past its reporting lease. That retained value is history, not +// evidence that the disk is hot now, so risk must not judge it. A source that +// predates collection state counts as collected. +func CollectedTemperature(temperature int, collection *diskinventory.CollectionStatus) int { + if collection == nil { + return temperature + } + switch collection.Temperature.State { + case "", diskinventory.FieldAvailable: + return temperature + default: + return 0 + } +} + func AssessPhysicalDisk(disk models.PhysicalDisk) Assessment { sample := Sample{ Model: disk.Model, Health: disk.Health, - Temperature: disk.Temperature, + Temperature: CollectedTemperature(disk.Temperature, disk.Collection), Wearout: disk.Wearout, WearoutKnown: WearoutReported(disk.Wearout, disk.Type), } @@ -96,7 +115,7 @@ func AssessHostSMARTDiskWithThresholds(disk models.HostDiskSMART, thresholds SMA sample := Sample{ Model: disk.Model, Health: disk.Health, - Temperature: disk.Temperature, + Temperature: CollectedTemperature(disk.Temperature, disk.Collection), Wearout: -1, } applySMARTAttributes(&sample, disk.Attributes) diff --git a/internal/storagehealth/risk_test.go b/internal/storagehealth/risk_test.go index 44bdde444d..e044189efd 100644 --- a/internal/storagehealth/risk_test.go +++ b/internal/storagehealth/risk_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) // --- AssessSample: comprehensive branch coverage --- @@ -538,3 +539,59 @@ func TestAssessPhysicalDisk_RotationalZeroIsNotRisk(t *testing.T) { t.Error("a rotational disk reporting 0 wearout reports no endurance at all and must not assess critical") } } + +func TestCollectedTemperatureKeepsOnlyCurrentReadings(t *testing.T) { + cases := []struct { + name string + collection *diskinventory.CollectionStatus + want int + }{ + {name: "no collection state", collection: nil, want: 72}, + {name: "empty state", collection: &diskinventory.CollectionStatus{}, want: 72}, + {name: "available", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}, want: 72}, + {name: "unavailable", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}, want: 0}, + {name: "unsupported", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "")}, want: 0}, + {name: "missing", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Missing("smartctl", "")}, want: 0}, + } + for _, tc := range cases { + if got := CollectedTemperature(72, tc.collection); got != tc.want { + t.Errorf("%s: CollectedTemperature = %d, want %d", tc.name, got, tc.want) + } + } +} + +func TestDiskAssessmentsIgnoreRetainedTemperature(t *testing.T) { + retained := &diskinventory.CollectionStatus{ + Temperature: diskinventory.Unavailable("host_agent", "host agent stopped reporting"), + } + collected := &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")} + + hasTemperatureReason := func(assessment Assessment) bool { + for _, reason := range assessment.Reasons { + if reason.Code == "temperature_high" { + return true + } + } + return false + } + + disk := models.PhysicalDisk{Model: "WDC WD80EFAX", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 72} + disk.Collection = collected + if got := AssessPhysicalDisk(disk); got.Level != RiskCritical || !hasTemperatureReason(got) { + t.Fatalf("collected 72C physical disk = %+v, want critical temperature_high", got) + } + disk.Collection = retained + if got := AssessPhysicalDisk(disk); got.Level != RiskHealthy || hasTemperatureReason(got) { + t.Fatalf("retained 72C physical disk = %+v, want healthy", got) + } + + smart := models.HostDiskSMART{Device: "/dev/sda", Model: "WDC WD80EFAX", Type: "sata", Health: "PASSED", Temperature: 72} + smart.Collection = collected + if got := AssessHostSMARTDisk(smart); got.Level != RiskCritical || !hasTemperatureReason(got) { + t.Fatalf("collected 72C host SMART disk = %+v, want critical temperature_high", got) + } + smart.Collection = retained + if got := AssessHostSMARTDisk(smart); got.Level != RiskHealthy || hasTemperatureReason(got) { + t.Fatalf("retained 72C host SMART disk = %+v, want healthy", got) + } +} diff --git a/internal/unifiedresources/physical_disk_risk.go b/internal/unifiedresources/physical_disk_risk.go index 2cb8007a46..444a62af1d 100644 --- a/internal/unifiedresources/physical_disk_risk.go +++ b/internal/unifiedresources/physical_disk_risk.go @@ -82,7 +82,7 @@ func physicalDiskAssessmentFromMeta(meta *PhysicalDiskMeta) storagehealth.Assess sample := storagehealth.Sample{ Model: meta.Model, Health: meta.Health, - Temperature: meta.Temperature, + Temperature: storagehealth.CollectedTemperature(meta.Temperature, meta.Collection), Wearout: meta.Wearout, WearoutKnown: storagehealth.WearoutReported(meta.Wearout, meta.DiskType), } diff --git a/internal/unifiedresources/registry_test.go b/internal/unifiedresources/registry_test.go index 8eacc8f37b..0256304351 100644 --- a/internal/unifiedresources/registry_test.go +++ b/internal/unifiedresources/registry_test.go @@ -10,6 +10,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) func TestRegistry_CachedReadsUseSharedLock(t *testing.T) { @@ -7033,3 +7034,155 @@ func TestRegistryProjectionSnapshotDetachesAndTracksMappings(t *testing.T) { t.Fatal("empty projection invented resources or targets") } } + +// A retained temperature is last-known history, not evidence that the disk is +// hot now. The risk verdict (and the warning status it drives) must judge only +// a temperature whose collection state says it was observed now. +func TestPhysicalDiskRiskIgnoresRetainedTemperature(t *testing.T) { + // A fresh check time keeps the registry's source-staleness sweep out of + // the status this test reads. + now := time.Now() + retained := diskinventory.Unavailable("host_agent", "host agent stopped reporting") + + cases := []struct { + name string + collection *diskinventory.CollectionStatus + wantHot bool + }{ + {name: "collected now", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("proxmox_node_smart")}, wantHot: true}, + {name: "source predates collection state", collection: nil, wantHot: true}, + {name: "state left empty", collection: &diskinventory.CollectionStatus{}, wantHot: true}, + {name: "retained after the agent went silent", collection: &diskinventory.CollectionStatus{Temperature: retained}, wantHot: false}, + {name: "retained while in standby", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}, wantHot: false}, + {name: "retained after the source stopped exposing it", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature")}, wantHot: false}, + {name: "retained while unexpectedly missing", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Missing("smartctl", "")}, wantHot: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + adapter := NewMonitorAdapter(NewRegistry(nil)) + adapter.PopulateFromSnapshot(models.StateSnapshot{ + PhysicalDisks: []models.PhysicalDisk{{ + ID: "homelab-pve--dev-sda", + Node: "pve", + Instance: "homelab", + DevPath: "/dev/sda", + Model: "WDC WD80EFAX", + Serial: "SERIAL-SDA", + Type: "sata", + Health: "PASSED", + Wearout: -1, + Temperature: 72, + Collection: tc.collection, + LastChecked: now, + }}, + }) + var disks []Resource + for _, resource := range adapter.GetAll() { + if resource.Type == ResourceTypePhysicalDisk { + disks = append(disks, resource) + } + } + if len(disks) != 1 || disks[0].PhysicalDisk == nil { + t.Fatalf("physical disks = %+v, want one", disks) + } + disk := disks[0] + if disk.PhysicalDisk.Temperature != 72 { + t.Fatalf("temperature = %d, want the reading kept at 72", disk.PhysicalDisk.Temperature) + } + hot := false + if disk.PhysicalDisk.Risk != nil { + for _, reason := range disk.PhysicalDisk.Risk.Reasons { + hot = hot || reason.Code == "temperature_high" + } + } + if hot != tc.wantHot { + t.Fatalf("temperature_high = %v, want %v (risk %+v)", hot, tc.wantHot, disk.PhysicalDisk.Risk) + } + if tc.wantHot && disk.Status != StatusWarning { + t.Fatalf("status = %q, want warning for a collected 72C", disk.Status) + } + if !tc.wantHot && (disk.PhysicalDisk.Risk != nil || disk.Status != StatusOnline) { + t.Fatalf("retained 72C: status = %q risk = %+v, want online with no risk", disk.Status, disk.PhysicalDisk.Risk) + } + }) + } + + // A host agent's SMART row merged with the Proxmox inventory row for the + // same disk: the agent's retained reading keeps its unavailable state + // through the registry merge and recompute, so it stays out of risk. + for _, tc := range []struct { + name string + state diskinventory.FieldStatus + wantHot bool + }{ + {name: "agent reading collected", state: diskinventory.Available("host_agent"), wantHot: true}, + {name: "agent reading retained", state: retained, wantHot: false}, + } { + t.Run("merged "+tc.name, func(t *testing.T) { + adapter := NewMonitorAdapter(NewRegistry(nil)) + adapter.PopulateFromSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-pve", + Hostname: "pve", + Status: "online", + LastSeen: now, + Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{ + Device: "/dev/sda", + Model: "WDC WD80EFAX", + Serial: "SERIAL-SDA", + Type: "sata", + Temperature: 72, + Health: "PASSED", + Collection: &diskinventory.CollectionStatus{Temperature: tc.state}, + }}}, + }}, + PhysicalDisks: []models.PhysicalDisk{{ + ID: "homelab-pve--dev-sda", + Node: "pve", + Instance: "homelab", + DevPath: "/dev/sda", + Model: "WDC WD80EFAX", + Serial: "SERIAL-SDA", + Type: "sata", + Health: "PASSED", + Wearout: -1, + Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature")}, + LastChecked: now, + }}, + }) + var disks []Resource + for _, resource := range adapter.GetAll() { + if resource.Type == ResourceTypePhysicalDisk { + disks = append(disks, resource) + } + } + if len(disks) != 1 || disks[0].PhysicalDisk == nil { + t.Fatalf("physical disks = %d, want the agent and Proxmox rows merged into one", len(disks)) + } + disk := disks[0] + if disk.PhysicalDisk.Temperature != 72 { + t.Fatalf("merged temperature = %d, want 72", disk.PhysicalDisk.Temperature) + } + hot := false + if disk.PhysicalDisk.Risk != nil { + for _, reason := range disk.PhysicalDisk.Risk.Reasons { + hot = hot || reason.Code == "temperature_high" + } + } + if hot != tc.wantHot { + t.Fatalf("merged temperature_high = %v, want %v (collection %+v, risk %+v)", hot, tc.wantHot, disk.PhysicalDisk.Collection, disk.PhysicalDisk.Risk) + } + }) + } + + // The registry recomputes risk from merged metadata after a source merge; + // that path follows the same rule. + meta := &PhysicalDiskMeta{Health: "PASSED", Temperature: 72, Collection: &diskinventory.CollectionStatus{Temperature: retained}} + if risk := physicalDiskRiskFromMeta(meta, nil); risk != nil { + t.Fatalf("meta recompute risk = %+v, want none for a retained temperature", risk) + } + meta.Collection = &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")} + if risk := physicalDiskRiskFromMeta(meta, nil); risk == nil || risk.Level != storagehealth.RiskCritical { + t.Fatalf("meta recompute risk = %+v, want critical for a collected 72C", risk) + } +} From 991948f28b4fc2c3766b3b272b1b4f1b563d9e23 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:32:31 +0100 Subject: [PATCH 18/53] Lead alert cards and Patrol with the live reading of held alerts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A threshold alert stays open below its trigger until the reading reaches the clear level and holds there for the recovery delay, while its message keeps the last breach. #2567 added the live status and moved node, guest and Mail Gateway drawers onto it, but the Alerts page card, the open incident in the History panel and Patrol's attention summary still said "Node temperature at 95.0°C" while the node read 72°C. The Alerts card now leads with the reading now and what clears the alert, keeps the last breach as hover text, and labels the alert and clear levels in the metric's unit, temperatures in the viewer's °C or °F. Without a live status it labels a level only for metric types whose unit is known, so a Mail Gateway queue-age alert no longer reads "limit: 30%" for a 30 minute threshold. The resource incident panel describes only the open occurrence whose alert id and start match a live alert, the incident store's own occurrence key, rechecked on the shared clock; closed occurrences keep their recorded message. Patrol's attention summary is built server-side because the Patrol inbox, the Assistant handoff and the mobile app all read plainLanguageSummary. A latched or recovering alert with a fresh status gets the live reading in the frontend helper's words; a breaching alert keeps its message, which tracks the breach, and a stale status falls back to it. Every disk alert on a guest carries the guest's name, so the live reading names the disk from metadata.label as the per-disk message does; without it two disks on one guest read the same. Recovery progress now rounds down, so 4m50s into a 5 minute delay no longer reads "5 minutes so far" while the alert is still open, and the server copy rounds the way the frontend does (toFixed and Math.round ties), so Patrol and the card agree. From the sm breakpoint the card's text column keeps a 16rem floor. Before, it shrank to its longest word beside one row of action buttons, so at a 1024px window every card's text sat in a 110-160px column; the buttons now wrap instead. Refs #2068 --- .../v6/internal/subsystems/ai-runtime.md | 15 ++ .../v6/internal/subsystems/alerts.md | 17 ++ .../subsystems/frontend-primitives.md | 12 ++ .../811c71b54c36777b.json | 61 ++++++ .../alerts/AlertOverviewAlertCard.tsx | 91 ++++++++- .../alerts/AlertResourceIncidentsPanel.tsx | 45 ++++- ...ertOverviewAlertCard.system-scope.test.tsx | 127 +++++++++++- .../AlertResourceIncidentsPanel.test.tsx | 95 +++++++++ .../features/alerts/__tests__/helpers.test.ts | 34 ++++ .../alerts/metricAlertPresentation.ts | 42 +++- .../features/alerts/useAlertHistoryState.ts | 3 + internal/ai/attention.go | 189 +++++++++++++++++- internal/ai/attention_test.go | 140 +++++++++++++ 13 files changed, 848 insertions(+), 23 deletions(-) create mode 100644 frontend-modern/browser-verification/811c71b54c36777b.json diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 1f82e68833..f2b5fa5736 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -4653,6 +4653,21 @@ Assistant receives a selected attention item only after the operator opens it. The handoff is explanation-only and cannot infer a capability, approve an action, or replace evidence and lifecycle truth. +A held threshold alert's attention summary leads with the live reading (issue +#2068). An alert's `Message` keeps the last breach while the alert waits below +its trigger for the clear level, so for a `MetricStatus` in the `latched` or +`recovering` phase that is under ten minutes old, the projection builds +`plainLanguageSummary` from the status instead: the reading now, then for a +latched alert the alert level it is back under and the clear level with its +hold, and for a recovering alert the clear level, delay and progress so far +(rounded down). The words and rounding follow the frontend's +`features/alerts/metricAlertPresentation.ts`, and a guest's disk is named +from `metadata.label`. Temperatures stay in °C like the alert messages, +because the server does not know a viewer's unit preference. A breaching +alert keeps `Message`, which tracks the breach, and a stale or missing status +falls back to it. The Patrol inbox, the investigation handoff and API clients +read that summary, so none of them re-derives the phase. + ### Patrol model-readiness evidence Keep the Patrol model-readiness advisor provider-neutral and evidence-based. diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 043df12ec8..9854905d3a 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -32,6 +32,23 @@ day is not auto-resolved as unmonitored. Frontend surfaces format the status through `features/alerts/metricAlertPresentation.ts` and never re-derive the phase or recovery timing. +### Alert card and open incident lead with the live reading — issue #2068 + +The Alerts overview card leads an open threshold alert with the presentation's +summary and clear rule, keeps the last breach as the summary's hover text, and +labels the alert and clear levels in the status's unit (temperatures follow +the viewer's °C/°F preference, as every frontend reading does). Without a live +status it keeps the message and labels an alert level only for metric types +whose unit is known; other thresholds (queue ages, counts) are stated in their +own message, so the card never guesses a percent. The resource incident panel +describes only the open occurrence whose `alertIdentifier` and `openedAt` +match an active alert's id and `startTime` (the incident store's occurrence +key), from that alert's live status, rechecked on the shared relative-time +clock so it turns stale when evaluations stop; closed occurrences keep their +recorded message. Every disk alert on a guest carries the guest's name, so the +presentation names the disk from `metadata.label`, as the per-disk message +does. + ### Retained guest observations are not alert evidence Guest memory marked `last-known`, `unavailable` or an unknown observation state diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 4286b59290..f1e42c6366 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -30,6 +30,18 @@ and `title`, and `getTemperatureTextClass` treats an open alert's severity as a floor: a reading that has dipped under an open alert's trigger keeps the alert's tone, while the reading still wins when it is worse. +### Alert card and incident panel format held alerts through one helper — issue #2068 + +`AlertOverviewAlertCard` and `AlertResourceIncidentsPanel` describe an open +threshold alert through `features/alerts/metricAlertPresentation.ts` +(summary, clear rule, `alertLevel` and `clearLevel`) instead of their own unit +guesses or the breach message. `useAlertHistoryState` exposes its active-alert +accessor so the incident panel matches its open occurrence to the live alert +without a second store read, and the panel reads `useRelativeTimeNow` so a +reading that stops updating turns stale on screen. From the `sm` breakpoint the +card's text column keeps a 16rem floor and breaks long words, so the action +buttons wrap instead of squeezing the reading to its longest word beside them. + ### Canonical drawer History preserves guest read provenance The shared resource drawer passes selected memory observation state/source/time diff --git a/frontend-modern/browser-verification/811c71b54c36777b.json b/frontend-modern/browser-verification/811c71b54c36777b.json new file mode 100644 index 0000000000..cfb96ffcd7 --- /dev/null +++ b/frontend-modern/browser-verification/811c71b54c36777b.json @@ -0,0 +1,61 @@ +{ + "version": 1, + "base_sha": "3ca8b9647ab992101adf9153b71b5ef1757ac82f", + "verified_at": "2026-10-06T19:34:00Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/features/alerts/AlertOverviewAlertCard.tsx", + "frontend-modern/src/features/alerts/AlertResourceIncidentsPanel.tsx", + "frontend-modern/src/features/alerts/metricAlertPresentation.ts", + "frontend-modern/src/features/alerts/useAlertHistoryState.ts" + ], + "content_sha256": { + "frontend-modern/src/features/alerts/AlertOverviewAlertCard.tsx": "bae52439900c395e8c7d33f87296e6c3d4f449e15fcf1dc69a0137111025d0ab", + "frontend-modern/src/features/alerts/AlertResourceIncidentsPanel.tsx": "6c8347cb65f80520711f9f4b66b2c46e09c51121bf67e9ec382bce67ca3468a6", + "frontend-modern/src/features/alerts/metricAlertPresentation.ts": "1b7314213d79cdfbad9f84b854ae8652802ad6c2d51955586c175475a024dbee", + "frontend-modern/src/features/alerts/useAlertHistoryState.ts": "5b9f10978f2d2d7101f92343e826bef9ae5e2677b36ccf431ff0e6dfa9d676a8" + }, + "routes": [ + "/alerts/overview", + "/alerts/history", + "/patrol" + ], + "viewports": [ + { + "width": 1280, + "height": 900 + }, + { + "width": 1024, + "height": 607 + }, + { + "width": 768, + "height": 1024 + }, + { + "width": 375, + "height": 812 + } + ], + "states": [ + "Private mock stack (sidecar build, uncommitted generator overlay pinning pve2 CPU temperature; alert level 80\u00b0C, clear level 75\u00b0C, 5 minute recovery delay)", + "Alerts overview card, breaching at 95\u00b0C: \"Temperature 95\u00b0C, above the 80\u00b0C alert level\", clear rule detail, \"Alert level 80\u00b0C\" and \"Clear level 75\u00b0C\"", + "Alerts overview card, held at 78\u00b0C: \"Temperature 78\u00b0C now, back under the 80\u00b0C alert level\" with hover \"Last reading at or above 80\u00b0C: 95\u00b0C\"; message \"Node temperature at 95.0\u00b0C\" no longer shown", + "Alerts overview card, recovering at 72\u00b0C: \"Temperature 72\u00b0C now, recovering\" and \"Clears after 5 minutes at 75\u00b0C or lower, 2 minutes so far.\" at 1280px and 375px (no horizontal scroll)", + "Fahrenheit preference: held card reads \"Temperature 172\u00b0F now, back under the 176\u00b0F alert level\", \"Alert level 176\u00b0F\", \"Clear level 167\u00b0F\", hover \"Last reading at or above 176\u00b0F: 203\u00b0F\"", + "Mail Gateway queue-age card keeps its message and no longer shows a guessed \"30%\" level; breaching memory card shows \"Memory 85%, above the 85% alert level\" with its levels", + "Card layout at 1024px: text column 256px with actions wrapped to two rows (was 110-160px beside one row); a 66-character resource name truncates with no overflow; 1280px unchanged (one row of actions); 768px text 256px, no overflow; 375px stacked, no horizontal scroll", + "Card removed after the alert resolved", + "History resource incidents panel: the open occurrence whose id and start match the live alert shows the live summary with the breach on hover; a cached open occurrence with an earlier start and closed occurrences keep their recorded messages (incident list extended in-page because mock mode never records open occurrences)", + "Patrol inbox row for pve2 shows the held summary from plainLanguageSummary; API shows recovering summary, breaching pve3 keeps its message, resolved pve2 keeps \"Resolved: Node temperature at 72.0\u00b0C\"" + ], + "interactions": [ + "Walked pve2 temperature 95 -> 78 -> 72 via the control file and reloaded the overview between phases; repeated 95 -> 78 after the final rebuild", + "Read card text and hover titles in each phase", + "Switched the temperature unit to Fahrenheit (localStorage temperatureUnit) and reloaded", + "Measured card column widths and overflow at 1280, 1024, 768 and 375 px, with a long resource name substituted in the DOM", + "Opened the pve2 Resource incidents panel from the History table row and pressed Refresh", + "Opened Patrol inbox and read the pve2 row" + ] +} diff --git a/frontend-modern/src/features/alerts/AlertOverviewAlertCard.tsx b/frontend-modern/src/features/alerts/AlertOverviewAlertCard.tsx index 9c82fe010e..870d095c84 100644 --- a/frontend-modern/src/features/alerts/AlertOverviewAlertCard.tsx +++ b/frontend-modern/src/features/alerts/AlertOverviewAlertCard.tsx @@ -28,12 +28,14 @@ import { import { alertTypeDisplayLabel } from './helpers'; import { describeAlertDeliveryStatus } from './deliveryDiagnosisPresentation'; import { getCanonicalAlertId } from './identity'; +import { getMetricAlertPresentation } from './metricAlertPresentation'; import type { AlertIncidentTimelineState } from './useAlertIncidentTimelineState'; import type { AlertOverviewState } from './useAlertOverviewState'; import { ResourceMonitoringPolicyAction } from './ResourceMonitoringPolicyAction'; import { AlertSnoozeAction } from './AlertSnoozeAction'; import { isAlertSnoozed } from './useAlertSnoozeState'; import { isPulseSystemAlert } from '@/utils/alertScope'; +import { formatTemperature } from '@/utils/temperature'; import { normalizeSourcePlatformQueryValue, resolvePlatformTypeFromSources, @@ -82,6 +84,23 @@ const PLATFORM_PAGE_PATHS: Record = { standalone: buildStandalonePath(), }; +// Units of the threshold metrics, matching the evaluator's metric status unit. +const THRESHOLD_UNIT_BY_ALERT_TYPE: Record = { + cpu: '%', + memory: '%', + disk: '%', + 'disk-usage': '%', + usage: '%', + temperature: '°C', + disk_temperature: '°C', + diskTemperature: '°C', + diskRead: ' MB/s', + diskWrite: ' MB/s', + networkIn: ' MB/s', + networkOut: ' MB/s', + 'disk-wearout': '%', +}; + const platformPagePath = (platform: string | undefined): string | undefined => { if (!platform) return undefined; const navId = PRIMARY_PLATFORM_NAV_IDS.find((id) => @@ -116,8 +135,35 @@ export function AlertOverviewAlertCard(props: AlertOverviewAlertCardProps) { return describeAlertDeliveryStatus(diagnosis, props.alert.acknowledged); }; const [moreOpen, setMoreOpen] = createSignal(false); + // A threshold alert can stay open below its trigger, so the card leads with + // the reading Pulse is evaluating now; the message keeps the last breach. + const metricPresentation = () => + isPulseSystemAlert(props.alert) + ? null + : getMetricAlertPresentation(props.alert, props.state.tick()); + const alertLevels = (): { alert: string; clear?: string } | null => { + const presentation = metricPresentation(); + if (presentation) { + return { + alert: presentation.alertLevel, + clear: + presentation.clearLevel !== presentation.alertLevel ? presentation.clearLevel : undefined, + }; + } + if (isPulseSystemAlert(props.alert) || !(props.alert.threshold > 0)) return null; + // Without a live status only the metric types carry a known unit; other + // thresholds (queue ages, counts) are stated in their own message. + const unit = THRESHOLD_UNIT_BY_ALERT_TYPE[props.alert.type]; + if (!unit) return null; + return { + alert: + unit === '°C' + ? formatTemperature(props.alert.threshold) + : `${props.alert.threshold}${unit}`, + }; + }; const hasMetaLine = () => - (!isPulseSystemAlert(props.alert) && props.alert.threshold > 0) || + Boolean(alertLevels()) || Boolean(deliveryStatusLine()) || (isAlertSnoozed(props.alert) && Boolean(props.alert.operationalRecord?.suppression?.expiresAt)); const timelineOpen = () => props.timelineState.expandedIncidents().has(alertKey()); @@ -203,7 +249,9 @@ export function AlertOverviewAlertCard(props: AlertOverviewAlertCardProps) { return (
-
+ {/* Without a floor the text column shrinks to its longest word and the + actions keep one row, squeezing the reading to ~110px beside them. */} +
{props.alert.acknowledged ? ( @@ -225,7 +273,7 @@ export function AlertOverviewAlertCard(props: AlertOverviewAlertCardProps) { )}
-
+
-

{props.alert.message}

+ + {props.alert.message} +

+ } + > + {(presentation) => ( + <> +

+ {presentation().summary} +

+

{presentation().detail}

+ + )} +
- 0}> - - limit: {props.alert.threshold} - {props.alert.type === 'temperature' || props.alert.type === 'diskTemperature' - ? '°C' - : '%'} - + + {(levels) => ( + <> + Alert level {levels().alert} + + Clear level {levels().clear} + + + )} {(selection) => { @@ -53,6 +60,20 @@ export function AlertResourceIncidentsPanel(props: AlertResourceIncidentsPanelPr const hasError = () => props.state.resourceIncidentError()[resourceId]; const lookupResource = () => props.getResource ?? props.state.getResource; const resource = () => lookupResource()?.(resourceId); + // An open occurrence's message is the breach that opened it. A + // threshold alert can stay open below its trigger, so describe the + // open occurrence from its live alert; closed ones keep their copy. + // An occurrence is its alert id plus its start, as the incident store + // keys it, so a cached row never takes a later recurrence's reading. + const liveAlertFor = (incident: Incident): Alert | undefined => { + if (normalizeAlertIncidentStatus(incident.status) !== 'open') return undefined; + const openedAt = Date.parse(incident.openedAt); + if (!Number.isFinite(openedAt)) return undefined; + return Object.values(props.state.activeAlerts?.() ?? {}).find( + (alert) => + alert?.id === incident.alertIdentifier && Date.parse(alert.startTime) === openedAt, + ); + }; const resourceDisplayName = () => { const current = resource(); if (current) { @@ -155,6 +176,10 @@ export function AlertResourceIncidentsPanel(props: AlertResourceIncidentsPanelPr filteredEvents().length > 0 ? filteredEvents()[filteredEvents().length - 1] : undefined; + const livePresentation = () => { + const alert = liveAlertFor(incident); + return alert ? getMetricAlertPresentation(alert, now()) : null; + }; const filteredLabel = () => filteredEvents().length !== events.length ? `${filteredEvents().length}/${events.length}` @@ -192,8 +217,24 @@ export function AlertResourceIncidentsPanel(props: AlertResourceIncidentsPanelPr {INCIDENT_HISTORY_PARTIAL}

- -

{incident.message}

+ +

+ {incident.message} +

+ + } + > + {(live) => ( +

+ {live().summary}. {live().detail} +

+ )}

diff --git a/frontend-modern/src/features/alerts/__tests__/AlertOverviewAlertCard.system-scope.test.tsx b/frontend-modern/src/features/alerts/__tests__/AlertOverviewAlertCard.system-scope.test.tsx index 31b9436d66..a3dddd8e8f 100644 --- a/frontend-modern/src/features/alerts/__tests__/AlertOverviewAlertCard.system-scope.test.tsx +++ b/frontend-modern/src/features/alerts/__tests__/AlertOverviewAlertCard.system-scope.test.tsx @@ -3,6 +3,7 @@ import { cleanup, fireEvent, render, screen } from '@solidjs/testing-library'; import type { AlertOverviewState } from '../useAlertOverviewState'; import type { AlertIncidentTimelineState } from '../useAlertIncidentTimelineState'; import { makeSystemAlert, SYSTEM_ALERT_TYPES } from '../__fixtures__/systemAlerts'; +import { temperatureStore } from '@/utils/temperature'; vi.mock('@solidjs/router', () => ({ A: (props: any) => ( @@ -67,7 +68,7 @@ describe('Pulse system-alert overview scope', () => { )); expect(screen.getByText('Pulse').closest('a')).toBeNull(); - expect(screen.queryByText(/limit:/)).toBeNull(); + expect(screen.queryByText(/Alert level/)).toBeNull(); fireEvent.click(screen.getByRole('button', { name: 'More' })); expect(screen.queryByText('Resource monitoring policy')).toBeNull(); }); @@ -87,13 +88,135 @@ describe('Pulse system-alert overview scope', () => { 'href', '/proxmox/overview', ); - expect(screen.getByText('limit: 80%')).toBeInTheDocument(); + expect(screen.getByText('Alert level 80%')).toBeInTheDocument(); // Monitoring policy is a secondary action behind the More disclosure. expect(screen.queryByText('Resource monitoring policy')).toBeNull(); fireEvent.click(screen.getByRole('button', { name: 'More' })); expect(screen.getByText('Resource monitoring policy')).toBeInTheDocument(); }); + it('leads a threshold alert held under its trigger with the reading now', () => { + // minipc on 2026-10-06: the card said "Node temperature at 80.0°C" while + // the node read 72°C, because the message keeps the last breach. + const now = Date.now(); + const alert = makeSystemAlert('temperature', { + id: 'homelab-minipc::metric-threshold:temperature', + resourceId: 'homelab-minipc', + resourceName: 'minipc', + node: 'minipc', + message: 'Node temperature at 80.0°C', + value: 80, + threshold: 80, + lastSeen: new Date(now - 5 * 60_000).toISOString(), + metadata: { resourceType: 'node' }, + metricStatus: { + phase: 'recovering', + value: 72, + unit: '°C', + observedAt: new Date(now).toISOString(), + trigger: 80, + recovery: 75, + recoveryDelaySeconds: 300, + recoveryElapsedSeconds: 120, + }, + }); + render(() => ( + + )); + expect(screen.getByText('Temperature 72°C now, recovering')).toHaveAttribute( + 'title', + expect.stringMatching(/^Last reading at or above 80°C: 80°C, /), + ); + expect( + screen.getByText('Clears after 5 minutes at 75°C or lower, 2 minutes so far.'), + ).toBeInTheDocument(); + expect(screen.queryByText('Node temperature at 80.0°C')).toBeNull(); + expect(screen.getByText('Alert level 80°C')).toBeInTheDocument(); + expect(screen.getByText('Clear level 75°C')).toBeInTheDocument(); + }); + + it('labels a rate alert level in its own unit, not a guessed percent', () => { + const alert = makeSystemAlert('diskWrite', { + id: 'vm-101::metric-threshold:diskWrite', + resourceId: 'vm-101', + resourceName: 'db-01', + message: 'VM diskWrite at 240.0 MB/s', + value: 240, + threshold: 200, + metadata: { resourceType: 'vm' }, + metricStatus: { + phase: 'breaching', + value: 240, + unit: 'MB/s', + observedAt: new Date().toISOString(), + trigger: 200, + recovery: 200, + }, + }); + render(() => ( + + )); + expect(screen.getByText('Disk Write 240 MB/s, above the 200 MB/s alert level')).toBeVisible(); + expect(screen.getByText('Alert level 200 MB/s')).toBeInTheDocument(); + // Clearing at the trigger itself needs no second level. + expect(screen.queryByText(/Clear level/)).toBeNull(); + }); + + it('does not guess a percent for a threshold stated in its own message', () => { + const alert = makeSystemAlert('queue-age', { + id: 'pmg-eu::queue-age', + resourceId: 'pmg-eu', + resourceName: 'mail-gateway-eu', + message: 'PMG mail-gateway-eu has messages queued for 57 minutes (threshold: 30 minutes)', + value: 57, + threshold: 30, + metadata: { resourceType: 'pmg' }, + }); + render(() => ( + + )); + expect(screen.getByText(alert.message)).toBeInTheDocument(); + expect(screen.queryByText(/Alert level/)).toBeNull(); + }); + + it('keeps the level label for a percent threshold that has no live status', () => { + const alert = makeSystemAlert('disk-wearout', { + id: 'disk-wearout-nvme0', + resourceId: 'host-1/disk:nvme0n1', + resourceName: 'nvme0n1', + message: 'SSD nvme0n1 has less than 10% life remaining', + value: 7, + threshold: 10, + metadata: { resourceType: 'physical_disk' }, + }); + render(() => ( + + )); + expect(screen.getByText('Alert level 10%')).toBeInTheDocument(); + }); + + it('shows a temperature level without a live status in the viewer unit', () => { + // Right after a restart an alert has no live status until it is evaluated. + temperatureStore.setUnit('fahrenheit'); + try { + const alert = makeSystemAlert('temperature', { + id: 'homelab-minipc::metric-threshold:temperature', + resourceId: 'homelab-minipc', + resourceName: 'minipc', + message: 'Node temperature at 80.0°C', + value: 80, + threshold: 80, + metadata: { resourceType: 'node' }, + }); + render(() => ( + + )); + expect(screen.getByText('Alert level 176°F')).toBeInTheDocument(); + } finally { + temperatureStore.setUnit('celsius'); + } + }); + it('links a vCenter alarm on a network to vSphere by its incident provider', () => { // The summary is the alarm's own name, so the message no longer says VMware. const alert = makeSystemAlert('resource-incident', { diff --git a/frontend-modern/src/features/alerts/__tests__/AlertResourceIncidentsPanel.test.tsx b/frontend-modern/src/features/alerts/__tests__/AlertResourceIncidentsPanel.test.tsx index 653d748759..2adfb51e4a 100644 --- a/frontend-modern/src/features/alerts/__tests__/AlertResourceIncidentsPanel.test.tsx +++ b/frontend-modern/src/features/alerts/__tests__/AlertResourceIncidentsPanel.test.tsx @@ -189,6 +189,101 @@ describe('AlertResourceIncidentsPanel', () => { ).toBeNull(); }); + it('describes only the open occurrence from its live alert', () => { + // The recorded message is the breach that opened the occurrence; a + // threshold alert held below its trigger reads lower now. + const alertId = 'homelab-minipc::metric-threshold:temperature'; + const occurrence = (overrides: Partial): Incident => ({ + id: 'occurrence', + alertIdentifier: alertId, + alertType: 'temperature', + level: 'warning', + resourceId: 'homelab-minipc', + resourceName: 'minipc', + status: 'open', + acknowledged: false, + openedAt: '2026-10-06T07:57:44Z', + message: 'Node temperature at 80.0°C', + events: [], + ...overrides, + }); + const now = Date.now(); + render(() => ( + ({ resourceId: 'homelab-minipc', resourceName: 'minipc' }), + resourceIncidents: () => ({ + 'homelab-minipc': [ + occurrence({ id: 'open' }), + // A cached row still marked open from an earlier start is a + // different occurrence and must not take this alert's reading. + occurrence({ + id: 'cached-earlier', + openedAt: '2026-10-04T09:00:00Z', + message: 'Node temperature at 81.0°C', + }), + occurrence({ + id: 'closed', + status: 'resolved', + openedAt: '2026-10-05T07:00:00Z', + closedAt: '2026-10-05T08:00:00Z', + message: 'Node temperature at 83.0°C', + }), + ], + }), + activeAlerts: () => ({ + [alertId]: { + id: alertId, + type: 'temperature', + level: 'warning', + resourceId: 'homelab-minipc', + resourceName: 'minipc', + node: 'minipc', + instance: 'homelab', + message: 'Node temperature at 80.0°C', + value: 80, + threshold: 80, + startTime: '2026-10-06T07:57:44Z', + lastSeen: new Date(now - 5 * 60_000).toISOString(), + acknowledged: false, + metricStatus: { + phase: 'latched', + value: 76, + unit: '°C', + observedAt: new Date(now).toISOString(), + trigger: 80, + recovery: 75, + recoveryDelaySeconds: 300, + }, + }, + }), + resourceIncidentError: () => ({}), + resourceIncidentLoading: () => ({ 'homelab-minipc': false }), + expandedResourceIncidentIds: () => new Set(), + resourceIncidentEventFilters: () => new Set(['opened']), + setResourceIncidentEventFilters: vi.fn(), + refreshResourceIncidentPanel: vi.fn(), + setResourceIncidentPanel: vi.fn(), + toggleResourceIncidentDetails: vi.fn(), + } as any + } + /> + )); + + const live = screen.getByText( + 'Temperature 76°C now, back under the 80°C alert level. Stays open until it reaches 75°C or lower and stays there for 5 minutes.', + ); + expect(live).toHaveAttribute( + 'title', + expect.stringMatching(/^Last reading at or above 80°C: 80°C, /), + ); + expect(screen.queryByText('Node temperature at 80.0°C')).toBeNull(); + expect(screen.getByText('Node temperature at 83.0°C')).toBeInTheDocument(); + expect(screen.getByText('Node temperature at 81.0°C')).toBeInTheDocument(); + expect(screen.getAllByText(/now, back under the 80°C alert level/)).toHaveLength(1); + }); + it('opens Assistant from a resource incident without carrying raw command details', () => { const openSpy = vi.spyOn(aiChatStore, 'open'); aiChatStore.setEnabled(true); diff --git a/frontend-modern/src/features/alerts/__tests__/helpers.test.ts b/frontend-modern/src/features/alerts/__tests__/helpers.test.ts index 676038b14f..d04919c3bd 100644 --- a/frontend-modern/src/features/alerts/__tests__/helpers.test.ts +++ b/frontend-modern/src/features/alerts/__tests__/helpers.test.ts @@ -288,6 +288,8 @@ describe('getMetricAlertPresentation', () => { summary: 'Temperature 76°C now, back under the 80°C alert level', detail: 'Stays open until it reaches 75°C or lower and stays there for 5 minutes.', phaseLabel: 'Alert still open', + alertLevel: '80°C', + clearLevel: '75°C', }); expect(presentation?.lastBreach).toMatch(/^Last reading at or above 80°C: 80°C, /); }); @@ -309,6 +311,13 @@ describe('getMetricAlertPresentation', () => { NOW, )?.detail, ).toBe('Clears after 5 minutes at 75°C or lower, 2 minutes so far.'); + // Progress rounds down: 4m50s into a 5 minute delay is not "5 minutes so far". + expect( + getMetricAlertPresentation( + minipcAlert({ phase: 'recovering', value: 72, recoveryElapsedSeconds: 290 }), + NOW, + )?.detail, + ).toBe('Clears after 5 minutes at 75°C or lower, 4 minutes so far.'); }); it('names the alert level while the reading is still over it', () => { @@ -360,6 +369,31 @@ describe('getMetricAlertPresentation', () => { }); }); + it('names the disk of a per-disk guest alert, as its message does', () => { + // Every disk alert on a guest carries the guest's name; only the message + // and metadata say which disk. + const alert: Alert = { + ...minipcAlert(), + type: 'disk', + resourceName: 'web-01', + message: 'VM disk (/var) at 92.0%', + metadata: { label: '/var', mountpoint: '/var' }, + metricStatus: { + phase: 'latched', + value: 87, + unit: '%', + observedAt: new Date(NOW).toISOString(), + trigger: 90, + recovery: 85, + }, + }; + expect(getMetricAlertPresentation(alert, NOW)).toMatchObject({ + summary: 'Disk (/var) 87% now, back under the 90% alert level', + alertLevel: '90%', + clearLevel: '85%', + }); + }); + it('returns null without a live status so callers keep the alert message', () => { expect(getMetricAlertPresentation(minipcAlert(), NOW)).toBeNull(); expect(getAlertAttentionCopy(minipcAlert(), NOW)).toEqual({ diff --git a/frontend-modern/src/features/alerts/metricAlertPresentation.ts b/frontend-modern/src/features/alerts/metricAlertPresentation.ts index 8d72d7c312..d0c3937aea 100644 --- a/frontend-modern/src/features/alerts/metricAlertPresentation.ts +++ b/frontend-modern/src/features/alerts/metricAlertPresentation.ts @@ -22,6 +22,10 @@ export interface MetricAlertPresentation { phaseLabel: string; /** The last reading that met the trigger, once the live one differs. */ lastBreach?: string; + /** The level that opens the alert, in the metric's unit. */ + alertLevel: string; + /** The level the reading must reach before the alert can clear. */ + clearLevel: string; } const formatMetricValue = (value: number, unit: string | undefined): string => { @@ -43,6 +47,14 @@ const formatSeconds = (seconds: number): string => { return `${hours} ${hours === 1 ? 'hour' : 'hours'}`; }; +// Progress rounds down, so a run four minutes and fifty seconds into a five +// minute delay does not read as finished while the alert is still open. +const formatElapsed = (seconds: number): string => { + if (seconds < 60) return formatSeconds(seconds); + if (seconds < 3600) return formatSeconds(Math.floor(seconds / 60) * 60); + return formatSeconds(Math.floor(seconds / 360) * 360); +}; + const describeReading = (label: string, status: MetricAlertStatus): string => { const value = formatMetricValue(status.value, status.unit); const window = status.evaluationWindowSeconds ?? 0; @@ -62,6 +74,14 @@ const describeClearRule = (status: MetricAlertStatus, lead: string): string => { : `${lead} ${recovery} or lower.`; }; +// A guest raises one disk alert per disk, all named after the guest, so the +// reading names the disk the way the alert message does ("VM disk (/var)"). +const readingLabel = (alert: Pick): string => { + const label = alertTypeDisplayLabel(alert.type); + const disk = alert.type === 'disk' ? alert.metadata?.label : undefined; + return typeof disk === 'string' && disk.trim() ? `${label} (${disk.trim()})` : label; +}; + const isUsableStatus = (status: MetricAlertStatus | undefined): status is MetricAlertStatus => !!status && Number.isFinite(status.value) && @@ -76,15 +96,15 @@ const isUsableStatus = (status: MetricAlertStatus | undefined): status is Metric * no live status (non-threshold alerts, or the moments after a restart). */ export function getMetricAlertPresentation( - alert: Pick, + alert: Pick, now: number = Date.now(), ): MetricAlertPresentation | null { const status = alert.metricStatus; if (!isUsableStatus(status)) return null; - const label = alertTypeDisplayLabel(alert.type); - const reading = describeReading(label, status); + const reading = describeReading(readingLabel(alert), status); const trigger = formatMetricValue(status.trigger, status.unit); + const recovery = formatMetricValue(status.recovery, status.unit); const observedAt = Date.parse(status.observedAt); const stale = !Number.isFinite(observedAt) || now - observedAt > METRIC_ALERT_STATUS_STALE_MS; @@ -105,12 +125,11 @@ export function getMetricAlertPresentation( case 'recovering': { const delay = status.recoveryDelaySeconds ?? 0; const elapsed = status.recoveryElapsedSeconds ?? 0; - const recovery = formatMetricValue(status.recovery, status.unit); summary = `${reading} now, recovering`; detail = delay > 0 ? `Clears after ${formatSeconds(delay)} at ${recovery} or lower${ - elapsed > 0 ? `, ${formatSeconds(Math.min(elapsed, delay))} so far` : '' + elapsed > 0 ? `, ${formatElapsed(Math.min(elapsed, delay))} so far` : '' }.` : `Clears at ${recovery} or lower.`; phaseLabel = 'Recovering'; @@ -131,7 +150,16 @@ export function getMetricAlertPresentation( lastBreach = `Last reading at or above ${trigger}: ${formatMetricValue(alert.value, status.unit)}${when}`; } - return { phase: status.phase, stale, summary, detail, phaseLabel, lastBreach }; + return { + phase: status.phase, + stale, + summary, + detail, + phaseLabel, + lastBreach, + alertLevel: trigger, + clearLevel: recovery, + }; } export interface AlertAttentionCopy { @@ -142,7 +170,7 @@ export interface AlertAttentionCopy { /** Copy for an alert row in a drawer's "Needs attention" list. */ export function getAlertAttentionCopy( - alert: Pick, + alert: Pick, now: number = Date.now(), ): AlertAttentionCopy { const presentation = getMetricAlertPresentation(alert, now); diff --git a/frontend-modern/src/features/alerts/useAlertHistoryState.ts b/frontend-modern/src/features/alerts/useAlertHistoryState.ts index 5ef14948ec..88d5d64381 100644 --- a/frontend-modern/src/features/alerts/useAlertHistoryState.ts +++ b/frontend-modern/src/features/alerts/useAlertHistoryState.ts @@ -361,6 +361,9 @@ export function useAlertHistoryState(props: UseAlertHistoryStateProps) { // Exposed so the inline resource panel can resolve a display name without // threading the lookup down through the table and mobile list. getResource: props.getResource, + // Exposed so the resource panel can describe its open occurrence from the + // live alert instead of the message recorded when it opened. + activeAlerts: props.activeAlerts, formatAlertRowTime, formatAlertRowTimestamp, openResourceIncidentPanel: resourceIncidentsState.openResourceIncidentPanel, diff --git a/internal/ai/attention.go b/internal/ai/attention.go index 68c115f692..b43400b45c 100644 --- a/internal/ai/attention.go +++ b/internal/ai/attention.go @@ -3,11 +3,14 @@ package ai import ( "errors" "fmt" + "math" "sort" + "strconv" "strings" "time" "github.com/rcourtman/pulse-go-rewrite/internal/alerts" + "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" recoverymodel "github.com/rcourtman/pulse-go-rewrite/internal/recovery/model" ) @@ -223,7 +226,7 @@ func projectAttentionAlert( freshness, completeness := summarizeAttentionEvidence(record.State, evidence, now) resourceName := firstAttentionText(alert.ResourceName, alert.Instance, record.SubjectResourceID) title := attentionTitle(alert, resourceName) - summary := firstAttentionText(alert.Message, record.ImpactSummary, title) + summary := firstAttentionText(heldMetricAlertSummary(alert, now), alert.Message, record.ImpactSummary, title) related := make([]AttentionResource, 0, len(record.RelatedResourceIDs)) for _, resourceID := range canonicalAttentionIDs(record.RelatedResourceIDs) { @@ -635,6 +638,190 @@ func attentionResourceType(alert alerts.Alert) string { return "" } +// attentionMetricStatusStaleAfter matches the frontend's +// METRIC_ALERT_STATUS_STALE_MS: an older evaluation is no longer "now". +const attentionMetricStatusStaleAfter = 10 * time.Minute + +// heldMetricAlertSummary describes a threshold alert that is still open below +// its trigger from the evaluator's live status, in the words the frontend's +// metric alert presentation uses. The alert's Message keeps the last reading +// that met the trigger, so on its own it reports a value the resource may no +// longer have. A breaching alert keeps its Message, which tracks the current +// breach, and a missing or stale status returns "" so the caller falls back +// to it. +func heldMetricAlertSummary(alert alerts.Alert, now time.Time) string { + status := alert.MetricStatus + if status == nil { + return "" + } + if status.Phase != models.MetricAlertPhaseLatched && status.Phase != models.MetricAlertPhaseRecovering { + return "" + } + if !attentionFinite(status.Value) || !attentionFinite(status.Trigger) || !attentionFinite(status.Recovery) { + return "" + } + if status.ObservedAt.IsZero() || now.Sub(status.ObservedAt) > attentionMetricStatusStaleAfter { + return "" + } + + reading := attentionMetricReading(alert, status) + recovery := formatAttentionMetricValue(status.Recovery, status.Unit) + delay := status.RecoveryDelaySeconds + if status.Phase == models.MetricAlertPhaseRecovering { + if delay <= 0 { + return fmt.Sprintf("%s now, recovering. Clears at %s or lower.", reading, recovery) + } + progress := "" + if elapsed := status.RecoveryElapsedSeconds; elapsed > 0 { + progress = ", " + formatAttentionElapsed(min(elapsed, delay)) + " so far" + } + return fmt.Sprintf("%s now, recovering. Clears after %s at %s or lower%s.", + reading, formatAttentionSeconds(delay), recovery, progress) + } + hold := "" + if delay > 0 { + hold = " and stays there for " + formatAttentionSeconds(delay) + } + return fmt.Sprintf("%s now, back under the %s alert level. Stays open until it reaches %s or lower%s.", + reading, formatAttentionMetricValue(status.Trigger, status.Unit), recovery, hold) +} + +func attentionMetricReading(alert alerts.Alert, status *models.MetricAlertStatus) string { + label := attentionMetricLabel(alert) + value := formatAttentionMetricValue(status.Value, status.Unit) + if status.EvaluationWindowSeconds <= 0 { + return label + " " + value + } + latest := "" + if status.RawValue != nil && attentionFinite(*status.RawValue) { + latest = ", latest " + formatAttentionMetricValue(*status.RawValue, status.Unit) + } + return fmt.Sprintf("%s averaged %s over %s%s", + label, value, formatAttentionSeconds(status.EvaluationWindowSeconds), latest) +} + +// attentionMetricLabel names the metric the way the alert type label does. A +// guest raises one disk alert per disk, all named after the guest, so a disk +// reading also names its disk, as the alert message does. +func attentionMetricLabel(alert alerts.Alert) string { + var label string + switch alert.Type { + case "cpu": + label = "CPU" + case "memory": + label = "Memory" + case "disk", "disk-usage": + label = "Disk" + case "usage": + label = "Usage" + case "diskRead": + label = "Disk Read" + case "diskWrite": + label = "Disk Write" + case "networkIn": + label = "Network In" + case "networkOut": + label = "Network Out" + case "temperature", "disk_temperature", "diskTemperature": + label = "Temperature" + default: + label = firstAttentionText(alert.Type, "Reading") + } + if alert.Type == "disk" { + if disk, ok := alert.Metadata["label"].(string); ok && strings.TrimSpace(disk) != "" { + label += " (" + strings.TrimSpace(disk) + ")" + } + } + return label +} + +func formatAttentionMetricValue(value float64, unit string) string { + switch unit { + case "°C": + return formatAttentionDecimal(attentionRoundHalfUp(value), 0) + "°C" + case "%": + if math.Abs(value) >= 10 { + return formatAttentionDecimal(attentionRoundHalfUp(value), 0) + "%" + } + return formatAttentionDecimal(value, 1) + "%" + case "": + return formatAttentionDecimal(value, 1) + default: + return formatAttentionDecimal(value, 1) + " " + unit + } +} + +// formatAttentionDecimal rounds like the frontend's Number(value.toFixed(n)): +// from the float's exact value, so 2.55 (stored just under it) reads 2.5, and +// an exact tie such as 4.25 rounds away from zero where FormatFloat would +// round it to even. No trailing ".0". +func formatAttentionDecimal(value float64, places int) string { + scale := math.Pow10(places) + scaled := value * scale + var rounded float64 + if math.FMA(value, scale, -scaled) == 0 && math.Abs(scaled-math.Trunc(scaled)) == 0.5 { + rounded = math.Round(scaled) / scale + } else { + parsed, err := strconv.ParseFloat(strconv.FormatFloat(value, 'f', places, 64), 64) + if err != nil { + parsed = value + } + rounded = parsed + } + if rounded == 0 { + rounded = 0 // a template literal prints -0 as "0" + } + return strconv.FormatFloat(rounded, 'f', -1, 64) +} + +// attentionRoundHalfUp rounds like JavaScript's Math.round: ties go toward +// positive infinity, so -10.5 rounds to -10 where math.Round gives -11. +func attentionRoundHalfUp(value float64) float64 { + floor := math.Floor(value) + if value-floor >= 0.5 { + return floor + 1 + } + return floor +} + +func formatAttentionSeconds(seconds int) string { + plural := func(n float64, unit string) string { + if n == 1 { + return "1 " + unit + } + return strconv.FormatFloat(n, 'f', -1, 64) + " " + unit + "s" + } + switch { + case seconds < 60: + return plural(float64(seconds), "second") + case seconds < 3600: + return plural(math.Round(float64(seconds)/60), "minute") + default: + hours, err := strconv.ParseFloat(formatAttentionDecimal(float64(seconds)/3600, 1), 64) + if err != nil { + hours = float64(seconds) / 3600 + } + return plural(hours, "hour") + } +} + +// formatAttentionElapsed rounds progress down, so a run four minutes and +// fifty seconds into a five minute delay does not read as finished. +func formatAttentionElapsed(seconds int) string { + switch { + case seconds < 60: + return formatAttentionSeconds(seconds) + case seconds < 3600: + return formatAttentionSeconds(seconds / 60 * 60) + default: + return formatAttentionSeconds(seconds / 360 * 360) + } +} + +func attentionFinite(value float64) bool { + return !math.IsNaN(value) && !math.IsInf(value, 0) +} + func firstAttentionText(values ...string) string { for _, value := range values { if normalized := strings.TrimSpace(value); normalized != "" { diff --git a/internal/ai/attention_test.go b/internal/ai/attention_test.go index ccd1230896..6e89d5fc08 100644 --- a/internal/ai/attention_test.go +++ b/internal/ai/attention_test.go @@ -5,6 +5,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/alerts" + "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" recoverymodel "github.com/rcourtman/pulse-go-rewrite/internal/recovery/model" ) @@ -242,6 +243,145 @@ func TestProjectAttentionItemsProducesCurrentCalmEvaluation(t *testing.T) { } } +func TestProjectAttentionItemsLeadsHeldMetricAlertsWithTheLiveReading(t *testing.T) { + // minipc on 2026-10-06: the alert opened at 80°C and stayed open while the + // node read 71-76°C, and Patrol kept saying "Node temperature at 80.0°C". + now := time.Date(2026, 10, 6, 10, 52, 0, 0, time.UTC) + raw := 61.0 + tests := []struct { + name string + alertType string + message string + metadata map[string]interface{} + status *models.MetricAlertStatus + want string + }{ + { + name: "held under the trigger", + alertType: "temperature", + message: "Node temperature at 80.0°C", + status: &models.MetricAlertStatus{ + Phase: models.MetricAlertPhaseLatched, Value: 76, Unit: "°C", + ObservedAt: now.Add(-10 * time.Second), Trigger: 80, Recovery: 75, RecoveryDelaySeconds: 300, + }, + want: "Temperature 76°C now, back under the 80°C alert level. Stays open until it reaches 75°C or lower and stays there for 5 minutes.", + }, + { + name: "recovering at the clear level", + alertType: "temperature", + message: "Node temperature at 80.0°C", + status: &models.MetricAlertStatus{ + Phase: models.MetricAlertPhaseRecovering, Value: 72, Unit: "°C", + ObservedAt: now.Add(-10 * time.Second), Trigger: 80, Recovery: 75, + RecoveryDelaySeconds: 300, RecoveryElapsedSeconds: 120, + }, + want: "Temperature 72°C now, recovering. Clears after 5 minutes at 75°C or lower, 2 minutes so far.", + }, + { + name: "recovery progress rounds down", + alertType: "temperature", + message: "Node temperature at 80.0°C", + status: &models.MetricAlertStatus{ + Phase: models.MetricAlertPhaseRecovering, Value: 72, Unit: "°C", + ObservedAt: now, Trigger: 80, Recovery: 75, + RecoveryDelaySeconds: 300, RecoveryElapsedSeconds: 290, + }, + want: "Temperature 72°C now, recovering. Clears after 5 minutes at 75°C or lower, 4 minutes so far.", + }, + { + name: "one of a guest's disks", + alertType: "disk", + message: "VM disk (/var) at 92.0%", + metadata: map[string]interface{}{"label": "/var"}, + status: &models.MetricAlertStatus{ + Phase: models.MetricAlertPhaseRecovering, Value: 84.6, Unit: "%", + ObservedAt: now, Trigger: 90, Recovery: 85, + }, + want: "Disk (/var) 85% now, recovering. Clears at 85% or lower.", + }, + { + name: "rolling-average rule", + alertType: "cpu", + message: "VM cpu at 93.0%", + status: &models.MetricAlertStatus{ + Phase: models.MetricAlertPhaseLatched, Value: 84.2, RawValue: &raw, EvaluationWindowSeconds: 600, + Unit: "%", ObservedAt: now, Trigger: 90, Recovery: 80, + }, + want: "CPU averaged 84% over 10 minutes, latest 61% now, back under the 90% alert level. Stays open until it reaches 80% or lower.", + }, + { + name: "breaching keeps the message that tracks the breach", + alertType: "temperature", + message: "Node temperature at 88.0°C", + status: &models.MetricAlertStatus{ + Phase: models.MetricAlertPhaseBreaching, Value: 88, Unit: "°C", + ObservedAt: now, Trigger: 80, Recovery: 75, + }, + want: "Node temperature at 88.0°C", + }, + { + name: "an old evaluation is not described as now", + alertType: "temperature", + message: "Node temperature at 80.0°C", + status: &models.MetricAlertStatus{ + Phase: models.MetricAlertPhaseLatched, Value: 76, Unit: "°C", + ObservedAt: now.Add(-attentionMetricStatusStaleAfter - time.Minute), Trigger: 80, Recovery: 75, + }, + want: "Node temperature at 80.0°C", + }, + { + name: "no live status", + alertType: "temperature", + message: "Node temperature at 80.0°C", + want: "Node temperature at 80.0°C", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + alert := attentionTestAlert("held", operationaltrust.OperationalOpen, operationaltrust.SeverityWarning, now.Add(-3*time.Hour), now) + alert.Type = tt.alertType + alert.Message = tt.message + alert.Value = 80 + alert.Metadata = tt.metadata + alert.MetricStatus = tt.status + projected := ProjectAttentionItems([]alerts.Alert{alert}, nil, nil, now) + if len(projected.Details) != 1 { + t.Fatalf("details = %d, want 1", len(projected.Details)) + } + if got := projected.Details[0].Item.PlainLanguageSummary; got != tt.want { + t.Fatalf("summary = %q, want %q", got, tt.want) + } + }) + } +} + +func TestAttentionMetricFormattingRoundsLikeTheFrontend(t *testing.T) { + // The frontend rounds with Number(value.toFixed(1)), which works from the + // float's exact value: 2.55 is stored just under 2.55 and reads 2.5. + cases := []struct{ got, want string }{ + {formatAttentionMetricValue(2.55, "MB/s"), "2.5 MB/s"}, + {formatAttentionMetricValue(4.25, "%"), "4.3%"}, + {formatAttentionMetricValue(0.75, "MB/s"), "0.8 MB/s"}, + {formatAttentionMetricValue(1.05, "MB/s"), "1.1 MB/s"}, + {formatAttentionMetricValue(12.5, "MB/s"), "12.5 MB/s"}, + {formatAttentionMetricValue(3.0, "MB/s"), "3 MB/s"}, + {formatAttentionMetricValue(84.5, "%"), "85%"}, + {formatAttentionMetricValue(71.6, "°C"), "72°C"}, + {formatAttentionMetricValue(-10.5, "%"), "-10%"}, + {formatAttentionMetricValue(-0.01, "MB/s"), "0 MB/s"}, + {formatAttentionMetricValue(0.49999999999999994, "°C"), "0°C"}, + {formatAttentionSeconds(4140), "1.1 hours"}, + {formatAttentionSeconds(3600), "1 hour"}, + {formatAttentionSeconds(90), "2 minutes"}, + {formatAttentionElapsed(290), "4 minutes"}, + } + for _, tc := range cases { + if tc.got != tc.want { + t.Errorf("formatted %q, want %q", tc.got, tc.want) + } + } +} + func TestAttentionTitleProducesHumanReadableLabels(t *testing.T) { tests := []struct { name string From cdae811bb7a0f243f88b2ed6a94309d88b5c7ed7 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:33:10 +0100 Subject: [PATCH 19/53] Chart Net and Disk I/O in the row history lens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hovering a guest row swapped CPU, memory and disk for history sparklines but left Net I/O and Disk I/O as static numbers, although they are the jumpiest readings in the row and the hover request already returns their series. The lens now charts all five cells on one synchronized cursor. I/O charts, on hover and in Trends, keep a compact current pair beside the chart (↓3.3M ↑2.2M, R1.4M W826K) instead of hiding the rates in a tooltip; the glyphs use the series colours so the pair is also the legend. Rate sparklines get a 100 KiB/s scale floor so an idle guest's few-KB/s chatter no longer draws the same full-height spike as a busy one. GuestRow now builds every sparkline from accessors. Each live snapshot used to remount the lens and Trends charts, dropping an open scrub tooltip; an open tooltip also re-reads the current series instead of holding the values from the last mouse move. --- .../subsystems/frontend-primitives.md | 13 +- .../subsystems/performance-and-scalability.md | 11 +- .../baa3a6b3d9fabd1a.json | 61 ++++ .../src/components/Workloads/GuestRow.tsx | 319 ++++++++---------- .../MetricDisplayModeSegmentedControl.tsx | 2 +- .../Workloads/MetricMiniSparkline.tsx | 60 +++- .../Workloads/__tests__/GuestRow.test.tsx | 166 ++++++++- .../__tests__/MetricMiniSparkline.test.tsx | 85 +++++ .../workloadMetricHistoryModel.test.ts | 32 ++ .../Workloads/workloadMetricHistoryModel.ts | 35 ++ .../src/utils/__tests__/format.test.ts | 21 ++ frontend-modern/src/utils/format.ts | 29 ++ 12 files changed, 642 insertions(+), 192 deletions(-) create mode 100644 frontend-modern/browser-verification/baa3a6b3d9fabd1a.json diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 710f4ef07e..a017298dca 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -666,13 +666,18 @@ existing picker unchanged. The default Workloads metric presentation keeps compact progress bars at rest. A fine-pointer preview or keyboard focus on one guest row replaces CPU, memory, -and disk together with the existing `MetricMiniSparkline` presentation without -changing row height; touch pointer entry does not trigger this transient lens, -and the persistent Trends View choice remains the touch-accessible fallback. +disk, Net I/O, and Disk I/O together with the existing `MetricMiniSparkline` +presentation without changing row height; touch pointer entry does not trigger +this transient lens, and the persistent Trends View choice remains the +touch-accessible fallback. Both I/O cells render through one rate renderer in +the lens and in Trends: the chart keeps a compact current pair beside it +(`MetricMiniSparklineRatePair`, the at-rest ↓/↑ and R/W glyphs coloured like +the series they label) while the full rates stay in the chart's accessible +label, and a stopped guest or unavailable I/O keeps its dash. The active chart owns its local tooltip while its normalized cursor position is shared across sibling charts in that guest row, so every guide represents the same relative point in the selected history range. Leaving the row clears the -cursor and restores all three bars together. The lens mounts with a short +cursor and restores the bars and I/O readouts together. The lens mounts with a short reduced-motion-safe fade and must not leave both bar and chart semantics in the accessibility tree simultaneously. Bar mode resolves history only for that active guest through its canonical diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index a26b7e22af..0863723717 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -2005,8 +2005,10 @@ Trend-mode and row-history-lens cells own their vertical scale through `getMetricMiniSparklineScale` in `frontend-modern/src/components/Workloads/workloadMetricHistoryModel.ts`. Percent series are zero-floored and scaled to their own observed peak with a -5% floor ceiling and a 100% cap, the same shape rule the network and disk I/O -series already follow. A fixed 0-100 window is not the contract, because inside +5% floor ceiling and a 100% cap. Network and disk I/O series follow the same +shape rule with a 100 KiB/s floor ceiling, so an idle guest's background +chatter stays near the axis instead of drawing the full-height spike a busy +guest draws. A fixed 0-100 window is not the contract, because inside a 16px table cell it draws low-domain series such as a guest's share of host memory or an idle guest's CPU on top of the axis rule, where a flat line is indistinguishable from missing history. Absolute level stays readable through @@ -2015,7 +2017,10 @@ work must not remove those. The scale is derived from the points already fetched for that cell and must not add history reads or a second pass over the Workloads hot path. Pointer scrubbing may synchronize one normalized cursor position across the active guest row, but it must remain row-local and operate -only on the already-fetched series. +only on the already-fetched series. `GuestRow` builds each chart from accessors, +so a live snapshot updates the mounted lens or Trends chart in place instead of +remounting it, and an open scrub tooltip re-reads the current series at its +pointer position rather than holding the values from the last mouse move. The Workloads table metric display mode is part of the protected Workloads hot path. Default bar mode keeps compact current-value bars at rest and swaps diff --git a/frontend-modern/browser-verification/baa3a6b3d9fabd1a.json b/frontend-modern/browser-verification/baa3a6b3d9fabd1a.json new file mode 100644 index 0000000000..32c74e7a1c --- /dev/null +++ b/frontend-modern/browser-verification/baa3a6b3d9fabd1a.json @@ -0,0 +1,61 @@ +{ + "version": 1, + "base_sha": "9aaac4d3e502c01913fb3637d0997b0ce03a8b7b", + "verified_at": "2026-10-06T19:32:29Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Workloads/GuestRow.tsx", + "frontend-modern/src/components/Workloads/MetricDisplayModeSegmentedControl.tsx", + "frontend-modern/src/components/Workloads/MetricMiniSparkline.tsx", + "frontend-modern/src/components/Workloads/workloadMetricHistoryModel.ts", + "frontend-modern/src/utils/format.ts" + ], + "content_sha256": { + "frontend-modern/src/components/Workloads/GuestRow.tsx": "3e7b1583b1c2ff5db07aaaa59006684f2e8c7d71d035f1ba29c03be28ae3bcd9", + "frontend-modern/src/components/Workloads/MetricDisplayModeSegmentedControl.tsx": "fc4af4a98de6ddae1b26033ad102398cb598975bae0619cba134610d8102e2d9", + "frontend-modern/src/components/Workloads/MetricMiniSparkline.tsx": "3afd872c9633430d60f8782b33501d941ccdef8f58f28a65cecf5ca7845c6523", + "frontend-modern/src/components/Workloads/workloadMetricHistoryModel.ts": "b2fbcd169b0d7bacb8045ad4c134e6ddb5addc7ce6c4d0ac4a3c6673b85f462f", + "frontend-modern/src/utils/format.ts": "30ce3873607bdf03e34307f56e561c191957304b7e07b780afb2bbb9bd554d98" + }, + "routes": [ + "/proxmox" + ], + "viewports": [ + { + "width": 1920, + "height": 1000 + }, + { + "width": 1600, + "height": 1000 + }, + { + "width": 1440, + "height": 1000 + }, + { + "width": 768, + "height": 1000 + } + ], + "states": [ + "Private mock stack (backend + Vite on 7735/5253) built from main plus this change; before captures taken on the same stack before the edit, and the 1440 behaviour re-checked with main's versions of these files swapped back in", + "Before, bars mode at 1600 dark: hovering artifact-cache-310 swapped CPU, memory and disk for synchronized sparklines while Net I/O and Disk I/O stayed as static numbers", + "After, bars mode at 1600 dark and 1920 light: the same hover swaps all five cells; the I/O cells show both series with a compact coloured pair (e.g. ↓3.2M ↑2.2M, R1.4M W826K) and the svg labels carry the full rates; row stays 32px tall", + "After, pointer scrub on the CPU chart moves one cursor across all five charts, including both I/O charts", + "After, live update while scrubbing: with the Net I/O tooltip open on artifact-cache-310, a websocket snapshot changed the compact pair (↓2.8M to ↓3.7M) while all five chart nodes stayed mounted and the tooltip stayed open (before this change every live update remounted the lens charts)", + "After, idle I/O (history intercepted to 1-6 KB/s with one 80 KB/s disk write flush): both I/O lines sit within a pixel of the axis and the flush still shows as a bump instead of every blip filling the cell", + "After, Trends (sparklines) mode at 1600 dark: every running guest's I/O cell shows chart plus compact pair (736 charts, 256 rate pairs, 0 clipped), stopped dev-portal-312 keeps its dash", + "After, 768 dark and light: I/O columns are hidden by the layout as before; the lens and Trends still render CPU/memory/disk with 0 clipped cells", + "Pre-existing, not changed here: at 1440 the first-use history hint wraps and its dismissal shifts the table 36px, so the first lens opens on the next row (same on main)" + ], + "interactions": [ + "Hovered guest rows with a fine pointer in bars mode and read data-history-lens-active, chart count and svg aria-labels", + "Moved the pointer across the CPU chart to scrub and confirmed cursor lines in all five charts", + "Tagged the five lens chart nodes, opened the Net I/O tooltip, waited for live updates and compared node tags, compact pair text and tooltip presence", + "Left the row and confirmed bars and I/O numbers returned", + "Switched View to sparklines via the metric display toggle and measured chart, pair and overflow counts", + "Intercepted /api/metrics-store/history for one guest to idle I/O values to inspect the scale floor", + "Traced pointerenter/pointerleave and row y at 1440 to explain the lens opening on the next row" + ] +} diff --git a/frontend-modern/src/components/Workloads/GuestRow.tsx b/frontend-modern/src/components/Workloads/GuestRow.tsx index adb2046f0a..ef0d60564d 100644 --- a/frontend-modern/src/components/Workloads/GuestRow.tsx +++ b/frontend-modern/src/components/Workloads/GuestRow.tsx @@ -1,4 +1,4 @@ -import { createMemo, createSignal, Show, splitProps } from 'solid-js'; +import { createMemo, createSignal, Show, splitProps, type JSX } from 'solid-js'; import BoxIcon from 'lucide-solid/icons/box'; import type { VM } from '@/types/api'; import type { WorkloadGuest } from '@/types/workloads'; @@ -22,6 +22,7 @@ import { getWorkloadCPUPercent, resolveWorkloadType } from '@/utils/workloads'; import { EnhancedCPUBar } from '@/components/Workloads/EnhancedCPUBar'; import { MetricMiniSparkline, + MetricMiniSparklineRatePair, type MetricMiniSparklineValueLabelContext, } from '@/components/Workloads/MetricMiniSparkline'; import { UpdateButton } from '@/components/shared/ContainerUpdateBadge'; @@ -38,6 +39,7 @@ import type { GuestRowProps } from './guestRowModel'; import { useGuestRowState } from './useGuestRowState'; import type { WorkloadMetricHistoryReader, + WorkloadRateMetric, WorkloadTableMetric, } from './workloadMetricHistoryModel'; @@ -200,29 +202,88 @@ export function GuestRow(props: GuestRowProps) { } return clampMetricPercent(fallbackUsage); }; + // Every input is an accessor so a live snapshot updates the mounted chart in + // place. Reading signals while building the chart would remount it on each + // update and drop an open scrub tooltip. const renderMetricSparkline = ( metric: WorkloadTableMetric, - valueLabel: string, - title: string, - unit = '%', - valueLabelMode: 'inline' | 'tooltip' | 'hidden' = 'inline', - formatValue?: (value: number) => string, - seriesOptions?: Parameters[2], - valueLabelContext?: MetricMiniSparklineValueLabelContext, + options: { + valueLabel: () => string; + title: () => string; + unit?: string; + formatValue: (value: number) => string; + seriesOptions?: () => Parameters[2]; + valueLabelContext?: () => MetricMiniSparklineValueLabelContext; + valueContent?: () => JSX.Element; + }, ) => ( ); + const renderCpuSparkline = () => + renderMetricSparkline('cpu', { + valueLabel: () => formatMetricPercent(cpuPercent()), + title: () => `${props.guest.name} CPU history`, + formatValue: formatMetricPercent, + }); + const renderMemorySparkline = () => + renderMetricSparkline('memory', { + valueLabel: () => + memoryDisplayUnavailable() + ? 'N/A' + : formatMetricPercent( + usagePercent( + props.guest.memory?.used, + memoryDisplayTotal(), + isHostMemoryBasis() ? undefined : props.guest.memory?.usage, + ), + ), + title: () => + isHostMemoryBasis() + ? `${props.guest.name} host memory share history` + : `${props.guest.name} memory history`, + formatValue: formatMetricPercent, + seriesOptions: () => ({ + memoryDisplayBasis: props.memoryDisplayBasis, + parentMemoryTotal: props.parentMemoryTotal, + }), + }); + const renderDiskSparkline = () => + renderMetricSparkline('disk', { + valueLabel: () => (hasDiskUsage() ? formatMetricPercent(props.guest.disk?.usage) : '—'), + title: () => `${props.guest.name} disk usage history`, + formatValue: formatMetricPercent, + valueLabelContext: () => diskValueLabelContext(), + }); + const renderRateSparkline = (metric: WorkloadRateMetric) => { + const isNetwork = metric === 'netIo'; + const live = () => isRunning() && telemetryAvailable(isNetwork ? 'networkIO' : 'diskIO'); + const values = (): [number, number] => + isNetwork ? [networkIn(), networkOut()] : [diskRead(), diskWrite()]; + return renderMetricSparkline(metric, { + valueLabel: () => + live() ? `${formatSpeed(values()[0])} / ${formatSpeed(values()[1])}` : '—', + title: () => `${props.guest.name} ${isNetwork ? 'network' : 'disk'} I/O history`, + unit: 'B/s', + formatValue: formatSpeed, + valueContent: () => + live() ? : undefined, + }); + }; const getDiskStatusTooltip = () => { if (!isVM(props.guest)) return 'Disk stats unavailable'; @@ -475,27 +536,13 @@ export function GuestRow(props: GuestRowProps) { } >

- {renderMetricSparkline( - 'cpu', - formatMetricPercent(cpuPercent()), - `${props.guest.name} CPU history`, - '%', - 'inline', - formatMetricPercent, - )} + {renderCpuSparkline()}
} > - {renderMetricSparkline( - 'cpu', - formatMetricPercent(cpuPercent()), - `${props.guest.name} CPU history`, - '%', - 'inline', - formatMetricPercent, - )} + {renderCpuSparkline()} @@ -545,55 +592,13 @@ export function GuestRow(props: GuestRowProps) { } >
- {renderMetricSparkline( - 'memory', - memoryDisplayUnavailable() - ? 'N/A' - : formatMetricPercent( - usagePercent( - props.guest.memory?.used, - memoryDisplayTotal(), - isHostMemoryBasis() ? undefined : props.guest.memory?.usage, - ), - ), - isHostMemoryBasis() - ? `${props.guest.name} host memory share history` - : `${props.guest.name} memory history`, - '%', - 'inline', - formatMetricPercent, - { - memoryDisplayBasis: props.memoryDisplayBasis, - parentMemoryTotal: props.parentMemoryTotal, - }, - )} + {renderMemorySparkline()}
} > - {renderMetricSparkline( - 'memory', - memoryDisplayUnavailable() - ? 'N/A' - : formatMetricPercent( - usagePercent( - props.guest.memory?.used, - memoryDisplayTotal(), - isHostMemoryBasis() ? undefined : props.guest.memory?.usage, - ), - ), - isHostMemoryBasis() - ? `${props.guest.name} host memory share history` - : `${props.guest.name} memory history`, - '%', - 'inline', - formatMetricPercent, - { - memoryDisplayBasis: props.memoryDisplayBasis, - parentMemoryTotal: props.parentMemoryTotal, - }, - )} + {renderMemorySparkline()} @@ -605,18 +610,7 @@ export function GuestRow(props: GuestRowProps) { data-workload-col="disk" title={diskReadStatus()?.message} > - - {renderMetricSparkline( - 'disk', - hasDiskUsage() ? formatMetricPercent(props.guest.disk?.usage) : '—', - `${props.guest.name} disk usage history`, - '%', - 'inline', - formatMetricPercent, - undefined, - diskValueLabelContext(), - )} - + {renderDiskSparkline()}
- {renderMetricSparkline( - 'disk', - formatMetricPercent(props.guest.disk?.usage), - `${props.guest.name} disk usage history`, - '%', - 'inline', - formatMetricPercent, - undefined, - diskValueLabelContext(), - )} + {renderDiskSparkline()}
@@ -916,18 +901,7 @@ export function GuestRow(props: GuestRowProps) { {/* Net I/O */} - - {renderMetricSparkline( - 'netIo', - isRunning() && telemetryAvailable('networkIO') - ? `${formatSpeed(networkIn())} / ${formatSpeed(networkOut())}` - : '—', - `${props.guest.name} network I/O history`, - 'B/s', - 'tooltip', - formatSpeed, - )} - + {renderRateSparkline('netIo')} } > -
- ↓ - - {formatSpeed(networkIn())} - - ↑ - - {formatSpeed(networkOut())} - -
+ + ↓ + + {formatSpeed(networkIn())} + + ↑ + + {formatSpeed(networkOut())} + +
+ } + > +
+ {renderRateSparkline('netIo')} +
+ @@ -971,18 +954,7 @@ export function GuestRow(props: GuestRowProps) { {/* Disk I/O */} - - {renderMetricSparkline( - 'diskIo', - isRunning() && telemetryAvailable('diskIO') - ? `${formatSpeed(diskRead())} / ${formatSpeed(diskWrite())}` - : '—', - `${props.guest.name} disk I/O history`, - 'B/s', - 'tooltip', - formatSpeed, - )} - + {renderRateSparkline('diskIo')} } > -
- R - - {formatSpeed(diskRead())} - - W - - {formatSpeed(diskWrite())} - -
+ + R + + {formatSpeed(diskRead())} + + W + + {formatSpeed(diskWrite())} + +
+ } + > +
+ {renderRateSparkline('diskIo')} +
+ diff --git a/frontend-modern/src/components/Workloads/MetricDisplayModeSegmentedControl.tsx b/frontend-modern/src/components/Workloads/MetricDisplayModeSegmentedControl.tsx index a6faaa2a3b..e5e5ee47d8 100644 --- a/frontend-modern/src/components/Workloads/MetricDisplayModeSegmentedControl.tsx +++ b/frontend-modern/src/components/Workloads/MetricDisplayModeSegmentedControl.tsx @@ -47,7 +47,7 @@ export const MetricHoverModeSegmentedControl: Component = (props) const showInlineValue = createMemo( () => valueLabelMode() === 'inline' || (valueLabelMode() === 'tooltip' && !hasLine()), ); - const [hoveredState, setHoveredState] = createSignal( - null, - ); + // Keep only where the pointer is. The tooltip's point and values derive from + // the current series, so a live update under an open tooltip re-reads it + // instead of showing the snapshot taken at the last mouse move. + const [pointer, setPointer] = createSignal(null); + const hoveredState = createMemo(() => { + const current = pointer(); + if (!current) return null; + const state = computeMetricMiniSparklineHoverState(props.series, current.cursorRatio, 1); + return state ? { ...state, tooltipX: current.tooltipX, tooltipY: current.tooltipY } : null; + }); const synchronizedState = createMemo(() => { const ratio = props.cursorRatio; if (ratio === null || ratio === undefined) return null; @@ -102,7 +117,7 @@ export const MetricMiniSparkline: Component = (props) ); const handleMouseMove: JSX.EventHandler = (event) => { if (!hasLine()) { - setHoveredState(null); + setPointer(null); props.onCursorRatioChange?.(null); return; } @@ -113,10 +128,10 @@ export const MetricMiniSparkline: Component = (props) event.clientX - rect.left, rect.width, ); - setHoveredState( + setPointer( next ? { - ...next, + cursorRatio: next.cursorRatio, tooltipX: event.clientX, tooltipY: rect.top, } @@ -125,7 +140,7 @@ export const MetricMiniSparkline: Component = (props) props.onCursorRatioChange?.(next?.cursorRatio ?? null); }; const handleMouseLeave = () => { - setHoveredState(null); + setPointer(null); props.onCursorRatioChange?.(null); }; const showTooltip = createMemo(() => (props.showTooltip ?? true) && hoveredState()); @@ -179,7 +194,7 @@ export const MetricMiniSparkline: Component = (props) - {displayLabel()} + {props.valueContent ?? displayLabel()} @@ -215,3 +230,28 @@ export const MetricMiniSparkline: Component = (props)
); }; + +interface MetricMiniSparklineRatePairProps { + metric: WorkloadRateMetric; + values: readonly [number, number]; +} + +/** + * Current in/out (or read/write) rates compact enough to sit beside a rate + * sparkline. The full rates belong in the chart's valueLabel, which carries + * them to assistive technology, so this abbreviation stays out of that tree. + */ +export const MetricMiniSparklineRatePair: Component = (props) => ( + +); diff --git a/frontend-modern/src/components/Workloads/__tests__/GuestRow.test.tsx b/frontend-modern/src/components/Workloads/__tests__/GuestRow.test.tsx index 39b59fb197..36317abf96 100644 --- a/frontend-modern/src/components/Workloads/__tests__/GuestRow.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/GuestRow.test.tsx @@ -541,9 +541,15 @@ describe('GuestRow', () => { expect(screen.queryByTestId('disk-bar')).not.toBeInTheDocument(); }); - it('keeps Net I/O and Disk I/O live labels out of sparkline table cells', () => { + it('pairs Net I/O and Disk I/O sparklines with compact current rates', () => { renderGuestRow({ - guest: makeGuest({ name: 'spark-io-vm', networkIn: 1024, networkOut: 2048 }), + guest: makeGuest({ + name: 'spark-io-vm', + networkIn: 3.32 * 1024 * 1024, + networkOut: 512 * 1024, + diskRead: 0, + diskWrite: 2048, + }), visibleColumnIds: ['name', 'netIo', 'diskIo'], metricDisplayMode: 'sparklines', metricHistory: { @@ -565,10 +571,160 @@ describe('GuestRow', () => { const sparklines = screen.getAllByTestId('metric-mini-sparkline'); expect(sparklines).toHaveLength(2); expect(sparklines.map((sparkline) => sparkline.dataset.valueLabelMode)).toEqual([ - 'tooltip', - 'tooltip', + 'inline', + 'inline', ]); - expect(screen.queryByText('1.00 KB/s / 2.00 KB/s')).toBeNull(); + // The full pair does not fit beside a chart; the compact pair does, and + // its glyphs double as the legend for the two lines. + expect(sparklines.map((sparkline) => sparkline.textContent)).toEqual([ + '↓3.3M↑512K', + 'R0W2.0K', + ]); + expect(screen.queryByText('3.32 MB/s / 512 KB/s')).toBeNull(); + expect( + sparklines.map((sparkline) => sparkline.querySelector('svg')?.getAttribute('aria-label')), + ).toEqual([ + 'spark-io-vm network I/O history, current 3.32 MB/s / 512 KB/s', + 'spark-io-vm disk I/O history, current 0 B/s / 2.00 KB/s', + ]); + }); + + it('swaps Net I/O and Disk I/O readouts into the row history lens', () => { + renderGuestRow({ + guest: makeGuest({ name: 'io-lens-vm', networkIn: 2048, networkOut: 4096 }), + visibleColumnIds: ['name', 'cpu', 'netIo', 'diskIo'], + metricDisplayMode: 'bars', + metricHistory: { + getGuestMetricSeries: (_guest, metric) => [ + { + id: metric, + label: metric, + color: '#8b5cf6', + points: [ + { timestamp: 1, value: 10 }, + { timestamp: 2, value: 25 }, + ], + }, + ], + getNodeMetricSeries: () => [], + }, + }); + + expect(screen.getByText('2.00 KB/s')).toBeInTheDocument(); + expect(screen.getByText('4.00 KB/s')).toBeInTheDocument(); + + const row = screen.getByText('io-lens-vm').closest('tr')!; + fireEvent.pointerEnter(row, { pointerType: 'mouse' }); + + const lensCharts = screen.getAllByTestId('metric-mini-sparkline'); + expect( + lensCharts.map((chart) => chart.querySelector('svg')?.getAttribute('aria-label')), + ).toEqual([ + 'io-lens-vm CPU history, current 25%', + 'io-lens-vm network I/O history, current 2.00 KB/s / 4.00 KB/s', + 'io-lens-vm disk I/O history, current 500 B/s / 600 B/s', + ]); + expect(screen.queryByText('2.00 KB/s')).toBeNull(); + expect(lensCharts[1].textContent).toBe('↓2.0K↑4.0K'); + expect(lensCharts[1].parentElement).toHaveClass('h-4', 'motion-reduce:animate-none'); + + const cpuSvg = lensCharts[0].querySelector('svg') as SVGSVGElement; + cpuSvg.getBoundingClientRect = () => + ({ + bottom: 18, + height: 18, + left: 0, + right: 96, + top: 0, + width: 96, + x: 0, + y: 0, + toJSON: () => ({}), + }) as DOMRect; + fireEvent.mouseMove(cpuSvg, { clientX: 48, clientY: 8 }); + expect(lensCharts.every((chart) => chart.querySelector('[data-metric-history-cursor]'))).toBe( + true, + ); + + fireEvent.pointerLeave(row, { pointerType: 'mouse' }); + expect(screen.queryByTestId('metric-mini-sparkline')).not.toBeInTheDocument(); + expect(screen.getByText('2.00 KB/s')).toBeInTheDocument(); + }); + + it('keeps lens charts mounted across live updates so scrub state survives', () => { + const [guest, setGuest] = createSignal( + makeGuest({ name: 'live-lens-vm', cpu: 0.2, networkIn: 2048, networkOut: 4096 }), + ); + renderGuestRow({ + get guest() { + return guest(); + }, + visibleColumnIds: ['name', 'cpu', 'netIo'], + metricDisplayMode: 'bars', + metricHistory: { + getGuestMetricSeries: (_guest, metric) => [ + { + id: metric, + label: metric, + color: '#8b5cf6', + points: [ + { timestamp: 1, value: 10 }, + { timestamp: 2, value: 25 }, + ], + }, + ], + getNodeMetricSeries: () => [], + }, + }); + + const row = screen.getByText('live-lens-vm').closest('tr')!; + fireEvent.pointerEnter(row, { pointerType: 'mouse' }); + const [cpuChart, netChart] = screen.getAllByTestId('metric-mini-sparkline'); + + setGuest(makeGuest({ name: 'live-lens-vm', cpu: 0.4, networkIn: 8192, networkOut: 1024 })); + + // A websocket snapshot while the operator is reading a chart must update + // its values in place: a remount drops the chart's open scrub tooltip. + const [nextCpuChart, nextNetChart] = screen.getAllByTestId('metric-mini-sparkline'); + expect(nextCpuChart).toBe(cpuChart); + expect(nextNetChart).toBe(netChart); + expect(nextNetChart.textContent).toBe('↓8.0K↑1.0K'); + expect(nextCpuChart.querySelector('svg')).toHaveAttribute( + 'aria-label', + 'live-lens-vm CPU history, current 40%', + ); + expect(nextNetChart.querySelector('svg')).toHaveAttribute( + 'aria-label', + 'live-lens-vm network I/O history, current 8.00 KB/s / 1.00 KB/s', + ); + }); + + it('leaves stopped guests on the I/O dash while the lens charts the rest of the row', () => { + renderGuestRow({ + guest: makeGuest({ name: 'io-stopped-vm', status: 'stopped' }), + visibleColumnIds: ['name', 'cpu', 'netIo', 'diskIo'], + metricDisplayMode: 'bars', + metricHistory: { + getGuestMetricSeries: (_guest, metric) => [ + { + id: metric, + label: metric, + color: '#8b5cf6', + points: [ + { timestamp: 1, value: 10 }, + { timestamp: 2, value: 25 }, + ], + }, + ], + getNodeMetricSeries: () => [], + }, + }); + + const row = screen.getByText('io-stopped-vm').closest('tr')!; + fireEvent.pointerEnter(row, { pointerType: 'mouse' }); + + expect(screen.getAllByTestId('metric-mini-sparkline')).toHaveLength(1); + expect(row.querySelector('[data-metric-rate-pair]')).toBeNull(); }); it('shows dash when disk data is unavailable', () => { diff --git a/frontend-modern/src/components/Workloads/__tests__/MetricMiniSparkline.test.tsx b/frontend-modern/src/components/Workloads/__tests__/MetricMiniSparkline.test.tsx index d145438de7..1d6a654f65 100644 --- a/frontend-modern/src/components/Workloads/__tests__/MetricMiniSparkline.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/MetricMiniSparkline.test.tsx @@ -4,6 +4,7 @@ import { fireEvent, render, screen } from '@solidjs/testing-library'; import { createSignal } from 'solid-js'; import { MetricMiniSparkline, + MetricMiniSparklineRatePair, type MetricMiniSparklineValueLabelContext, } from '../MetricMiniSparkline'; @@ -146,4 +147,88 @@ describe('MetricMiniSparkline', () => { expect(screen.getByText('Out')).toBeInTheDocument(); expect(screen.getByText('200 B/s')).toBeInTheDocument(); }); + + it('shows a compact rate pair beside the chart while announcing the full rates', () => { + render(() => ( + + } + series={[ + { + id: 'diskread', + label: 'Read', + color: '#3b82f6', + points: [ + { timestamp: 1_000, value: 10 }, + { timestamp: 2_000, value: 20 }, + ], + }, + ]} + /> + )); + + const sparkline = screen.getByTestId('metric-mini-sparkline'); + expect(sparkline.dataset.valueLabelMode).toBe('inline'); + expect(sparkline.textContent).toBe('R3.3MW512K'); + expect(screen.queryByText('3.32 MB/s / 512 KB/s')).not.toBeInTheDocument(); + expect(sparkline.querySelector('svg')).toHaveAttribute( + 'aria-label', + 'Disk I/O history, current 3.32 MB/s / 512 KB/s', + ); + + const pair = sparkline.querySelector('[data-metric-rate-pair]') as HTMLElement; + expect(pair).toHaveAttribute('aria-hidden', 'true'); + // Glyph colours match the read/write series they label. + expect(screen.getByText('R')).toHaveStyle({ color: '#3b82f6' }); + expect(screen.getByText('W')).toHaveStyle({ color: '#f59e0b' }); + }); + + it('re-reads an open tooltip from the current series when history updates', () => { + const series = (inbound: number) => [ + { + id: 'netin', + label: 'In', + color: '#10b981', + points: [ + { timestamp: 1_000, value: 10 }, + { timestamp: 2_000, value: inbound }, + { timestamp: 3_000, value: 30 }, + ], + }, + ]; + const [current, setCurrent] = createSignal(series(20)); + render(() => ( + `${value} B/s`} + series={current()} + /> + )); + + const svg = screen.getByTestId('metric-mini-sparkline').querySelector('svg') as SVGSVGElement; + svg.getBoundingClientRect = () => + ({ + bottom: 38, + height: 18, + left: 0, + right: 96, + top: 20, + width: 96, + x: 0, + y: 20, + toJSON: () => ({}), + }) as DOMRect; + fireEvent.mouseMove(svg, { clientX: 48, clientY: 24 }); + expect(screen.getByText('20 B/s')).toBeInTheDocument(); + + setCurrent(series(25)); + + expect(screen.queryByText('20 B/s')).not.toBeInTheDocument(); + expect(screen.getByText('25 B/s')).toBeInTheDocument(); + }); }); diff --git a/frontend-modern/src/components/Workloads/__tests__/workloadMetricHistoryModel.test.ts b/frontend-modern/src/components/Workloads/__tests__/workloadMetricHistoryModel.test.ts index 9814659315..4a1bfe826b 100644 --- a/frontend-modern/src/components/Workloads/__tests__/workloadMetricHistoryModel.test.ts +++ b/frontend-modern/src/components/Workloads/__tests__/workloadMetricHistoryModel.test.ts @@ -13,6 +13,7 @@ import { getNodeChartKeyCandidates, getWorkloadChartKeyCandidates, isWorkloadTableMetricHistoryRange, + MIN_RATE_SCALE_CEILING, normalizeWorkloadChartKey, WORKLOAD_TABLE_HISTORY_DEFAULT_RANGE, WORKLOAD_TABLE_HISTORY_MAX_POINTS, @@ -191,6 +192,37 @@ describe('workloadMetricHistoryModel', () => { expect(scale).toEqual({ minValue: 0, maxValue: 100 }); }); + it('keeps idle I/O chatter near the axis instead of drawing it as a full-height spike', () => { + const rateSeries = (inbound: number[], outbound: number[]) => [ + { + id: 'netin', + label: 'In', + color: '#10b981', + points: inbound.map((value, index) => ({ timestamp: index + 1, value })), + }, + { + id: 'netout', + label: 'Out', + color: '#fb923c', + points: outbound.map((value, index) => ({ timestamp: index + 1, value })), + }, + ]; + + const idle = rateSeries([2 * 1024, 6 * 1024, 3 * 1024], [1024, 1536, 1024]); + const idleScale = getMetricMiniSparklineScale(idle, 'B/s'); + expect(idleScale).toEqual({ minValue: 0, maxValue: MIN_RATE_SCALE_CEILING }); + // The 6 KB/s blip stays within a pixel of the axis rule at y=16 instead of reaching the top. + expect(buildMetricMiniSparklinePath(idle[0].points, idleScale)).toBe( + 'M1.00,15.72 L48.00,15.16 L95.00,15.58', + ); + + const busy = rateSeries([400 * 1024 ** 2, 500 * 1024 ** 2], [1024, 2048]); + expect(getMetricMiniSparklineScale(busy, 'B/s')).toEqual({ + minValue: 0, + maxValue: 500 * 1024 ** 2 * 1.15, + }); + }); + it('uses a shared time range and resolves nearest hover values for paired I/O sparklines', () => { const series = [ { diff --git a/frontend-modern/src/components/Workloads/workloadMetricHistoryModel.ts b/frontend-modern/src/components/Workloads/workloadMetricHistoryModel.ts index d2fa35b55a..054a51971b 100644 --- a/frontend-modern/src/components/Workloads/workloadMetricHistoryModel.ts +++ b/frontend-modern/src/components/Workloads/workloadMetricHistoryModel.ts @@ -80,8 +80,36 @@ export const WORKLOAD_TABLE_HISTORY_DEFAULT_RANGE: WorkloadTableMetricHistoryRan // when an operator changes range on a large estate. export const WORKLOAD_TABLE_HISTORY_MAX_POINTS = 36; export const MIN_PERCENT_SCALE_CEILING = 5; +// Rate sparklines scale to their own peak, so without a floor a guest moving +// between 2 KB/s and 6 KB/s draws the same full-height spike as one hitting +// 500 MB/s. Background chatter stays near the axis below this ceiling. +export const MIN_RATE_SCALE_CEILING = 100 * 1024; export const WORKLOAD_TABLE_HISTORY_POLL_MS = 30_000; +export type WorkloadRateMetric = Extract; + +export interface WorkloadRateMetricGlyph { + glyph: string; + color: string; + mono: boolean; +} + +// The glyphs the rate cells already use at rest, coloured like the series they +// label, so a compact value pair beside a chart also serves as its legend. +export const WORKLOAD_RATE_METRIC_GLYPHS: Record< + WorkloadRateMetric, + readonly [WorkloadRateMetricGlyph, WorkloadRateMetricGlyph] +> = { + netIo: [ + { glyph: '↓', color: NET_IN_COLOR, mono: false }, + { glyph: '↑', color: NET_OUT_COLOR, mono: false }, + ], + diskIo: [ + { glyph: 'R', color: DISK_READ_COLOR, mono: true }, + { glyph: 'W', color: DISK_WRITE_COLOR, mono: true }, + ], +}; + export const WORKLOAD_TABLE_HISTORY_INFRA_METRICS = [ 'cpu', 'memory', @@ -324,6 +352,13 @@ export const getMetricMiniSparklineScale = ( }; } + if (unit === 'B/s') { + return { + minValue: 0, + maxValue: Math.max(MIN_RATE_SCALE_CEILING, maxValue * 1.15), + }; + } + return { minValue: 0, maxValue: Math.max(1, maxValue * 1.15), diff --git a/frontend-modern/src/utils/__tests__/format.test.ts b/frontend-modern/src/utils/__tests__/format.test.ts index 417c38b56e..944ccdc882 100644 --- a/frontend-modern/src/utils/__tests__/format.test.ts +++ b/frontend-modern/src/utils/__tests__/format.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'; import { formatBytes, formatSpeed, + formatCompactSpeed, formatObservedSpeed, formatPercent, formatCpuPercent, @@ -69,6 +70,26 @@ describe('formatSpeed', () => { }); }); +describe('formatCompactSpeed', () => { + it.each([ + [0, '0'], + [-1024, '0'], + [Number.NaN, '0'], + [512, '<1K'], + [1024, '1.0K'], + [3.32 * 1024 * 1024, '3.3M'], + [9.94 * 1024, '9.9K'], + [9.96 * 1024, '10K'], + [512 * 1024, '512K'], + [1023.6 * 1024, '1.0M'], + [2.5 * 1024 ** 3, '2.5G'], + [2 ** 64, '16E'], + [Number.MAX_VALUE, '>1023E'], + ])('formats %s B/s as %s', (value, expected) => { + expect(formatCompactSpeed(value)).toBe(expected); + }); +}); + describe('formatCpuPercent', () => { it.each([ [0, '0%'], diff --git a/frontend-modern/src/utils/format.ts b/frontend-modern/src/utils/format.ts index b6f3ba94ec..0abbaf3b8c 100644 --- a/frontend-modern/src/utils/format.ts +++ b/frontend-modern/src/utils/format.ts @@ -67,6 +67,35 @@ export function formatSpeed(bytesPerSecond: number, decimals: number | 'auto' = return `${formatBytes(bytesPerSecond, decimals)}/s`; } +const COMPACT_SPEED_UNITS = ['K', 'M', 'G', 'T', 'P', 'E']; + +/** + * Transfer rate for a dense cell that shares its width with a chart: binary + * unit letter, no "/s", one decimal below 10 ("3.3M", "512K"). Sub-kilobyte + * rates read "<1K" so idle chatter is not dressed up as a measurement. + */ +export function formatCompactSpeed(bytesPerSecond: number): string { + if (!Number.isFinite(bytesPerSecond) || bytesPerSecond <= 0) return '0'; + if (bytesPerSecond < 1024) return '<1K'; + + let value = bytesPerSecond / 1024; + let unit = 0; + while (value >= 1024 && unit < COMPACT_SPEED_UNITS.length - 1) { + value /= 1024; + unit += 1; + } + // Exabytes per second is already a wrapped counter, not a measurement; keep + // anything beyond it to the slot's width rather than printing an exponent. + if (unit === COMPACT_SPEED_UNITS.length - 1 && value >= 1023.5) return '>1023E'; + if (value < 9.95) return `${value.toFixed(1)}${COMPACT_SPEED_UNITS[unit]}`; + + const rounded = Math.round(value); + if (rounded >= 1024 && unit < COMPACT_SPEED_UNITS.length - 1) { + return `1.0${COMPACT_SPEED_UNITS[unit + 1]}`; + } + return `${rounded}${COMPACT_SPEED_UNITS[unit]}`; +} + export function formatObservedSpeed(bytesPerSecond: number | null | undefined): string { return typeof bytesPerSecond === 'number' && Number.isFinite(bytesPerSecond) && From 2265d28242a234273b76d605deabdd0523f9e4b0 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:13:36 +0100 Subject: [PATCH 20/53] Remove the never-mounted unified resource table PlatformResourceTable was the only renderer of UnifiedResourceTable, and nothing imports it. Every platform page (Proxmox, Docker, Kubernetes, TrueNAS, vSphere, Machines) renders its own native tables, and /infrastructure only shows the opening-workspace placeholder. A production build of main already tree-shook the whole cluster (none of its strings ship), yet it still carried 11 source files, 7 test files, registry ownership in two subsystems and contract rules for a table nobody sees. Removed: PlatformResourceTable and its PLATFORM_STATUS_FILTER_OPTIONS; UnifiedResourceTable with its host, service, PBS and PMG cards, source badge cell, state hook, viewport sync and models; and ClusterDeployBanner, which only the host card rendered (the live cluster deploy flow is the Settings ClusterAgentDeployDialog). Their tests go with them. Coverage for live code those tests also exercised is kept: the detail mapper no-fetch boundary and the online boolean sensor case move to resourceDetailMappers.test.ts, the ResourceFacetSummary render cases move to their own test file, and both files become registered proofs for the policies that govern their subjects. Guardrail tests, the shared template registry, registry.json, status.json evidence and the performance, unified-resources, frontend-primitives and cloud-paid contracts drop the deleted paths and the rules that only governed them. Rollup used to name the large shared chunk after sharedPlatformPage; it now names it MetricBar, so the bundle baseline entry is re-keyed at its measured size. --- docs/release-control/v6/internal/status.json | 17 +- .../v6/internal/subsystems/cloud-paid.md | 5 +- .../subsystems/frontend-primitives.md | 20 +- .../subsystems/performance-and-scalability.md | 165 +- .../v6/internal/subsystems/registry.json | 115 +- .../internal/subsystems/unified-resources.md | 131 +- frontend-modern/.bundlesize.json | 4 +- .../d494df7afd0adcb5.json | 93 ++ .../scripts/shared-template-registry.json | 16 - .../Infrastructure/ClusterDeployBanner.tsx | 74 - .../UnifiedResourceHostTableCard.tsx | 799 ---------- .../UnifiedResourcePBSTableSection.tsx | 353 ----- .../UnifiedResourcePMGTableSection.tsx | 393 ----- ...ifiedResourceServiceInfrastructureCard.tsx | 35 - .../UnifiedResourceSourceBadgeCell.tsx | 72 - .../Infrastructure/UnifiedResourceTable.tsx | 25 - .../__tests__/ClusterDeployBanner.test.tsx | 235 --- .../__tests__/ResourceFacetSummary.test.tsx | 69 + ...esourceTable.performance.contract.test.tsx | 1341 ----------------- ...edResourceTable.policy-governance.test.tsx | 103 -- ...ifiedResourceTable.workloads-link.test.tsx | 293 ---- .../__tests__/resourceDetailMappers.test.ts | 37 + ...ifiedResourceTableModel.branchcov2.test.ts | 340 ----- ...rceTableStateModel.branchcov0724pm.test.ts | 173 --- .../unifiedResourceTableStateModel.test.ts | 226 --- .../unifiedResourceTableModel.ts | 108 -- .../unifiedResourceTableStateModel.ts | 449 ------ .../useUnifiedResourceTableState.ts | 267 ---- .../useUnifiedResourceTableViewportSync.ts | 78 - .../SharedPrimitives.guardrails.test.ts | 59 +- .../platformOverviewLayout.guardrails.test.ts | 1 - .../platformPage/sharedPlatformPage.tsx | 76 - .../frontendResourceTypeBoundaries.test.ts | 51 - .../release_control/subsystem_lookup_test.py | 6 +- .../tests/68-platform-pages-shell.spec.ts | 2 +- 35 files changed, 261 insertions(+), 5970 deletions(-) create mode 100644 frontend-modern/browser-verification/d494df7afd0adcb5.json delete mode 100644 frontend-modern/src/components/Infrastructure/ClusterDeployBanner.tsx delete mode 100644 frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx delete mode 100644 frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx delete mode 100644 frontend-modern/src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx delete mode 100644 frontend-modern/src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx delete mode 100644 frontend-modern/src/components/Infrastructure/UnifiedResourceSourceBadgeCell.tsx delete mode 100644 frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx delete mode 100644 frontend-modern/src/components/Infrastructure/__tests__/ClusterDeployBanner.test.tsx create mode 100644 frontend-modern/src/components/Infrastructure/__tests__/ResourceFacetSummary.test.tsx delete mode 100644 frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.performance.contract.test.tsx delete mode 100644 frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.policy-governance.test.tsx delete mode 100644 frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.workloads-link.test.tsx delete mode 100644 frontend-modern/src/components/Infrastructure/__tests__/unifiedResourceTableModel.branchcov2.test.ts delete mode 100644 frontend-modern/src/components/Infrastructure/__tests__/unifiedResourceTableStateModel.branchcov0724pm.test.ts delete mode 100644 frontend-modern/src/components/Infrastructure/__tests__/unifiedResourceTableStateModel.test.ts delete mode 100644 frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts delete mode 100644 frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts delete mode 100644 frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts delete mode 100644 frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts diff --git a/docs/release-control/v6/internal/status.json b/docs/release-control/v6/internal/status.json index a77903e369..42f0bdec05 100644 --- a/docs/release-control/v6/internal/status.json +++ b/docs/release-control/v6/internal/status.json @@ -95,11 +95,6 @@ "path": "cmd/pulse-agent/service_windows.go", "kind": "file" }, - { - "repo": "pulse", - "path": "frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.workloads-link.test.tsx", - "kind": "file" - }, { "repo": "pulse", "path": "frontend-modern/src/hooks/__tests__/useUnifiedResources.test.ts", @@ -5666,11 +5661,6 @@ "performance-and-scalability" ], "evidence": [ - { - "repo": "pulse", - "path": "frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.performance.contract.test.tsx", - "kind": "file" - }, { "repo": "pulse", "path": "frontend-modern/src/components/Infrastructure/infrastructureSelectors.ts", @@ -5681,11 +5671,6 @@ "path": "frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts", "kind": "file" }, - { - "repo": "pulse", - "path": "frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx", - "kind": "file" - }, { "repo": "pulse", "path": "frontend-modern/src/components/Workloads/__tests__/workloadInventorySourceIssues.test.ts", @@ -5923,7 +5908,7 @@ }, { "repo": "pulse", - "path": "frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx", + "path": "frontend-modern/src/hooks/useUnifiedResources.ts", "kind": "file" }, { diff --git a/docs/release-control/v6/internal/subsystems/cloud-paid.md b/docs/release-control/v6/internal/subsystems/cloud-paid.md index 34c5094087..693ed47bcc 100644 --- a/docs/release-control/v6/internal/subsystems/cloud-paid.md +++ b/docs/release-control/v6/internal/subsystems/cloud-paid.md @@ -1168,9 +1168,8 @@ hands-on Patrol modes, issue investigation, verified fixes, and longer history`. aliases such as `/settings/workloads/docker` or nested `/settings/infrastructure/platforms/*` paths. Each platform page must remain chrome-only: routing plus sub-tab navigation that embeds the - canonical `WorkloadsSurface`, `StorageSurface`, `RecoverySurface`, or - `UnifiedResourceTable` in `embedded tableOnly` mode with a forced - platform/source filter. The shell must not introduce dashboard cards, + canonical `WorkloadsSurface`, `StorageSurface`, or `RecoverySurface` in + `embedded tableOnly` mode with a forced platform/source filter. The shell must not introduce dashboard cards, bespoke per-family tables, synthetic placeholder data, or reintroduce Infrastructure as a primary navigation entry without a governed contract decision recorded by the frontend-primitives owner. diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 710f4ef07e..aee3dc3bfd 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -3276,9 +3276,9 @@ default` instead of fusing provider and badge text such as Platform-first top-level pages registered through `frontend-modern/src/App.tsx` must stay chrome-only and route through the canonical app shell: each per-platform surface owns navigation and sub-tab - chrome, then embeds the canonical `WorkloadsSurface`, `StorageSurface`, - `RecoverySurface`, or `UnifiedResourceTable` in `embedded tableOnly` mode - with a forced platform or source filter. Per-platform features must not + chrome, then embeds the canonical `WorkloadsSurface`, `StorageSurface`, or + `RecoverySurface` in `embedded tableOnly` mode with a forced platform or + source filter. Per-platform features must not fork their own table primitives, header layouts, or summary cards when a shared canonical surface already exists; new shared platform-page primitives live under `frontend-modern/src/features/platformPage/` so the @@ -6284,16 +6284,10 @@ layering on the default. Tables therefore state the padding they want in their own classes or presentation constants instead of carrying `!px-*` overrides to beat the primitive. `Table.test.tsx` pins the base, single-side, important, `p-*`, prefixed-only and reactive cases. -That same shared table boundary now owns CSP-safe sizing for infrastructure -tables and metric bars. `frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts` -and `frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts` -must express table layout and column sizing as shared class/attribute -presentation instead of inline `style=` maps, and -`frontend-modern/src/components/shared/ProgressBar.tsx` must render fill width -through DOM attributes rather than inline width styles. Infrastructure host and -service tables may still vary by breakpoint and column family, but they must do -so through the shared presentation owner instead of lane-local style objects -that break the public demo CSP. +That same shared table boundary now owns CSP-safe fill rendering for metric +bars: `frontend-modern/src/components/shared/ProgressBar.tsx` must render fill +width through DOM attributes rather than inline width styles that break the +public demo CSP. That same shared-boundary rule applies to summary density. The shared compact mode on `SummaryPanel.tsx` and `SummaryMetricCard.tsx` exists for genuinely dense monitoring surfaces, but pages that are trying to align with the normal diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index a26b7e22af..2b6cfd9607 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -474,15 +474,6 @@ in-place click and chevron activation over a selection. 64. `frontend-modern/src/components/Workloads/__tests__/workloadRouteStateModel.test.ts` 65. `frontend-modern/src/components/Workloads/__tests__/workloadUrlSyncModel.test.ts` 66. `frontend-modern/src/components/Workloads/__tests__/workloadTopology.test.ts` -67. `frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx` -68. `frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts` -69. `frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts` -70. `frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts` -71. `frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx` -72. `frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx` -73. `frontend-modern/src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx` -74. `frontend-modern/src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx` -75. `frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts` 76. `frontend-modern/src/components/Infrastructure/infrastructureSelectors.ts` 77. `frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts` 78. `frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx` @@ -504,7 +495,6 @@ in-place click and chevron activation over a selection. 94. `frontend-modern/src/components/Workloads/__tests__/workloadMetricHistoryModel.test.ts` 95. `frontend-modern/src/components/Workloads/GuestDrawer.test.tsx` 96. `frontend-modern/src/components/Workloads/__tests__/useGroupedTableWindowing.test.ts` -97. `frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.performance.contract.test.tsx` 98. `frontend-modern/src/components/Workloads/useWorkloadViewportSync.ts` 99. `frontend-modern/src/components/Workloads/__tests__/useWorkloadViewportSync.test.tsx` 100. `frontend-modern/src/utils/workloadsSummaryCache.ts` @@ -548,16 +538,7 @@ or a measured fleet performance improvement. 1. `frontend-modern/src/components/Infrastructure/infrastructureSelectors.ts` shared with `unified-resources`: the infrastructure selector pipeline is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. 2. `frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts` shared with `unified-resources`: resource detail mappers are both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -3. `frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx` shared with `unified-resources`: the unified resource host table card is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -4. `frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx` shared with `unified-resources`: the unified resource PBS section is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -5. `frontend-modern/src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx` shared with `unified-resources`: the unified resource PMG section is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -6. `frontend-modern/src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx` shared with `unified-resources`: the unified resource service infrastructure card is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -7. `frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx` shared with `unified-resources`: the unified resource table is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -8. `frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts` shared with `unified-resources`: unified resource service row shaping and I/O emphasis are both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -9. `frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts` shared with `unified-resources`: unified resource table state derivation, sort-cycle policy, service sorting, and responsive column layout are both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. 10. `frontend-modern/src/components/Infrastructure/useTableWindowing.ts` shared with `frontend-primitives`: the shared bounded table-window controller is both a canonical frontend rendering primitive and a fleet-scale scrolling hot-path boundary. -10. `frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts` shared with `unified-resources`: unified resource table state, grouping, and windowing are both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -11. `frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts` shared with `unified-resources`: unified resource table viewport sync and selected-row reveal are both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. 12. `frontend-modern/src/features/platformPage/PlatformWindowedList.tsx` shared with `frontend-primitives`: the shared bounded list renderer is both a canonical platform-page primitive and a fleet-scale mounted-DOM performance boundary. 13. `frontend-modern/src/features/platformPage/PlatformWindowedRows.tsx` shared with `frontend-primitives`: the shared bounded table-row renderer is both a canonical platform-page primitive and a fleet-scale mounted-DOM performance boundary. 14. `frontend-modern/src/features/platformPage/usePlatformWindowedItems.ts` shared with `frontend-primitives`: the platform windowing controller is both a canonical frontend scroll primitive and a directional-runway performance hot path. @@ -1060,16 +1041,6 @@ change may globally weaken the Task 03 lifecycle-state idempotency invariant. `/auth/trial-activate` and `POST /api/license/trial/start` from public-path and CSRF inventories must stay as constant-time route-table absence rather than replacing the old callback with persistence-backed router probes. - Infrastructure host-table health explanations follow the same hot-path - rule. `UnifiedResourceHostTableCard.tsx` may render compact issue labels - from the already-materialized resource incident and storage-risk fields, but - it must not introduce per-row API reads, broad resource scans, storage - topology recomputation, or layout-measuring work just to explain warning or - degraded rows. Infrastructure table source derivation follows the same - bounded-work rule: `unifiedResourceTableStateModel.ts` must read the - already-materialized top-level `resource.sources` array before legacy - `platformData.sources` hints, and must not reconstruct platform identity by - scanning sibling resources or making row-time API calls. 5. Extend workload hot-path filter, sort, grouping, and stats math through `frontend-modern/src/components/Workloads/workloadSelectors.ts`, and extend workload identity, discovery routing, and node-topology helpers through `frontend-modern/src/components/Workloads/workloadTopology.ts`, rather than duplicating selector or topology logic in `frontend-modern/src/components/Workloads/WorkloadsSurface.tsx` Platform-owned Workloads surfaces may force a platform scope, but that scope must be applied before deriving host, Kubernetes context/namespace, @@ -1749,22 +1720,16 @@ without contention or cross-delivery. 2. Update this contract when a new protected hot path is adopted 3. Route runtime changes through the explicit performance proof policies in `registry.json`; default fallback proof routing is not allowed 4. Record the evidence source for any claimed performance improvement -5. Keep wide-desktop infrastructure table layout proof on the shared owner. - Changes to `frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts` - that affect host, PBS, or PMG desktop column widths must ship with shared - model verification plus desktop Playwright proof that full-width shells - distribute surplus width across peer columns instead of stretching only the - `Resource` column. -6. Keep entitlement and runtime capability lookups on shared router/settings +5. Keep entitlement and runtime capability lookups on shared router/settings hot paths bounded and request-local. AI control-level clamping in `internal/api/router.go` may consult the already-wired runtime entitlement service, but it must not add broad persistence scans, metrics fan-out, or external network calls to protected settings or chat request paths. -7. Initialize and cache each organization RBAC manager once per router +6. Initialize and cache each organization RBAC manager once per router lifecycle. Request handlers may reuse the cached manager but must not reopen or remigrate SQLite on each role or assignment request, and shutdown must close the owned manager set. -8. Guest Overview filesystem provenance is local presentation over the already +7. Guest Overview filesystem provenance is local presentation over the already supplied snapshot: no additional query, timer, poll, History fetch or guest command may be added to establish it. Its existing linear filesystem mapping may label retained values and omit current utilization bars; it must preserve @@ -1823,8 +1788,8 @@ definitions with four concurrent command or REST executions. Group/subgroup ordering and numeric, boolean, or timestamp rendering remain one bounded sort-and-map over that collection and stay outside table-row and virtualized viewport hot paths. -`UnifiedResourceTable.performance.contract.test.tsx` pins the resource-local -projection and no-fetch boundary. +`resourceDetailMappers.test.ts` pins the resource-local projection and no-fetch +boundary. ### Workload refreshes retain one coherent paged generation @@ -2717,15 +2682,6 @@ re-accumulating discovery heuristics back into the performance hot-path mapper. That mapper may add simple host-agent thermals rows from already-loaded sensor metadata, including `thermalState`, but it must not perform additional resource lookups, history reads, or polling work to decorate the drawer. -The PBS service-table hot path now follows that same split: raw job arrays may -cross the transport boundary, but -`frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts` -must collapse them into one shared `Activity` presentation via the canonical -service-model helpers, while -`frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx` -stays a thin render shell. Future running-task visibility changes must extend -through the shared model and accepted performance proof file instead of adding -per-row PBS status parsing or inline job scans in the table render path. The Workloads disk list now follows the same pattern: the shell stays in `frontend-modern/src/components/Workloads/DiskList.tsx`, while disk-row presentation derivations and fallback tooltip/runtime wiring live in @@ -2889,48 +2845,11 @@ precedence, reinterpret ratios above an arbitrary threshold, or replace an established provider history series; source authority and canonical identity convergence belong upstream in the unified-resource and monitoring contracts. -The unified resource table hot path is now also governed as explicit -performance-owned runtime, with shared ownership against the unified-resource -consumer boundary. The remaining performance work is no longer top-level -ownership ambiguity on the main Infrastructure or Workloads tables. -The table's reactive runtime, grouping, and row-windowing owner now lives in -`frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts`, -while pure table-state derivation, service sorting, sort-cycle policy, and -responsive column layout now live in -`frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts`, -and viewport-sync plus selected-row reveal behavior now live in -`frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts`, -so future hot-path table-state changes must not fold selector derivation, -layout policy, and scroll coordination back into one mixed owner or the render -shell. -The same shared layout policy now treats the phone branch as a zero-overflow -fixed-width table rather than preserving a 640-pixel intrinsic floor. The -shared state model must select one identity track fixed at 30 percent plus five to seven -source-relevant health or activity tracks, normalize those widths to the -available container, and keep the document and table shell at scroll-width -parity. That denser phone projection remains pure model/class work: it must not -add row-time measurement, duplicate tables, viewport listeners, or per-row -responsive signals to the infrastructure hot path. -That same hot-path boundary now also owns CSP-safe table sizing. Infrastructure -host, PBS, and PMG table shells must take their layout and column sizing from -the shared presentation owner in -`frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts` -and apply those values through classes and width/height attributes, not inline -`style=` maps on the live table DOM. Future hot-path table work must not -reintroduce inline width, min/max width, or row-height styles into the render -shell just to land a local layout tweak. -That same shared sizing contract now also owns wide-desktop width -distribution. In full-width shells, host, PBS, and PMG tables must keep an -explicit desktop `Resource` column width in the shared presentation owner so -surplus width is redistributed across peer metric, source, uptime, and action -columns instead of being dumped into the first column and wasting operator -visible table density. -That hot-path contract now includes policy badge rendering on resource rows. -Policy-rich table rows must only surface a single inline summary chip for -blocking `local-only`/`restricted` posture; non-blocking `sensitive` + +Resource table rows that surface policy posture must show at most a single +inline summary chip for blocking `local-only`/`restricted` posture; non-blocking `sensitive` + `local-first` and redaction-only metadata belongs in Data Handling, detail, and AI/governance surfaces instead of spending default row visual budget. -Agentless availability evidence belongs on that same bounded row path. +Agentless availability evidence belongs on the bounded row path. Any resource row carrying an `AvailabilityData` facet—whether a standalone `network-endpoint` or a known guest that inherited the facet through explicit link or IP correlation—may replace otherwise empty host metric slots with one @@ -2960,40 +2879,28 @@ The summary API feeding that hot path must also normalize mixed-resolution history into equal-time summary buckets before it reaches the shell/runtime owners, so long-range cards do not bunch recent higher-resolution samples at the right edge. -It now also includes compact resource-facet summary chips rendered next to -policy metadata, and table-row uses must set an explicit visible chip limit -with overflow disclosure instead of allowing mock-rich resource rows to wrap -an unbounded badge list. Those chips must stay within the same bounded -windowing and mounted-row budget proved by -`UnifiedResourceTable.performance.contract.test.tsx`. -The same facet summary contract applies to the service-resource rows inside -the unified table as well, so PBS and PMG entries must keep the same bounded -presentation and verification surface as the primary fleet rows. The shared -`ResourceFacetSummary` component now owns that chip rendering path, so any -future summary changes must preserve the same bounded row budget instead of -forking separate table-only presentation logic. That component now also +Compact resource-facet summary chips render through the shared +`ResourceFacetSummary` component, whose production consumer is the detail +drawer's change history. A table-row use must set an explicit visible chip +limit with overflow disclosure instead of letting mock-rich rows wrap an +unbounded badge list, and must stay within the table's bounded windowing and +mounted-row budget rather than forking separate table-only presentation +logic. That component now also consumes the shared `frontend-modern/src/utils/resourceChangePresentation.ts` label helper for canonical change kinds, source types, and adapter provenance so the chip wording stays consistent without adding extra hot-path branching. -Host, PBS, and PMG summary rows also declare themselves as the compact touch -disclosure target through `SummaryRowActionButton`; the shared primitive hides -the redundant visible mobile chevron while preserving accessible disclosure -semantics, so provider sections do not pay a separate cell-width or tap-target -cost for an action already owned by the whole row. The same infrastructure hot path now also depends on the shared `frontend-modern/src/components/shared/ProgressBar.tsx` primitive for metric fill rendering. Performance-sensitive metric bars may vary by value and color, but they must render width through shared attribute-driven progress geometry instead of per-row inline width styles that break the hosted demo CSP. -The default table hot path now scopes those summary chips to timeline and -change-provenance badges only. Generic capability and relationship badges are -removed from the default row surface entirely until the underlying data is -proven populated, which preserves the fleet-table scan path and avoids -spending hot-path visual budget on model nouns that do not yet clear the -product bar. -Row summaries now also prefer canonical `facetCounts` on each resource when -they are available, so the hot path can stay within the same budget while -still reading totals from the shared resource contract. The drawer history +Those summary chips are scoped to timeline and change-provenance badges only. +Generic capability and relationship badges stay off the default surface until +the underlying data is proven populated, so no surface spends visual budget on +model nouns that do not yet clear the product bar. +Facet summaries fall back to canonical `facetCounts` on each resource when no +fetched facet counts are available, reading totals from the shared resource +contract. The drawer history surface reuses the same governed resource route helpers for relationship and related-resource links, so cross-resource navigation stays within the existing infrastructure surface rather than branching into custom detail-only routing. @@ -3007,23 +2914,17 @@ investigation-first product contract instead of rendering low-signal generic facet sections by default. Governance metadata such as sensitivity and routing scope may be visible in the table, but it must remain on the same bounded row-windowing and mounted-row -budget proved by `UnifiedResourceTable.performance.contract.test.tsx` rather -than creating a separate unbounded rendering path for policy-rich fleets. -Infrastructure cluster-header presentation must stay inside that same bounded -row model. `unifiedResourceTableStateModel.ts` owns whether the compact -`Cluster` chip is rendered, including suppressing it when the visible group -name already ends in `Cluster`, so label cleanup does not add a second -row-rendering path or per-page DOM workaround. -The shared table and detail drawer now also render governed resource labels -through the shared identity/display contract, which routes policy-aware -resources through the canonical policy-aware helper and suppresses the raw -alternate name when policy requires governed handling. That keeps the -policy-aware label path inside the same hot-row rendering budget instead of -adding a second display branch for redacted fleets, and the proof for that -behavior lives in `UnifiedResourceTable.performance.contract.test.tsx`. -The shared table now also passes the same canonical resource-label resolver -into the detail drawer so related-resource chips in the timeline/history path -can resolve through the canonical catalog without adding a separate +budget rather than creating a separate unbounded rendering path for +policy-rich fleets. +The detail drawer now also renders governed resource labels through the +shared identity/display contract, which routes policy-aware resources through +the canonical policy-aware helper and suppresses the raw alternate name when +policy requires governed handling. That keeps the policy-aware label path +inside the same rendering budget instead of adding a second display branch for +redacted fleets. +Platform tables pass the canonical resource-label resolver into the detail +drawer through `PlatformResourceDetailTableRow` so related-resource chips in +the timeline/history path can resolve through the canonical catalog without adding a separate detail-only lookup branch to the hot-row path. The same detail drawer also uses that resolver for correlation dependency and dependent chips, so the investigation path does not fall back to raw IDs in diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index 35fe44427d..c2d7d797ec 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -153,62 +153,6 @@ "unified-resources" ] }, - { - "path": "frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx", - "rationale": "the unified resource host table card is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary", - "subsystems": [ - "performance-and-scalability", - "unified-resources" - ] - }, - { - "path": "frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx", - "rationale": "the unified resource PBS section is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary", - "subsystems": [ - "performance-and-scalability", - "unified-resources" - ] - }, - { - "path": "frontend-modern/src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx", - "rationale": "the unified resource PMG section is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary", - "subsystems": [ - "performance-and-scalability", - "unified-resources" - ] - }, - { - "path": "frontend-modern/src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx", - "rationale": "the unified resource service infrastructure card is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary", - "subsystems": [ - "performance-and-scalability", - "unified-resources" - ] - }, - { - "path": "frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx", - "rationale": "the unified resource table is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary", - "subsystems": [ - "performance-and-scalability", - "unified-resources" - ] - }, - { - "path": "frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts", - "rationale": "unified resource service row shaping and I/O emphasis are both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary", - "subsystems": [ - "performance-and-scalability", - "unified-resources" - ] - }, - { - "path": "frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts", - "rationale": "unified resource table state derivation, sort-cycle policy, service sorting, and responsive column layout are both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary", - "subsystems": [ - "performance-and-scalability", - "unified-resources" - ] - }, { "path": "frontend-modern/src/components/Infrastructure/useTableWindowing.ts", "rationale": "the shared bounded table-window controller is both a canonical frontend rendering primitive and a fleet-scale scrolling hot-path boundary", @@ -217,22 +161,6 @@ "performance-and-scalability" ] }, - { - "path": "frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts", - "rationale": "unified resource table state, grouping, and windowing are both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary", - "subsystems": [ - "performance-and-scalability", - "unified-resources" - ] - }, - { - "path": "frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts", - "rationale": "unified resource table viewport sync and selected-row reveal are both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary", - "subsystems": [ - "performance-and-scalability", - "unified-resources" - ] - }, { "path": "frontend-modern/src/components/Settings/AgentIntegrationsPanel.tsx", "rationale": "the External agents settings panel is the optional settings-shell projection of Pulse MCP onboarding, the AI runtime connected-agent onboarding surface, and a presentation consumer of the shared agent capabilities frontend client", @@ -7117,16 +7045,7 @@ "owned_files": [ "frontend-modern/src/components/Infrastructure/infrastructureSelectors.ts", "frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts", - "frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx", - "frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts", - "frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts", "frontend-modern/src/components/Infrastructure/useTableWindowing.ts", - "frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts", - "frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts", "frontend-modern/src/components/Workloads/DiskList.tsx", "frontend-modern/src/components/Workloads/diskListModel.ts", "frontend-modern/src/components/Workloads/EnhancedCPUBar.tsx", @@ -7286,16 +7205,7 @@ "match_files": [ "frontend-modern/src/components/Infrastructure/infrastructureSelectors.ts", "frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts", - "frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx", - "frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts", - "frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts", "frontend-modern/src/components/Infrastructure/useTableWindowing.ts", - "frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts", - "frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts", "frontend-modern/src/components/Workloads/DiskList.tsx", "frontend-modern/src/components/Workloads/diskListModel.ts", "frontend-modern/src/components/Workloads/EnhancedCPUBar.tsx", @@ -7365,8 +7275,7 @@ "test_prefixes": [], "exact_files": [ "frontend-modern/src/components/Infrastructure/__tests__/infrastructureSelectors.test.ts", - "frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.performance.contract.test.tsx", - "frontend-modern/src/components/Infrastructure/__tests__/unifiedResourceTableStateModel.test.ts", + "frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts", "frontend-modern/src/components/Infrastructure/__tests__/useTableWindowing.test.ts", "frontend-modern/src/components/Workloads/__tests__/AvailabilityProbeStatusCard.test.tsx", "frontend-modern/src/components/Workloads/__tests__/DiskList.test.tsx", @@ -8110,19 +8019,10 @@ "frontend-modern/src/components/Infrastructure/ResourceFacetSummary.tsx", "frontend-modern/src/components/Infrastructure/ResourceOperatorStateSection.tsx", "frontend-modern/src/components/Infrastructure/ResourcePolicySummary.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx", - "frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts", - "frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts", "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts", "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDockerActionsState.ts", "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerHistoryState.ts", "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerState.ts", - "frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts", - "frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts", "frontend-modern/src/components/Kubernetes/K8sDeploymentsDrawer.tsx", "frontend-modern/src/components/Kubernetes/K8sNamespacesDrawer.tsx", "frontend-modern/src/features/docker/DockerAlertsTable.tsx", @@ -8318,19 +8218,10 @@ "frontend-modern/src/components/Infrastructure/ResourceFacetSummary.tsx", "frontend-modern/src/components/Infrastructure/ResourceOperatorStateSection.tsx", "frontend-modern/src/components/Infrastructure/ResourcePolicySummary.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx", - "frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx", - "frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts", - "frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts", "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts", "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDockerActionsState.ts", "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerHistoryState.ts", "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerState.ts", - "frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts", - "frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts", "frontend-modern/src/components/Kubernetes/K8sDeploymentsDrawer.tsx", "frontend-modern/src/components/Kubernetes/K8sNamespacesDrawer.tsx", "frontend-modern/src/features/docker/DockerAlertsTable.tsx", @@ -8419,11 +8310,9 @@ "frontend-modern/src/components/Infrastructure/__tests__/resourceDetailDrawerOperationalModel.test.ts", "frontend-modern/src/components/Infrastructure/__tests__/resourceDetailDrawerTrueNASModel.test.ts", "frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts", + "frontend-modern/src/components/Infrastructure/__tests__/ResourceFacetSummary.test.tsx", "frontend-modern/src/components/Infrastructure/__tests__/ResourceOperatorStateSection.test.ts", "frontend-modern/src/components/Infrastructure/__tests__/ResourcePolicySummary.test.tsx", - "frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.performance.contract.test.tsx", - "frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.workloads-link.test.tsx", - "frontend-modern/src/components/Infrastructure/__tests__/unifiedResourceTableStateModel.test.ts", "frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts", "frontend-modern/src/features/docker/__tests__/DockerAlertsTable.test.tsx", "frontend-modern/src/features/docker/__tests__/dockerContainerLifecycleActions.test.ts", diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 5c1dfdb1bb..40136fb3ac 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -826,12 +826,6 @@ Compact Coverage age cells use the existing metadata-size text and reclaim numer 49. `frontend-modern/src/components/Infrastructure/ResourceCorrelationSummary.tsx` 50. `frontend-modern/src/components/Infrastructure/ResourceOperatorStateSection.tsx` 50a. `frontend-modern/src/components/Infrastructure/ResourcePolicySummary.tsx` -51. `frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx` -52. `frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx` -53. `frontend-modern/src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx` -54. `frontend-modern/src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx` -55. `frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts` -56. `frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts` 57. `frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts` 58. `frontend-modern/src/components/Infrastructure/resourceDetailDrawerServiceModel.ts` 59. `frontend-modern/src/components/Infrastructure/resourceDetailDrawerVmwareModel.ts` @@ -841,8 +835,6 @@ Compact Coverage age cells use the existing metadata-size text and reclaim numer 62. `frontend-modern/src/components/Infrastructure/useResourceDetailDrawerHistoryState.ts` 63. `frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDockerActionsState.ts` 64. `frontend-modern/src/components/Infrastructure/useResourceDetailDrawerState.ts` -65. `frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts` -66. `frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts` 67. `frontend-modern/src/components/Discovery/discoveryReadiness.ts` 68. `frontend-modern/src/components/Discovery/DiscoveryTab.tsx` 69. `frontend-modern/src/components/Discovery/useDiscoveryTabState.ts` @@ -1706,15 +1698,6 @@ container inventory table. 1. `frontend-modern/src/components/Infrastructure/infrastructureSelectors.ts` shared with `performance-and-scalability`: the infrastructure selector pipeline is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. 2. `frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts` shared with `performance-and-scalability`: resource detail mappers are both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -3. `frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx` shared with `performance-and-scalability`: the unified resource host table card is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -4. `frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx` shared with `performance-and-scalability`: the unified resource PBS section is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -5. `frontend-modern/src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx` shared with `performance-and-scalability`: the unified resource PMG section is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -6. `frontend-modern/src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx` shared with `performance-and-scalability`: the unified resource service infrastructure card is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -7. `frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx` shared with `performance-and-scalability`: the unified resource table is both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -8. `frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts` shared with `performance-and-scalability`: unified resource service row shaping and I/O emphasis are both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -9. `frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts` shared with `performance-and-scalability`: unified resource table state derivation, sort-cycle policy, service sorting, and responsive column layout are both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -10. `frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts` shared with `performance-and-scalability`: unified resource table state, grouping, and windowing are both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. -11. `frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts` shared with `performance-and-scalability`: unified resource table viewport sync and selected-row reveal are both a canonical unified-resource consumer surface and a fleet-scale performance hot-path boundary. 12. `frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsx` shared with `storage-recovery`: Proxmox backup server table rows are both a storage/recovery backup-health surface and a unified-resource platform-table consumer boundary. The table is composed on the Proxmox Backups tab and must not be duplicated on Overview. The Backups placement keeps the canonical PBS row and drawer @@ -2347,18 +2330,7 @@ provider identity or governed safe-summary posture when that context helps an operator, but the rendered labels must stay product-neutral and use `Analysis`, `Analysis Reasoning`, and `Safe Summary` rather than reviving generic `AI` or `AI-Safe` branding inside the resource drawer or discovery -shell. 14. Keep the operator-facing unified resource table width-aware at the table -surface, not just at the browser viewport. `frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx` -must route its root ref through `frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts`, -and `frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts` -owns the column-priority breakpoints for host and service infrastructure -rows. When the app shell leaves tablet-sized space during live resize, the -table hides lower-priority metadata first. At phone width, the state model -must remove the old 640-pixel floor, preserve identity at exactly 30 percent -of the table, and allocate the remaining width across the bounded -source-relevant health and activity columns. Both the document and table -shell must remain free of horizontal overflow; desktop and tablet stages -retain their existing complete column contracts. 15. Keep shared policy-posture framing on the unified-resource card owner. +shell. 15. Keep shared policy-posture framing on the unified-resource card owner. `frontend-modern/src/components/Infrastructure/ResourcePolicySummary.tsx` may accept caller-owned subtitle or resource-count wording when Patrol or another shared surface needs to explain how the same governed policy counts @@ -2570,67 +2542,13 @@ application resource-provider or WebSocket lifecycle. counts or capacity must use the shared `AnimatedNumber` primitive rather than page-local counter state, so readout motion stays presentation-only and canonical unified-resource identity and scope stay unchanged. -12. Keep infrastructure chart hover non-destructive to the unified-resource - table. If the hovered resource row is already visible in - `frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx`, - the row may highlight in place through the shared active-resource id; if it - is off-screen, the page must offer an explicit `Jump to row` affordance - rather than auto-scrolling or collapsing the table on hover. - 12a. Keep infrastructure summary visibility as display preference, not a +12. Keep infrastructure summary visibility as display preference, not a unified-resource filter. Platform/runtime pages and shared infrastructure summary consumers may hide or restore chart sections through shared presentation controls, but those controls must not mutate resource identity, table membership, source scope, or summary-hover state. The retired top-level `/infrastructure` page and its saved-view/route-state machinery must not be reintroduced for this purpose. -13. Keep infrastructure cluster headers as canonical summary scope. Grouped - headers in `frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx` - must publish cluster scope from the same `ResourceGroup` / unified-resource - ids that power the table rows, and - `frontend-modern/src/components/Infrastructure/useInfrastructureSummaryState.ts` - must consume that scope through the shared page/group/entity interaction - contract rather than inventing infrastructure-local summary filters or - route-backed cluster hover state. Host and service infrastructure table - card frames must consume the frontend-primitives-owned `TableCard` wrapper; - unified-resource ownership remains on resource identity, grouping, and row - semantics rather than forking a table border/background shell. Deliberate - cluster focus must also stay - on the canonical infrastructure route through the shared `summaryGroup` - query state, so pinned scope is shareable, reversible, and owned by the - same route-backed summary contract as row focus. Infrastructure must stay - row-first here: the pinned cluster header remains the visible scoped - state, and explicit clearing belongs to the shared infrastructure table - card header action plus the shared `Escape` reset path rather than a - search-row fallback widget, page-level scope strip, or a second - scope/pinned pill inside the cluster row chrome. Background whitespace - clearing may remain a convenience, but infrastructure must not rely on it - as the only reversible control. -14. Keep infrastructure row emphasis on the shared frontend presentation - contract. Host, PBS, and PMG table sections may decide whether a resource - is contextually active, but they must expose that state through - `data-summary-row-active` and rely on the shared row presentation owned by - `frontend-modern/src/index.css` instead of provider-specific background - classes that drift across resource tables or hide inline metric bars. - Cluster-member rows must also expose shared preview-versus-pinned group - emphasis through `data-summary-group-member-active`, so the whole cluster - block reads as the active scope without inventing a second infrastructure- - local outline or banner treatment. - Static grouped cluster-header emphasis must route through - `frontend-modern/src/components/shared/groupedTableRowPresentation.ts` and - the shared `.grouped-table-row` CSS contract in `frontend-modern/src/index.css`, - rather than infrastructure-local background or hover-fill classes. - Summary-linked infrastructure rows and cluster headers must also route - pointer preview and focus preview through - `frontend-modern/src/components/shared/summaryInteractionA11y.ts`, while - deliberate expand/scope ownership must route through - `frontend-modern/src/components/shared/SummaryRowActionButton.tsx`, so the - unified-resource table does not fork mouse-only hover logic, focusable-row - button shims, touch-hostile synthetic hover, or provider-specific control - handling across host, PBS, and PMG sections. Those three resource rows are - themselves the compact touch disclosure target and must explicitly enable - the shared mobile-chevron suppression; their accessible row/button state - remains available without spending a visible summary-cell slot on a - duplicate control. 15. Keep infrastructure search aligned with the governed display label. Shared infrastructure filtering through `frontend-modern/src/components/Infrastructure/infrastructureSelectors.ts` @@ -4015,8 +3933,6 @@ when the opened drawer would otherwise fall below the fold. That reveal must scroll only enough of the infrastructure table to keep the row header plus the start of the detail visible, not leave the drawer clipped and not hard-center the selected row. -`useUnifiedResourceTableViewportSync.ts` must stay viewport-only; it may not -grow a second selected-row reveal path or a resource-local centering rule. That same unified-resource boundary now also owns stored metrics-target continuity for provider-backed resources. When registry rebuild cannot derive a fresh metrics target from raw source facets, `internal/unifiedresources/registry.go` @@ -4392,9 +4308,8 @@ raw backup, sync, verify, prune, and garbage job arrays travel through the unified-resource metadata contract in `frontend-modern/src/types/resource.ts` and `frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts`, `resourceDetailDrawerServiceModel.ts` owns active-task status classification and -shared activity wording, and both -`frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx` -and `frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx` +shared activity wording, and +`frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx` must render from that shared projection instead of rescanning raw job arrays or inventing local PBS status heuristics, `resourceDetailDrawerIdentityModel.ts` owns the pure identity-card, @@ -4562,30 +4477,9 @@ The shared node-state adapter also routes Proxmox cluster labels through that same helper, so infrastructure summary projections keep the same canonical cluster name as the rest of the unified resource model instead of rewriting the label locally. -The unified resource table now routes reactive table-state composition, -grouping, and row-windowing through -`frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts`, -while pure table-state derivation, service sorting, sort-cycle policy, and -responsive column layout now route through -`frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts`, -and viewport reveal plus scroll synchronization now route through -`frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts`, -so the shared consumer model is no longer interleaving selector derivation, -layout policy, and DOM viewport coordination inside one mixed state boundary. -That viewport controller must consume -`frontend-modern/src/components/shared/windowedPageScroll.ts` for scroll- -ancestor selection, wheel normalization, and listener lifecycle. Wheel may -prewarm the bounded row runway, but touch remains compositor-native and updates -the runway only through the passive native page-scroll event; unified-resource -tables must not register their own touch listener. -The mobile shell class from that shared state model now uses `min-w-full` for -the phone stage and restores the existing wider-stage floor above it. Mobile -column weights must sum to the available table width, retain the prioritized -identity and operational set, and avoid both local and document-level -horizontal scrolling. This is a phone projection only; wider stages keep the -established desktop distribution. -The native provider rollout now applies the same projection to Proxmox backup, -coverage, recoverable, Ceph, Mail Gateway, and replication rows; Docker native +Native provider tables keep a phone projection: prioritized identity and +operational columns sized to the available width, with no local or +document-level horizontal scrolling, for Proxmox backup, coverage, recoverable, Ceph, Mail Gateway, and replication rows; Docker native and Swarm inventories; Kubernetes native inventories; TrueNAS systems and workflows; vSphere hosts and workflows; and Standalone machines. The shared `platform-table-phone-hidden` marker removes secondary or duplicate tracks at @@ -4595,17 +4489,6 @@ demotion while promoting identity to a 40-percent track. Docker rows whose full labels, image references, or placement values cannot fit now expose those values in the same touch- and keyboard-operable inline detail pattern used by the other platform tables rather than relying on hover titles. -That same unified-resource consumer contract now also owns CSP-safe table -presentation for infrastructure rows. Host, PBS, and PMG table sections must -consume the shared column presentation owner and render canonical table sizing -through classes plus DOM width/height attributes rather than lane-local inline -style objects, so the same unified-resource dataset can reach the public demo -without transport-specific DOM drift. -That same consumer contract now also owns full-width desktop balance for the -infrastructure tables. The shared column presentation owner must publish an -explicit desktop `Resource` width for host, PBS, and PMG sections so wide -shells redistribute surplus width across the remaining columns instead of -turning the first column into blank filler that hides metric density. The canonical unified-resource change and relationship presenters now also share the same elapsed-time and "ago" wording utilities, so `observed`, `last seen`, and `ago` fragments stay consistent without each formatter diff --git a/frontend-modern/.bundlesize.json b/frontend-modern/.bundlesize.json index cea9cb20ad..e4c858092d 100644 --- a/frontend-modern/.bundlesize.json +++ b/frontend-modern/.bundlesize.json @@ -7,8 +7,8 @@ "index": { "baselineGzip": 150212 }, - "sharedPlatformPage": { - "baselineGzip": 120645 + "MetricBar": { + "baselineGzip": 120887 }, "Alerts": { "baselineGzip": 120045 diff --git a/frontend-modern/browser-verification/d494df7afd0adcb5.json b/frontend-modern/browser-verification/d494df7afd0adcb5.json new file mode 100644 index 0000000000..7873545ecb --- /dev/null +++ b/frontend-modern/browser-verification/d494df7afd0adcb5.json @@ -0,0 +1,93 @@ +{ + "version": 1, + "base_sha": "9aaac4d3e502c01913fb3637d0997b0ce03a8b7b", + "verified_at": "2026-10-06T19:05:00Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Infrastructure/ClusterDeployBanner.tsx", + "frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx", + "frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx", + "frontend-modern/src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx", + "frontend-modern/src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx", + "frontend-modern/src/components/Infrastructure/UnifiedResourceSourceBadgeCell.tsx", + "frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx", + "frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts", + "frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts", + "frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts", + "frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts", + "frontend-modern/src/features/platformPage/sharedPlatformPage.tsx" + ], + "content_sha256": { + "frontend-modern/src/components/Infrastructure/ClusterDeployBanner.tsx": "deleted", + "frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx": "deleted", + "frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx": "deleted", + "frontend-modern/src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx": "deleted", + "frontend-modern/src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx": "deleted", + "frontend-modern/src/components/Infrastructure/UnifiedResourceSourceBadgeCell.tsx": "deleted", + "frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx": "deleted", + "frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts": "deleted", + "frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts": "deleted", + "frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts": "deleted", + "frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts": "deleted", + "frontend-modern/src/features/platformPage/sharedPlatformPage.tsx": "3232e48158a03f69e98474731da49ceb5dd050e99fa167972849b7ccb61d8643" + }, + "routes": [ + "/proxmox/overview", + "/proxmox/storage", + "/proxmox/replication", + "/proxmox/backups", + "/proxmox/ceph", + "/proxmox/mail", + "/docker/overview", + "/docker/images", + "/docker/storage", + "/docker/networks", + "/docker/swarm", + "/kubernetes/overview", + "/kubernetes/nodes", + "/kubernetes/workloads", + "/kubernetes/services", + "/kubernetes/storage", + "/kubernetes/configuration", + "/kubernetes/events", + "/truenas/overview", + "/truenas/storage", + "/truenas/services", + "/truenas/apps", + "/truenas/vms", + "/truenas/shares", + "/truenas/protection", + "/vmware/overview", + "/vmware/storage", + "/vmware/networks", + "/vmware/health", + "/vmware/activity", + "/standalone/machines", + "/standalone/availability" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "production vite build embedded in a private mock-mode backend (50 Proxmox nodes, Docker, Swarm, 3 Kubernetes clusters, TrueNAS, vSphere, agents)", + "every platform sub-tab mounts its page root and populated tables at desktop and phone width with zero document horizontal overflow", + "no element with data-testid infrastructure-table-surface renders on any route, before or after the change", + "no-match search empty state on Machines and Docker images", + "Machines and Docker images inline row detail open", + "before/after pair: the same row-detail interaction on the origin/main build shows the identical 8 pre-existing style-src-attr CSP violations from ResourceDetailDrawer overflow-anchor styles, so none come from this change" + ], + "interactions": [ + "signed in with the hot-dev credentials", + "visited every platform sub-tab discovered from each platform nav at 1440x900 and 390x844", + "typed a row-name prefix into the shared platform toolbar search on Machines and Docker images and confirmed the row count narrowed", + "typed a non-matching query and confirmed the no-match empty state, then cleared it and confirmed every row returned", + "clicked the first row to open the inline resource detail" + ] +} diff --git a/frontend-modern/scripts/shared-template-registry.json b/frontend-modern/scripts/shared-template-registry.json index e4cb1a9da1..44cb11003f 100644 --- a/frontend-modern/scripts/shared-template-registry.json +++ b/frontend-modern/scripts/shared-template-registry.json @@ -118,15 +118,6 @@ { "path": "src/components/Alerts/AlertResourceTableRow.tsx" }, - { - "path": "src/components/Infrastructure/UnifiedResourceHostTableCard.tsx" - }, - { - "path": "src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx" - }, - { - "path": "src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx" - }, { "path": "src/features/docker/DockerNativeTableShared.tsx" }, @@ -212,9 +203,6 @@ "export": "InlineDetailTableRow" }, "requiredConsumers": [ - { "path": "src/components/Infrastructure/UnifiedResourceHostTableCard.tsx" }, - { "path": "src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx" }, - { "path": "src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx" }, { "path": "src/components/Workloads/WorkloadPanel.tsx" }, { "path": "src/features/docker/DockerAlertsTable.tsx" }, { "path": "src/features/docker/DockerHostsTable.tsx" }, @@ -413,8 +401,6 @@ "export": "TableCard" }, "requiredConsumers": [ - { "path": "src/components/Infrastructure/UnifiedResourceHostTableCard.tsx" }, - { "path": "src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx" }, { "path": "src/components/Storage/StorageContentCard.tsx" }, { "path": "src/components/Workloads/WorkloadsSurface.tsx" }, { "path": "src/components/Workloads/WorkloadsTable.tsx" }, @@ -435,8 +421,6 @@ "export": "TableCardHeader" }, "requiredConsumers": [ - { "path": "src/components/Infrastructure/UnifiedResourceHostTableCard.tsx" }, - { "path": "src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx" }, { "path": "src/components/Storage/StorageContentCard.tsx" }, { "path": "src/features/platformPage/sharedPlatformPage.tsx" } ], diff --git a/frontend-modern/src/components/Infrastructure/ClusterDeployBanner.tsx b/frontend-modern/src/components/Infrastructure/ClusterDeployBanner.tsx deleted file mode 100644 index 6f9df6aa52..0000000000 --- a/frontend-modern/src/components/Infrastructure/ClusterDeployBanner.tsx +++ /dev/null @@ -1,74 +0,0 @@ -import { Component, Show, createMemo } from 'solid-js'; -import type { ResourceGroup } from '@/components/Infrastructure/infrastructureSelectors'; -import InfoIcon from 'lucide-solid/icons/info'; -import RocketIcon from 'lucide-solid/icons/rocket'; - -interface ClusterDeployBannerProps { - group: ResourceGroup; - onDeploy: (clusterId: string, clusterName: string) => void; -} - -/** - * Compact inline banner shown in cluster group headers when reachable PVE - * nodes lack a Pulse Agent and a source agent is available to deploy from. - */ -export const ClusterDeployBanner: Component = (props) => { - // Visibility conditions: - // 1. Non-empty cluster name (not Standalone) - // 2. At least one resource has platformType === 'proxmox-pve' - // 3. At least one resource has agent?.agentId (source agent exists) - // 4. At least one *reachable* PVE node does NOT have an agent - // - // Offline nodes are excluded: we can't deploy a Pulse Unified Agent to - // a node we can't reach, and a node going temporarily offline is not - // the same situation as a never-onboarded cluster peer. - const deployInfo = createMemo(() => { - const resources = props.group.resources; - const cluster = props.group.cluster; - - if (!cluster) return null; - - // Only consider actual PVE node resources (not VMs/containers). - const pveNodes = resources.filter( - (r) => r.type === 'agent' && r.platformType === 'proxmox-pve', - ); - if (pveNodes.length === 0) return null; - - const hasSourceAgent = pveNodes.some((r) => r.agent?.agentId); - if (!hasSourceAgent) return null; - - const deployableCount = pveNodes.filter( - (r) => !r.agent?.agentId && r.status !== 'offline', - ).length; - if (deployableCount === 0) return null; - - return { cluster, deployableCount }; - }); - - return ( - - {(info) => ( -
-
- - - {info().deployableCount} node{info().deployableCount !== 1 ? 's' : ''} ready for Pulse - Agent - -
- -
- )} - - ); -}; diff --git a/frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx b/frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx deleted file mode 100644 index ce2fea9148..0000000000 --- a/frontend-modern/src/components/Infrastructure/UnifiedResourceHostTableCard.tsx +++ /dev/null @@ -1,799 +0,0 @@ -import { For, Show, createMemo } from 'solid-js'; -import type { Component } from 'solid-js'; -import { - formatBytes, - formatSpeed, - formatObservedSpeed, - formatUptime, - getResourceDiskSummary, - normalizeDiskArray, -} from '@/utils/format'; -import { formatTemperature, getTemperatureTextClass } from '@/utils/temperature'; -import { - GROUPED_TABLE_ROW_BADGE_CLASS, - GROUPED_TABLE_ROW_META_CLASS, - getGroupedTableRowCellClass, - getInteractiveGroupedTableRowClass, -} from '@/components/shared/groupedTableRowPresentation'; -import { InlineDetailTableRow } from '@/components/shared/InlineDetailTableRow'; -import { TableCardHeader } from '@/components/shared/TableCardHeader'; -import { TableCard } from '@/components/shared/TableCard'; -import { hostOverrideIdCandidates } from '@/features/alerts/alertOverridesModel'; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from '@/components/shared/Table'; -import { StatusDot } from '@/components/shared/StatusDot'; -import { ResponsiveMetricCell } from '@/components/shared/responsive'; -import { StackedDiskBar } from '@/components/Workloads/StackedDiskBar'; -import { StackedMemoryBar } from '@/components/Workloads/StackedMemoryBar'; -import { buildMetricKeyForUnifiedResource } from '@/utils/metricsKeys'; -import { - getInfrastructureSystemIdentityBadges, - getInfrastructureSystemTitleBadges, - getPlatformBadge, - getSourceBadge, - getUnifiedSourceBadges, -} from '@/utils/resourceBadgePresentation'; -import { getAgentStatusIndicator } from '@/utils/status'; -import { getPreferredInfrastructureDisplayName } from '@/utils/resourceIdentity'; -import { - getResourcePolicyTableBadges, - shouldShowResourceAlternateName, -} from '@/utils/resourcePolicyPresentation'; -import { getAvailabilityProbePresentation } from '@/utils/availabilityProbePresentation'; -import { ResourceDetailDrawer } from './ResourceDetailDrawer'; -import { getResourceHealthIssuePresentation } from './resourceHealthPresentation'; -import { ClusterDeployBanner } from './ClusterDeployBanner'; -import { ResourceFacetSummary } from './ResourceFacetSummary'; -import { UnifiedResourceSourceBadgeCell } from './UnifiedResourceSourceBadgeCell'; -import type { SummarySeriesGroupScope } from '@/components/shared/summaryCardInteraction'; -import { resolveSummaryGroupMemberInteractionState } from '@/components/shared/summaryCardInteraction'; -import { - buildSummaryDisclosureControlsId, - createSummaryInteractiveRowPreviewHandlers, -} from '@/components/shared/summaryInteractionA11y'; -import { SummaryRowActionButton } from '@/components/shared/SummaryRowActionButton'; -import { ResourceNameWithWebInterfaceLink } from '@/components/shared/WebInterfaceLink'; -import { - type UnifiedResourceTableProps, - type UnifiedResourceTableState, -} from './useUnifiedResourceTableState'; -import { shouldShowClusterGroupTypeLabel } from './unifiedResourceTableStateModel'; -import { getOutlierEmphasis, isResourceOnline } from './unifiedResourceTableModel'; -import { type Resource, getCpuPercent, getMemoryPercent } from '@/types/resource'; - -interface UnifiedResourceHostTableCardProps { - tableProps: UnifiedResourceTableProps; - table: UnifiedResourceTableState; -} - -type ThermalPressurePresentation = { - label: string; - title: string; - className: string; -}; - -const getThermalPressurePresentation = (resource: Resource): ThermalPressurePresentation | null => { - const thermalState = resource.agent?.sensors?.thermalState; - if (!thermalState) return null; - - const pressure = thermalState?.pressure?.trim().toLowerCase(); - if (!pressure) return null; - - const label = - pressure === 'nominal' ? 'Nominal' : pressure === 'constrained' ? 'Constrained' : 'Unknown'; - const className = - pressure === 'nominal' - ? 'text-emerald-600 dark:text-emerald-400' - : pressure === 'constrained' - ? 'text-amber-600 dark:text-amber-400' - : 'text-slate-500 dark:text-slate-400'; - const limits = Object.entries(thermalState.limitsPercent ?? {}) - .filter(([, value]) => Number.isFinite(value) && value < 100) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([key, value]) => `${key.replace(/_/g, ' ')} ${Math.round(value)}%`); - const source = thermalState.source ? ` via ${thermalState.source}` : ''; - const limitSummary = limits.length > 0 ? ` · limits: ${limits.join(', ')}` : ''; - - return { - label, - className, - title: `Thermal pressure ${label.toLowerCase()}${source}${limitSummary}`, - }; -}; - -export const UnifiedResourceHostTableCard: Component = ( - props, -) => { - const { table, tableProps } = props; - - return ( - - - - - - - table.handleSort('name')} - > - {table.headerLabels().resource} {table.renderSortIndicator('name')} - - table.handleSort('cpu')} - > - {table.headerLabels().cpu} {table.renderSortIndicator('cpu')} - - table.handleSort('memory')} - > - {table.headerLabels().memory} {table.renderSortIndicator('memory')} - - table.handleSort('disk')} - > - {table.headerLabels().disk} {table.renderSortIndicator('disk')} - - table.handleSort('network')} - > - {table.headerLabels().network} {table.renderSortIndicator('network')} - - table.handleSort('diskio')} - > - {table.headerLabels().diskIo} {table.renderSortIndicator('diskio')} - - table.handleSort('source')} - > - {table.headerLabels().source} {table.renderSortIndicator('source')} - - table.handleSort('uptime')} - > - {table.headerLabels().uptime} {table.renderSortIndicator('uptime')} - - table.handleSort('temp')} - > - {table.headerLabels().temp} {table.renderSortIndicator('temp')} - - - - - 0}> - - - - - {(item) => { - if (item.type === 'header') { - const group = item.group; - const groupSummaryScope = createMemo(() => - table.buildHostSummaryGroupScope(group), - ); - const isSummaryGroupHighlighted = createMemo( - () => tableProps.activeSummaryGroupScope?.id === groupSummaryScope()?.id, - ); - const handleGroupHoverChange = (next: SummarySeriesGroupScope | null) => { - tableProps.onGroupHoverChange?.(next); - }; - const handleGroupFocusToggle = () => { - const nextScope = groupSummaryScope(); - tableProps.onGroupFocusChange?.( - nextScope && tableProps.focusedSummaryGroupId === nextScope.id - ? null - : (nextScope?.id ?? null), - ); - }; - const groupRowInteraction = createSummaryInteractiveRowPreviewHandlers({ - onPreview: () => handleGroupHoverChange(groupSummaryScope()), - onPreviewClear: () => handleGroupHoverChange(null), - }); - return ( - - -
- Standalone} - > - {group.cluster} - - Cluster - - - - {group.resources.length}{' '} - {group.resources.length === 1 ? 'resource' : 'resources'} - -
- - - - - - ); - } - - const resource = item.resource; - const isExpanded = createMemo(() => tableProps.expandedResourceId === resource.id); - const isHighlighted = createMemo( - () => tableProps.highlightedResourceId === resource.id, - ); - const displayName = createMemo(() => - getPreferredInfrastructureDisplayName(resource), - ); - const summaryGroupMemberState = createMemo(() => - resolveSummaryGroupMemberInteractionState({ - seriesId: resource.id, - hoveredGroupScope: tableProps.hoveredSummaryGroupScope, - focusedGroupScope: tableProps.focusedSummaryGroupScope, - }), - ); - const statusIndicator = createMemo(() => - getAgentStatusIndicator({ status: resource.status }), - ); - const healthIssue = createMemo(() => getResourceHealthIssuePresentation(resource)); - const metricsKey = createMemo(() => buildMetricKeyForUnifiedResource(resource)); - const alertResourceIdCandidates = createMemo(() => - hostOverrideIdCandidates(resource), - ); - const cpuThresholds = createMemo(() => - table.getMetricThresholds('agent', 'cpu', alertResourceIdCandidates()), - ); - const memoryThresholds = createMemo(() => - table.getMetricThresholds('agent', 'memory', alertResourceIdCandidates()), - ); - const diskThresholds = createMemo(() => - table.getMetricThresholds('agent', 'disk', alertResourceIdCandidates()), - ); - const temperatureThresholds = createMemo(() => - table.getMetricThresholds('node', 'temperature', alertResourceIdCandidates()), - ); - const detailControlsId = createMemo(() => - buildSummaryDisclosureControlsId(resource.id), - ); - - const cpuPercentValue = createMemo(() => - resource.cpu ? Math.round(getCpuPercent(resource)) : null, - ); - const memoryPercentValue = createMemo(() => - resource.memory ? Math.round(getMemoryPercent(resource)) : null, - ); - const diskSummary = createMemo(() => getResourceDiskSummary(resource)); - const diskPercentValue = createMemo(() => { - const summary = diskSummary(); - return summary ? Math.round(summary.usage) : null; - }); - - const memorySublabel = createMemo(() => { - if ( - !resource.memory || - resource.memory.used === undefined || - resource.memory.total === undefined - ) - return undefined; - return `${formatBytes(resource.memory.used)}/${formatBytes(resource.memory.total)}`; - }); - - const diskSublabel = createMemo(() => { - const summary = diskSummary(); - if (!summary) return undefined; - return `${formatBytes(summary.used)}/${formatBytes(summary.total)}`; - }); - const networkTotal = createMemo( - () => (resource.network?.rxBytes ?? 0) + (resource.network?.txBytes ?? 0), - ); - const networkEmphasis = createMemo(() => - getOutlierEmphasis(networkTotal(), table.ioScale().network), - ); - const diskIOTotal = createMemo(() => - resource.diskIO?.readRate !== undefined && - resource.diskIO?.writeRate !== undefined - ? resource.diskIO.readRate + resource.diskIO.writeRate - : NaN, - ); - const diskIOEmphasis = createMemo(() => - getOutlierEmphasis(diskIOTotal(), table.ioScale().diskIO), - ); - - const rowClass = createMemo(() => { - const baseHover = - 'cursor-pointer transition-all duration-200 relative group hover:bg-surface-hover'; - - if (isExpanded()) { - return 'cursor-pointer transition-all duration-200 relative z-10 group bg-blue-50 dark:bg-blue-900/25'; - } - - let className = baseHover; - if (!isResourceOnline(resource)) { - className += ' opacity-60'; - } - - return className; - }); - const platformBadge = createMemo(() => getPlatformBadge(resource.platformType)); - const sourceBadge = createMemo(() => getSourceBadge(resource.sourceType)); - const unifiedSources = createMemo(() => table.getUnifiedSources(resource)); - const sourceBadges = createMemo(() => getUnifiedSourceBadges(unifiedSources())); - const systemBadges = createMemo(() => - getInfrastructureSystemIdentityBadges(resource), - ); - const systemTitleBadges = createMemo(() => - getInfrastructureSystemTitleBadges(systemBadges(), sourceBadges()), - ); - const policyBadges = createMemo(() => - getResourcePolicyTableBadges(resource.policy), - ); - const availabilityProbe = createMemo(() => - getAvailabilityProbePresentation(resource), - ); - const thermalPressure = createMemo(() => getThermalPressurePresentation(resource)); - const resourceRowInteraction = createSummaryInteractiveRowPreviewHandlers({ - onPreview: () => tableProps.onHoverChange?.(resource.id), - onPreviewClear: () => tableProps.onHoverChange?.(null), - }); - - return ( - <> - table.toggleExpand(resource.id)} - {...resourceRowInteraction} - > - -
- table.toggleExpand(resource.id)} - onPreviewClear={() => tableProps.onHoverChange?.(null)} - /> - -
-
- - - {(issue) => ( - - )} - - - {(probe) => ( - - {probe().methodLabel} - - )} - - - - - - {(badge) => ( - - {badge.label} - - )} - - - -
-
-
- - - - - - - } - > - - - - - - - - - } - > -
- -
- - - - - - - - } - > -
- -
- - - - - - - - } - > - {(probe) => ( -
- - {probe().resultLabel} - - } - > - {(targetLabel) => ( - <> - - {targetLabel()}: - - - {probe().resultLabel} - - - )} - - - {probe().freshnessLabel} - -
- )} - - } - > -
- ↓ - - {formatSpeed(resource.network!.rxBytes)} - - ↑ - - {formatSpeed(resource.network!.txBytes)} - -
- - - - - - - - } - > -
- - R - - - {formatObservedSpeed(resource.diskIO!.readRate)} - - - W - - - {formatObservedSpeed(resource.diskIO!.writeRate)} - -
- - - - - - - - -
- -
- - - -
- - } - > - - {formatTemperature(resource.temperature)} - - -
- - - - - tableProps.onExpandedResourceChange(null)} - /> - - - - ); - }} - - - 0}> - - - -
- - - ); -}; diff --git a/frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx b/frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx deleted file mode 100644 index 7e638e9263..0000000000 --- a/frontend-modern/src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx +++ /dev/null @@ -1,353 +0,0 @@ -import { Show, createMemo } from 'solid-js'; -import type { Component } from 'solid-js'; -import { formatUptime } from '@/utils/format'; -import { InlineDetailTableRow } from '@/components/shared/InlineDetailTableRow'; -import { StatusDot } from '@/components/shared/StatusDot'; -import { - buildSummaryDisclosureControlsId, - createSummaryInteractiveRowPreviewHandlers, -} from '@/components/shared/summaryInteractionA11y'; -import { SummaryRowActionButton } from '@/components/shared/SummaryRowActionButton'; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from '@/components/shared/Table'; -import { - getInfrastructureSystemIdentityBadges, - getInfrastructureSystemTitleBadges, - getPlatformBadge, - getSourceBadge, - getUnifiedSourceBadges, -} from '@/utils/resourceBadgePresentation'; -import { getAgentStatusIndicator } from '@/utils/status'; -import { getServiceHealthSummaryPresentation } from '@/utils/serviceHealthPresentation'; -import { getPreferredInfrastructureDisplayName } from '@/utils/resourceIdentity'; -import { shouldShowResourceAlternateName } from '@/utils/resourcePolicyPresentation'; -import { ResourceDetailDrawer } from './ResourceDetailDrawer'; -import { ResourceFacetSummary } from './ResourceFacetSummary'; -import { UnifiedResourceSourceBadgeCell } from './UnifiedResourceSourceBadgeCell'; -import { - type UnifiedResourceTableProps, - type UnifiedResourceTableState, -} from './useUnifiedResourceTableState'; -import { getPBSTableRow, isResourceOnline } from './unifiedResourceTableModel'; -import { buildServiceDetailLinks } from './serviceDetailLinks'; -import { ResourceNameWithWebInterfaceLink } from '@/components/shared/WebInterfaceLink'; -import { PlatformWindowedRows } from '@/features/platformPage/PlatformWindowedRows'; - -interface UnifiedResourcePBSTableSectionProps { - tableProps: UnifiedResourceTableProps; - table: UnifiedResourceTableState; -} - -export const UnifiedResourcePBSTableSection: Component = ( - props, -) => { - const { table, tableProps } = props; - const isServiceMobileColumnVisible = (priority: 'primary' | 'secondary') => - table.isMobile() || table.isServiceVisible(priority); - - return ( - 0}> -
- PBS Services -
- - - - - {table.headerLabels().resource} - - - {table.headerLabels().datastores} - - - {table.headerLabels().activity} - - - {table.headerLabels().health} - - - {table.headerLabels().source} - - - {table.headerLabels().uptime} - - - {table.headerLabels().action} - - - - - - {(resource) => { - const isExpanded = createMemo(() => tableProps.expandedResourceId === resource.id); - const isHighlighted = createMemo( - () => tableProps.highlightedResourceId === resource.id, - ); - const displayName = createMemo(() => getPreferredInfrastructureDisplayName(resource)); - const serviceLink = createMemo(() => buildServiceDetailLinks(resource)[0] ?? null); - const statusIndicator = createMemo(() => - getAgentStatusIndicator({ status: resource.status }), - ); - const pbsRow = createMemo(() => getPBSTableRow(resource)); - const platformBadge = createMemo(() => getPlatformBadge(resource.platformType)); - const sourceBadge = createMemo(() => getSourceBadge(resource.sourceType)); - const unifiedSources = createMemo(() => table.getUnifiedSources(resource)); - const sourceBadges = createMemo(() => getUnifiedSourceBadges(unifiedSources())); - const systemBadges = createMemo(() => - getInfrastructureSystemIdentityBadges(resource), - ); - const systemTitleBadges = createMemo(() => - getInfrastructureSystemTitleBadges(systemBadges(), sourceBadges()), - ); - const healthClass = createMemo( - () => - getServiceHealthSummaryPresentation(resource.status, pbsRow()?.health).textClass, - ); - const activityClass = createMemo(() => - (pbsRow()?.activeTaskCount || 0) > 0 - ? 'text-[11px] font-semibold text-emerald-700 dark:text-emerald-300' - : 'text-xs text-base-content', - ); - - const rowClass = createMemo(() => { - const baseBorder = 'border-b border-border-subtle'; - const baseHover = `cursor-pointer transition-all duration-200 relative hover:shadow-xs group ${baseBorder}`; - - if (isExpanded()) { - return `cursor-pointer transition-all duration-200 relative hover:shadow-xs z-10 group bg-blue-50 dark:bg-blue-900/25 ${baseBorder}`; - } - - let className = baseHover; - if (!isResourceOnline(resource)) { - className += ' opacity-60'; - } - - return className; - }); - const detailControlsId = createMemo(() => - buildSummaryDisclosureControlsId(resource.id), - ); - const resourceRowInteraction = createSummaryInteractiveRowPreviewHandlers({ - onPreview: () => tableProps.onHoverChange?.(resource.id), - onPreviewClear: () => tableProps.onHoverChange?.(null), - }); - - return ( - <> - table.toggleExpand(resource.id)} - {...resourceRowInteraction} - > - -
- table.toggleExpand(resource.id)} - onPreviewClear={() => tableProps.onHoverChange?.(null)} - /> - - - - - -
- - - -
- -
- - - -
- -
- - - -
- -
- - - - - - - -
- -
- - - -
- -
- - - - - tableProps.onExpandedResourceChange(null)} - /> - - - - ); - }} - - -
- - ); -}; diff --git a/frontend-modern/src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx b/frontend-modern/src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx deleted file mode 100644 index 79f38dc613..0000000000 --- a/frontend-modern/src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx +++ /dev/null @@ -1,393 +0,0 @@ -import { Show, createMemo } from 'solid-js'; -import type { Component } from 'solid-js'; -import { formatUptime } from '@/utils/format'; -import { InlineDetailTableRow } from '@/components/shared/InlineDetailTableRow'; -import { StatusDot } from '@/components/shared/StatusDot'; -import { - buildSummaryDisclosureControlsId, - createSummaryInteractiveRowPreviewHandlers, -} from '@/components/shared/summaryInteractionA11y'; -import { SummaryRowActionButton } from '@/components/shared/SummaryRowActionButton'; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from '@/components/shared/Table'; -import { - getInfrastructureSystemIdentityBadges, - getInfrastructureSystemTitleBadges, - getPlatformBadge, - getSourceBadge, - getUnifiedSourceBadges, -} from '@/utils/resourceBadgePresentation'; -import { getAgentStatusIndicator } from '@/utils/status'; -import { getServiceHealthSummaryPresentation } from '@/utils/serviceHealthPresentation'; -import { getPreferredInfrastructureDisplayName } from '@/utils/resourceIdentity'; -import { shouldShowResourceAlternateName } from '@/utils/resourcePolicyPresentation'; -import { ResourceDetailDrawer } from './ResourceDetailDrawer'; -import { ResourceFacetSummary } from './ResourceFacetSummary'; -import { UnifiedResourceSourceBadgeCell } from './UnifiedResourceSourceBadgeCell'; -import { - type UnifiedResourceTableProps, - type UnifiedResourceTableState, -} from './useUnifiedResourceTableState'; -import { getPMGTableRow, isResourceOnline } from './unifiedResourceTableModel'; -import { buildServiceDetailLinks } from './serviceDetailLinks'; -import { ResourceNameWithWebInterfaceLink } from '@/components/shared/WebInterfaceLink'; -import { PlatformWindowedRows } from '@/features/platformPage/PlatformWindowedRows'; - -interface UnifiedResourcePMGTableSectionProps { - tableProps: UnifiedResourceTableProps; - table: UnifiedResourceTableState; -} - -export const UnifiedResourcePMGTableSection: Component = ( - props, -) => { - const { table, tableProps } = props; - const isServiceMobileColumnVisible = (priority: 'primary' | 'secondary') => - table.isMobile() || table.isServiceVisible(priority); - - return ( - 0}> -
- PMG Services -
- - - - - {table.headerLabels().resource} - - - {table.headerLabels().queue} - - - {table.headerLabels().deferred} - - - {table.headerLabels().hold} - - - {table.headerLabels().nodes} - - - {table.headerLabels().health} - - - {table.headerLabels().source} - - - {table.headerLabels().uptime} - - - {table.headerLabels().action} - - - - - - {(resource) => { - const isExpanded = createMemo(() => tableProps.expandedResourceId === resource.id); - const isHighlighted = createMemo( - () => tableProps.highlightedResourceId === resource.id, - ); - const displayName = createMemo(() => getPreferredInfrastructureDisplayName(resource)); - const serviceLink = createMemo(() => buildServiceDetailLinks(resource)[0] ?? null); - const statusIndicator = createMemo(() => - getAgentStatusIndicator({ status: resource.status }), - ); - const pmgRow = createMemo(() => getPMGTableRow(resource)); - const platformBadge = createMemo(() => getPlatformBadge(resource.platformType)); - const sourceBadge = createMemo(() => getSourceBadge(resource.sourceType)); - const unifiedSources = createMemo(() => table.getUnifiedSources(resource)); - const sourceBadges = createMemo(() => getUnifiedSourceBadges(unifiedSources())); - const systemBadges = createMemo(() => - getInfrastructureSystemIdentityBadges(resource), - ); - const systemTitleBadges = createMemo(() => - getInfrastructureSystemTitleBadges(systemBadges(), sourceBadges()), - ); - const healthClass = createMemo( - () => - getServiceHealthSummaryPresentation(resource.status, pmgRow()?.health).textClass, - ); - const queueClass = createMemo(() => - (pmgRow()?.deferred || 0) + (pmgRow()?.hold || 0) > 0 - ? 'text-amber-600 dark:text-amber-400' - : 'text-base-content', - ); - - const rowClass = createMemo(() => { - const baseBorder = 'border-b border-border-subtle'; - const baseHover = `cursor-pointer transition-all duration-200 relative hover:shadow-xs group ${baseBorder}`; - - if (isExpanded()) { - return `cursor-pointer transition-all duration-200 relative hover:shadow-xs z-10 group bg-blue-50 dark:bg-blue-900/25 ${baseBorder}`; - } - - let className = baseHover; - if (!isResourceOnline(resource)) { - className += ' opacity-60'; - } - - return className; - }); - const detailControlsId = createMemo(() => - buildSummaryDisclosureControlsId(resource.id), - ); - const resourceRowInteraction = createSummaryInteractiveRowPreviewHandlers({ - onPreview: () => tableProps.onHoverChange?.(resource.id), - onPreviewClear: () => tableProps.onHoverChange?.(null), - }); - - return ( - <> - table.toggleExpand(resource.id)} - {...resourceRowInteraction} - > - -
- table.toggleExpand(resource.id)} - onPreviewClear={() => tableProps.onHoverChange?.(null)} - /> - - - - - -
- - - -
- -
- - - -
- -
- - - -
- -
- - - -
- -
- - - -
- -
- - - - - - - -
- -
- - - -
- -
- - - - - tableProps.onExpandedResourceChange(null)} - /> - - - - ); - }} - - -
- - ); -}; diff --git a/frontend-modern/src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx b/frontend-modern/src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx deleted file mode 100644 index c1a479867b..0000000000 --- a/frontend-modern/src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import { Show } from 'solid-js'; -import type { Component } from 'solid-js'; -import { TableCardHeader } from '@/components/shared/TableCardHeader'; -import { TableCard } from '@/components/shared/TableCard'; -import { - type UnifiedResourceTableProps, - type UnifiedResourceTableState, -} from './useUnifiedResourceTableState'; -import { UnifiedResourcePBSTableSection } from './UnifiedResourcePBSTableSection'; -import { UnifiedResourcePMGTableSection } from './UnifiedResourcePMGTableSection'; - -interface UnifiedResourceServiceInfrastructureCardProps { - tableProps: UnifiedResourceTableProps; - table: UnifiedResourceTableState; -} - -export const UnifiedResourceServiceInfrastructureCard: Component< - UnifiedResourceServiceInfrastructureCardProps -> = (props) => { - const { table, tableProps } = props; - - return ( - 0 || table.sortedPMGResources().length > 0}> - - - - - - - ); -}; diff --git a/frontend-modern/src/components/Infrastructure/UnifiedResourceSourceBadgeCell.tsx b/frontend-modern/src/components/Infrastructure/UnifiedResourceSourceBadgeCell.tsx deleted file mode 100644 index 3e72eaa983..0000000000 --- a/frontend-modern/src/components/Infrastructure/UnifiedResourceSourceBadgeCell.tsx +++ /dev/null @@ -1,72 +0,0 @@ -import { For, Show, createMemo } from 'solid-js'; -import type { Component } from 'solid-js'; -import type { ResourceBadge } from '@/utils/resourceBadgePresentation'; -import type { UnifiedResourceTableLayoutMode } from './unifiedResourceTableStateModel'; - -interface UnifiedResourceSourceBadgeCellProps { - unifiedBadges: ResourceBadge[]; - platformBadge: ResourceBadge | null; - sourceBadge: ResourceBadge | null; - titleBadges?: ResourceBadge[]; - layoutMode: UnifiedResourceTableLayoutMode; -} - -const getVisibleSourceBadgeLimit = (layoutMode: UnifiedResourceTableLayoutMode): number => - layoutMode === 'wide' ? 3 : 2; - -export const UnifiedResourceSourceBadgeCell: Component = ( - props, -) => { - const badges = createMemo(() => { - if (props.unifiedBadges.length > 0) return props.unifiedBadges; - return [props.platformBadge, props.platformBadge ? null : props.sourceBadge].filter( - (badge): badge is ResourceBadge => Boolean(badge), - ); - }); - const titleBadges = createMemo(() => - props.titleBadges && props.titleBadges.length > 0 ? props.titleBadges : badges(), - ); - const visibleBadges = createMemo(() => - badges().slice(0, getVisibleSourceBadgeLimit(props.layoutMode)), - ); - const hiddenBadges = createMemo(() => badges().slice(visibleBadges().length)); - const hiddenBadgeCount = createMemo(() => Math.max(0, badges().length - visibleBadges().length)); - const hiddenBadgeLabel = createMemo(() => - hiddenBadgeCount() === 1 ? `+${hiddenBadges()[0]?.label ?? ''}` : `+${hiddenBadgeCount()}`, - ); - const title = createMemo(() => - titleBadges() - .map((badge) => badge.title ?? badge.label) - .join(', '), - ); - - return ( -
- - {(badge) => ( - - {badge.label} - - )} - - 0}> - badge.title ?? badge.label) - .join(', ')}`} - title={title()} - > - {hiddenBadgeLabel()} - - -
- ); -}; diff --git a/frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx b/frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx deleted file mode 100644 index 6c5a45ce13..0000000000 --- a/frontend-modern/src/components/Infrastructure/UnifiedResourceTable.tsx +++ /dev/null @@ -1,25 +0,0 @@ -import type { Component } from 'solid-js'; -import { - useUnifiedResourceTableState, - type UnifiedResourceTableProps, -} from './useUnifiedResourceTableState'; -import { UnifiedResourceHostTableCard } from './UnifiedResourceHostTableCard'; -import { UnifiedResourceServiceInfrastructureCard } from './UnifiedResourceServiceInfrastructureCard'; - -export const UnifiedResourceTable: Component = (props) => { - const table = useUnifiedResourceTableState(props); - - return ( -
- - -
- ); -}; - -export default UnifiedResourceTable; diff --git a/frontend-modern/src/components/Infrastructure/__tests__/ClusterDeployBanner.test.tsx b/frontend-modern/src/components/Infrastructure/__tests__/ClusterDeployBanner.test.tsx deleted file mode 100644 index 9530f564a6..0000000000 --- a/frontend-modern/src/components/Infrastructure/__tests__/ClusterDeployBanner.test.tsx +++ /dev/null @@ -1,235 +0,0 @@ -import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'; -import { render, screen, fireEvent, cleanup } from '@solidjs/testing-library'; - -/* ── hoisted mocks ────────────────────────────────────────────── */ -vi.mock('lucide-solid/icons/info', () => ({ - default: () => , -})); - -vi.mock('lucide-solid/icons/rocket', () => ({ - default: () => , -})); - -/* ── component import (after mocks) ──────────────────────────── */ -import { ClusterDeployBanner } from '../ClusterDeployBanner'; -import type { ResourceGroup } from '../infrastructureSelectors'; -import type { Resource } from '@/types/resource'; - -/* ── helpers ──────────────────────────────────────────────────── */ - -/** Minimal PVE node resource with optional agent and overridable status */ -function makePveNode( - id: string, - agentId?: string, - status: Resource['status'] = 'online', -): Resource { - return { - id, - type: 'agent', - name: id, - displayName: id, - platformId: 'pve1', - platformType: 'proxmox-pve', - sourceType: 'api', - status, - agent: agentId ? { agentId } : undefined, - } as Resource; -} - -/** Non-PVE resource (e.g. a VM) */ -function makeVm(id: string): Resource { - return { - id, - type: 'vm', - name: id, - displayName: id, - platformId: 'pve1', - platformType: 'proxmox-pve', - sourceType: 'api', - status: 'running', - } as Resource; -} - -/** Non-PVE platform node (e.g. k8s) with optional agent */ -function makeK8sNode(id: string, agentId?: string): Resource { - return { - id, - type: 'k8s-node', - name: id, - displayName: id, - platformId: 'k8s1', - platformType: 'kubernetes', - sourceType: 'agent', - status: 'online', - agent: agentId ? { agentId } : undefined, - } as Resource; -} - -function makeGroup(cluster: string, resources: Resource[]): ResourceGroup { - return { cluster, resources }; -} - -/* ── lifecycle ────────────────────────────────────────────────── */ -beforeEach(() => { - vi.clearAllMocks(); -}); - -afterEach(() => { - cleanup(); -}); - -/* ── tests ────────────────────────────────────────────────────── */ -describe('ClusterDeployBanner', () => { - describe('visibility conditions', () => { - it('renders nothing when group has no cluster name', () => { - const group = makeGroup('', [makePveNode('node1', 'agent-1'), makePveNode('node2')]); - render(() => ); - expect(screen.queryByText('Review & Deploy')).not.toBeInTheDocument(); - }); - - it('renders nothing when there are no PVE nodes', () => { - const group = makeGroup('my-cluster', [makeVm('vm1'), makeK8sNode('k8s-node1')]); - render(() => ); - expect(screen.queryByText('Review & Deploy')).not.toBeInTheDocument(); - }); - - it('renders nothing when no PVE node has a source agent', () => { - const group = makeGroup('my-cluster', [ - makePveNode('node1'), // no agent - makePveNode('node2'), // no agent - ]); - render(() => ); - expect(screen.queryByText('Review & Deploy')).not.toBeInTheDocument(); - }); - - it('renders nothing when all PVE nodes already have agents', () => { - const group = makeGroup('my-cluster', [ - makePveNode('node1', 'agent-1'), - makePveNode('node2', 'agent-2'), - ]); - render(() => ); - expect(screen.queryByText('Review & Deploy')).not.toBeInTheDocument(); - }); - - it('renders when there is a mix of monitored and unmonitored PVE nodes', () => { - const group = makeGroup('my-cluster', [ - makePveNode('node1', 'agent-1'), // source agent - makePveNode('node2'), // unmonitored - ]); - render(() => ); - expect(screen.getByText('1 node ready for Pulse Agent')).toBeInTheDocument(); - expect(screen.getByText('Review & Deploy')).toBeInTheDocument(); - }); - - it('does not count offline no-agent nodes (cannot deploy to an unreachable host)', () => { - const group = makeGroup('my-cluster', [ - makePveNode('node1', 'agent-1'), // source agent - makePveNode('offline-node', undefined, 'offline'), // offline, no agent - ]); - render(() => ); - expect(screen.queryByText(/ready for Pulse Agent/)).not.toBeInTheDocument(); - expect(screen.queryByText('Review & Deploy')).not.toBeInTheDocument(); - }); - - it('still counts online no-agent nodes when other no-agent nodes are offline', () => { - const group = makeGroup('my-cluster', [ - makePveNode('node1', 'agent-1'), - makePveNode('online-no-agent'), - makePveNode('offline-no-agent', undefined, 'offline'), - ]); - render(() => ); - expect(screen.getByText('1 node ready for Pulse Agent')).toBeInTheDocument(); - }); - }); - - describe('deployable count display', () => { - it('shows singular "node" for 1 deployable', () => { - const group = makeGroup('cluster-a', [makePveNode('node1', 'agent-1'), makePveNode('node2')]); - render(() => ); - expect(screen.getByText('1 node ready for Pulse Agent')).toBeInTheDocument(); - }); - - it('shows plural "nodes" for multiple deployable', () => { - const group = makeGroup('cluster-a', [ - makePveNode('node1', 'agent-1'), - makePveNode('node2'), - makePveNode('node3'), - makePveNode('node4'), - ]); - render(() => ); - expect(screen.getByText('3 nodes ready for Pulse Agent')).toBeInTheDocument(); - }); - }); - - describe('deploy button', () => { - it('calls onDeploy with cluster name when clicked', () => { - const onDeploy = vi.fn(); - const group = makeGroup('prod-cluster', [ - makePveNode('node1', 'agent-1'), - makePveNode('node2'), - ]); - render(() => ); - fireEvent.click(screen.getByText('Review & Deploy')); - expect(onDeploy).toHaveBeenCalledTimes(1); - expect(onDeploy).toHaveBeenCalledWith('prod-cluster', 'prod-cluster'); - }); - - it('stops event propagation on click', () => { - const parentHandler = vi.fn(); - const group = makeGroup('my-cluster', [ - makePveNode('node1', 'agent-1'), - makePveNode('node2'), - ]); - render(() => ( -
- -
- )); - fireEvent.click(screen.getByText('Review & Deploy')); - expect(parentHandler).not.toHaveBeenCalled(); - }); - }); - - describe('edge cases', () => { - it('ignores VMs when computing PVE node counts', () => { - const group = makeGroup('my-cluster', [ - makePveNode('node1', 'agent-1'), - makeVm('vm1'), - makePveNode('node2'), - ]); - render(() => ); - expect(screen.getByText('1 node ready for Pulse Agent')).toBeInTheDocument(); - }); - - it('ignores non-PVE platform nodes', () => { - const group = makeGroup('my-cluster', [ - makePveNode('pve-node1', 'agent-1'), - makeK8sNode('k8s-node1'), - makePveNode('pve-node2'), - ]); - render(() => ); - expect(screen.getByText('1 node ready for Pulse Agent')).toBeInTheDocument(); - }); - - it('does not treat non-PVE agent as a PVE source agent', () => { - // A k8s node with an agent should NOT satisfy the source-agent requirement for PVE - const group = makeGroup('my-cluster', [ - makePveNode('pve-node1'), // PVE node, no agent - makePveNode('pve-node2'), // PVE node, no agent - makeK8sNode('k8s-node1', 'k8s-agent-1'), // non-PVE node with agent - ]); - render(() => ); - expect(screen.queryByText('Review & Deploy')).not.toBeInTheDocument(); - }); - - it('renders icons', () => { - const group = makeGroup('my-cluster', [ - makePveNode('node1', 'agent-1'), - makePveNode('node2'), - ]); - render(() => ); - expect(screen.getByTestId('info-icon')).toBeInTheDocument(); - expect(screen.getByTestId('rocket-icon')).toBeInTheDocument(); - }); - }); -}); diff --git a/frontend-modern/src/components/Infrastructure/__tests__/ResourceFacetSummary.test.tsx b/frontend-modern/src/components/Infrastructure/__tests__/ResourceFacetSummary.test.tsx new file mode 100644 index 0000000000..f79396fd58 --- /dev/null +++ b/frontend-modern/src/components/Infrastructure/__tests__/ResourceFacetSummary.test.tsx @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest'; +import { render } from '@solidjs/testing-library'; +import { ResourceFacetSummary } from '@/components/Infrastructure/ResourceFacetSummary'; + +describe('ResourceFacetSummary', () => { + it('renders timeline-only badges for canonical resource counts', () => { + const { getByText, queryByText } = render(() => ( + + )); + + expect(getByText('Timeline 3')).toBeInTheDocument(); + expect(getByText('Restart 2')).toBeInTheDocument(); + expect(getByText('Config update 1')).toBeInTheDocument(); + expect(getByText('Anomaly 1')).toBeInTheDocument(); + expect(getByText('Platform event 1')).toBeInTheDocument(); + expect(getByText('Pulse diff 2')).toBeInTheDocument(); + expect(getByText('Docker adapter 2')).toBeInTheDocument(); + expect(getByText('Proxmox adapter 1')).toBeInTheDocument(); + expect(queryByText('Capabilities 1')).toBeNull(); + expect(queryByText('Relationships 1')).toBeNull(); + }); + + it('collapses badges behind an overflow chip when a visible limit is set', () => { + const { getByText, queryByText } = render(() => ( + + )); + + expect(getByText('Timeline 3')).toBeInTheDocument(); + expect(getByText('+7')).toBeInTheDocument(); + expect(queryByText('Config update 1')).toBeNull(); + }); +}); diff --git a/frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.performance.contract.test.tsx b/frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.performance.contract.test.tsx deleted file mode 100644 index 9231ffcd19..0000000000 --- a/frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.performance.contract.test.tsx +++ /dev/null @@ -1,1341 +0,0 @@ -import { readFileSync } from 'node:fs'; -import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { render, waitFor, within } from '@solidjs/testing-library'; -import type { Resource } from '@/types/resource'; -import { UnifiedResourceTable } from '@/components/Infrastructure/UnifiedResourceTable'; -import { ResourceFacetSummary } from '@/components/Infrastructure/ResourceFacetSummary'; -import { - buildCustomSensorRows, - buildTemperatureRows, - formatSensorName, -} from '@/components/Infrastructure/resourceDetailMappers'; -import { getPreferredInfrastructureDisplayName } from '@/utils/resourceIdentity'; -import unifiedResourceTableSource from '@/components/Infrastructure/UnifiedResourceTable.tsx?raw'; -import unifiedResourceTableStateSource from '@/components/Infrastructure/useUnifiedResourceTableState.ts?raw'; -import unifiedResourceTableViewportSyncSource from '@/components/Infrastructure/useUnifiedResourceTableViewportSync.ts?raw'; -import unifiedResourceTableModelSource from '@/components/Infrastructure/unifiedResourceTableModel.ts?raw'; -import unifiedResourceTableStateModelSource from '@/components/Infrastructure/unifiedResourceTableStateModel.ts?raw'; -import unifiedResourceHostTableCardSource from '@/components/Infrastructure/UnifiedResourceHostTableCard.tsx?raw'; -import unifiedResourceServiceInfrastructureCardSource from '@/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx?raw'; -import unifiedResourcePBSTableSectionSource from '@/components/Infrastructure/UnifiedResourcePBSTableSection.tsx?raw'; -import unifiedResourcePMGTableSectionSource from '@/components/Infrastructure/UnifiedResourcePMGTableSection.tsx?raw'; -import summaryRowActionButtonSource from '@/components/shared/SummaryRowActionButton.tsx?raw'; -import summaryInteractionA11ySource from '@/components/shared/summaryInteractionA11y.ts?raw'; -import resourceDetailMappersSource from '@/components/Infrastructure/resourceDetailMappers.ts?raw'; -import resourceDetailDiscoveryModelSource from '@/components/Infrastructure/resourceDetailDiscoveryModel.ts?raw'; -import { - buildStatusOptions, - filterResources, - matchesSearch, - sortResources, - groupResources, - splitPrimaryAndServiceResources, - computeIOScale, -} from '@/components/Infrastructure/infrastructureSelectors'; -const resourceDetailDrawerMock = vi.hoisted(() => vi.fn()); -const frontendIndexCssSource = readFileSync(join(process.cwd(), 'src/index.css'), 'utf8'); -const resizeObserverCallbacks: ResizeObserverCallback[] = []; -globalThis.ResizeObserver = class ResizeObserver { - constructor(callback: ResizeObserverCallback) { - resizeObserverCallbacks.push(callback); - } - observe() {} - unobserve() {} - disconnect() {} -} as unknown as typeof ResizeObserver; -const emitResizeObserverWidth = (width: number) => { - for (const callback of resizeObserverCallbacks) { - callback([{ contentRect: { width } } as ResizeObserverEntry], {} as ResizeObserver); - } -}; -if (typeof Element.prototype.scrollIntoView !== 'function') { - Element.prototype.scrollIntoView = vi.fn(); -} - -vi.mock('@/hooks/useBreakpoint', () => ({ - useBreakpoint: () => ({ - isMobile: () => false, - isVisible: () => true, - }), -})); - -vi.mock('@/components/shared/responsive', () => ({ - ResponsiveMetricCell: () =>
metric
, -})); - -vi.mock('@/components/Infrastructure/ResourceDetailDrawer', () => ({ - ResourceDetailDrawer: (props: Record) => { - resourceDetailDrawerMock(props); - return
drawer
; - }, -})); - -const makeResource = (i: number, overrides?: Partial): Resource => ({ - id: `resource-${i}`, - type: i % 3 === 0 ? 'docker-host' : 'agent', - name: `host-${i}`, - displayName: `Host ${i}`, - platformId: `platform-${i}`, - platformType: 'proxmox-pve', - sourceType: 'api', - status: i % 10 === 0 ? 'offline' : 'online', - cpu: { current: (i % 100) / 100 }, - memory: { - current: (i % 80) / 100, - total: 16 * 1024 * 1024 * 1024, - used: ((i % 80) / 100) * 16 * 1024 * 1024 * 1024, - }, - disk: { current: 0.3 }, - lastSeen: Date.now(), - platformData: { sources: ['proxmox'] }, - ...overrides, -}); - -const PROFILES = { - S: 250, - M: 1000, - L: 3000, -} as const; - -const makeResources = (count: number, overrides?: (i: number) => Partial): Resource[] => - Array.from({ length: count }, (_, i) => makeResource(i, overrides?.(i))); - -const getTypeDistribution = (resources: Resource[]) => - resources.reduce( - (acc, resource) => { - if (resource.type === 'agent') acc.agent += 1; - if (resource.type === 'docker-host') acc.dockerHost += 1; - return acc; - }, - { agent: 0, dockerHost: 0 }, - ); - -const expectedTypeDistribution = (count: number) => { - const dockerHost = Math.floor((count - 1) / 3) + 1; - const agent = count - dockerHost; - return { agent, dockerHost }; -}; - -const getBodyRowCount = (container: HTMLElement) => container.querySelectorAll('tbody tr').length; - -describe('UnifiedResourceTable performance contract', () => { - beforeEach(() => { - vi.clearAllMocks(); - resizeObserverCallbacks.length = 0; - localStorage.clear(); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - describe('Fixture profile validation', () => { - for (const [profile, count] of Object.entries(PROFILES)) { - it(`builds profile ${profile} with stable size and type distribution`, () => { - const resources = makeResources(count); - const distribution = getTypeDistribution(resources); - const expectedDistribution = expectedTypeDistribution(count); - - expect(resources).toHaveLength(count); - expect(distribution).toEqual(expectedDistribution); - }); - } - }); - - describe('Baseline structural contracts', () => { - it('renders Profile S table structure within the mounted-row budget', async () => { - const resources = makeResources(PROFILES.S); - const { container } = render(() => ( - - )); - - await waitFor(() => { - expect(container.querySelector('table')).toBeInTheDocument(); - }); - await waitFor(() => { - expect(getBodyRowCount(container)).toBeGreaterThan(0); - }); - await waitFor(() => { - expect(getBodyRowCount(container)).toBeGreaterThan(0); - expect(getBodyRowCount(container)).toBeLessThanOrEqual(140); - }); - }); - - it('keeps the infrastructure table shell CSP-safe without inline style attributes', async () => { - const resources = [ - makeResource(1), - makeResource(2, { type: 'pbs', platformType: 'proxmox-pbs' }), - makeResource(3, { type: 'pmg', platformType: 'proxmox-pmg' }), - ]; - - const { container } = render(() => ( - - )); - - await waitFor(() => { - expect(container.querySelector('table')).toBeInTheDocument(); - }); - - expect(container.querySelector('[style]')).toBeNull(); - }); - - it('renders thermal pressure in the temperature column when Celsius is unavailable', async () => { - const resources = [ - makeResource(1, { - type: 'agent', - platformType: 'agent', - sourceType: 'agent', - temperature: undefined, - agent: { - platform: 'macos', - sensors: { - thermalState: { - source: 'pmset', - pressure: 'nominal', - }, - }, - }, - }), - ]; - - const { getByText } = render(() => ( - - )); - - await waitFor(() => { - expect(getByText('Nominal')).toBeInTheDocument(); - }); - expect(getByText('Nominal')).toHaveAttribute('title', 'Thermal pressure nominal via pmset'); - }); - - it('renders reported host identity instead of container runtime as the primary system badge', async () => { - const resources = [ - makeResource(1, { - type: 'docker-host', - displayName: 'tower', - platformType: 'docker', - sourceType: 'hybrid', - platformData: { - sources: ['agent', 'docker'], - agent: { - platform: 'linux', - hostProfile: 'unraid', - osName: 'Unraid', - osVersion: '7.1.0', - }, - }, - }), - ]; - const { getByText, queryByText } = render(() => ( - - )); - - await waitFor(() => { - expect(getByText('System')).toBeInTheDocument(); - expect(getByText('Unraid 7.1.0')).toBeInTheDocument(); - }); - expect(queryByText('Docker')).toBeNull(); - }); - - it('keeps versioned platform identity from repeating as a plain source in system titles', async () => { - const resources = [ - makeResource(1, { - type: 'agent', - displayName: 'pi', - platformType: 'proxmox-pve', - sourceType: 'hybrid', - sources: ['proxmox', 'agent'], - platformData: { - sources: ['proxmox', 'agent'], - proxmox: { - pveVersion: '', - }, - agent: { - platform: 'raspbian', - osName: 'Proxmox VE', - osVersion: '8.3.3', - }, - }, - }), - ]; - - const { container, getByText } = render(() => ( - - )); - - await waitFor(() => { - expect(getByText('PVE 8.3.3')).toBeInTheDocument(); - }); - expect(container.querySelector('[aria-label="System: PVE 8.3.3, Agent"]')).toBeTruthy(); - expect(container.querySelector('[aria-label="System: PVE 8.3.3, PVE, Agent"]')).toBeNull(); - }); - - it('renders compact canonical health reasons without replacing host identity', async () => { - const resources = [ - makeResource(1, { - type: 'agent', - displayName: 'Tower', - status: 'degraded', - agent: { - storagePostureSummary: 'Unraid array is running without parity protection', - rebuildSummary: 'Unraid array is running check', - }, - }), - ]; - - const { getByText } = render(() => ( - - )); - - await waitFor(() => { - expect(getByText('Tower')).toBeInTheDocument(); - expect(getByText('No parity')).toBeInTheDocument(); - }); - }); - - it('surfaces availability probe evidence directly in network endpoint rows', async () => { - vi.spyOn(Date, 'now').mockReturnValue(new Date('2026-05-06T13:00:20Z').getTime()); - const resources = [ - makeResource(1, { - type: 'network-endpoint', - name: 'MQTT power meter', - displayName: 'MQTT power meter', - platformId: 'mock-availability-mqtt-meter', - platformType: 'availability', - sourceType: 'api', - status: 'online', - cpu: undefined, - memory: undefined, - disk: undefined, - platformData: { - sources: ['availability'], - availability: { - protocol: 'tcp', - port: 1883, - available: true, - latencyMillis: 7, - lastChecked: '2026-05-06T13:00:06Z', - }, - }, - }), - ]; - - const { getByText, getAllByText, queryByText } = render(() => ( - - )); - - await waitFor(() => { - expect(getByText('1883:')).toBeInTheDocument(); - expect(getByText('7 ms')).toBeInTheDocument(); - }); - expect(queryByText('TCP 1883:')).toBeNull(); - expect(queryByText('TCP 1883 - 7 ms - checked 14s ago')).toBeNull(); - expect(getAllByText('7 ms')).toHaveLength(1); - }); - - it('renders availability badge for agent resources with ICMP protocol', async () => { - const resources = [ - makeResource(1, { - type: 'agent', - platformType: 'agent', - sourceType: 'agent', - platformData: { - sources: ['agent'], - availability: { - protocol: 'icmp', - available: true, - latencyMillis: 5, - lastChecked: new Date().toISOString(), - }, - }, - }), - ]; - - const { container } = render(() => ( - - )); - - await waitFor(() => { - expect(container.textContent).toContain('ICMP'); - }); - }); - - it('adapts host columns from the measured table surface width instead of the window width', async () => { - const resources = [ - makeResource(1, { - network: { rxBytes: 1200, txBytes: 3400 }, - diskIO: { readRate: 2000, writeRate: 800 }, - uptime: 86400, - temperature: 42, - }), - ]; - const { getByText } = render(() => ( - - )); - - await waitFor(() => { - expect(getByText('Net I/O')).toBeInTheDocument(); - }); - - emitResizeObserverWidth(760); - - await waitFor(() => { - expect(getByText('Net').closest('th')).not.toHaveClass('hidden'); - expect(getByText('Sys').closest('th')).not.toHaveClass('hidden'); - expect(getByText('I/O').closest('th')).toHaveClass('hidden'); - expect(getByText('Up').closest('th')).toHaveClass('hidden'); - }); - expect(getByText('CPU').closest('th')).not.toHaveClass('hidden'); - expect(getByText('Mem')).toBeInTheDocument(); - expect(getByText('Disk').closest('th')).not.toHaveClass('hidden'); - - emitResizeObserverWidth(820); - - await waitFor(() => { - expect(getByText('Net').closest('th')).not.toHaveClass('hidden'); - expect(getByText('I/O').closest('th')).not.toHaveClass('hidden'); - expect(getByText('Sys').closest('th')).not.toHaveClass('hidden'); - expect(getByText('Up').closest('th')).toHaveClass('hidden'); - }); - - emitResizeObserverWidth(980); - - await waitFor(() => { - expect(getByText('Net I/O').closest('th')).not.toHaveClass('hidden'); - expect(getByText('Disk I/O').closest('th')).not.toHaveClass('hidden'); - expect(getByText('System').closest('th')).not.toHaveClass('hidden'); - expect(getByText('Up').closest('th')).not.toHaveClass('hidden'); - expect(getByText('Temp').closest('th')).not.toHaveClass('hidden'); - }); - - emitResizeObserverWidth(1200); - - await waitFor(() => { - expect(getByText('Memory')).toBeInTheDocument(); - expect(getByText('System').closest('th')).not.toHaveClass('hidden'); - expect(getByText('Uptime').closest('th')).not.toHaveClass('hidden'); - }); - }); - - it('keeps service columns denser than host columns before hiding them', async () => { - const resources = [ - makeResource(2, { - type: 'pbs', - name: 'pbs-docker', - displayName: 'pbs-docker', - platformType: 'proxmox-pbs', - sourceType: 'hybrid', - platformData: { sources: ['proxmox-pbs', 'agent'] }, - uptime: 86400, - }), - ]; - const { container, getByText, queryByText } = render(() => ( - - )); - - await waitFor(() => { - expect(getByText('Stores')).toBeInTheDocument(); - }); - - emitResizeObserverWidth(600); - - await waitFor(() => { - expect(getByText('Store').closest('th')).not.toHaveClass('hidden'); - expect(getByText('Jobs').closest('th')).not.toHaveClass('hidden'); - expect(getByText('Sys').closest('th')).not.toHaveClass('hidden'); - expect(getByText('Up').closest('th')).toHaveClass('hidden'); - }); - expect(getByText('PBS')).toBeInTheDocument(); - expect(queryByText('Agent')).toBeNull(); - expect(queryByText('PBS+Agent')).toBeNull(); - expect(queryByText('+1')).toBeNull(); - expect( - container.querySelectorAll('[aria-label^="System:"] .h-1\\.5.w-1\\.5.rounded-full'), - ).toHaveLength(0); - - emitResizeObserverWidth(660); - - await waitFor(() => { - expect(getByText('Up').closest('th')).not.toHaveClass('hidden'); - }); - }); - - it('renders the shared facet summary component in timeline-only mode for canonical resource counts', async () => { - const { getByText, queryByText } = render(() => ( - - )); - - expect(getByText('Timeline 3')).toBeInTheDocument(); - expect(getByText('Restart 2')).toBeInTheDocument(); - expect(getByText('Config update 1')).toBeInTheDocument(); - expect(getByText('Anomaly 1')).toBeInTheDocument(); - expect(getByText('Platform event 1')).toBeInTheDocument(); - expect(getByText('Pulse diff 2')).toBeInTheDocument(); - expect(getByText('Docker adapter 2')).toBeInTheDocument(); - expect(getByText('Proxmox adapter 1')).toBeInTheDocument(); - expect(queryByText('Capabilities 1')).toBeNull(); - expect(queryByText('Relationships 1')).toBeNull(); - }); - - it('collapses resource facet badges behind an overflow chip when a table row sets a visible limit', async () => { - const { getByText, queryByText } = render(() => ( - - )); - - expect(getByText('Timeline 3')).toBeInTheDocument(); - expect(getByText('+7')).toBeInTheDocument(); - expect(queryByText('Config update 1')).toBeNull(); - }); - - it('formats sensor labels through the shared resource detail mapper helper', () => { - expect(formatSensorName('fan1_cpu_temp')).toBe('Cpu Temp'); - expect(formatSensorName('psu_temp')).toBe('Temp'); - expect( - buildTemperatureRows({ - temperatureCelsius: {}, - thermalState: { - source: 'pmset', - pressure: 'constrained', - limitsPercent: { cpu_speed_limit: 72, scheduler_limit: 100 }, - }, - }), - ).toEqual([ - { label: 'Thermal pressure', value: 'Constrained', valueTitle: 'Constrained via pmset' }, - { label: 'Speed Limit', value: '72%', valueTitle: 'Speed Limit 72%' }, - ]); - expect(resourceDetailMappersSource).toContain('titleCaseDelimitedLabel'); - expect(resourceDetailMappersSource).not.toContain('export const toDiscoveryConfig'); - expect(resourceDetailDiscoveryModelSource).toContain('export const toDiscoveryConfig'); - }); - - it('keeps typed GPU detail rows bounded to the selected resource payload', () => { - expect( - buildTemperatureRows({ - temperatureCelsius: { - cpu_package: 48, - gpu_nvidia_0: 63, - }, - gpu: [ - { - id: '0', - name: 'NVIDIA RTX A6000', - temperatureCelsius: 63, - utilizationPercent: 0, - memoryUsedBytes: 2 * 1024 * 1024 * 1024, - memoryTotalBytes: 48 * 1024 * 1024 * 1024, - }, - ], - }), - ).toEqual([ - { - label: 'GPU 0', - value: 'NVIDIA RTX A6000 · 63°C · 0% · 2.00 GB / 48.0 GB', - valueTitle: 'NVIDIA RTX A6000 · 63°C · 0% · 2.00 GB / 48.0 GB', - }, - { - label: 'Package', - value: '48°C', - valueTitle: '48.0°C', - }, - ]); - expect(resourceDetailMappersSource).not.toContain('nvidia-smi'); - expect(resourceDetailMappersSource).not.toContain('fetch('); - }); - - it('keeps custom metric formatting bounded to the selected resource payload', () => { - expect( - buildCustomSensorRows({ - custom: [ - { - id: 'queue_depth', - name: 'Queue depth', - unit: 'items', - value: 12.5, - status: 'warning', - observedAt: '2026-07-30T19:00:00Z', - }, - { - id: 'backup_age', - name: 'Last backup', - group: 'Main server', - subgroup: 'Backup', - kind: 'timestamp', - value: 7200, - status: 'warning', - observedAt: '2026-07-30T20:00:00Z', - }, - { - id: 'service_online', - name: 'Checkout', - group: 'Main server', - subgroup: 'Services', - kind: 'boolean', - value: 1, - status: 'ok', - observedAt: '2026-07-30T20:00:00Z', - }, - ], - }), - ).toEqual([ - { - label: 'Main server / Backup / Last backup', - value: '2h ago', - valueTitle: '2h ago · Warning', - }, - { - label: 'Main server / Services / Checkout', - value: 'Online', - valueTitle: 'Online · OK', - }, - { - label: 'Queue depth', - value: '12.5 items', - valueTitle: '12.5 items · Warning', - }, - ]); - expect(resourceDetailMappersSource).not.toContain('fetch('); - }); - - it('keeps power and fan detail rows bounded to the selected resource payload', () => { - expect( - buildTemperatureRows({ - additional: { vrm_temp: 55.6 }, - fanRpm: { chassis_fan: 1199.6 }, - powerWatts: { cpu_package: 82.4 }, - }), - ).toEqual([ - { - label: 'VRM Temp', - value: '56°C', - valueTitle: '55.6°C', - }, - { - label: 'Chassis Fan', - value: '1,200 RPM', - valueTitle: 'Chassis Fan 1,200 RPM', - }, - { - label: 'CPU Package Power', - value: '82.4 W', - valueTitle: 'CPU Package Power 82.4 W', - }, - ]); - expect(resourceDetailMappersSource).not.toContain('powercap'); - expect(resourceDetailMappersSource).not.toContain('fetch('); - }); - - it('keeps hot-path table state and windowing in the shared table state owner', () => { - expect(unifiedResourceTableSource).toContain('useUnifiedResourceTableState'); - expect(unifiedResourceTableSource).toContain('UnifiedResourceHostTableCard'); - expect(unifiedResourceTableSource).toContain('UnifiedResourceServiceInfrastructureCard'); - expect(unifiedResourceTableSource).toContain('data-summary-clear-surface'); - expect(unifiedResourceTableSource).not.toContain( - 'const sortedPBSResources = createMemo(() =>', - ); - expect(unifiedResourceTableSource).not.toContain('const getOutlierEmphasis ='); - expect(unifiedResourceTableSource).not.toContain('const getPBSTableRow ='); - expect(unifiedResourceTableStateSource).toContain("from './unifiedResourceTableStateModel'"); - expect(unifiedResourceTableStateSource).toContain('buildHostTableItems'); - expect(unifiedResourceTableStateSource).toContain( - 'getUnifiedResourceTableColumnPresentations', - ); - expect(unifiedResourceTableStateSource).toContain('getUnifiedResourceTableHeaderLabels'); - expect(unifiedResourceTableStateSource).toContain('getUnifiedResourceTableLayoutMode'); - expect(unifiedResourceTableStateSource).toContain('isUnifiedResourceServiceColumnVisible'); - expect(unifiedResourceTableStateSource).toContain('getUnifiedResourceTableShellClass'); - expect(unifiedResourceTableStateSource).toContain('useTableWindowing'); - expect(unifiedResourceTableStateSource).toContain('useUnifiedResourceTableViewportSync'); - expect(unifiedResourceTableStateSource).toContain('clearPinnedSummaryScope?: () => void;'); - expect(unifiedResourceTableStateSource).toContain('showHostClearAction'); - expect(unifiedResourceTableStateSource).toContain('showServiceClearAction'); - expect(unifiedResourceTableStateSource).toContain('useAlertsActivation'); - expect(unifiedResourceTableStateSource).toContain('getMetricThresholds'); - expect(unifiedResourceTableStateSource).not.toContain( - 'const resourceColumnStyle = createMemo(() =>', - ); - expect(unifiedResourceHostTableCardSource).toContain( - "table.getMetricThresholds('agent', 'cpu', alertResourceIdCandidates())", - ); - expect(unifiedResourceHostTableCardSource).toContain( - "table.getMetricThresholds('agent', 'memory', alertResourceIdCandidates())", - ); - expect(unifiedResourceHostTableCardSource).toContain( - "table.getMetricThresholds('agent', 'disk', alertResourceIdCandidates())", - ); - expect(unifiedResourceHostTableCardSource).toContain('thresholds={cpuThresholds()}'); - expect(unifiedResourceHostTableCardSource).toContain('thresholds={memoryThresholds()}'); - expect(unifiedResourceHostTableCardSource).toContain('thresholds={diskThresholds()}'); - expect(unifiedResourceHostTableCardSource).not.toContain('style={'); - expect(unifiedResourcePBSTableSectionSource).not.toContain('style={'); - expect(unifiedResourcePMGTableSectionSource).not.toContain('style={'); - expect(unifiedResourceTableStateSource).not.toContain( - "const showGroupHeaders = props.groupingMode === 'grouped'", - ); - expect(unifiedResourceTableStateSource).not.toContain('const items: HostTableItem[] = [];'); - expect(unifiedResourceTableStateSource).not.toContain('window.addEventListener'); - expect(unifiedResourceTableStateSource).not.toContain('getBoundingClientRect'); - expect(unifiedResourceTableViewportSyncSource).toContain('bindWindowedPageScrollEvents'); - expect(unifiedResourceTableViewportSyncSource).toContain('getBoundingClientRect'); - expect(unifiedResourceTableViewportSyncSource).not.toContain('scrollIntoView'); - expect(unifiedResourceTableViewportSyncSource).not.toContain("addEventListener('touch"); - expect(unifiedResourceTableViewportSyncSource).toContain('hostWindowing.onScroll'); - expect(unifiedResourceTableModelSource).toContain('export const getPBSTableRow'); - expect(unifiedResourceTableModelSource).toContain('export const getPMGTableRow'); - expect(unifiedResourceTableModelSource).toContain('export const getOutlierEmphasis'); - expect(unifiedResourceTableStateModelSource).toContain('export const buildHostTableItems'); - expect(unifiedResourceTableStateModelSource).toContain( - 'export const getUnifiedResourceTableColumnPresentations', - ); - expect(unifiedResourceTableStateModelSource).toContain( - 'export const getUnifiedResourceTableHeaderLabels', - ); - expect(unifiedResourceTableStateModelSource).toContain( - 'export const getUnifiedResourceTableLayoutMode', - ); - expect(unifiedResourceTableStateModelSource).toContain( - 'export const isUnifiedResourceServiceColumnVisible', - ); - expect(unifiedResourceTableStateModelSource).toContain( - 'export const getUnifiedResourceTableShellClass', - ); - expect(unifiedResourceTableStateModelSource).toContain( - 'export const getNextUnifiedResourceTableSortState', - ); - }); - - it('keeps PBS activity projection in the shared table-model owner', () => { - expect(unifiedResourceTableModelSource).toContain('getPbsActivitySummary'); - expect(unifiedResourcePBSTableSectionSource).toContain('headerLabels().activity'); - expect(unifiedResourcePBSTableSectionSource).not.toContain('backupJobs'); - expect(unifiedResourcePBSTableSectionSource).not.toContain('syncJobs'); - expect(unifiedResourcePBSTableSectionSource).not.toContain('getPbsActivitySummary'); - }); - - it('keeps contextual resource-row emphasis on the shared active-row presentation contract', () => { - expect(frontendIndexCssSource).toContain("tr[data-summary-row-active='true'] > td"); - expect(frontendIndexCssSource).toContain('--color-summary-row-bg'); - expect(frontendIndexCssSource).toContain('--color-summary-row-accent'); - expect(frontendIndexCssSource).toContain('tr.grouped-table-row > td'); - expect(frontendIndexCssSource).toContain('--color-grouped-table-row-bg'); - expect(summaryInteractionA11ySource).toContain('createSummaryInteractiveRowPreviewHandlers'); - expect(summaryInteractionA11ySource).toContain( - 'createSummaryInteractiveActionKeydownHandler', - ); - expect(summaryRowActionButtonSource).toContain('SummaryRowActionButton'); - expect(summaryRowActionButtonSource).toContain('aria-controls'); - expect(summaryRowActionButtonSource).toContain('aria-expanded'); - expect(summaryRowActionButtonSource).toContain('aria-pressed'); - - for (const source of [ - unifiedResourceHostTableCardSource, - unifiedResourcePBSTableSectionSource, - unifiedResourcePMGTableSectionSource, - ]) { - expect(source).toContain('data-summary-row-active'); - expect(source).toContain('createSummaryInteractiveRowPreviewHandlers'); - expect(source).toContain('SummaryRowActionButton'); - expect(source).toContain('hideWhenRowTappableOnMobile'); - expect(source).not.toContain('bg-blue-50 dark:bg-blue-900/25 ring-1 ring-blue-300'); - expect(source).not.toContain('bg-blue-100 dark:bg-blue-800'); - } - - expect(unifiedResourceHostTableCardSource).not.toContain('kind="scope"'); - expect(unifiedResourceHostTableCardSource).toContain('getInteractiveGroupedTableRowClass'); - expect(unifiedResourceHostTableCardSource).toContain('getGroupedTableRowCellClass'); - expect(unifiedResourceHostTableCardSource).not.toContain('cursor-pointer bg-surface-alt'); - expect(unifiedResourceHostTableCardSource).toContain('TableCard'); - expect(unifiedResourceHostTableCardSource).toContain('TableCardHeader'); - expect(unifiedResourceHostTableCardSource).toContain( - 'onClear={tableProps.clearPinnedSummaryScope}', - ); - for (const source of [ - unifiedResourceHostTableCardSource, - unifiedResourcePBSTableSectionSource, - unifiedResourcePMGTableSectionSource, - ]) { - expect(source).toMatch(/)/); - expect(source).not.toContain('
'); - expect(source).not.toContain('
'); - } - expect(unifiedResourceServiceInfrastructureCardSource).toContain('TableCard'); - expect(unifiedResourceServiceInfrastructureCardSource).toContain('TableCardHeader'); - expect(unifiedResourceServiceInfrastructureCardSource).toContain('showClearAction'); - expect(unifiedResourceServiceInfrastructureCardSource).toContain( - 'onClear={tableProps.clearPinnedSummaryScope}', - ); - }); - - it('keeps source filtering on the shared canonical source-platform helper', () => { - const resources = [ - makeResource(0, { platformData: { sources: ['proxmox'] } }), - makeResource(1, { platformData: { sources: ['docker'] } }), - ]; - - const filtered = filterResources(resources, new Set(['proxmox-pve']), new Set(), []); - - expect(filtered).toHaveLength(1); - expect(filtered[0]?.platformData?.sources).toEqual(['proxmox']); - }); - - it('searches policy-redacted resources by their raw infrastructure display name', () => { - // Local search; redaction is a transmission-boundary policy (docs/PRIVACY.md). - const governedResource = makeResource(9, { - name: 'secret-host-9', - displayName: 'secret-host-9', - policy: { - sensitivity: 'restricted', - routing: { scope: 'local-only', redact: ['hostname'] }, - }, - aiSafeSummary: 'Production Host', - }); - - expect(matchesSearch(governedResource, 'secret-host-9')).toBe(true); - expect(matchesSearch(governedResource, 'Production')).toBe(false); - }); - - it('suppresses non-blocking policy posture in host-table rows while preserving blocking policy badges', async () => { - const resources = [ - makeResource(0, { - name: 'default-policy-host', - displayName: 'Default Policy Host', - policy: { - sensitivity: 'internal', - routing: { scope: 'cloud-summary' }, - }, - }), - makeResource(1, { - name: 'governed-host', - displayName: 'Governed Host', - policy: { - sensitivity: 'sensitive', - routing: { scope: 'local-first', redact: ['hostname'] }, - }, - }), - makeResource(2, { - name: 'local-only-host', - displayName: 'Local Only Host', - policy: { - sensitivity: 'restricted', - routing: { scope: 'local-only', redact: ['hostname'] }, - }, - }), - ]; - - const { container } = render(() => ( - - )); - - await waitFor(() => { - expect(container.querySelector('table')).toBeInTheDocument(); - }); - - const body = container.querySelector('tbody'); - expect(body).not.toBeNull(); - const bodyQueries = within(body as HTMLElement); - - expect(bodyQueries.queryByText('Internal')).not.toBeInTheDocument(); - expect(bodyQueries.queryByText('Cloud Summary')).not.toBeInTheDocument(); - expect(bodyQueries.queryByText('Sensitive')).not.toBeInTheDocument(); - expect(bodyQueries.queryByText('Local First')).not.toBeInTheDocument(); - const policyBadge = bodyQueries.getByText('Local Only'); - expect(policyBadge).toBeInTheDocument(); - expect(policyBadge.closest('[title]')?.getAttribute('title')).toContain('Restricted'); - }); - - it('renders facet summary badges without changing the Profile S row budget', async () => { - const resources = makeResources(PROFILES.S, (i) => - i === 1 - ? { - capabilities: [ - { - name: 'restart', - type: 'native', - description: 'Restart the host', - minimumApprovalLevel: 'admin', - }, - ], - relationships: [ - { - sourceId: 'resource-1', - targetId: 'storage-1', - type: 'depends_on', - confidence: 0.91, - active: true, - discoverer: 'proxmox_adapter', - observedAt: new Date().toISOString(), - lastSeenAt: new Date().toISOString(), - }, - ], - facetCounts: { - recentChanges: 3, - recentChangeKinds: { - restart: 2, - config_update: 1, - metric_anomaly: 1, - }, - recentChangeSourceTypes: { - platform_event: 1, - pulse_diff: 2, - }, - recentChangeSourceAdapters: { - docker_adapter: 2, - proxmox_adapter: 1, - }, - }, - recentChanges: [], - } - : {}, - ); - const { container, getByText } = render(() => ( - - )); - - await waitFor(() => { - expect(container.querySelector('table')).toBeInTheDocument(); - }); - await waitFor(() => { - expect(getByText('Timeline 3')).toBeInTheDocument(); - expect(getByText('+7')).toBeInTheDocument(); - }); - expect(container.querySelector('tbody')?.textContent).not.toContain('Config update 1'); - await waitFor(() => { - expect(getBodyRowCount(container)).toBeGreaterThan(0); - expect(getBodyRowCount(container)).toBeLessThanOrEqual(140); - }); - }); - - it('renders service facet summary badges without changing the dedicated service row budget', async () => { - const resources: Resource[] = [ - makeResource(0, { - id: 'pbs-service', - type: 'pbs', - displayName: 'pbs-service', - name: 'pbs-service', - platformType: 'proxmox-pbs', - sourceType: 'api', - platformData: { - sources: ['pbs'], - pbs: { - datastoreCount: 1, - backupJobCount: 1, - }, - }, - recentChanges: [ - { - id: 'pbs-change-1', - observedAt: new Date().toISOString(), - resourceId: 'pbs-service', - kind: 'state_transition', - sourceType: 'pulse_diff', - confidence: 'high', - }, - ], - }), - makeResource(1, { - id: 'pmg-service', - type: 'pmg', - displayName: 'pmg-service', - name: 'pmg-service', - platformType: 'proxmox-pmg', - sourceType: 'api', - platformData: { - sources: ['pmg'], - pmg: { - queueTotal: 12, - nodeCount: 1, - }, - }, - recentChanges: [ - { - id: 'pmg-change-1', - observedAt: new Date().toISOString(), - resourceId: 'pmg-service', - kind: 'metric_anomaly', - sourceType: 'heuristic', - confidence: 'medium', - }, - ], - }), - ]; - const { container, getByText } = render(() => ( - - )); - - await waitFor(() => { - expect(container.querySelector('table')).toBeInTheDocument(); - }); - await waitFor(() => { - expect(getBodyRowCount(container)).toBe(2); - }); - - const pbsRow = getByText('pbs-service').closest('tr'); - expect(pbsRow).toBeTruthy(); - if (pbsRow) { - const row = within(pbsRow); - expect(row.getByText('Timeline 1')).toBeInTheDocument(); - expect(row.queryByText('Capabilities 1')).toBeNull(); - expect(row.queryByText('Relationships 1')).toBeNull(); - } - - const pmgRow = getByText('pmg-service').closest('tr'); - expect(pmgRow).toBeTruthy(); - if (pmgRow) { - const row = within(pmgRow); - expect(row.getByText('Timeline 1')).toBeInTheDocument(); - expect(row.queryByText('Capabilities 1')).toBeNull(); - expect(row.queryByText('Relationships 1')).toBeNull(); - } - }); - - it('passes the canonical resource-label resolver into the drawer for related-resource chips', async () => { - resourceDetailDrawerMock.mockClear(); - const resources = [ - makeResource(0, { - id: 'resource-0', - displayName: 'Host 0', - name: 'host-0', - }), - ]; - const { container } = render(() => ( - - )); - - await waitFor(() => { - expect(container.querySelector('table')).toBeInTheDocument(); - }); - await waitFor(() => { - expect(resourceDetailDrawerMock).toHaveBeenCalled(); - }); - - const drawerProps = resourceDetailDrawerMock.mock.calls.at(-1)?.[0] as { - resolveResourceLabel?: (resourceId: string) => string | undefined; - }; - - expect(drawerProps.resolveResourceLabel?.('resource-0')).toBe('Host 0'); - }); - }); - - describe('Row windowing contracts', () => { - it('Profile M: preserves row windowing with policy badge rendering enabled', async () => { - const resources = makeResources(PROFILES.M, (i) => - i === 0 - ? { - displayName: 'Sensitive Host', - name: 'sensitive-host', - aiSafeSummary: 'restricted host summary safe for remote AI consumption', - policy: { - sensitivity: 'restricted', - routing: { - scope: 'local-only', - redact: ['hostname', 'alias'], - }, - }, - } - : { - policy: { - sensitivity: i % 2 === 0 ? 'restricted' : 'internal', - routing: { - scope: i % 3 === 0 ? 'local-only' : 'local-first', - redact: ['hostname', 'alias'], - }, - }, - }, - ); - const { container, getAllByText } = render(() => ( - - )); - - await waitFor(() => { - expect(container.querySelector('table')).toBeInTheDocument(); - }); - await waitFor( - () => { - const rowCount = getBodyRowCount(container); - expect(rowCount).toBeGreaterThan(0); - expect(rowCount).toBeLessThanOrEqual(140); - }, - { timeout: 15000 }, - ); - await waitFor( - () => { - expect(getAllByText('Local Only').length).toBeGreaterThan(0); - }, - { timeout: 15000 }, - ); - expect(getPreferredInfrastructureDisplayName(resources[0]!)).toBe('Sensitive Host'); - }, 30000); - - it('Profile M: caps mounted rows when windowing is active', async () => { - const resources = makeResources(PROFILES.M); - const { container } = render(() => ( - - )); - - await waitFor(() => { - expect(container.querySelector('table')).toBeInTheDocument(); - }); - await waitFor( - () => { - const rowCount = getBodyRowCount(container); - expect(rowCount).toBeGreaterThan(0); - expect(rowCount).toBeLessThanOrEqual(140); - }, - { timeout: 15000 }, - ); - }, 30000); - - it('Profile L: keeps mounted rows capped under large load', async () => { - const resources = makeResources(PROFILES.L); - const { container } = render(() => ( - - )); - - await waitFor(() => { - expect(container.querySelector('table')).toBeInTheDocument(); - }); - await waitFor( - () => { - const rowCount = getBodyRowCount(container); - expect(rowCount).toBeGreaterThan(0); - expect(rowCount).toBeLessThanOrEqual(140); - }, - { timeout: 15000 }, - ); - }, 20000); - }); - - describe('Grouped vs flat mode contract', () => { - it('renders grouped mode with profile S resources', async () => { - const resources = makeResources(PROFILES.S, (i) => ({ - clusterId: i % 2 === 0 ? 'cluster-a' : 'cluster-b', - })); - const { container, getByText } = render(() => ( - - )); - - await waitFor( - () => { - expect(getByText('cluster-a')).toBeInTheDocument(); - expect(getByText('cluster-b')).toBeInTheDocument(); - }, - { timeout: 15000 }, - ); - await waitFor( - () => { - expect(getBodyRowCount(container)).toBeGreaterThan(0); - expect(getBodyRowCount(container)).toBeLessThanOrEqual(140); - }, - { timeout: 15000 }, - ); - }, 30000); - - it('renders flat mode with profile S resources', async () => { - const resources = makeResources(PROFILES.S, (i) => ({ - clusterId: i % 2 === 0 ? 'cluster-a' : 'cluster-b', - })); - const { container, queryByText } = render(() => ( - - )); - - await waitFor( - () => { - expect(container.querySelector('table')).toBeInTheDocument(); - }, - { timeout: 15000 }, - ); - await waitFor( - () => { - expect(getBodyRowCount(container)).toBeGreaterThan(0); - expect(getBodyRowCount(container)).toBeLessThanOrEqual(140); - }, - { timeout: 15000 }, - ); - expect(queryByText('cluster-a')).not.toBeInTheDocument(); - expect(queryByText('cluster-b')).not.toBeInTheDocument(); - }, 30000); - }); - - describe('Infrastructure derivation contracts', () => { - it('filterResources returns correct count after source filter on Profile S', () => { - const resources = makeResources(PROFILES.S); - const proxmoxOnly = filterResources(resources, new Set(['proxmox-pve']), new Set(), []); - expect(proxmoxOnly.length).toBe(PROFILES.S); - }); - - it('sortResources preserves array length for all sort keys', () => { - const resources = makeResources(PROFILES.M); - for (const key of ['default', 'name', 'cpu', 'memory', 'disk']) { - const sorted = sortResources(resources, key, 'asc'); - expect(sorted).toHaveLength(PROFILES.M); - } - }); - - it('buildStatusOptions keeps canonical statuses ahead of unknown filters', () => { - expect(buildStatusOptions(new Set(['custom-state', 'paused', 'degraded']))).toEqual([ - { key: 'degraded', label: 'Degraded' }, - { key: 'paused', label: 'Paused' }, - { key: 'custom-state', label: 'custom-state' }, - ]); - }); - - it('groupResources produces correct total resource count in grouped mode', () => { - const resources = makeResources(PROFILES.S, (i) => ({ - clusterId: `cluster-${i % 3}`, - })); - const grouped = groupResources(resources, 'grouped'); - const total = grouped.reduce((sum, group) => sum + group.resources.length, 0); - expect(total).toBe(PROFILES.S); - expect(grouped.length).toBe(3); - }); - - it('splitPrimaryAndServiceResources partitions without losing resources', () => { - const resources = makeResources(PROFILES.S); - const { primaryResources, services } = splitPrimaryAndServiceResources(resources); - expect(primaryResources.length + services.length).toBe(PROFILES.S); - }); - - it('computeIOScale produces valid stats for Profile M', () => { - const resources = makeResources(PROFILES.M, (i) => ({ - network: { rxBytes: i * 100, txBytes: i * 50 }, - diskIO: { readRate: i * 10, writeRate: i * 5 }, - })); - const scale = computeIOScale(resources); - expect(scale.network.count).toBeGreaterThan(0); - expect(scale.diskIO.count).toBeGreaterThan(0); - expect(scale.network.median).toBeGreaterThan(0); - }); - }); -}); diff --git a/frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.policy-governance.test.tsx b/frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.policy-governance.test.tsx deleted file mode 100644 index 90a471152d..0000000000 --- a/frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.policy-governance.test.tsx +++ /dev/null @@ -1,103 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { render } from '@solidjs/testing-library'; -import type { Resource } from '@/types/resource'; -import { UnifiedResourceTable } from '@/components/Infrastructure/UnifiedResourceTable'; - -if (typeof globalThis.ResizeObserver === 'undefined') { - globalThis.ResizeObserver = class ResizeObserver { - observe() {} - unobserve() {} - disconnect() {} - } as unknown as typeof ResizeObserver; -} - -vi.mock('@/hooks/useBreakpoint', () => ({ - useBreakpoint: () => ({ - isMobile: () => false, - isVisible: () => true, - }), -})); - -vi.mock('@/components/shared/responsive', () => ({ - ResponsiveMetricCell: () =>
metric
, -})); - -vi.mock('@/components/Infrastructure/ResourceDetailDrawer', () => ({ - ResourceDetailDrawer: () =>
drawer
, -})); - -const resource: Resource = { - id: 'resource-1', - type: 'agent', - name: 'sensitive-host', - displayName: 'Sensitive Host', - platformId: 'platform-1', - platformType: 'agent', - sourceType: 'agent', - status: 'online', - lastSeen: Date.now(), - platformData: { sources: ['agent'] }, - policy: { - sensitivity: 'restricted', - routing: { - scope: 'local-only', - redact: ['hostname', 'alias'], - }, - }, - aiSafeSummary: 'restricted host summary safe for remote AI consumption', - recentChanges: [ - { - id: 'change-1', - observedAt: new Date().toISOString(), - resourceId: 'resource-1', - kind: 'state_transition', - sourceType: 'pulse_diff', - confidence: 'high', - }, - ], -}; - -describe('UnifiedResourceTable governance presentation', () => { - it('surfaces blocking policy posture in the resource row', () => { - const { getByText, queryByText } = render(() => ( - - )); - - expect(getByText('Local Only')).toBeInTheDocument(); - expect(queryByText('Restricted')).toBeNull(); - expect(getByText('Sensitive Host')).toBeInTheDocument(); - expect(queryByText('restricted host summary safe for remote AI consumption')).toBeNull(); - expect(queryByText('(sensitive-host)')).toBeNull(); - }); - - it('surfaces resource facet counts in the resource row', () => { - const facetResource: Resource = { - ...resource, - policy: { - sensitivity: 'sensitive', - routing: { - scope: 'local-first', - redact: ['hostname'], - }, - }, - }; - - const { getByText, queryByText } = render(() => ( - - )); - - expect(getByText('Timeline 1')).toBeInTheDocument(); - expect(queryByText('Capabilities 2')).toBeNull(); - expect(queryByText('Relationships 1')).toBeNull(); - }); -}); diff --git a/frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.workloads-link.test.tsx b/frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.workloads-link.test.tsx deleted file mode 100644 index 493b6e0a71..0000000000 --- a/frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.workloads-link.test.tsx +++ /dev/null @@ -1,293 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { fireEvent, render, screen, within } from '@solidjs/testing-library'; -import type { Resource } from '@/types/resource'; -import { UnifiedResourceTable } from '@/components/Infrastructure/UnifiedResourceTable'; - -const responsiveMetricCellMock = vi.fn(); - -if (typeof globalThis.ResizeObserver === 'undefined') { - globalThis.ResizeObserver = class ResizeObserver { - observe() {} - unobserve() {} - disconnect() {} - } as unknown as typeof ResizeObserver; -} - -vi.mock('@/hooks/useBreakpoint', () => ({ - useBreakpoint: () => ({ - isMobile: () => false, - isVisible: () => true, - }), -})); - -vi.mock('@/components/shared/responsive', () => ({ - ResponsiveMetricCell: (props: unknown) => { - responsiveMetricCellMock(props); - return
metric
; - }, -})); - -vi.mock('@/components/Infrastructure/ResourceDetailDrawer', () => ({ - ResourceDetailDrawer: () =>
drawer
, -})); - -const baseResource = (overrides: Partial): Resource => ({ - id: 'resource-1', - type: 'agent', - name: 'pve1', - displayName: 'pve1', - platformId: 'pve1', - platformType: 'proxmox-pve', - sourceType: 'api', - status: 'online', - lastSeen: Date.now(), - platformData: { sources: ['proxmox'] }, - ...overrides, -}); - -describe('UnifiedResourceTable workloads links', () => { - it('passes canonical metrics keys to metric cells for docker-host resources', async () => { - responsiveMetricCellMock.mockClear(); - - render(() => ( - - )); - - expect(responsiveMetricCellMock).toHaveBeenCalled(); - expect( - responsiveMetricCellMock.mock.calls.some( - ([props]) => (props as { resourceId?: string }).resourceId === 'dockerHost:docker-host-1', - ), - ).toBe(true); - expect(screen.getByText('Tower').tagName).toBe('SPAN'); - expect(screen.getByRole('link', { name: 'Open web interface for Tower' })).toHaveAttribute( - 'href', - 'https://tower.internal:9443', - ); - }); - - it('no longer renders host-row workloads cross-jump links in the platform-first layout', () => { - const resources: Resource[] = [ - baseResource({ - id: 'node-1', - type: 'agent', - platformData: { - sources: ['proxmox'], - proxmox: { nodeName: 'pve1' }, - }, - }), - baseResource({ - id: 'k8s-cluster-1', - type: 'k8s-cluster', - name: 'cluster-a', - displayName: 'cluster-a', - clusterId: 'cluster-a', - platformType: 'kubernetes', - sourceType: 'api', - platformData: { - sources: ['kubernetes'], - kubernetes: { - clusterName: 'cluster-a', - }, - }, - }), - ]; - - const { queryAllByRole } = render(() => ( - - )); - - // Legacy cross-jump from a host row to /workloads?... was retired with - // the standalone Workloads page; the platform pages expose workloads as - // adjacent sub-tabs instead. - expect(queryAllByRole('link', { name: /view workloads/i })).toHaveLength(0); - }); - - it('renders PBS and PMG resources in a dedicated service table with service-native columns', async () => { - const onExpandedResourceChange = vi.fn(); - const resources: Resource[] = [ - baseResource({ - id: 'pbs-1', - type: 'pbs', - name: 'pbs-main', - displayName: 'pbs-main', - customUrl: 'https://pbs-main.internal:8007', - platformType: 'proxmox-pbs', - sourceType: 'api', - platformData: { - sources: ['pbs'], - pbs: { - datastoreCount: 2, - backupJobCount: 2, - syncJobCount: 1, - backupJobs: [ - { - id: 'backup-nightly', - store: 'fast', - type: 'vm', - vmid: '100', - lastBackup: '', - nextRun: '', - status: 'running', - error: '', - }, - { - id: 'backup-weekly', - store: 'archive', - type: 'ct', - vmid: '200', - lastBackup: '', - nextRun: '', - status: 'ok', - error: '', - }, - ], - syncJobs: [ - { - id: 'sync-remote', - store: 'fast', - remote: 'offsite', - status: 'queued', - lastSync: '', - nextRun: '', - error: '', - }, - ], - }, - }, - recentChanges: [ - { - id: 'pbs-change-1', - observedAt: new Date().toISOString(), - resourceId: 'pbs-1', - kind: 'state_transition', - sourceType: 'pulse_diff', - confidence: 'high', - }, - ], - }), - baseResource({ - id: 'pmg-1', - type: 'pmg', - name: 'pmg-main', - displayName: 'pmg-main', - customUrl: 'https://pmg-main.internal:8006', - platformType: 'proxmox-pmg', - sourceType: 'api', - platformData: { - sources: ['pmg'], - pmg: { - queueTotal: 519, - nodeCount: 1, - }, - }, - recentChanges: [ - { - id: 'pmg-change-1', - observedAt: new Date().toISOString(), - resourceId: 'pmg-1', - kind: 'metric_anomaly', - sourceType: 'heuristic', - confidence: 'medium', - }, - ], - }), - ]; - - const { getByText, getByRole, getAllByText } = render(() => ( - - )); - - expect(getByText('Service Infrastructure')).toBeInTheDocument(); - expect(getByText('PBS Services')).toBeInTheDocument(); - expect(getByText('PMG Services')).toBeInTheDocument(); - // Service Infrastructure dual-table renders the compact column label. - expect(getByText('Stores')).toBeInTheDocument(); - expect(getByText('Activity')).toBeInTheDocument(); - // Compact layout uses 'Action' for the action column header. - expect(getAllByText('Action').length).toBeGreaterThan(0); - expect(getByText('Queue')).toBeInTheDocument(); - // Compact layout abbreviates Deferred to 'Def'. - expect(getByText('Def')).toBeInTheDocument(); - expect(getByText('Hold')).toBeInTheDocument(); - expect(getByText('Nodes')).toBeInTheDocument(); - expect(getByText('pbs-main').tagName).toBe('SPAN'); - expect(getByText('pmg-main').tagName).toBe('SPAN'); - expect(getByRole('link', { name: 'Open web interface for pbs-main' })).toHaveAttribute( - 'href', - 'https://pbs-main.internal:8007', - ); - expect(getByRole('link', { name: 'Open web interface for pmg-main' })).toHaveAttribute( - 'href', - 'https://pmg-main.internal:8006', - ); - - const pbsRow = getByText('pbs-main').closest('tr'); - expect(pbsRow).toBeTruthy(); - if (pbsRow) { - const row = within(pbsRow); - expect(row.getByText('2')).toBeInTheDocument(); - expect(row.getByText('2 active')).toBeInTheDocument(); - expect(row.getByText('3 total')).toBeInTheDocument(); - expect(row.getByText('Timeline 1')).toBeInTheDocument(); - expect(row.queryByText('Capabilities 1')).toBeNull(); - expect(row.queryByText('Relationships 1')).toBeNull(); - } - - const pmgRow = getByText('pmg-main').closest('tr'); - expect(pmgRow).toBeTruthy(); - if (pmgRow) { - const row = within(pmgRow); - expect(row.getByText('519')).toBeInTheDocument(); - expect(row.getByText('1')).toBeInTheDocument(); - expect(row.getByText('Timeline 1')).toBeInTheDocument(); - expect(row.queryByText('Capabilities 1')).toBeNull(); - expect(row.queryByText('Relationships 1')).toBeNull(); - } - - const pmgLink = getByRole('link', { name: /open pmg thresholds/i }); - expect(pmgLink).toHaveTextContent('Thresholds'); - expect(pmgLink).toHaveAttribute('href', '/alerts/thresholds/mail-gateway'); - pmgLink.addEventListener('click', (event) => event.preventDefault()); - await fireEvent.click(pmgLink); - expect(onExpandedResourceChange).not.toHaveBeenCalled(); - }); -}); diff --git a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts index e46e554bff..0df9376977 100644 --- a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts +++ b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts @@ -8,6 +8,8 @@ import { toAgentFromResource, toNodeFromProxmox, } from '@/components/Infrastructure/resourceDetailMappers'; +import resourceDetailMappersSource from '@/components/Infrastructure/resourceDetailMappers.ts?raw'; +import resourceDetailDiscoveryModelSource from '@/components/Infrastructure/resourceDetailDiscoveryModel.ts?raw'; import type { Resource } from '@/types/resource'; const createHybridHostResource = (): Resource => @@ -304,6 +306,41 @@ describe('resourceDetailMappers', () => { }, ]); }); + + it('formats an online boolean metric with an OK status', () => { + expect( + buildCustomSensorRows({ + custom: [ + { + id: 'service_online', + name: 'Checkout', + group: 'Main server', + subgroup: 'Services', + kind: 'boolean', + value: 1, + status: 'ok', + observedAt: '2026-07-30T20:00:00Z', + }, + ], + }), + ).toEqual([ + { + label: 'Main server / Services / Checkout', + value: 'Online', + valueTitle: 'Online · OK', + }, + ]); + }); + }); + + describe('detail row projection boundary', () => { + it('formats temperature, GPU, power, fan and custom rows from the selected resource payload only', () => { + expect(resourceDetailMappersSource).not.toContain('fetch('); + expect(resourceDetailMappersSource).not.toContain('nvidia-smi'); + expect(resourceDetailMappersSource).not.toContain('powercap'); + expect(resourceDetailMappersSource).not.toContain('export const toDiscoveryConfig'); + expect(resourceDetailDiscoveryModelSource).toContain('export const toDiscoveryConfig'); + }); }); describe('toNodeFromProxmox', () => { diff --git a/frontend-modern/src/components/Infrastructure/__tests__/unifiedResourceTableModel.branchcov2.test.ts b/frontend-modern/src/components/Infrastructure/__tests__/unifiedResourceTableModel.branchcov2.test.ts deleted file mode 100644 index e69fea5320..0000000000 --- a/frontend-modern/src/components/Infrastructure/__tests__/unifiedResourceTableModel.branchcov2.test.ts +++ /dev/null @@ -1,340 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { Resource } from '@/types/resource'; -import type { IODistributionStats } from '@/components/Infrastructure/infrastructureSelectors'; -import { - getOutlierEmphasis, - getPBSTableRow, - getPMGTableRow, -} from '@/components/Infrastructure/unifiedResourceTableModel'; - -/** - * Branch-coverage suite for the still-uncovered exported functions in - * unifiedResourceTableModel.ts: getPBSTableRow, getPMGTableRow, getOutlierEmphasis. - * Each test drives a specific branch (guard, ternary arm, optional-chain outcome) - * and asserts on the concrete returned shape — never truthiness alone. - */ - -const makeResource = (overrides: Partial & { id: string }): Resource => - ({ - type: 'agent', - name: `name-${overrides.id}`, - displayName: `Display ${overrides.id}`, - platformId: `platform-${overrides.id}`, - platformType: 'proxmox-pve', - sourceType: 'hybrid', - status: 'online', - lastSeen: 1_700_000_000_000, - platformData: {}, - ...overrides, - }) as unknown as Resource; - -const makeStats = (overrides: Partial = {}): IODistributionStats => ({ - median: 0, - mad: 0, - max: 100, - p97: 90, - p99: 95, - count: 10, - ...overrides, -}); - -describe('getPBSTableRow', () => { - it('returns null when resource.type is not pbs (guard)', () => { - expect(getPBSTableRow(makeResource({ id: 'a', type: 'pmg' }))).toBeNull(); - expect(getPBSTableRow(makeResource({ id: 'b', type: 'agent' }))).toBeNull(); - }); - - it('nulls every derived field when platformData has no pbs block (optional-chain miss)', () => { - const row = getPBSTableRow( - makeResource({ id: 'pbs-empty', type: 'pbs', status: 'online', platformData: {} }), - ); - expect(row).toEqual({ - datastores: null, - jobs: null, - activity: null, - activityDetail: null, - activeTaskCount: 0, - health: null, - tone: 'ok', - }); - }); - - it('surfaces a positive datastoreCount and jobs total, trims health, and derives tone from health', () => { - const row = getPBSTableRow( - makeResource({ - id: 'pbs-full', - type: 'pbs', - status: 'degraded', - platformData: { - pbs: { datastoreCount: 3, backupJobCount: 2, connectionHealth: ' Healthy ' }, - }, - }), - ); - expect(row).toEqual({ - datastores: 3, - jobs: 2, - activity: '2 jobs', - activityDetail: null, - activeTaskCount: 0, - health: 'Healthy', - tone: 'ok', - }); - }); - - it('nulls datastores/jobs for zero counts and maps whitespace health + offline status to a warning tone', () => { - const row = getPBSTableRow( - makeResource({ - id: 'pbs-zero', - type: 'pbs', - status: 'offline', - platformData: { pbs: { datastoreCount: 0, connectionHealth: ' ' } }, - }), - ); - expect(row).toEqual({ - datastores: null, - jobs: null, - activity: null, - activityDetail: null, - activeTaskCount: 0, - health: null, - tone: 'warning', - }); - }); -}); - -describe('getPMGTableRow', () => { - it('returns null when resource.type is not pmg (guard)', () => { - expect(getPMGTableRow(makeResource({ id: 'a', type: 'pbs' }))).toBeNull(); - expect(getPMGTableRow(makeResource({ id: 'b', type: 'agent' }))).toBeNull(); - }); - - it('nulls every derived field when platformData has no pmg block (optional-chain miss)', () => { - const row = getPMGTableRow( - makeResource({ id: 'pmg-empty', type: 'pmg', status: 'online', platformData: {} }), - ); - expect(row).toEqual({ - queue: null, - deferred: null, - hold: null, - nodes: null, - health: null, - tone: 'ok', - }); - }); - - it('reports positive queue/deferred/hold/nodes and forces a warning tone when backlog > 0', () => { - const row = getPMGTableRow( - makeResource({ - id: 'pmg-full', - type: 'pmg', - status: 'online', - platformData: { - pmg: { - queueTotal: 12, - queueDeferred: 3, - queueHold: 2, - nodeCount: 4, - connectionHealth: 'degraded', - }, - }, - }), - ); - expect(row).toEqual({ - queue: 12, - deferred: 3, - hold: 2, - nodes: 4, - health: 'degraded', - tone: 'warning', - }); - }); - - it('nulls zero counts, trims whitespace health to null, and derives tone from status when backlog is 0', () => { - const row = getPMGTableRow( - makeResource({ - id: 'pmg-zero', - type: 'pmg', - status: 'warning', - platformData: { - pmg: { - queueTotal: 5, - queueDeferred: 0, - queueHold: 0, - nodeCount: 0, - connectionHealth: ' ', - }, - }, - }), - ); - expect(row).toEqual({ - queue: 5, - deferred: null, - hold: null, - nodes: null, - health: null, - tone: 'warning', - }); - }); - - it('nulls queue when queueTotal is explicitly 0 with a present pmg block', () => { - const row = getPMGTableRow( - makeResource({ - id: 'pmg-no-queue', - type: 'pmg', - status: 'online', - platformData: { pmg: { queueTotal: 0, nodeCount: 1 } }, - }), - ); - expect(row).toEqual({ - queue: null, - deferred: null, - hold: null, - nodes: 1, - health: null, - tone: 'ok', - }); - }); -}); - -describe('getOutlierEmphasis', () => { - describe('guard branch (!isFinite(value) || value <= 0 || stats.max <= 0)', () => { - it('mutes a non-finite value (NaN)', () => { - expect(getOutlierEmphasis(Number.NaN, makeStats({ max: 100 }))).toEqual({ - className: 'text-muted', - showOutlierHint: false, - }); - }); - - it('mutes a non-finite value (Infinity)', () => { - expect(getOutlierEmphasis(Number.POSITIVE_INFINITY, makeStats({ max: 100 }))).toEqual({ - className: 'text-muted', - showOutlierHint: false, - }); - }); - - it('mutes a value of exactly 0', () => { - expect(getOutlierEmphasis(0, makeStats({ max: 100 }))).toEqual({ - className: 'text-muted', - showOutlierHint: false, - }); - }); - - it('mutes a negative value', () => { - expect(getOutlierEmphasis(-5, makeStats({ max: 100 }))).toEqual({ - className: 'text-muted', - showOutlierHint: false, - }); - }); - - it('mutes when stats.max is 0', () => { - expect(getOutlierEmphasis(50, makeStats({ max: 0 }))).toEqual({ - className: 'text-muted', - showOutlierHint: false, - }); - }); - - it('mutes when stats.max is negative', () => { - expect(getOutlierEmphasis(50, makeStats({ max: -1 }))).toEqual({ - className: 'text-muted', - showOutlierHint: false, - }); - }); - }); - - describe('small-sample branch (stats.count < 4)', () => { - it('emphasizes when value/max ratio >= 0.995 (exact boundary)', () => { - expect(getOutlierEmphasis(99.5, makeStats({ max: 100, count: 3 }))).toEqual({ - className: 'text-base-content font-medium', - showOutlierHint: true, - }); - }); - - it('emphasizes a ratio of 1.0', () => { - expect(getOutlierEmphasis(100, makeStats({ max: 100, count: 3 }))).toEqual({ - className: 'text-base-content font-medium', - showOutlierHint: true, - }); - }); - - it('mutes when value/max ratio < 0.995', () => { - expect(getOutlierEmphasis(80, makeStats({ max: 100, count: 3 }))).toEqual({ - className: 'text-muted', - showOutlierHint: false, - }); - }); - }); - - describe('mad-positive branch (stats.count >= 4, stats.mad > 0)', () => { - it('returns semibold for a severe outlier (modifiedZ >= 6.5 and value >= p99)', () => { - // modifiedZ = 0.6745 * (200 - 50) / 2 = 50.5875 (>= 6.5); value 200 >= p99 180 - const stats = makeStats({ median: 50, mad: 2, max: 200, p97: 150, p99: 180, count: 10 }); - expect(getOutlierEmphasis(200, stats)).toEqual({ - className: 'text-base-content font-semibold', - showOutlierHint: true, - }); - }); - - it('returns medium when modifiedZ is in [5.5, 6.5) and value >= p97 but < p99', () => { - // modifiedZ = 0.6745 * (90 - 0) / 10 = 6.0705 (in [5.5, 6.5)); value 90 >= p97 80, < p99 200 - const stats = makeStats({ median: 0, mad: 10, max: 300, p97: 80, p99: 200, count: 10 }); - expect(getOutlierEmphasis(90, stats)).toEqual({ - className: 'text-base-content font-medium', - showOutlierHint: true, - }); - }); - - it('mutes a non-outlier whose modifiedZ is below 5.5', () => { - // modifiedZ = 0.6745 * (55 - 50) / 10 = 0.337 (< 5.5) - const stats = makeStats({ median: 50, mad: 10, max: 200, p97: 80, p99: 180, count: 10 }); - expect(getOutlierEmphasis(55, stats)).toEqual({ - className: 'text-muted', - showOutlierHint: false, - }); - }); - - it('falls through to medium when modifiedZ >= 6.5 but value < p99 (skips severe, hits moderate)', () => { - // modifiedZ = 0.6745 * (70 - 50) / 2 = 6.745 (>= 6.5); value 70 >= p99 80 is FALSE; - // value 70 >= p97 60 is TRUE -> moderate path. - const stats = makeStats({ median: 50, mad: 2, max: 200, p97: 60, p99: 80, count: 10 }); - expect(getOutlierEmphasis(70, stats)).toEqual({ - className: 'text-base-content font-medium', - showOutlierHint: true, - }); - }); - }); - - describe('mad-zero fallback branch (stats.count >= 4, stats.mad <= 0)', () => { - it('returns semibold when value >= p99', () => { - const stats = makeStats({ mad: 0, max: 300, p97: 120, p99: 150, count: 10 }); - expect(getOutlierEmphasis(200, stats)).toEqual({ - className: 'text-base-content font-semibold', - showOutlierHint: true, - }); - }); - - it('returns medium when value >= p97 but < p99', () => { - const stats = makeStats({ mad: 0, max: 300, p97: 120, p99: 150, count: 10 }); - expect(getOutlierEmphasis(130, stats)).toEqual({ - className: 'text-base-content font-medium', - showOutlierHint: true, - }); - }); - - it('mutes a positive value below p97', () => { - const stats = makeStats({ mad: 0, max: 300, p97: 120, p99: 150, count: 10 }); - expect(getOutlierEmphasis(50, stats)).toEqual({ - className: 'text-muted', - showOutlierHint: false, - }); - }); - - it('honors an explicitly negative mad identically to mad === 0', () => { - // Defensive: mad <= 0 takes the fallback path; -1 must behave like 0. - const stats = makeStats({ mad: -1, max: 300, p97: 120, p99: 150, count: 10 }); - expect(getOutlierEmphasis(200, stats)).toEqual({ - className: 'text-base-content font-semibold', - showOutlierHint: true, - }); - }); - }); -}); diff --git a/frontend-modern/src/components/Infrastructure/__tests__/unifiedResourceTableStateModel.branchcov0724pm.test.ts b/frontend-modern/src/components/Infrastructure/__tests__/unifiedResourceTableStateModel.branchcov0724pm.test.ts deleted file mode 100644 index b224899b81..0000000000 --- a/frontend-modern/src/components/Infrastructure/__tests__/unifiedResourceTableStateModel.branchcov0724pm.test.ts +++ /dev/null @@ -1,173 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { Resource } from '@/types/resource'; -import { - UNIFIED_RESOURCE_TABLE_DEFAULT_LAYOUT_WIDTH, - buildHostRowIndexById, - buildHostTableItems, - getHostRevealTargetIndex, - getHostSpacerHeights, - getNextUnifiedResourceTableSortState, - normalizeUnifiedResourceTableLayoutWidth, - shouldShowClusterGroupTypeLabel, - sortServiceResources, -} from '@/components/Infrastructure/unifiedResourceTableStateModel'; - -/** - * Branch-coverage additions for unifiedResourceTableStateModel.ts targeting the - * arms left cold by the existing happy-path spec - * (unifiedResourceTableStateModel.test.ts). - * - * Each test drives a specific measured-cold branch (default fallback, map miss, - * nullish coalescing RHS, non-windowed early return, ternary consequent) and - * asserts on the concrete returned value — never a tautology. - * - * The makeResource fixture mirrors the existing spec's builder so import style - * and field shape stay consistent. - */ - -const makeResource = (id: string, overrides: Partial = {}): Resource => - ({ - id, - type: 'agent', - name: `name-${id}`, - displayName: `Display ${id}`, - platformId: `platform-${id}`, - platformType: 'proxmox-pve', - sourceType: 'api', - status: 'online', - lastSeen: Date.now(), - platformData: { sources: ['proxmox'] }, - ...overrides, - }) as Resource; - -describe('normalizeUnifiedResourceTableLayoutWidth — default-fallback arm', () => { - // Drives branch 6 arm 0 (lines 69-71): the trailing - // `return UNIFIED_RESOURCE_TABLE_DEFAULT_LAYOUT_WIDTH;` reached only when the - // width argument is invalid AND the fallback is also invalid. The existing - // spec only exercises a valid width or an invalid width with a valid - // fallback, so the final default return is never reached. - it('falls back to the default constant when the width is absent and the fallback is NaN', () => { - expect(normalizeUnifiedResourceTableLayoutWidth(null, Number.NaN)).toBe( - UNIFIED_RESOURCE_TABLE_DEFAULT_LAYOUT_WIDTH, - ); - }); - - it('falls back to the default constant when the width is absent and the fallback is 0', () => { - expect(normalizeUnifiedResourceTableLayoutWidth(null, 0)).toBe( - UNIFIED_RESOURCE_TABLE_DEFAULT_LAYOUT_WIDTH, - ); - }); -}); - -describe('getHostRevealTargetIndex — map-miss returns null', () => { - // Drives branch 31 arm 0 (line 150: `rowIndexById.get(targetId) ?? null`): - // the LHS of `??` is undefined because targetId is absent from the map. The - // existing spec always resolves a present id, so the nullish-coalescing RHS - // is never taken. - const items = buildHostTableItems( - [{ cluster: 'cluster-a', resources: [makeResource('a'), makeResource('b')] }], - 'grouped', - ); - const rowIndexById = buildHostRowIndexById(items); - - it('returns null when expandedResourceId is not present in the row index', () => { - // rowIndexById only holds 'a' (1) and 'b' (2); 'missing' is absent. - expect(getHostRevealTargetIndex(rowIndexById, 'missing', null)).toBeNull(); - }); - - it('returns null when revealedResourceId is the chosen target but absent from the row index', () => { - // expandedResourceId is null so revealedResourceId 'ghost' wins the chain, - // then misses the map lookup. - expect(getHostRevealTargetIndex(rowIndexById, null, null, 'ghost')).toBeNull(); - }); -}); - -describe('shouldShowClusterGroupTypeLabel — nullish clusterLabel arm', () => { - // Drives branch 35 arm 0 (line 166: `(clusterLabel ?? '')`): the RHS of `??` - // is only evaluated when clusterLabel is null/undefined. The existing spec - // passes only concrete strings, so the coalescing RHS stays cold. - it('returns false for a null clusterLabel without throwing', () => { - expect(shouldShowClusterGroupTypeLabel(null)).toBe(false); - }); - - it('returns false for an undefined clusterLabel without throwing', () => { - expect(shouldShowClusterGroupTypeLabel(undefined)).toBe(false); - }); -}); - -describe('getHostSpacerHeights — non-windowed early-return arm', () => { - // Drives branch 38 arm 0 (lines 177-179): the `if (!isWindowed)` consequent - // that returns `{ top: 0, bottom: 0 }`. The existing spec always calls with - // isWindowed = true, so the zero-spacer early return is never produced. - it('returns zero spacers regardless of window bounds when windowing is disabled', () => { - const result = getHostSpacerHeights(100, 5, 20, false); - expect(result).toEqual({ top: 0, bottom: 0 }); - }); - - it('returns zero spacers even when a custom row height is supplied', () => { - // Discriminator: a custom estimatedRowHeight (40) would otherwise scale the - // top spacer to 5 * 40 = 200; the early return suppresses it entirely. - const result = getHostSpacerHeights(100, 5, 20, false, 40); - expect(result).toEqual({ top: 0, bottom: 0 }); - expect(result.top).not.toBe(200); - }); -}); - -describe('getNextUnifiedResourceTableSortState — asc default for name/source', () => { - // Drives branch 43 arm 0 (line 204: the `'asc'` consequent of - // `nextKey === 'name' || nextKey === 'source' ? 'asc' : 'desc'`). The existing - // spec only enters the function with currentKey === nextKey (so it never - // reaches line 204) or with nextKey 'cpu' (which takes the 'desc' alternate). - it('initializes a name sort to ascending when switching from a different key', () => { - expect(getNextUnifiedResourceTableSortState('default', 'asc', 'name')).toEqual({ - key: 'name', - direction: 'asc', - }); - }); - - it('initializes a source sort to ascending when switching from a different key', () => { - // Also drives the second operand of the `||` (`nextKey === 'source'`) true. - expect(getNextUnifiedResourceTableSortState('cpu', 'desc', 'source')).toEqual({ - key: 'source', - direction: 'asc', - }); - }); -}); - -describe('sortServiceResources — uncovered exported function', () => { - // Drives fn 15 (sortServiceResources, line 217): the only exported function - // the existing spec never invokes. It must (a) filter to the requested - // service type and (b) delegate to sortResources with 'default'/'asc', which - // ranks online resources first then by display name. Assertions are made on - // the concrete id sequence so they cannot pass against an unfiltered or - // unsorted list. - const services: Resource[] = [ - makeResource('zebra', { type: 'pbs', displayName: 'Zeta', status: 'offline' }), - makeResource('alpha', { type: 'pbs', displayName: 'Alpha', status: 'online' }), - makeResource('mango', { type: 'pbs', displayName: 'Mango', status: 'online' }), - makeResource('pmg-only', { type: 'pmg', displayName: 'Should Be Filtered', status: 'online' }), - makeResource('agent-only', { type: 'agent', displayName: 'Also Filtered', status: 'online' }), - ]; - - it('returns only pbs resources, online-first then by display name', () => { - const result = sortServiceResources(services, 'pbs'); - expect(result.map((r) => r.id)).toEqual(['alpha', 'mango', 'zebra']); - expect(result.every((r) => r.type === 'pbs')).toBe(true); - }); - - it('returns only pmg resources when type is pmg', () => { - const result = sortServiceResources(services, 'pmg'); - expect(result.map((r) => r.id)).toEqual(['pmg-only']); - expect(result.every((r) => r.type === 'pmg')).toBe(true); - }); - - it('returns an empty array when the input list is empty', () => { - expect(sortServiceResources([], 'pbs')).toEqual([]); - }); - - it('does not mutate the input array', () => { - const snapshot = services.map((r) => r.id); - sortServiceResources(services, 'pbs'); - expect(services.map((r) => r.id)).toEqual(snapshot); - }); -}); diff --git a/frontend-modern/src/components/Infrastructure/__tests__/unifiedResourceTableStateModel.test.ts b/frontend-modern/src/components/Infrastructure/__tests__/unifiedResourceTableStateModel.test.ts deleted file mode 100644 index 1e028a9764..0000000000 --- a/frontend-modern/src/components/Infrastructure/__tests__/unifiedResourceTableStateModel.test.ts +++ /dev/null @@ -1,226 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { Resource } from '@/types/resource'; -import type { ResourceGroup } from '@/components/Infrastructure/infrastructureSelectors'; -import { - buildHostRowIndexById, - buildHostTableItems, - buildResourceLabelById, - getHostRevealTargetIndex, - getHostSpacerHeights, - getNextUnifiedResourceTableSortState, - getUnifiedResourceTableColumnPresentations, - getUnifiedResourceTableHeaderLabels, - getUnifiedResourceTableLayoutMode, - getUnifiedResourceTableShellClass, - getUnifiedResourceTableSortIndicator, - getUnifiedSources, - getVisibleHostTableItems, - isUnifiedResourceHostDiskIoVisible, - isUnifiedResourceServiceColumnVisible, - isUnifiedResourceTableColumnVisible, - normalizeUnifiedResourceTableLayoutWidth, - shouldShowClusterGroupTypeLabel, - shouldShowUnifiedResourceHostTable, - shouldUseUnifiedResourceTableMobileLayout, -} from '@/components/Infrastructure/unifiedResourceTableStateModel'; - -const makeResource = (id: string, overrides: Partial = {}): Resource => - ({ - id, - type: 'agent', - name: `name-${id}`, - displayName: `Display ${id}`, - platformId: `platform-${id}`, - platformType: 'proxmox-pve', - sourceType: 'api', - status: 'online', - lastSeen: Date.now(), - platformData: { sources: ['proxmox'] }, - ...overrides, - }) as Resource; - -describe('unifiedResourceTableStateModel', () => { - it('builds resource labels from the canonical preferred display name', () => { - const labels = buildResourceLabelById([ - makeResource('a', { displayName: 'Alpha' }), - makeResource('b', { displayName: 'Beta' }), - ]); - - expect(labels.get('a')).toBe('Alpha'); - expect(labels.get('b')).toBe('Beta'); - }); - - it('builds host table items with group headers only in grouped mode', () => { - const groups: ResourceGroup[] = [ - { - cluster: 'cluster-a', - resources: [makeResource('a'), makeResource('b')], - }, - ]; - - expect(buildHostTableItems(groups, 'grouped').map((item) => item.type)).toEqual([ - 'header', - 'row', - 'row', - ]); - expect(buildHostTableItems(groups, 'flat').map((item) => item.type)).toEqual(['row', 'row']); - }); - - it('suppresses duplicate cluster type labels when the group name already ends with cluster', () => { - expect(shouldShowClusterGroupTypeLabel('Core Fabric')).toBe(true); - expect(shouldShowClusterGroupTypeLabel('Production Cluster')).toBe(false); - expect(shouldShowClusterGroupTypeLabel('production cluster')).toBe(false); - expect(shouldShowClusterGroupTypeLabel('')).toBe(false); - }); - - it('tracks reveal indexes only for row items', () => { - const items = buildHostTableItems( - [{ cluster: 'cluster-a', resources: [makeResource('a'), makeResource('b')] }], - 'grouped', - ); - const rowIndexById = buildHostRowIndexById(items); - - expect(getHostRevealTargetIndex(rowIndexById, 'b', null)).toBe(2); - expect(getHostRevealTargetIndex(rowIndexById, null, 'a')).toBe(1); - expect(getHostRevealTargetIndex(rowIndexById, null, null)).toBeNull(); - }); - - it('computes visible items and spacer heights from the active window', () => { - const items = buildHostTableItems( - [{ cluster: '', resources: [makeResource('a'), makeResource('b'), makeResource('c')] }], - 'flat', - ); - - expect(getVisibleHostTableItems(items, false, 1, 2)).toEqual(items); - expect( - getVisibleHostTableItems(items, true, 1, 3).map((item) => - item.type === 'row' ? item.resource.id : item.type, - ), - ).toEqual(['b', 'c']); - expect(getHostSpacerHeights(items.length, 1, 3, true, 40)).toEqual({ - top: 40, - bottom: 0, - }); - }); - - it('cycles sort state through asc, desc, then default', () => { - expect(getNextUnifiedResourceTableSortState('default', 'asc', 'cpu')).toEqual({ - key: 'cpu', - direction: 'desc', - }); - expect(getNextUnifiedResourceTableSortState('cpu', 'desc', 'cpu')).toEqual({ - key: 'default', - direction: 'asc', - }); - expect(getNextUnifiedResourceTableSortState('name', 'asc', 'name')).toEqual({ - key: 'name', - direction: 'desc', - }); - expect(getUnifiedResourceTableSortIndicator('name', 'asc', 'name')).toBe('▲'); - expect(getUnifiedResourceTableSortIndicator('cpu', 'desc', 'cpu')).toBe('▼'); - expect(getUnifiedResourceTableSortIndicator('default', 'asc', 'cpu')).toBeNull(); - }); - - it('derives responsive column presentations and host-table visibility as pure layout policy', () => { - const mobileColumns = getUnifiedResourceTableColumnPresentations('mobile'); - const narrowTabletColumns = getUnifiedResourceTableColumnPresentations('tablet', 760); - const tabletColumns = getUnifiedResourceTableColumnPresentations('tablet', 820); - const compactColumns = getUnifiedResourceTableColumnPresentations('compact'); - const wideColumns = getUnifiedResourceTableColumnPresentations('wide'); - - expect(getUnifiedResourceTableShellClass('mobile')).toContain('table-fixed'); - expect(getUnifiedResourceTableShellClass('mobile')).toContain('min-w-full'); - expect(getUnifiedResourceTableShellClass('mobile')).not.toContain('min-w-[640px]'); - expect(getUnifiedResourceTableShellClass('compact')).toContain('min-w-[640px]'); - expect(getUnifiedResourceTableShellClass('wide')).toContain('min-w-[640px]'); - expect(getUnifiedResourceTableShellClass('wide')).not.toContain('min-w-max'); - // Mobile keeps the prioritized column set but preserves a readable width - // floor inside the shared horizontal-scroll shell. Wider modes keep all - // host columns visible while compressing their tracks before any - // lower-priority columns are dropped. - expect(mobileColumns.resourceColumn.width).toBe('34%'); - expect(mobileColumns.metricColumn.width).toBe('14%'); - expect(mobileColumns.ioColumn.width).toBe('12%'); - expect(mobileColumns.uptimeColumn.width).toBe('12%'); - expect(mobileColumns.serviceResourceColumn.width).toBe('28%'); - expect(mobileColumns.serviceSourceColumn.width).toBe('10%'); - expect(mobileColumns.serviceCountColumn.width).toBe('11%'); - expect(mobileColumns.serviceHealthColumn.width).toBe('13%'); - expect(mobileColumns.serviceActionColumn.width).toBe('14%'); - expect(narrowTabletColumns.resourceColumn.width).toBe('34%'); - expect(narrowTabletColumns.ioColumn.width).toBe('14%'); - expect(narrowTabletColumns.sourceColumn.width).toBe('13%'); - expect(isUnifiedResourceHostDiskIoVisible(799)).toBe(false); - expect(isUnifiedResourceHostDiskIoVisible(800)).toBe(true); - expect(tabletColumns.resourceColumn.width).toBe('28%'); - expect(tabletColumns.serviceResourceColumn.width).toBe('24%'); - expect(tabletColumns.metricColumn.width).toBe('12%'); - expect(tabletColumns.ioColumn.width).toBe('12%'); - expect(tabletColumns.sourceColumn.width).toBe('12%'); - expect(tabletColumns.serviceSourceColumn.width).toBe('8%'); - expect(compactColumns.resourceColumn.width).toBe('16%'); - expect(compactColumns.serviceResourceColumn.width).toBe('18%'); - expect(compactColumns.metricColumn.width).toBe('10%'); - expect(compactColumns.sourceColumn.width).toBe('13%'); - expect(compactColumns.serviceSourceColumn.width).toBe('9.5%'); - expect(compactColumns.tempColumn.width).toBe('6.5%'); - expect(wideColumns.resourceColumn.width).toBe('18%'); - expect(wideColumns.serviceResourceColumn.width).toBe('18%'); - expect(wideColumns.serviceSourceColumn.width).toBe('10%'); - expect(wideColumns.ioColumn.width).toBe('12.5%'); - expect(wideColumns.serviceActionColumn.width).toBe('16%'); - expect(getUnifiedResourceTableHeaderLabels('wide').memory).toBe('Memory'); - expect(getUnifiedResourceTableHeaderLabels('wide').source).toBe('System'); - expect(getUnifiedResourceTableHeaderLabels('compact').memory).toBe('Mem'); - expect(getUnifiedResourceTableHeaderLabels('compact').source).toBe('System'); - expect(getUnifiedResourceTableHeaderLabels('tablet').network).toBe('Net'); - expect(getUnifiedResourceTableHeaderLabels('tablet').source).toBe('Sys'); - expect(getUnifiedResourceTableHeaderLabels('mobile').datastores).toBe('Store'); - expect(shouldShowUnifiedResourceHostTable(0, 0)).toBe(true); - expect(shouldShowUnifiedResourceHostTable(0, 2)).toBe(false); - expect(shouldShowUnifiedResourceHostTable(3, 2)).toBe(true); - }); - - it('derives infrastructure table breakpoints from the measured table surface width', () => { - expect(normalizeUnifiedResourceTableLayoutWidth(820.4)).toBe(820); - expect(normalizeUnifiedResourceTableLayoutWidth(null, 700)).toBe(700); - expect(getUnifiedResourceTableLayoutMode(699)).toBe('mobile'); - expect(getUnifiedResourceTableLayoutMode(700)).toBe('tablet'); - expect(getUnifiedResourceTableLayoutMode(899)).toBe('tablet'); - expect(getUnifiedResourceTableLayoutMode(900)).toBe('compact'); - expect(getUnifiedResourceTableLayoutMode(1159)).toBe('compact'); - expect(getUnifiedResourceTableLayoutMode(1160)).toBe('wide'); - expect(shouldUseUnifiedResourceTableMobileLayout(699)).toBe(true); - expect(shouldUseUnifiedResourceTableMobileLayout(700)).toBe(false); - expect(isUnifiedResourceTableColumnVisible('primary', 640)).toBe(true); - expect(isUnifiedResourceTableColumnVisible('secondary', 699)).toBe(false); - expect(isUnifiedResourceTableColumnVisible('secondary', 700)).toBe(true); - expect(isUnifiedResourceTableColumnVisible('supplementary', 899)).toBe(false); - expect(isUnifiedResourceTableColumnVisible('supplementary', 900)).toBe(true); - expect(isUnifiedResourceTableColumnVisible('detailed', 1159)).toBe(false); - expect(isUnifiedResourceTableColumnVisible('detailed', 1160)).toBe(true); - expect(isUnifiedResourceServiceColumnVisible('primary', 499)).toBe(false); - expect(isUnifiedResourceServiceColumnVisible('primary', 500)).toBe(true); - expect(isUnifiedResourceServiceColumnVisible('secondary', 579)).toBe(false); - expect(isUnifiedResourceServiceColumnVisible('secondary', 580)).toBe(true); - expect(isUnifiedResourceServiceColumnVisible('supplementary', 639)).toBe(false); - expect(isUnifiedResourceServiceColumnVisible('supplementary', 640)).toBe(true); - expect(isUnifiedResourceServiceColumnVisible('detailed', 899)).toBe(false); - expect(isUnifiedResourceServiceColumnVisible('detailed', 900)).toBe(true); - }); - - it('reads unified source tags from platform data without hook state', () => { - expect( - getUnifiedSources(makeResource('a', { platformData: { sources: ['proxmox', 'agent'] } })), - ).toEqual(['proxmox', 'agent']); - expect( - getUnifiedSources( - makeResource('b', { - sources: ['agent', 'docker'], - platformData: { sources: ['kubernetes'] }, - }), - ), - ).toEqual(['agent', 'docker']); - expect(getUnifiedSources(makeResource('c', { platformData: {} }))).toEqual([]); - }); -}); diff --git a/frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts b/frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts deleted file mode 100644 index 5567ad386a..0000000000 --- a/frontend-modern/src/components/Infrastructure/unifiedResourceTableModel.ts +++ /dev/null @@ -1,108 +0,0 @@ -import type { Resource } from '@/types/resource'; -import { getServiceHealthSummaryPresentation } from '@/utils/serviceHealthPresentation'; -import type { IODistributionStats } from '@/components/Infrastructure/infrastructureSelectors'; -import type { PBSPlatformData, PMGPlatformData } from './resourceDetailMappers'; -import { getPbsActivitySummary, getPbsJobTotal } from './resourceDetailDrawerServiceModel'; - -type PBSServiceData = PBSPlatformData; -type PMGServiceData = PMGPlatformData; - -export type PBSTableRow = { - datastores: number | null; - jobs: number | null; - activity: string | null; - activityDetail: string | null; - activeTaskCount: number; - health: string | null; - tone: ReturnType['tone']; -}; - -export type PMGTableRow = { - queue: number | null; - deferred: number | null; - hold: number | null; - nodes: number | null; - health: string | null; - tone: ReturnType['tone']; -}; - -export interface IOEmphasis { - className: string; - showOutlierHint: boolean; -} - -export const isResourceOnline = (resource: Resource) => { - const status = resource.status?.toLowerCase(); - return status !== 'offline' && status !== 'stopped'; -}; - -export const getPBSTableRow = (resource: Resource): PBSTableRow | null => { - if (resource.type !== 'pbs') return null; - const platformData = resource.platformData as - { pbs?: PBSServiceData; pmg?: PMGServiceData } | undefined; - const pbs = platformData?.pbs; - const totalJobs = getPbsJobTotal(pbs); - const activitySummary = getPbsActivitySummary(pbs); - const health = pbs?.connectionHealth?.trim() || null; - - return { - datastores: (pbs?.datastoreCount || 0) > 0 ? pbs?.datastoreCount || 0 : null, - jobs: totalJobs > 0 ? totalJobs : null, - activity: activitySummary.label, - activityDetail: activitySummary.detail, - activeTaskCount: activitySummary.activeTaskCount, - health, - tone: getServiceHealthSummaryPresentation(resource.status, health).tone, - }; -}; - -export const getPMGTableRow = (resource: Resource): PMGTableRow | null => { - if (resource.type !== 'pmg') return null; - const platformData = resource.platformData as - { pbs?: PBSServiceData; pmg?: PMGServiceData } | undefined; - const pmg = platformData?.pmg; - const health = pmg?.connectionHealth?.trim() || null; - const backlog = (pmg?.queueDeferred || 0) + (pmg?.queueHold || 0); - - return { - queue: (pmg?.queueTotal || 0) > 0 ? pmg?.queueTotal || 0 : null, - deferred: (pmg?.queueDeferred || 0) > 0 ? pmg?.queueDeferred || 0 : null, - hold: (pmg?.queueHold || 0) > 0 ? pmg?.queueHold || 0 : null, - nodes: (pmg?.nodeCount || 0) > 0 ? pmg?.nodeCount || 0 : null, - health, - tone: - backlog > 0 ? 'warning' : getServiceHealthSummaryPresentation(resource.status, health).tone, - }; -}; - -export const getOutlierEmphasis = (value: number, stats: IODistributionStats): IOEmphasis => { - if (!Number.isFinite(value) || value <= 0 || stats.max <= 0) { - return { className: 'text-muted', showOutlierHint: false }; - } - - if (stats.count < 4) { - const ratio = value / stats.max; - if (ratio >= 0.995) { - return { className: 'text-base-content font-medium', showOutlierHint: true }; - } - return { className: 'text-muted', showOutlierHint: false }; - } - - if (stats.mad > 0) { - const modifiedZ = (0.6745 * (value - stats.median)) / stats.mad; - if (modifiedZ >= 6.5 && value >= stats.p99) { - return { className: 'text-base-content font-semibold', showOutlierHint: true }; - } - if (modifiedZ >= 5.5 && value >= stats.p97) { - return { className: 'text-base-content font-medium', showOutlierHint: true }; - } - return { className: 'text-muted', showOutlierHint: false }; - } - - if (value >= stats.p99) - return { className: 'text-base-content font-semibold', showOutlierHint: true }; - if (value >= stats.p97) - return { className: 'text-base-content font-medium', showOutlierHint: true }; - if (value > 0) return { className: 'text-muted', showOutlierHint: false }; - return { className: 'text-muted', showOutlierHint: false }; -}; diff --git a/frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts b/frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts deleted file mode 100644 index 86ae9b087b..0000000000 --- a/frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts +++ /dev/null @@ -1,449 +0,0 @@ -import type { Resource } from '@/types/resource'; -import type { ColumnPriority } from '@/hooks/useBreakpoint'; -import { getPreferredInfrastructureDisplayName } from '@/utils/resourceIdentity'; -import { - sortResources, - type ResourceGroup, -} from '@/components/Infrastructure/infrastructureSelectors'; - -export type UnifiedResourceTableSortKey = - | 'default' - | 'name' - | 'uptime' - | 'cpu' - | 'memory' - | 'disk' - | 'network' - | 'diskio' - | 'source' - | 'temp'; - -export type UnifiedResourceTableSortDirection = 'asc' | 'desc'; - -export type HostTableHeaderItem = { - type: 'header'; - group: ResourceGroup; -}; - -export type HostTableResourceItem = { - type: 'row'; - group: ResourceGroup; - resource: Resource; -}; - -export type HostTableItem = HostTableHeaderItem | HostTableResourceItem; - -export const HOST_TABLE_ESTIMATED_ROW_HEIGHT = 40; -export const HOST_TABLE_WINDOW_SIZE = 137; -export const UNIFIED_RESOURCE_TABLE_DEFAULT_LAYOUT_WIDTH = 1024; -export const UNIFIED_RESOURCE_TABLE_MOBILE_LAYOUT_WIDTH = 700; -export const UNIFIED_RESOURCE_TABLE_TABLET_DISK_IO_LAYOUT_WIDTH = 800; -export const UNIFIED_RESOURCE_TABLE_COMPACT_LAYOUT_WIDTH = 900; -export const UNIFIED_RESOURCE_TABLE_WIDE_LAYOUT_WIDTH = 1160; -export const UNIFIED_RESOURCE_TABLE_COLUMN_BREAKPOINTS: Record = { - essential: 0, - primary: 640, - secondary: UNIFIED_RESOURCE_TABLE_MOBILE_LAYOUT_WIDTH, - supplementary: UNIFIED_RESOURCE_TABLE_COMPACT_LAYOUT_WIDTH, - detailed: UNIFIED_RESOURCE_TABLE_WIDE_LAYOUT_WIDTH, -}; -export const UNIFIED_RESOURCE_SERVICE_COLUMN_BREAKPOINTS: Record = { - essential: 0, - primary: 500, - secondary: 580, - supplementary: 640, - detailed: 900, -}; - -export type UnifiedResourceTableLayoutMode = 'mobile' | 'tablet' | 'compact' | 'wide'; - -export const normalizeUnifiedResourceTableLayoutWidth = ( - width: number | null | undefined, - fallback: number = UNIFIED_RESOURCE_TABLE_DEFAULT_LAYOUT_WIDTH, -): number => { - if (typeof width === 'number' && Number.isFinite(width) && width > 0) { - return Math.round(width); - } - if (Number.isFinite(fallback) && fallback > 0) { - return Math.round(fallback); - } - return UNIFIED_RESOURCE_TABLE_DEFAULT_LAYOUT_WIDTH; -}; - -export const getUnifiedResourceTableLayoutMode = ( - layoutWidth: number, -): UnifiedResourceTableLayoutMode => { - const width = normalizeUnifiedResourceTableLayoutWidth(layoutWidth); - - if (width < UNIFIED_RESOURCE_TABLE_MOBILE_LAYOUT_WIDTH) return 'mobile'; - if (width < UNIFIED_RESOURCE_TABLE_COMPACT_LAYOUT_WIDTH) return 'tablet'; - if (width < UNIFIED_RESOURCE_TABLE_WIDE_LAYOUT_WIDTH) return 'compact'; - return 'wide'; -}; - -export const shouldUseUnifiedResourceTableMobileLayout = (layoutWidth: number): boolean => - getUnifiedResourceTableLayoutMode(layoutWidth) === 'mobile'; - -export const isUnifiedResourceTableColumnVisible = ( - priority: ColumnPriority, - layoutWidth: number, -): boolean => - normalizeUnifiedResourceTableLayoutWidth(layoutWidth) >= - UNIFIED_RESOURCE_TABLE_COLUMN_BREAKPOINTS[priority]; - -export const isUnifiedResourceServiceColumnVisible = ( - priority: ColumnPriority, - layoutWidth: number, -): boolean => - normalizeUnifiedResourceTableLayoutWidth(layoutWidth) >= - UNIFIED_RESOURCE_SERVICE_COLUMN_BREAKPOINTS[priority]; - -export const isUnifiedResourceHostDiskIoVisible = (layoutWidth: number): boolean => - normalizeUnifiedResourceTableLayoutWidth(layoutWidth) >= - UNIFIED_RESOURCE_TABLE_TABLET_DISK_IO_LAYOUT_WIDTH; - -export const buildResourceLabelById = (resources: Resource[]): Map => { - const map = new Map(); - for (const resource of resources) { - map.set(resource.id, getPreferredInfrastructureDisplayName(resource)); - } - return map; -}; - -export const buildHostTableItems = ( - groupedResources: ResourceGroup[], - groupingMode: 'grouped' | 'flat' | undefined, -): HostTableItem[] => { - const items: HostTableItem[] = []; - const showGroupHeaders = groupingMode === 'grouped'; - - for (const group of groupedResources) { - if (showGroupHeaders) { - items.push({ type: 'header', group }); - } - for (const resource of group.resources) { - items.push({ type: 'row', group, resource }); - } - } - - return items; -}; - -export const buildHostRowIndexById = (items: HostTableItem[]): Map => { - const map = new Map(); - items.forEach((item, index) => { - if (item.type === 'row') { - map.set(item.resource.id, index); - } - }); - return map; -}; - -export const getHostRevealTargetIndex = ( - rowIndexById: Map, - expandedResourceId: string | null, - highlightedResourceId: string | null | undefined, - revealedResourceId?: string | null, -): number | null => { - const targetId = expandedResourceId ?? revealedResourceId ?? highlightedResourceId ?? null; - if (!targetId) return null; - return rowIndexById.get(targetId) ?? null; -}; - -export const getVisibleHostTableItems = ( - items: HostTableItem[], - isWindowed: boolean, - startIndex: number, - endIndex: number, -): HostTableItem[] => { - if (!isWindowed) return items; - return items.slice(startIndex, endIndex); -}; - -export const shouldShowClusterGroupTypeLabel = ( - clusterLabel: string | null | undefined, -): boolean => { - const normalized = (clusterLabel ?? '').trim().replace(/\s+/g, ' ').toLowerCase(); - return Boolean(normalized) && !/\bcluster$/.test(normalized); -}; - -export const getHostSpacerHeights = ( - totalCount: number, - startIndex: number, - endIndex: number, - isWindowed: boolean, - estimatedRowHeight: number = HOST_TABLE_ESTIMATED_ROW_HEIGHT, -): { top: number; bottom: number } => { - if (!isWindowed) { - return { top: 0, bottom: 0 }; - } - - return { - top: startIndex * estimatedRowHeight, - bottom: Math.max(0, (totalCount - endIndex) * estimatedRowHeight), - }; -}; - -export const getNextUnifiedResourceTableSortState = ( - currentKey: UnifiedResourceTableSortKey, - currentDirection: UnifiedResourceTableSortDirection, - nextKey: Exclude, -): { - key: UnifiedResourceTableSortKey; - direction: UnifiedResourceTableSortDirection; -} => { - if (currentKey === nextKey) { - if (currentDirection === 'asc') { - return { key: nextKey, direction: 'desc' }; - } - return { key: 'default', direction: 'asc' }; - } - - return { - key: nextKey, - direction: nextKey === 'name' || nextKey === 'source' ? 'asc' : 'desc', - }; -}; - -export const getUnifiedResourceTableSortIndicator = ( - activeKey: UnifiedResourceTableSortKey, - activeDirection: UnifiedResourceTableSortDirection, - key: UnifiedResourceTableSortKey, -): '▲' | '▼' | null => { - if (activeKey !== key) return null; - return activeDirection === 'asc' ? '▲' : '▼'; -}; - -export const sortServiceResources = (services: Resource[], type: 'pbs' | 'pmg'): Resource[] => - sortResources( - services.filter((resource) => resource.type === type), - 'default', - 'asc', - ); - -export const shouldShowUnifiedResourceHostTable = ( - primaryResourceCount: number, - serviceResourceCount: number, -): boolean => primaryResourceCount > 0 || serviceResourceCount === 0; - -export const getUnifiedSources = (resource: Resource): string[] => { - if (Array.isArray(resource.sources) && resource.sources.length > 0) { - return resource.sources; - } - const platformData = resource.platformData as { sources?: string[] } | undefined; - return platformData?.sources ?? []; -}; - -export interface UnifiedResourceTableColumnPresentation { - className: string; - width?: string | number; -} - -export type UnifiedResourceTableColumnPresentations = { - resourceColumn: UnifiedResourceTableColumnPresentation; - serviceResourceColumn: UnifiedResourceTableColumnPresentation; - metricColumn: UnifiedResourceTableColumnPresentation; - ioColumn: UnifiedResourceTableColumnPresentation; - sourceColumn: UnifiedResourceTableColumnPresentation; - serviceSourceColumn: UnifiedResourceTableColumnPresentation; - uptimeColumn: UnifiedResourceTableColumnPresentation; - tempColumn: UnifiedResourceTableColumnPresentation; - serviceCountColumn: UnifiedResourceTableColumnPresentation; - serviceQueueColumn: UnifiedResourceTableColumnPresentation; - serviceHealthColumn: UnifiedResourceTableColumnPresentation; - serviceActionColumn: UnifiedResourceTableColumnPresentation; -}; - -export type UnifiedResourceTableHeaderLabels = { - resource: string; - cpu: string; - memory: string; - disk: string; - network: string; - diskIo: string; - source: string; - uptime: string; - temp: string; - datastores: string; - activity: string; - queue: string; - deferred: string; - hold: string; - nodes: string; - health: string; - action: string; -}; - -const buildUnifiedResourceTableColumnPresentation = ( - className: string, - width?: string | number, -): UnifiedResourceTableColumnPresentation => ({ - className, - width, -}); - -export const getUnifiedResourceTableShellClass = ( - layoutMode: UnifiedResourceTableLayoutMode, -): string => - `table-fixed ${layoutMode === 'mobile' ? 'min-w-full' : 'min-w-[640px] sm:min-w-full'}${ - layoutMode === 'wide' ? '' : ' text-[11px] sm:text-xs' - }`; - -export const getUnifiedResourceTableHeaderLabels = ( - layoutMode: UnifiedResourceTableLayoutMode, -): UnifiedResourceTableHeaderLabels => { - if (layoutMode === 'wide') { - return { - resource: 'Resource', - cpu: 'CPU', - memory: 'Memory', - disk: 'Disk', - network: 'Net I/O', - diskIo: 'Disk I/O', - source: 'System', - uptime: 'Uptime', - temp: 'Temp', - datastores: 'Datastores', - activity: 'Activity', - queue: 'Queue', - deferred: 'Deferred', - hold: 'Hold', - nodes: 'Nodes', - health: 'Health', - action: 'Action', - }; - } - - if (layoutMode === 'compact') { - return { - resource: 'Resource', - cpu: 'CPU', - memory: 'Mem', - disk: 'Disk', - network: 'Net I/O', - diskIo: 'Disk I/O', - source: 'System', - uptime: 'Up', - temp: 'Temp', - datastores: 'Stores', - activity: 'Activity', - queue: 'Queue', - deferred: 'Def', - hold: 'Hold', - nodes: 'Nodes', - health: 'Health', - action: 'Action', - }; - } - - return { - resource: 'Resource', - cpu: 'CPU', - memory: 'Mem', - disk: 'Disk', - network: 'Net', - diskIo: 'I/O', - source: 'Sys', - uptime: 'Up', - temp: 'Temp', - datastores: 'Store', - activity: 'Jobs', - queue: 'Queue', - deferred: 'Def', - hold: 'Hold', - nodes: 'Nodes', - health: 'Health', - action: 'Action', - }; -}; - -export const getUnifiedResourceTableColumnPresentations = ( - layoutMode: UnifiedResourceTableLayoutMode, - layoutWidth?: number, -): UnifiedResourceTableColumnPresentations => { - if (layoutMode === 'mobile') { - // Mobile widths are percentages so the prioritized host columns fill the - // viewport without horizontal scroll. Hidden columns keep placeholder - // widths that never render. - return { - resourceColumn: buildUnifiedResourceTableColumnPresentation('', '34%'), - serviceResourceColumn: buildUnifiedResourceTableColumnPresentation('', '28%'), - metricColumn: buildUnifiedResourceTableColumnPresentation('', '14%'), - ioColumn: buildUnifiedResourceTableColumnPresentation('', '12%'), - sourceColumn: buildUnifiedResourceTableColumnPresentation('', '20%'), - serviceSourceColumn: buildUnifiedResourceTableColumnPresentation('', '10%'), - uptimeColumn: buildUnifiedResourceTableColumnPresentation('', '12%'), - tempColumn: buildUnifiedResourceTableColumnPresentation('', '15%'), - serviceCountColumn: buildUnifiedResourceTableColumnPresentation('', '11%'), - serviceQueueColumn: buildUnifiedResourceTableColumnPresentation('', '8%'), - serviceHealthColumn: buildUnifiedResourceTableColumnPresentation('', '13%'), - serviceActionColumn: buildUnifiedResourceTableColumnPresentation('', '14%'), - }; - } - - if (layoutMode === 'tablet') { - if (typeof layoutWidth === 'number' && isUnifiedResourceHostDiskIoVisible(layoutWidth)) { - return { - resourceColumn: buildUnifiedResourceTableColumnPresentation('', '28%'), - serviceResourceColumn: buildUnifiedResourceTableColumnPresentation('', '24%'), - metricColumn: buildUnifiedResourceTableColumnPresentation('', '12%'), - ioColumn: buildUnifiedResourceTableColumnPresentation('', '12%'), - sourceColumn: buildUnifiedResourceTableColumnPresentation('', '12%'), - serviceSourceColumn: buildUnifiedResourceTableColumnPresentation('', '8%'), - uptimeColumn: buildUnifiedResourceTableColumnPresentation('', '7%'), - tempColumn: buildUnifiedResourceTableColumnPresentation('', '6%'), - serviceCountColumn: buildUnifiedResourceTableColumnPresentation('', '8%'), - serviceQueueColumn: buildUnifiedResourceTableColumnPresentation('', '7.5%'), - serviceHealthColumn: buildUnifiedResourceTableColumnPresentation('', '14%'), - serviceActionColumn: buildUnifiedResourceTableColumnPresentation('', '13%'), - }; - } - - return { - resourceColumn: buildUnifiedResourceTableColumnPresentation('', '34%'), - serviceResourceColumn: buildUnifiedResourceTableColumnPresentation('', '24%'), - metricColumn: buildUnifiedResourceTableColumnPresentation('', '13%'), - ioColumn: buildUnifiedResourceTableColumnPresentation('', '14%'), - sourceColumn: buildUnifiedResourceTableColumnPresentation('', '13%'), - serviceSourceColumn: buildUnifiedResourceTableColumnPresentation('', '8%'), - uptimeColumn: buildUnifiedResourceTableColumnPresentation('', '7%'), - tempColumn: buildUnifiedResourceTableColumnPresentation('', '6%'), - serviceCountColumn: buildUnifiedResourceTableColumnPresentation('', '8%'), - serviceQueueColumn: buildUnifiedResourceTableColumnPresentation('', '7.5%'), - serviceHealthColumn: buildUnifiedResourceTableColumnPresentation('', '14%'), - serviceActionColumn: buildUnifiedResourceTableColumnPresentation('', '13%'), - }; - } - - if (layoutMode === 'compact') { - return { - resourceColumn: buildUnifiedResourceTableColumnPresentation('', '16%'), - serviceResourceColumn: buildUnifiedResourceTableColumnPresentation('', '18%'), - metricColumn: buildUnifiedResourceTableColumnPresentation('', '10%'), - ioColumn: buildUnifiedResourceTableColumnPresentation('', '12.5%'), - sourceColumn: buildUnifiedResourceTableColumnPresentation('', '13%'), - serviceSourceColumn: buildUnifiedResourceTableColumnPresentation('', '9.5%'), - uptimeColumn: buildUnifiedResourceTableColumnPresentation('', '7.5%'), - tempColumn: buildUnifiedResourceTableColumnPresentation('', '6.5%'), - serviceCountColumn: buildUnifiedResourceTableColumnPresentation('', '9.5%'), - serviceQueueColumn: buildUnifiedResourceTableColumnPresentation('', '9.5%'), - serviceHealthColumn: buildUnifiedResourceTableColumnPresentation('', '14%'), - serviceActionColumn: buildUnifiedResourceTableColumnPresentation('', '12%'), - }; - } - - return { - resourceColumn: buildUnifiedResourceTableColumnPresentation('', '18%'), - serviceResourceColumn: buildUnifiedResourceTableColumnPresentation('', '18%'), - metricColumn: buildUnifiedResourceTableColumnPresentation('', '10.5%'), - ioColumn: buildUnifiedResourceTableColumnPresentation('', '12.5%'), - sourceColumn: buildUnifiedResourceTableColumnPresentation('', '10%'), - serviceSourceColumn: buildUnifiedResourceTableColumnPresentation('', '10%'), - uptimeColumn: buildUnifiedResourceTableColumnPresentation('', '8%'), - tempColumn: buildUnifiedResourceTableColumnPresentation('', '7.5%'), - serviceCountColumn: buildUnifiedResourceTableColumnPresentation('', '8.5%'), - serviceQueueColumn: buildUnifiedResourceTableColumnPresentation('', '8.5%'), - serviceHealthColumn: buildUnifiedResourceTableColumnPresentation('', '14%'), - serviceActionColumn: buildUnifiedResourceTableColumnPresentation('', '16%'), - }; -}; diff --git a/frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts b/frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts deleted file mode 100644 index b8006f13c2..0000000000 --- a/frontend-modern/src/components/Infrastructure/useUnifiedResourceTableState.ts +++ /dev/null @@ -1,267 +0,0 @@ -import { createEffect, createMemo, createSignal, onCleanup } from 'solid-js'; -import type { Resource } from '@/types/resource'; -import type { ColumnPriority } from '@/hooks/useBreakpoint'; -import { useBreakpoint } from '@/hooks/useBreakpoint'; -import { useAlertsActivation } from '@/stores/alertsActivation'; -import type { AlertThresholdScope, DisplayMetricType } from '@/utils/metricThresholds'; -import { - buildInfrastructureSummaryGroupScope, - splitPrimaryAndServiceResources, - sortResources, - groupResources, - computeIOScale, -} from '@/components/Infrastructure/infrastructureSelectors'; -import type { SummarySeriesGroupScope } from '@/components/shared/summaryCardInteraction'; -import { useTableWindowing } from './useTableWindowing'; -import { useUnifiedResourceTableViewportSync } from './useUnifiedResourceTableViewportSync'; -import { - buildHostRowIndexById, - buildHostTableItems, - buildResourceLabelById, - getHostRevealTargetIndex, - getHostSpacerHeights, - getNextUnifiedResourceTableSortState, - getUnifiedResourceTableColumnPresentations, - getUnifiedResourceTableHeaderLabels, - getUnifiedResourceTableLayoutMode, - isUnifiedResourceHostDiskIoVisible, - isUnifiedResourceServiceColumnVisible, - isUnifiedResourceTableColumnVisible, - getUnifiedResourceTableShellClass, - getUnifiedResourceTableSortIndicator, - getUnifiedSources, - getVisibleHostTableItems, - HOST_TABLE_ESTIMATED_ROW_HEIGHT, - HOST_TABLE_WINDOW_SIZE, - normalizeUnifiedResourceTableLayoutWidth, - shouldShowUnifiedResourceHostTable, - shouldUseUnifiedResourceTableMobileLayout, - sortServiceResources, - type HostTableItem, - type UnifiedResourceTableSortKey as SortKey, -} from './unifiedResourceTableStateModel'; - -export interface UnifiedResourceTableProps { - resources: Resource[]; - expandedResourceId: string | null; - clearPinnedSummaryScope?: () => void; - highlightedResourceId?: string | null; - revealedResourceId?: string | null; - hoveredResourceId?: string | null; - activeSummaryGroupScope?: SummarySeriesGroupScope | null; - hoveredSummaryGroupScope?: SummarySeriesGroupScope | null; - focusedSummaryGroupScope?: SummarySeriesGroupScope | null; - focusedSummaryGroupId?: string | null; - onExpandedResourceChange: (id: string | null) => void; - onHoverChange?: (id: string | null) => void; - onGroupFocusChange?: (groupId: string | null) => void; - onGroupHoverChange?: (scope: SummarySeriesGroupScope | null) => void; - groupingMode?: 'grouped' | 'flat'; - onDeployCluster?: (clusterId: string, clusterName: string) => void; - setTableRootRef?: (element: HTMLDivElement | undefined) => void; -} - -export function useUnifiedResourceTableState(props: UnifiedResourceTableProps) { - const breakpoint = useBreakpoint(); - const alertsActivation = useAlertsActivation(); - const [tableContainerWidth, setTableContainerWidth] = createSignal(null); - const [tableRootElement, setTableRootElement] = createSignal(null); - const [sortKey, setSortKey] = createSignal('default'); - const [sortDirection, setSortDirection] = createSignal<'asc' | 'desc'>('asc'); - - const updateTableContainerWidth = (width: number | null | undefined) => { - if (typeof width === 'number' && Number.isFinite(width) && width > 0) { - setTableContainerWidth(Math.round(width)); - } - }; - - const setTableRootRef = (element: HTMLDivElement | undefined) => { - setTableRootElement(element ?? null); - props.setTableRootRef?.(element); - if (element) { - updateTableContainerWidth(element.clientWidth); - } else { - setTableContainerWidth(null); - } - }; - - createEffect(() => { - const element = tableRootElement(); - if (!element || typeof ResizeObserver === 'undefined') return; - - const observer = new ResizeObserver((entries) => { - const entryWidth = entries[0]?.contentRect.width; - updateTableContainerWidth(entryWidth ?? element.clientWidth); - }); - observer.observe(element); - onCleanup(() => observer.disconnect()); - }); - - const fallbackViewportWidth = () => { - const widthAccessor = breakpoint.width as (() => number) | undefined; - const width = widthAccessor?.(); - return normalizeUnifiedResourceTableLayoutWidth( - width, - typeof window !== 'undefined' ? window.innerWidth : undefined, - ); - }; - const tableLayoutWidth = createMemo(() => - normalizeUnifiedResourceTableLayoutWidth(tableContainerWidth(), fallbackViewportWidth()), - ); - const layoutMode = createMemo(() => getUnifiedResourceTableLayoutMode(tableLayoutWidth())); - const isMobile = createMemo(() => shouldUseUnifiedResourceTableMobileLayout(tableLayoutWidth())); - const isHostDiskIoVisible = createMemo(() => - isUnifiedResourceHostDiskIoVisible(tableLayoutWidth()), - ); - const isVisible = (priority: ColumnPriority) => - isUnifiedResourceTableColumnVisible(priority, tableLayoutWidth()); - const isServiceVisible = (priority: ColumnPriority) => - isUnifiedResourceServiceColumnVisible(priority, tableLayoutWidth()); - - const split = createMemo(() => splitPrimaryAndServiceResources(props.resources)); - const primaryResources = createMemo(() => split().primaryResources); - const serviceResources = createMemo(() => split().services); - const primaryResourceIds = createMemo( - () => new Set(primaryResources().map((resource) => resource.id)), - ); - const serviceResourceIds = createMemo( - () => new Set(serviceResources().map((resource) => resource.id)), - ); - - const sortedResources = createMemo(() => - sortResources(primaryResources(), sortKey(), sortDirection()), - ); - const resourceLabelById = createMemo(() => buildResourceLabelById(props.resources)); - const resolveResourceLabel = (resourceId: string): string | undefined => - resourceLabelById().get(resourceId); - - const groupedResources = createMemo(() => - groupResources(sortedResources(), props.groupingMode ?? 'grouped'), - ); - const hostTableItems = createMemo(() => - buildHostTableItems(groupedResources(), props.groupingMode), - ); - const hostRowIndexById = createMemo(() => buildHostRowIndexById(hostTableItems())); - const hostRevealTargetIndex = createMemo(() => - getHostRevealTargetIndex( - hostRowIndexById(), - props.expandedResourceId, - props.highlightedResourceId, - props.revealedResourceId, - ), - ); - - const hostWindowing = useTableWindowing({ - totalCount: () => hostTableItems().length, - windowSize: HOST_TABLE_WINDOW_SIZE, - revealIndex: hostRevealTargetIndex, - }); - - const visibleHostTableItems = createMemo(() => - getVisibleHostTableItems( - hostTableItems(), - hostWindowing.isWindowed(), - hostWindowing.startIndex(), - hostWindowing.endIndex(), - ), - ); - const hostSpacerHeights = createMemo(() => - getHostSpacerHeights( - hostTableItems().length, - hostWindowing.startIndex(), - hostWindowing.endIndex(), - hostWindowing.isWindowed(), - HOST_TABLE_ESTIMATED_ROW_HEIGHT, - ), - ); - const viewportSync = useUnifiedResourceTableViewportSync({ - totalCount: () => hostTableItems().length, - estimatedRowHeight: HOST_TABLE_ESTIMATED_ROW_HEIGHT, - hostWindowing, - }); - - const sortedPBSResources = createMemo(() => sortServiceResources(serviceResources(), 'pbs')); - const sortedPMGResources = createMemo(() => sortServiceResources(serviceResources(), 'pmg')); - - const ioScale = createMemo(() => computeIOScale(primaryResources())); - - const handleSort = (key: Exclude) => { - const nextSort = getNextUnifiedResourceTableSortState(sortKey(), sortDirection(), key); - setSortKey(nextSort.key); - setSortDirection(nextSort.direction); - }; - - const renderSortIndicator = (key: SortKey) => - getUnifiedResourceTableSortIndicator(sortKey(), sortDirection(), key); - - const getMetricThresholds = ( - scope: AlertThresholdScope, - metric: DisplayMetricType, - resourceIds?: string | string[], - ) => alertsActivation.getMetricThresholds(scope, metric, resourceIds); - - const toggleExpand = (resourceId: string) => { - props.onExpandedResourceChange(props.expandedResourceId === resourceId ? null : resourceId); - }; - - const tableShellClass = createMemo(() => getUnifiedResourceTableShellClass(layoutMode())); - const columnPresentations = createMemo(() => - getUnifiedResourceTableColumnPresentations(layoutMode(), tableLayoutWidth()), - ); - const headerLabels = createMemo(() => getUnifiedResourceTableHeaderLabels(layoutMode())); - const showHostTable = createMemo(() => - shouldShowUnifiedResourceHostTable(primaryResources().length, serviceResources().length), - ); - const showHostClearAction = createMemo(() => - Boolean( - props.focusedSummaryGroupId || - (props.expandedResourceId && primaryResourceIds().has(props.expandedResourceId)), - ), - ); - const showServiceClearAction = createMemo(() => - Boolean(props.expandedResourceId && serviceResourceIds().has(props.expandedResourceId)), - ); - - return { - layoutMode, - isMobile, - isHostDiskIoVisible, - isVisible, - isServiceVisible, - getMetricThresholds, - handleSort, - renderSortIndicator, - resolveResourceLabel, - setTableRootRef, - visibleHostTableItems, - hostTopSpacerHeight: () => hostSpacerHeights().top, - hostBottomSpacerHeight: () => hostSpacerHeights().bottom, - hostWindowing, - sortedPBSResources, - sortedPMGResources, - ioScale, - ...viewportSync, - tableShellClass, - headerLabels, - resourceColumn: () => columnPresentations().resourceColumn, - serviceResourceColumn: () => columnPresentations().serviceResourceColumn, - metricColumn: () => columnPresentations().metricColumn, - ioColumn: () => columnPresentations().ioColumn, - sourceColumn: () => columnPresentations().sourceColumn, - serviceSourceColumn: () => columnPresentations().serviceSourceColumn, - uptimeColumn: () => columnPresentations().uptimeColumn, - tempColumn: () => columnPresentations().tempColumn, - showHostTable, - showHostClearAction, - showServiceClearAction, - serviceCountColumn: () => columnPresentations().serviceCountColumn, - serviceQueueColumn: () => columnPresentations().serviceQueueColumn, - serviceHealthColumn: () => columnPresentations().serviceHealthColumn, - serviceActionColumn: () => columnPresentations().serviceActionColumn, - toggleExpand, - buildHostSummaryGroupScope: buildInfrastructureSummaryGroupScope, - getUnifiedSources, - }; -} - -export type UnifiedResourceTableState = ReturnType; diff --git a/frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts b/frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts deleted file mode 100644 index 70c7ffe008..0000000000 --- a/frontend-modern/src/components/Infrastructure/useUnifiedResourceTableViewportSync.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { createEffect, createSignal, onCleanup, untrack, type Accessor } from 'solid-js'; -import { - bindWindowedPageScrollEvents, - findWindowedPageScrollContainer, - isWindowedSurfaceHidden, - wheelDeltaInPixels, -} from '@/components/shared/windowedPageScroll'; -import { useTableWindowing } from './useTableWindowing'; - -interface UseUnifiedResourceTableViewportSyncOptions { - totalCount: Accessor; - estimatedRowHeight: number; - hostWindowing: ReturnType; -} - -export function useUnifiedResourceTableViewportSync( - options: UseUnifiedResourceTableViewportSyncOptions, -) { - const { totalCount, estimatedRowHeight, hostWindowing } = options; - const [hostBodyRef, setHostBodyRef] = createSignal(null); - - const syncHostWindowToViewport = (projectedScrollDelta = 0) => { - if (!hostWindowing.isWindowed() || typeof window === 'undefined') return; - const body = hostBodyRef(); - if (!body || isWindowedSurfaceHidden(body)) return; - const rect = body.getBoundingClientRect(); - const scrollContainer = findWindowedPageScrollContainer(body); - if (scrollContainer) { - const containerRect = scrollContainer.getBoundingClientRect(); - hostWindowing.onScroll( - Math.max(0, containerRect.top - rect.top + projectedScrollDelta), - scrollContainer.clientHeight || window.innerHeight, - estimatedRowHeight, - ); - return; - } - hostWindowing.onScroll( - Math.max(0, -rect.top + projectedScrollDelta), - window.innerHeight, - estimatedRowHeight, - ); - }; - - createEffect(() => { - if (typeof window === 'undefined') return; - totalCount(); - if (!hostWindowing.isWindowed()) return; - if (!hostBodyRef()) return; - - const body = hostBodyRef()!; - const scrollContainer = findWindowedPageScrollContainer(body); - const scrollTarget: HTMLElement | Window = scrollContainer ?? window; - const viewportHeight = () => scrollContainer?.clientHeight || window.innerHeight; - const handleViewportChange = () => syncHostWindowToViewport(); - const handleWheel = (event: Event) => { - const wheelEvent = event as WheelEvent; - if (wheelEvent.deltaY === 0) return; - syncHostWindowToViewport(wheelDeltaInPixels(wheelEvent, viewportHeight())); - }; - // The initial measurement reads the current window bounds and may move the - // window. Keep those reads outside this setup effect's dependency graph; - // otherwise a far-away reveal target and the viewport can repeatedly move - // the window back and forth until Solid exhausts the call stack. - untrack(handleViewportChange); - onCleanup( - bindWindowedPageScrollEvents({ - scrollTarget, - onScroll: handleViewportChange, - onWheel: handleWheel, - onResize: handleViewportChange, - }), - ); - }); - - return { - setHostBodyRef, - }; -} diff --git a/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts b/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts index 527c07f433..4fceba7599 100644 --- a/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts +++ b/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts @@ -101,7 +101,6 @@ import summaryTableFocusSource from '@/components/shared/summaryTableFocus.ts?ra import tableCardSource from '@/components/shared/TableCard.tsx?raw'; import groupedTableModeSegmentedControlSource from '@/components/shared/GroupedTableModeSegmentedControl.tsx?raw'; import groupedTableRowPresentationSource from '@/components/shared/groupedTableRowPresentation.ts?raw'; -import unifiedResourceTableViewportSyncSource from '@/components/Infrastructure/useUnifiedResourceTableViewportSync.ts?raw'; import storagePoolsTableWindowingSource from '@/components/Storage/useStoragePoolsTableWindowing.ts?raw'; import workloadViewportSyncSource from '@/components/Workloads/useWorkloadViewportSync.ts?raw'; import platformWindowedItemsSource from '@/features/platformPage/usePlatformWindowedItems.ts?raw'; @@ -223,10 +222,6 @@ import vsphereHostsTableSource from '@/features/vmware/VsphereHostsTable.tsx?raw import availabilityChecksTableSource from '@/features/standalone/AvailabilityChecksTable.tsx?raw'; import agentsMachinesTableSource from '@/features/standalone/AgentsMachinesTable.tsx?raw'; import agentMachineTableModelSource from '@/features/standalone/agentMachineTableModel.ts?raw'; -import unifiedResourceHostTableCardSource from '@/components/Infrastructure/UnifiedResourceHostTableCard.tsx?raw'; -import unifiedResourceServiceInfrastructureCardSource from '@/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx?raw'; -import unifiedResourcePBSTableSectionSource from '@/components/Infrastructure/UnifiedResourcePBSTableSection.tsx?raw'; -import unifiedResourcePMGTableSectionSource from '@/components/Infrastructure/UnifiedResourcePMGTableSection.tsx?raw'; import proxmoxMailGatewayDrawerSource from '@/features/proxmox/ProxmoxMailGatewayDrawer.tsx?raw'; import swarmServicesDrawerSource from '@/components/Docker/SwarmServicesDrawer.tsx?raw'; import k8sDeploymentsDrawerSource from '@/components/Kubernetes/K8sDeploymentsDrawer.tsx?raw'; @@ -1057,8 +1052,6 @@ describe('shared primitive guardrails', () => { expect(registeredRule?.canonical?.path).toBe('src/components/shared/TableCardHeader.tsx'); expect(registeredRule?.canonical?.export).toBe('TableCardHeader'); expect(registeredRule?.requiredConsumers?.map((consumer) => consumer.path)).toEqual([ - 'src/components/Infrastructure/UnifiedResourceHostTableCard.tsx', - 'src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx', 'src/components/Storage/StorageContentCard.tsx', 'src/features/platformPage/sharedPlatformPage.tsx', ]); @@ -1080,11 +1073,7 @@ describe('shared primitive guardrails', () => { expect(tableCardHeaderSource).toContain('TABLE_CARD_HEADER_CLASS'); expect(tableCardHeaderSource).not.toContain('Pinned to'); expect(tableCardHeaderSource).not.toContain('Scoped to'); - for (const source of [ - unifiedResourceHostTableCardSource, - unifiedResourceServiceInfrastructureCardSource, - storageContentCardSource, - ]) { + for (const source of [storageContentCardSource]) { expect(source).toContain('TableCardHeader'); expect(source).not.toContain('SummaryTableCardHeader'); } @@ -1116,8 +1105,6 @@ describe('shared primitive guardrails', () => { expect(registeredRule?.canonical?.path).toBe('src/components/shared/TableCard.tsx'); expect(registeredRule?.canonical?.export).toBe('TableCard'); expect(registeredRule?.requiredConsumers?.map((consumer) => consumer.path)).toEqual([ - 'src/components/Infrastructure/UnifiedResourceHostTableCard.tsx', - 'src/components/Infrastructure/UnifiedResourceServiceInfrastructureCard.tsx', 'src/components/Storage/StorageContentCard.tsx', 'src/components/Workloads/WorkloadsSurface.tsx', 'src/components/Workloads/WorkloadsTable.tsx', @@ -1149,8 +1136,6 @@ describe('shared primitive guardrails', () => { workloadsSurfaceSource, alertHistoryTableSectionSource, sharedPlatformPageSource, - unifiedResourceHostTableCardSource, - unifiedResourceServiceInfrastructureCardSource, storageContentCardSource, ]) { expect(source).toContain('TableCard'); @@ -1394,13 +1379,7 @@ describe('shared primitive guardrails', () => { expect(guestRowStateSource).not.toContain('bg-sky-50/70'); expect(guestRowStateSource).not.toContain('ring-sky-400/25'); - for (const source of [ - storagePoolRowSource, - diskListSource, - unifiedResourceHostTableCardSource, - unifiedResourcePBSTableSectionSource, - unifiedResourcePMGTableSectionSource, - ]) { + for (const source of [storagePoolRowSource, diskListSource]) { expect(source).toContain('data-summary-row-active'); expect(source).not.toContain('bg-sky-50/70'); expect(source).not.toContain('ring-sky-400/25'); @@ -1416,8 +1395,6 @@ describe('shared primitive guardrails', () => { expect(nodeGroupHeaderSource).toContain('getGroupedTableRowCellClass'); expect(workloadPanelSource).toContain('getInteractiveGroupedTableRowClass'); expect(workloadPanelSource).toContain('getGroupedTableRowCellClass'); - expect(unifiedResourceHostTableCardSource).toContain('getInteractiveGroupedTableRowClass'); - expect(unifiedResourceHostTableCardSource).toContain('getGroupedTableRowCellClass'); expect(alertHistoryTableGroupRowSource).toContain('getGroupedTableRowClass'); expect(alertHistoryTableGroupRowSource).toContain('getGroupedTableRowCellClass'); expect(alertHistoryTableGroupRowSource).not.toContain('class="bg-surface-alt"'); @@ -1426,7 +1403,6 @@ describe('shared primitive guardrails', () => { expect(infrastructureSourceManagerSource).toContain('getGroupedTableRowClass'); expect(infrastructureSourceManagerSource).toContain('getGroupedTableRowCellClass'); expect(infrastructureSourceManagerSource).not.toContain('bg-base hover:bg-base'); - expect(unifiedResourceHostTableCardSource).toContain('data-summary-group-member-active'); }); it('routes Proxmox node version presentation through the shared formatter', () => { @@ -1463,7 +1439,6 @@ describe('shared primitive guardrails', () => { expect(windowedPageScrollSource).not.toContain("addEventListener('touch"); for (const source of [ platformWindowedItemsSource, - unifiedResourceTableViewportSyncSource, storagePoolsTableWindowingSource, workloadViewportSyncSource, ]) { @@ -1487,9 +1462,6 @@ describe('shared primitive guardrails', () => { expect(tableCardHeaderSource).toContain('TABLE_CARD_HEADER_CLASS'); for (const source of [ - unifiedResourceHostTableCardSource, - unifiedResourcePBSTableSectionSource, - unifiedResourcePMGTableSectionSource, alertHistoryTableSectionSource, workloadsTableSource, storagePoolsTableSource, @@ -1509,7 +1481,6 @@ describe('shared primitive guardrails', () => { } for (const source of [ - unifiedResourceHostTableCardSource, alertHistoryTableSectionSource, workloadsTableSource, storageContentCardSource, @@ -1517,9 +1488,7 @@ describe('shared primitive guardrails', () => { expect(source).toContain(' { expect(platformResourceDetailTableRowSource).toContain('SummaryRowActionButton'); expect(platformResourceDetailTableRowSource).toContain('hideWhenRowTappableOnMobile'); - for (const source of [ - guestRowSource, - storagePoolRowSource, - diskListSource, - unifiedResourceHostTableCardSource, - unifiedResourcePBSTableSectionSource, - unifiedResourcePMGTableSectionSource, - ]) { + for (const source of [guestRowSource, storagePoolRowSource, diskListSource]) { expect(source).toContain('createSummaryInteractiveRowPreviewHandlers'); expect(source).toContain('SummaryRowActionButton'); expect(source).toContain('hideWhenRowTappableOnMobile'); @@ -1740,7 +1702,6 @@ describe('shared primitive guardrails', () => { expect(workloadPanelSource).toContain('createSummaryInteractiveRowPreviewHandlers'); expect(workloadPanelSource).not.toContain('kind="scope"'); expect(storageGroupRowSource).not.toContain('kind="scope"'); - expect(unifiedResourceHostTableCardSource).not.toContain('kind="scope"'); expect(nodeGroupHeaderSource).toContain('ResourceNameWithWebInterfaceLink'); expect(webInterfaceLinkSource).toContain('event.stopPropagation()'); @@ -5394,9 +5355,6 @@ describe('shared primitive guardrails', () => { 'src/features/proxmox/ProxmoxNodesTable.tsx', 'src/components/Alerts/AlertResourceGroupHeader.tsx', 'src/components/Alerts/AlertResourceTableRow.tsx', - 'src/components/Infrastructure/UnifiedResourceHostTableCard.tsx', - 'src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx', - 'src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx', 'src/features/docker/DockerNativeTableShared.tsx', 'src/features/docker/DockerContainersTable.tsx', 'src/features/kubernetes/KubernetesClustersTable.tsx', @@ -5413,9 +5371,6 @@ describe('shared primitive guardrails', () => { expect(proxmoxNodesTableSource).toContain('ResourceNameWithWebInterfaceLink'); expect(alertResourceGroupHeaderSource).toContain('ResourceNameWithWebInterfaceLink'); expect(alertResourceTableRowSource).toContain('ResourceNameWithWebInterfaceLink'); - expect(unifiedResourceHostTableCardSource).toContain('ResourceNameWithWebInterfaceLink'); - expect(unifiedResourcePBSTableSectionSource).toContain('ResourceNameWithWebInterfaceLink'); - expect(unifiedResourcePMGTableSectionSource).toContain('ResourceNameWithWebInterfaceLink'); expect(nodeGroupHeaderSource).not.toContain('target="_blank"'); expect(guestRowSource).not.toContain('target="_blank"'); expect(agentsMachinesTableSource).not.toContain('target="_blank"'); @@ -7766,9 +7721,6 @@ describe('shared primitive guardrails', () => { expect(inlineDetailTableRowSource).toContain('.focus({ preventScroll: true })'); expect(registeredRule?.requiredConsumers?.map((consumer) => consumer.path)).toEqual( expect.arrayContaining([ - 'src/components/Infrastructure/UnifiedResourceHostTableCard.tsx', - 'src/components/Infrastructure/UnifiedResourcePBSTableSection.tsx', - 'src/components/Infrastructure/UnifiedResourcePMGTableSection.tsx', 'src/components/Workloads/WorkloadPanel.tsx', 'src/features/docker/DockerHostsTable.tsx', 'src/features/docker/DockerNetworksTable.tsx', @@ -7796,9 +7748,6 @@ describe('shared primitive guardrails', () => { dockerHostsTableSource, proxmoxCoverageTableSource, proxmoxNodesTableSource, - unifiedResourceHostTableCardSource, - unifiedResourcePBSTableSectionSource, - unifiedResourcePMGTableSectionSource, workloadPanelSource, ]) { expect(source).toContain('InlineDetailTableRow'); diff --git a/frontend-modern/src/features/platformPage/__tests__/platformOverviewLayout.guardrails.test.ts b/frontend-modern/src/features/platformPage/__tests__/platformOverviewLayout.guardrails.test.ts index ade759b1d2..8d6b99b8e3 100644 --- a/frontend-modern/src/features/platformPage/__tests__/platformOverviewLayout.guardrails.test.ts +++ b/frontend-modern/src/features/platformPage/__tests__/platformOverviewLayout.guardrails.test.ts @@ -235,7 +235,6 @@ describe('platform overview layout guardrails', () => { expect(source).not.toContain('const [search, '); } - expect(sharedPlatformPageSource).toContain('searchSuggestions={tableState.searchSuggestions}'); expect(platformSearchSuggestionsSource).toContain('buildPlatformResourceSearchSuggestions'); expect(platformSearchSuggestionsSource).not.toContain('platformData'); }); diff --git a/frontend-modern/src/features/platformPage/sharedPlatformPage.tsx b/frontend-modern/src/features/platformPage/sharedPlatformPage.tsx index 182ab56400..3b1ab5cfcd 100644 --- a/frontend-modern/src/features/platformPage/sharedPlatformPage.tsx +++ b/frontend-modern/src/features/platformPage/sharedPlatformPage.tsx @@ -36,7 +36,6 @@ import { TableCardHeader } from '@/components/shared/TableCardHeader'; import { useActiveHorizontalRailItemVisibility } from '@/components/shared/useActiveHorizontalRailItemVisibility'; import { useBreakpoint } from '@/hooks/useBreakpoint'; import { usePersistentSignal } from '@/hooks/usePersistentSignal'; -import { UnifiedResourceTable } from '@/components/Infrastructure/UnifiedResourceTable'; import type { Resource } from '@/types/resource'; import { formatBytes, formatRelativeTime, formatUptime } from '@/utils/format'; import { useRelativeTimeNow } from '@/utils/relativeTimeClock'; @@ -1103,14 +1102,6 @@ export type PlatformResourceStatusFilter = 'all' | 'online' | 'degraded' | 'offl const statusDot = filterChipStatusDot; -export const PLATFORM_STATUS_FILTER_OPTIONS: PlatformTableFilterOption[] = - [ - { value: 'all', label: 'All' }, - { value: 'online', label: 'Online', tone: 'success', leading: statusDot('bg-emerald-500') }, - { value: 'degraded', label: 'Degraded', tone: 'warning', leading: statusDot('bg-amber-500') }, - { value: 'offline', label: 'Offline', tone: 'danger', leading: statusDot('bg-red-500') }, - ]; - export const PLATFORM_HEALTH_FILTER_OPTIONS: PlatformTableFilterOption[] = [ { value: 'all', label: 'All' }, @@ -1455,70 +1446,3 @@ export function PlatformTableToolbar(props: { /> ); } - -export const PlatformResourceTable: Component<{ - resources: Resource[]; - emptyIcon: JSX.Element; - emptyTitle: string; - emptyDescription: string; - groupingMode?: 'grouped' | 'flat'; - searchPlaceholder?: string; -}> = (props) => { - const [expandedResourceId, setExpandedResourceId] = createSignal(null); - const tableState = createPlatformTableFilterState({ - resources: () => props.resources, - initialStatus: 'all' as PlatformResourceStatusFilter, - filter: filterPlatformResources, - }); - - return ( - 0} - fallback={ - - } - > -
- - 0} - fallback={ - } - /> - } - > - - -
- - ); -}; diff --git a/frontend-modern/src/utils/__tests__/frontendResourceTypeBoundaries.test.ts b/frontend-modern/src/utils/__tests__/frontendResourceTypeBoundaries.test.ts index dd056e05e3..46e94e4c22 100644 --- a/frontend-modern/src/utils/__tests__/frontendResourceTypeBoundaries.test.ts +++ b/frontend-modern/src/utils/__tests__/frontendResourceTypeBoundaries.test.ts @@ -261,11 +261,6 @@ import resourceDetailDrawerOperationalModelSource from '@/components/Infrastruct import resourceDetailDrawerServiceModelSource from '@/components/Infrastructure/resourceDetailDrawerServiceModel.ts?raw'; import resourceDetailDrawerDockerActionsStateSource from '@/components/Infrastructure/useResourceDetailDrawerDockerActionsState.ts?raw'; import resourceDetailDrawerStateSource from '@/components/Infrastructure/useResourceDetailDrawerState.ts?raw'; -import unifiedResourceTableSource from '@/components/Infrastructure/UnifiedResourceTable.tsx?raw'; -import unifiedResourceTableStateSource from '@/components/Infrastructure/useUnifiedResourceTableState.ts?raw'; -import unifiedResourceTableViewportSyncSource from '@/components/Infrastructure/useUnifiedResourceTableViewportSync.ts?raw'; -import unifiedResourceTableStateModelSource from '@/components/Infrastructure/unifiedResourceTableStateModel.ts?raw'; -import unifiedResourceTableModelSource from '@/components/Infrastructure/unifiedResourceTableModel.ts?raw'; import useUnifiedResourcesSource from '@/hooks/useUnifiedResources.ts?raw'; import useWorkloadsSource from '@/hooks/useWorkloads.ts?raw'; import findingsPanelSource from '@/components/AI/FindingsPanel.tsx?raw'; @@ -2499,52 +2494,6 @@ describe('frontend resource type boundaries', () => { expect(resourceDetailMappersSource).not.toContain('export const toDiscoveryConfig'); expect(resourceDetailMappersSource).not.toContain('export const normalizeHealthLabel'); expect(resourceDetailMappersSource).not.toContain('export const healthToneClass'); - expect(unifiedResourceTableSource).toContain('useUnifiedResourceTableState'); - expect(unifiedResourceTableSource).toContain('UnifiedResourceHostTableCard'); - expect(unifiedResourceTableSource).toContain('UnifiedResourceServiceInfrastructureCard'); - expect(unifiedResourceTableSource).not.toContain('const split = createMemo(() =>'); - expect(unifiedResourceTableSource).not.toContain('const sortedPBSResources = createMemo(() =>'); - expect(unifiedResourceTableSource).not.toContain( - 'const resourceColumnStyle = createMemo(() =>', - ); - expect(unifiedResourceTableSource).not.toContain('getServiceHealthSummaryPresentation'); - expect(unifiedResourceTableSource).not.toContain('const getOutlierEmphasis ='); - expect(unifiedResourceTableSource).not.toContain('const summarizeServiceHealthTone ='); - expect(unifiedResourceTableStateSource).toContain( - 'export function useUnifiedResourceTableState', - ); - expect(unifiedResourceTableStateSource).toContain("from './unifiedResourceTableStateModel'"); - expect(unifiedResourceTableStateSource).toContain('buildHostTableItems'); - expect(unifiedResourceTableStateSource).toContain('getUnifiedResourceTableColumnPresentations'); - expect(unifiedResourceTableStateSource).toContain('getUnifiedResourceTableShellClass'); - expect(unifiedResourceTableStateSource).toContain('useTableWindowing'); - expect(unifiedResourceTableStateSource).toContain('useUnifiedResourceTableViewportSync'); - expect(unifiedResourceTableStateSource).not.toContain( - 'const resourceColumnStyle = createMemo(() =>', - ); - expect(unifiedResourceTableStateSource).not.toContain( - "const showGroupHeaders = props.groupingMode === 'grouped'", - ); - expect(unifiedResourceTableStateSource).not.toContain('const items: HostTableItem[] = [];'); - expect(unifiedResourceTableStateSource).not.toContain('window.addEventListener'); - expect(unifiedResourceTableStateSource).not.toContain('getBoundingClientRect'); - expect(unifiedResourceTableStateModelSource).toContain('export const buildHostTableItems'); - expect(unifiedResourceTableStateModelSource).toContain( - 'export const getUnifiedResourceTableColumnPresentations', - ); - expect(unifiedResourceTableStateModelSource).toContain( - 'export const getUnifiedResourceTableShellClass', - ); - expect(unifiedResourceTableStateModelSource).toContain( - 'export const getNextUnifiedResourceTableSortState', - ); - expect(unifiedResourceTableModelSource).toContain('getServiceHealthSummaryPresentation'); - expect(unifiedResourceTableModelSource).toContain('export const getOutlierEmphasis'); - expect(unifiedResourceTableViewportSyncSource).toContain('bindWindowedPageScrollEvents'); - expect(unifiedResourceTableViewportSyncSource).not.toContain('window.addEventListener'); - expect(unifiedResourceTableViewportSyncSource).toContain('getBoundingClientRect'); - expect(unifiedResourceTableViewportSyncSource).toContain('hostWindowing.onScroll'); - expect(unifiedResourceTableViewportSyncSource).not.toContain('scrollIntoView'); expect(discoveryTabSource).toContain('getDiscoverySuggestedURLReason'); expect(discoveryTabSource).toContain('getDiscoveryObservedSourceLabel'); expect(discoveryTabSource).toContain('getDiscoveryAnalysisProviderBadgeClass'); diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.py index 28aeddf79d..ae75d43aa3 100644 --- a/scripts/release_control/subsystem_lookup_test.py +++ b/scripts/release_control/subsystem_lookup_test.py @@ -378,11 +378,11 @@ class SubsystemLookupTest(unittest.TestCase): match["verification_requirement"]["exact_files"], ) - def test_lookup_paths_assigns_unified_resource_table_state_model_to_shared_infrastructure_hot_path( + def test_lookup_paths_assigns_resource_detail_mappers_to_shared_infrastructure_hot_path( self, ) -> None: result = lookup_paths( - ["frontend-modern/src/components/Infrastructure/unifiedResourceTableStateModel.ts"] + ["frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts"] ) self.assertEqual(result["unowned_runtime_files"], []) self.assertEqual( @@ -426,7 +426,7 @@ class SubsystemLookupTest(unittest.TestCase): "resource-consumers", ) self.assertIn( - "frontend-modern/src/components/Infrastructure/__tests__/unifiedResourceTableStateModel.test.ts", + "frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts", unified_resources_match["verification_requirement"]["exact_files"], ) diff --git a/tests/integration/tests/68-platform-pages-shell.spec.ts b/tests/integration/tests/68-platform-pages-shell.spec.ts index 3a8bd1b6ec..bc21e15ed9 100644 --- a/tests/integration/tests/68-platform-pages-shell.spec.ts +++ b/tests/integration/tests/68-platform-pages-shell.spec.ts @@ -136,7 +136,7 @@ const PLATFORM_PAGES: readonly PlatformPageCase[] = [ ]; // Every populated inventory tab — embedded canonical Workloads/Storage and -// UnifiedResourceTable-backed infrastructure views — owns an operator search +// platform-native infrastructure tables — owns an operator search // input. Overview stacks are summary surfaces and intentionally suppress their // table toolbars. Keep each route as an independent browser test so one slow // navigation cannot consume the time budget or state of the remaining routes. From 844e1dbb97de867bb99f75dde7015bf1ff192b39 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:36:49 +0100 Subject: [PATCH 21/53] Say a reviewed Patrol issue stays reviewed when suppression ends Ending a suppression puts the alert in Acknowledged when it is acknowledged and in Open otherwise (unsuppressOperationalAlert), and Acknowledged is not active attention. The attention detail still promised otherwise for an issue marked reviewed and then suppressed: its early-end button read "Return to active attention", the handled-view notice read "Returned to decision inbox", and the suppression form offered "Return it to active attention after", while the issue went back to Reviewed. The duration label and the early-end button now follow the detail's operationalRecord.acknowledgement, which the backend derives from alert.Acknowledged: "End suppression after" and "End suppression" for a reviewed issue, the existing copy for an unreviewed one. The notice reads the item's state from the reload that follows the change instead of that snapshot, so it reports "Suppression ended. It stays reviewed." only when the item really is Acknowledged, even if the review was removed elsewhere after the detail loaded. Contract-Neutral: No api-contracts or frontend-primitives delta: Patrol lifecycle copy only, no API payload or shared primitive change; patrol-intelligence.md carries the note --- .../subsystems/patrol-intelligence.md | 17 +- .../fb946589c643778e.json | 45 +++++ .../patrol/PatrolAttentionWorkbench.tsx | 47 ++++- .../PatrolAttentionWorkbench.test.tsx | 183 ++++++++++++++++++ 4 files changed, 281 insertions(+), 11 deletions(-) create mode 100644 frontend-modern/browser-verification/fb946589c643778e.json diff --git a/docs/release-control/v6/internal/subsystems/patrol-intelligence.md b/docs/release-control/v6/internal/subsystems/patrol-intelligence.md index 251bfc8252..3ac898ea7a 100644 --- a/docs/release-control/v6/internal/subsystems/patrol-intelligence.md +++ b/docs/release-control/v6/internal/subsystems/patrol-intelligence.md @@ -1361,8 +1361,21 @@ remove settled items and append genuinely new work. Acknowledging an occurrence is presented as `Mark reviewed`: on success the item leaves the active inbox, Patrol announces the remaining decision count, and advances to the next current item. Temporary suppression uses the same advancement loop and explicitly says -that the occurrence will return at the chosen time. Clearing the final item -returns to the calm inbox with an honest completion announcement. +when it ends and which state it hands the occurrence back to. Clearing the +final item returns to the calm inbox with an honest completion announcement. +Ending a suppression, early or at its expiry, puts the alert in Acknowledged +when it is acknowledged and in Open otherwise (`unsuppressOperationalAlert`), +and Acknowledged is not active attention. So for an issue marked reviewed +(`operationalRecord.acknowledgement` present) the suppression form's duration +reads `End suppression after` and the early-end control reads `End +suppression`; an unreviewed issue keeps `Return it to active attention after` +and `Return to active attention`. The handled-view notice after ending it reads +the state the post-change reload reports for the item, `Suppression ended. It +stays reviewed.` for Acknowledged and `Returned to decision inbox` otherwise, +so a detail read before the review changed elsewhere cannot misstate the +outcome; it falls back to the detail's acknowledgement only when the item is +gone from the reload. `getAttentionSuppressionEndPresentation` owns that copy +and `PatrolAttentionWorkbench.test.tsx` pins all three cases. The attention queue orders genuine operator decisions by severity, actionable approval/review state, and latest observation time. Collapsed rows expose the diff --git a/frontend-modern/browser-verification/fb946589c643778e.json b/frontend-modern/browser-verification/fb946589c643778e.json new file mode 100644 index 0000000000..772fcdb973 --- /dev/null +++ b/frontend-modern/browser-verification/fb946589c643778e.json @@ -0,0 +1,45 @@ +{ + "version": 1, + "base_sha": "9aaac4d3e502c01913fb3637d0997b0ce03a8b7b", + "verified_at": "2026-10-06T19:36:25Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx" + ], + "content_sha256": { + "frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx": "11fbba8f8a4e3a62aaceaed796ff9d67a09e6248bfe8eb95163a92c3dd85e728" + }, + "routes": [ + "/patrol" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 800, + "height": 600 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack (PULSE_MOCK_MODE) built from this tree on origin/main 9aaac4d3e5, Vite on the same checkout; real alert-manager lifecycle, no injected data", + "Reviewed then suppressed issue (operationalRecord.acknowledgement and suppression both set, state suppressed): Lifecycle shows the acknowledgement line, the suppression reason and an End suppression button, and no Return to active attention button (1440, 800 and 390 widths)", + "Suppression form on a Reviewed issue labels its duration End suppression after; on an open issue it still reads Return it to active attention after", + "After ending a reviewed issue's suppression: notice Suppression ended. It stays reviewed. N handled issues remain., list row reads Reviewed, and the attention API reports state acknowledged with acknowledgement kept and the active count unchanged", + "After ending an unreviewed issue's suppression: Return to active attention button, notice Returned to decision inbox. N handled issues remain., API state open (1440 and 390 widths)", + "Stale detail: a reviewed, suppressed issue whose review was removed through POST /api/ai/patrol/attention/{id}/unacknowledge while its detail stayed open; the button still read End suppression, and after clicking it the notice read Returned to decision inbox. 5 handled issues remain., matching the API state open", + "The suppression expiry line still reads Returns to active attention just now on main; fix/future-relative-time changes that line separately, and it was left untouched here", + "No page errors in any run" + ], + "interactions": [ + "Browser pane at 800x600: opened west-d-service-pool (local-zfs) \u00b7 ZFS pool state, Mark reviewed, Reviewed and suppressed (1), reopened it, Suppress temporarily with a typed reason and 24 hours, then End suppression", + "Playwright 1440x900: reviewed case (Mark reviewed, handled view, suppress, End suppression) on mail-gateway-eu and esxi-07.lab.local; unreviewed case (suppress from the inbox, handled view, Return to active attention) on backend-seafile and edge-cold-iscsi; stale-detail case on Utility Network", + "Playwright 390x844: reviewed case on postgres-replica-18 and Edge Stateful, unreviewed case on dev-portal-01 and archive-tier, reopening items through Back to handled issues after the queue advanced", + "Read each item's resulting state through GET /api/ai/patrol/attention/{id} and the attention summary after every lifecycle change" + ] +} diff --git a/frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx b/frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx index 72ac52cd43..3b34ff89bc 100644 --- a/frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx +++ b/frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx @@ -142,6 +142,23 @@ export function sortPatrolAttentionDecisions( }); } +// Ending a suppression, early or at its expiry, puts the alert in +// Acknowledged when it is acknowledged, which is not active attention, and +// otherwise in Open, back in the decision inbox. +function getAttentionSuppressionEndPresentation(reviewed: boolean) { + return reviewed + ? { + durationLabel: 'End suppression after', + actionLabel: 'End suppression', + successLabel: 'Suppression ended. It stays reviewed', + } + : { + durationLabel: 'Return it to active attention after', + actionLabel: 'Return to active attention', + successLabel: 'Returned to decision inbox', + }; +} + export function PatrolAttentionWorkbench( props: { autonomyLevel?: PatrolAutonomyLevel; @@ -311,7 +328,7 @@ export function PatrolAttentionWorkbench( }; const changeLifecycle = async ( operation: () => Promise, - options: { advanceAfter?: boolean; successLabel?: string } = {}, + options: { advanceAfter?: boolean; successLabel?: string | (() => string) } = {}, ) => { if (lifecycleBusy()) return; const selected = selectedItemId(); @@ -328,7 +345,10 @@ export function PatrolAttentionWorkbench( const remaining = orderedDecisions(); const next = remaining.find((decision) => decision.item.id === nextCandidateId) ?? remaining[0]; - const successLabel = options.successLabel ?? 'Decision updated'; + const successLabel = + (typeof options.successLabel === 'function' + ? options.successLabel() + : options.successLabel) ?? 'Decision updated'; setReviewNotice( attentionView() === 'handled' ? remaining.length > 0 @@ -565,10 +585,17 @@ export function PatrolAttentionWorkbench( successLabel: 'Suppressed temporarily', }) } - onUnsuppress={(itemId) => + onUnsuppress={(itemId, reviewed) => changeLifecycle(() => unsuppressPatrolAttention(itemId), { advanceAfter: attentionView() === 'handled', - successLabel: 'Returned to decision inbox', + // Announce the state the reload reports, so a detail read + // before the review changed elsewhere cannot misstate it. + successLabel: () => { + const after = patrolAttentionStore.items().find((entry) => entry.id === itemId); + return getAttentionSuppressionEndPresentation( + after ? after.state === 'acknowledged' : reviewed, + ).successLabel; + }, }) } onOpenFindings={props.onOpenFindings} @@ -835,7 +862,7 @@ function AttentionDetail(props: { onAcknowledge: (itemId: string) => Promise; onUnacknowledge: (itemId: string) => Promise; onSuppress: (itemId: string, reason: string, expiresAt: string) => Promise; - onUnsuppress: (itemId: string) => Promise; + onUnsuppress: (itemId: string, reviewed: boolean) => Promise; onOpenFindings?: (item: AttentionItem) => void; findings?: () => UnifiedFinding[]; }) { @@ -1254,12 +1281,14 @@ function AttentionLifecycleControls(props: { onAcknowledge: (itemId: string) => Promise; onUnacknowledge: (itemId: string) => Promise; onSuppress: (itemId: string, reason: string, expiresAt: string) => Promise; - onUnsuppress: (itemId: string) => Promise; + onUnsuppress: (itemId: string, reviewed: boolean) => Promise; }) { const [showSuppression, setShowSuppression] = createSignal(false); const [reason, setReason] = createSignal(''); const [durationMs, setDurationMs] = createSignal(SUPPRESSION_DURATIONS[1].value); const state = () => props.detail.item.state; + const reviewed = () => Boolean(props.detail.operationalRecord.acknowledgement); + const suppressionEnd = () => getAttentionSuppressionEndPresentation(reviewed()); const canAcknowledge = () => ['open', 'stale', 'unknown', 'resolving'].includes(state()); const canSuppress = () => ['open', 'acknowledged', 'stale', 'unknown', 'resolving'].includes(state()); @@ -1333,9 +1362,9 @@ function AttentionLifecycleControls(props: { size="sm" class="min-h-11 sm:min-h-0" isLoading={props.busy} - onClick={() => void props.onUnsuppress(props.detail.item.id)} + onClick={() => void props.onUnsuppress(props.detail.item.id, reviewed())} > - Return to active attention + {suppressionEnd().actionLabel} @@ -1376,7 +1405,7 @@ function AttentionLifecycleControls(props: { /> { ).toBeInTheDocument(); }); + it('keeps a reviewed issue reviewed when its suppression ends early', async () => { + // Ending the suppression restores Acknowledged, not active attention, so + // nothing on the way in or out may promise the decision inbox. + const reviewed = item({ state: 'acknowledged' }); + const suppressed = item({ state: 'suppressed' }); + const acknowledgement = { at: evaluatedAt, by: 'admin' }; + const reviewedDetail = detail(reviewed); + reviewedDetail.operationalRecord.acknowledgement = acknowledgement; + const suppressedDetail = detail(suppressed); + suppressedDetail.operationalRecord.acknowledgement = acknowledgement; + suppressedDetail.operationalRecord.suppression = { + at: evaluatedAt, + by: 'admin', + reason: 'Planned storage maintenance', + expiresAt: '2026-07-20T08:00:00Z', + }; + let isSuppressed = false; + apiMocks.getList.mockImplementation((filter: string) => { + const responseSummary = summary( + isSuppressed ? { suppressedCount: 1 } : { acknowledgedCount: 1 }, + ); + return Promise.resolve( + listResponse( + filter === 'all' ? [isSuppressed ? suppressed : reviewed] : [], + responseSummary, + ), + ); + }); + apiMocks.getDetail.mockImplementation(() => + Promise.resolve(isSuppressed ? suppressedDetail : reviewedDetail), + ); + apiMocks.suppress.mockImplementation(() => { + isSuppressed = true; + return Promise.resolve({ success: true }); + }); + apiMocks.unsuppress.mockImplementation(() => { + isSuppressed = false; + return Promise.resolve({ success: true }); + }); + renderWorkbench(); + + fireEvent.click(await screen.findByRole('button', { name: 'Reviewed and suppressed (1)' })); + fireEvent.click( + await screen.findByRole('button', { + name: 'Open Database VM · Disk pressure', + }), + ); + fireEvent.click(await screen.findByRole('button', { name: 'Suppress temporarily' })); + expect( + screen.queryByRole('combobox', { name: 'Return it to active attention after' }), + ).not.toBeInTheDocument(); + fireEvent.input( + screen.getByRole('textbox', { + name: 'Why is this safe to hide from active attention?', + }), + { target: { value: 'Planned storage maintenance' } }, + ); + fireEvent.change(screen.getByRole('combobox', { name: 'End suppression after' }), { + target: { value: String(60 * 60 * 1000) }, + }); + fireEvent.click(screen.getByRole('button', { name: 'Suppress temporarily' })); + await waitFor(() => + expect(screen.getByRole('status')).toHaveTextContent( + 'Suppressed temporarily. 1 handled issue remains.', + ), + ); + + const endSuppression = await screen.findByRole('button', { name: 'End suppression' }); + expect( + screen.queryByRole('button', { name: 'Return to active attention' }), + ).not.toBeInTheDocument(); + fireEvent.click(endSuppression); + + await waitFor(() => expect(apiMocks.unsuppress).toHaveBeenCalledWith('record-1')); + await waitFor(() => + expect(screen.getByRole('status')).toHaveTextContent( + 'Suppression ended. It stays reviewed. 1 handled issue remains.', + ), + ); + expect(screen.getByRole('status')).not.toHaveTextContent('decision inbox'); + expect( + screen.getByRole('heading', { name: '1 reviewed or suppressed issue' }), + ).toBeInTheDocument(); + expect( + await screen.findByRole('button', { name: 'Return to decision inbox' }), + ).toBeInTheDocument(); + }); + + it('returns an unreviewed issue to the decision inbox when its suppression ends early', async () => { + const active = item(); + const suppressed = item({ state: 'suppressed' }); + const suppressedDetail = detail(suppressed); + suppressedDetail.operationalRecord.suppression = { + at: evaluatedAt, + by: 'admin', + reason: 'Planned storage maintenance', + expiresAt: '2026-07-20T08:00:00Z', + }; + let isSuppressed = true; + apiMocks.getList.mockImplementation((filter: string) => { + if (isSuppressed) { + return Promise.resolve( + listResponse(filter === 'all' ? [suppressed] : [], summary({ suppressedCount: 1 })), + ); + } + return Promise.resolve( + listResponse([active], summary({ activeCount: 1, openCount: 1, calm: false })), + ); + }); + apiMocks.getDetail.mockImplementation(() => + Promise.resolve(isSuppressed ? suppressedDetail : detail(active)), + ); + apiMocks.unsuppress.mockImplementation(() => { + isSuppressed = false; + return Promise.resolve({ success: true }); + }); + renderWorkbench(); + + fireEvent.click(await screen.findByRole('button', { name: 'Reviewed and suppressed (1)' })); + fireEvent.click( + await screen.findByRole('button', { + name: 'Open Database VM · Disk pressure', + }), + ); + const returnToAttention = await screen.findByRole('button', { + name: 'Return to active attention', + }); + expect(screen.queryByRole('button', { name: 'End suppression' })).not.toBeInTheDocument(); + fireEvent.click(returnToAttention); + + await waitFor(() => expect(apiMocks.unsuppress).toHaveBeenCalledWith('record-1')); + expect(await screen.findByRole('status')).toHaveTextContent( + 'Returned to decision inbox. No other handled issues remain.', + ); + expect( + screen.getByRole('heading', { name: 'No reviewed or suppressed issues' }), + ).toBeInTheDocument(); + }); + + it('announces the state the reload reports when the review changed after the detail loaded', async () => { + // The detail still carries the acknowledgement, but the review was removed + // elsewhere, so ending the suppression reopens the issue. + const active = item(); + const suppressed = item({ state: 'suppressed' }); + const staleDetail = detail(suppressed); + staleDetail.operationalRecord.acknowledgement = { at: evaluatedAt, by: 'admin' }; + staleDetail.operationalRecord.suppression = { + at: evaluatedAt, + by: 'admin', + reason: 'Planned storage maintenance', + expiresAt: '2026-07-20T08:00:00Z', + }; + let isSuppressed = true; + apiMocks.getList.mockImplementation((filter: string) => + Promise.resolve( + isSuppressed + ? listResponse(filter === 'all' ? [suppressed] : [], summary({ suppressedCount: 1 })) + : listResponse([active], summary({ activeCount: 1, openCount: 1, calm: false })), + ), + ); + apiMocks.getDetail.mockImplementation(() => + Promise.resolve(isSuppressed ? staleDetail : detail(active)), + ); + apiMocks.unsuppress.mockImplementation(() => { + isSuppressed = false; + return Promise.resolve({ success: true }); + }); + renderWorkbench(); + + fireEvent.click(await screen.findByRole('button', { name: 'Reviewed and suppressed (1)' })); + fireEvent.click( + await screen.findByRole('button', { + name: 'Open Database VM · Disk pressure', + }), + ); + fireEvent.click(await screen.findByRole('button', { name: 'End suppression' })); + + await waitFor(() => expect(apiMocks.unsuppress).toHaveBeenCalledWith('record-1')); + expect(await screen.findByRole('status')).toHaveTextContent( + 'Returned to decision inbox. No other handled issues remain.', + ); + }); + it('offers every lasting Patrol decision on the finding that mirrors the alert', async () => { const active = item(); apiMocks.getList.mockResolvedValue( From 8c9000ac63930f3fcca74128e213346851e3fc34 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:09:11 +0100 Subject: [PATCH 22/53] Keep node temperatures off same-named nodes at other sites Two Proxmox connections can each have a node with the same name, which is common for MSPs running one px1 per site. The node temperature lookup fell back to the first host agent whose hostname matched the node name, even when that agent was linked to the other site's node, and even when the name was ambiguous enough that the agent linker had refused to link it. Site B's px1 then showed, alerted on and recorded history for site A's sensors. The cluster sensor cache leaked the same way: an agent's SSH readings for its cluster siblings were keyed by bare node name, so a site A sibling reading fed site B's node. The poller now hands the lookup the polled node, and the lookup works on that node's slot. The state folds the views of one machine reached through two connections (a cluster added twice, or a multi-homed host added by each address) into one node held under one view's ID, and the agent link follows it. Those views used to find their agent only through the loose hostname match, so a polled view the read state does not hold also takes the one same-named node proven to be the same machine. models.NodeObservationsSameMachine and models.HostAgentBridgesNodeViews give that proof from the evidence UpdateNodesForInstance folds on, and the state's agent corroboration check now shares its rule with them. Ambiguous candidates are declined, as the state declines an ambiguous alias. An agent linked to the slot is the node's agent. The hostname fallback only takes an agent with no link when no node outside the slot and no second unlinked agent has the name. Cluster sensor cache entries record their reporting agent and serve a node only when that agent's linked node shares one of the slot's connections, or belongs to a connection whose configured view of the node has a matching TLS fingerprint. LinkedNodeID holds the Proxmox source node ID for automatic, manual and continuity-restored links, so the link match needs no normalization. The physical-disk SMART merge already matches agents by node.LinkedAgentID within one instance and is unchanged. --- .../v6/internal/subsystems/agent-lifecycle.md | 13 + .../v6/internal/subsystems/monitoring.md | 40 +++ internal/models/models.go | 16 +- internal/models/node_machine_identity.go | 82 +++++ internal/models/node_machine_identity_test.go | 124 ++++++++ internal/monitoring/host_agent_temps.go | 291 ++++++++++++++---- internal/monitoring/host_agent_temps_test.go | 234 +++++++------- internal/monitoring/monitor.go | 6 +- .../monitoring/monitor_additional_test.go | 210 +++++++++++++ internal/monitoring/monitor_agents.go | 15 +- .../monitoring/monitor_host_agents_test.go | 33 ++ .../monitor_polling_node_helpers.go | 8 +- internal/monitoring/monitor_pve.go | 2 +- 13 files changed, 881 insertions(+), 193 deletions(-) create mode 100644 internal/models/node_machine_identity.go create mode 100644 internal/models/node_machine_identity_test.go diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 6813b04986..3d384e6479 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -288,6 +288,19 @@ not disable the local host, SMART, Ceph, or Proxmox reporting modules, alter agent enrollment or identity, or add remote command authority. The disabled path is pinned by `TestCollectClusterSensors_Disabled`; configuration parsing is pinned by `TestLoadConfigDisableClusterPeerSensorsFlag`. +Pulse attributes peer readings to the agent that reported them. Peers are +named by bare Proxmox node name, which a node of another connection can share, +so monitoring keys its cluster sensor cache by reporting agent and node name. +A reading serves a node only when the agent's currently linked node belongs to +that node's connection, or to another connection with a fingerprint-proven view +of the node (a cluster added twice); attribution follows the agent's link like +its own sensors. An unlinked reporter's peer readings serve no node, nor do +those of a reporter whose manual link names a node ID the read state does not +hold. This reads +the existing agent link and adds no identity, enrollment, link or command +authority; the ingest path is pinned by +`TestApplyHostReportScopesClusterSensorsToReportersConnection` in +`internal/monitoring/monitor_host_agents_test.go`. On supported Linux systemd hosts, the opt-in safe runtime is a root-owned, unprivileged monitoring collector plus the no-network typed helper, with remediation installed only as the separate root-owned `pulse-agent-runner`. diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 576a27650b..0e2bda7c3d 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -3789,6 +3789,46 @@ exists or the agent payload has no usable positive reading. Identity-only or zero-temperature SMART rows do not count as usable by themselves, but the runtime must not keep probing legacy SSH solely to augment an otherwise healthy agent temperature payload with SMART data. +A host agent's sensors feed the node the agent is linked to. Several Proxmox +connections can each have a node of the same name (one `px1` per site), and +`Host.LinkedNodeID` always holds the Proxmox source node ID, whether the link is +automatic, manual or restored from host continuity. The poller therefore hands +`getHostAgentTemperatureForNode` in `internal/monitoring/host_agent_temps.go` the +polled node itself, and the lookup works on that node's slot. A node the read +state holds under its own source ID is its own slot. The state folds the views +of one machine reached through two connections (a cluster added twice, or a +multi-homed host added by each address) into one node kept under one view's ID, +chosen by its merge preference, and an automatic agent link follows that node. +So a polled view the read state does not hold also takes the one same-named +node proven to be the same machine, by `models.NodeObservationsSameMachine` or, +for the agent linked to that node, `models.HostAgentBridgesNodeViews` (the agent +reports both endpoints and the views' TLS fingerprints do not contradict), both +in `internal/models/node_machine_identity.go`. These are pairwise forms of +evidence `UpdateNodesForInstance` folds on, and it shares the agent +corroboration rule with them; +`TestNodeObservationsSameMachineMatchesStateFold` and +`TestHostAgentBridgesNodeViewsMatchesStateFold` pin them against the state's +fold for the covered cases. Two or more such candidates are ambiguous and none +is taken, as the state declines an ambiguous alias. An agent linked to a slot ID +is the node's agent. A manual link stays pinned to the source ID the operator +chose, because the state never transfers operator intent to a replacement +provider identity, so while the state keeps that machine under the other view's +ID the node has no linked agent and no agent reading. Otherwise the hostname +fallback takes only an agent with no node, VM or container link whose hostname +is exactly the node name, and only when no Proxmox node outside the slot and no +second unlinked agent has that name. Cluster-sibling readings an agent collects over SSH arrive keyed by bare +node name, so the cluster sensor cache records the reporting agent. A reading +serves the node only when the node that agent is currently linked to belongs to +one of the slot's connections, or to a connection whose configured view of the +node carries a matching TLS fingerprint (the other connection of a cluster added +twice). The reading follows the agent's link the way the agent's own sensors do, +and a reading from an unlinked agent serves no node. Without these bounds, site +B's `px1` showed, alerted on and recorded history for site A's agent or +cluster-sibling sensors. `TestNodeTemperatureStaysWithinSameNamedNodesSite` in +`internal/monitoring/monitor_additional_test.go` pins the poller lookup, +including the folded-view cases, and +`TestApplyHostReportScopesClusterSensorsToReportersConnection` in +`internal/monitoring/monitor_host_agents_test.go` pins the agent ingest side. A node temperature presented as available must describe the node now, because node alert evaluation, node history writes, reporting, and the UI all read it as a live measurement. A host agent linked to a Proxmox node merges into one host diff --git a/internal/models/models.go b/internal/models/models.go index 30aaa665e6..0656616c9a 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -4051,7 +4051,7 @@ func (s *State) UpdateNodesForInstance(instanceName string, nodes []Node) { hostAgentByHostname := make(map[string]map[string]struct{}) // lowercase hostname -> hostAgentIDs hostAgentByIP := make(map[string]map[string]struct{}) // normalized ip -> hostAgentIDs validHostAgentIDs := make(map[string]bool) // set of existing host agent IDs - hostHostnameByID := make(map[string]string) // hostAgentID -> normalized full hostname + hostByID := make(map[string]Host) // hostAgentID -> host addHostAlias := func(name, hostID string) { name = strings.TrimSpace(strings.ToLower(name)) if name == "" || hostID == "" { @@ -4086,7 +4086,7 @@ func (s *State) UpdateNodesForInstance(instanceName string, nodes []Node) { for _, host := range s.Hosts { if host.ID != "" { validHostAgentIDs[host.ID] = true - hostHostnameByID[host.ID] = strings.TrimSpace(strings.ToLower(host.Hostname)) + hostByID[host.ID] = host addHostAlias(host.Hostname, host.ID) // Also index by short hostname if idx := strings.Index(host.Hostname, "."); idx > 0 { @@ -4109,16 +4109,8 @@ func (s *State) UpdateNodesForInstance(instanceName string, nodes []Node) { // Preserve legitimate split views when the same agent strongly bridges // both endpoints through an exact IP or a full (dotted) hostname. hostStronglyCorroboratesNode := func(hostID string, node Node) bool { - endpoint := extractHostEndpoint(node.Host) - if endpoint == "" { - return false - } - if ip := normalizeIPAddress(endpoint); ip != "" { - _, ok := hostIPsByID[hostID][ip] - return ok - } - hostname := hostHostnameByID[hostID] - return strings.Contains(endpoint, ".") && endpoint == hostname + host, ok := hostByID[hostID] + return ok && hostReportsNodeEndpoint(host, node) } sharedAgentProvesNodePair := func(hostID string, existing, candidate Node) bool { if nodeCrossViewMergeProven(existing, candidate) { diff --git a/internal/models/node_machine_identity.go b/internal/models/node_machine_identity.go new file mode 100644 index 0000000000..98ad82a30a --- /dev/null +++ b/internal/models/node_machine_identity.go @@ -0,0 +1,82 @@ +package models + +import "strings" + +// NodeObservationsSameMachine reports whether two observations of a Proxmox +// node are one machine by the evidence UpdateNodesForInstance folds node views +// into one slot on: the same logical key, or the same node name with no +// provider identity conflict and either positive cross-view proof (the same +// connection, the same cluster with matching TLS fingerprints, matching +// fingerprints, or for unclustered views the same endpoint) or a shared +// endpoint address. A cluster added through two connections is such a pair. +// Callers that see a polled node before it is merged, such as the node's host +// agent lookup, use this so they agree with the state on which views are the +// same node. +func NodeObservationsSameMachine(a, b Node) bool { + if key := nodeLogicalKey(a); key != "" && key == nodeLogicalKey(b) { + return true + } + name := normalizeNodeIdentityPart(a.Name) + if name == "" || name != normalizeNodeIdentityPart(b.Name) || nodeProviderIdentityConflicts(a, b) { + return false + } + if nodeCrossViewMergeProven(a, b) { + return true + } + // An address alias folds on the conflict check alone (resolveNodeMergeKey). + otherAliases := nodeEndpointMergeAliases(b) + for _, alias := range nodeEndpointMergeAliases(a) { + if !strings.HasPrefix(alias, "endpoint-ip:") { + continue + } + for _, other := range otherAliases { + if alias == other { + return true + } + } + } + return false +} + +// HostAgentBridgesNodeViews reports whether a host agent linked to view a proves +// view b is the same machine, the way UpdateNodesForInstance lets that agent +// fold b into a's slot: neither view names a cluster, their TLS fingerprints do +// not contradict each other (the state rejects an agent for a node whose known +// fingerprint differs from its linked node's), and the agent's own report names +// both endpoints, by exact address or full hostname. A host with several +// addresses added through two standalone connections is such a pair. +func HostAgentBridgesNodeViews(host Host, a, b Node) bool { + if normalizeNodeIdentityPart(a.ClusterName) != "" || normalizeNodeIdentityPart(b.ClusterName) != "" { + return false + } + fpA := normalizeNodeTLSFingerprint(a.TLSFingerprint) + fpB := normalizeNodeTLSFingerprint(b.TLSFingerprint) + if fpA != "" && fpB != "" && fpA != fpB { + return false + } + return hostReportsNodeEndpoint(host, a) && hostReportsNodeEndpoint(host, b) +} + +// hostReportsNodeEndpoint reports whether a host agent's own report names the +// node's endpoint: the endpoint address among the addresses it reports, or the +// endpoint's full (dotted) hostname as its hostname. +func hostReportsNodeEndpoint(host Host, node Node) bool { + endpoint := extractHostEndpoint(node.Host) + if endpoint == "" { + return false + } + if ip := normalizeIPAddress(endpoint); ip != "" { + if normalizeIPAddress(host.ReportIP) == ip { + return true + } + for _, iface := range host.NetworkInterfaces { + for _, address := range iface.Addresses { + if normalizeIPAddress(address) == ip { + return true + } + } + } + return false + } + return strings.Contains(endpoint, ".") && endpoint == strings.TrimSpace(strings.ToLower(host.Hostname)) +} diff --git a/internal/models/node_machine_identity_test.go b/internal/models/node_machine_identity_test.go new file mode 100644 index 0000000000..65c06cba98 --- /dev/null +++ b/internal/models/node_machine_identity_test.go @@ -0,0 +1,124 @@ +package models + +import "testing" + +// NodeObservationsSameMachine must agree with the fold UpdateNodesForInstance +// applies to two views of a node from different connections, so callers that +// see a polled node before it is merged treat the same views as one node. +func TestNodeObservationsSameMachineMatchesStateFold(t *testing.T) { + clusterNode := func(instance, cluster, host, fingerprint string) Node { + return Node{ + ID: instance + "-pve01", NodeIdentity: instance + "-pve01", Name: "pve01", Instance: instance, + ClusterName: cluster, IsClusterMember: cluster != "", Host: host, TLSFingerprint: fingerprint, Status: "online", + } + } + for _, tc := range []struct { + name string + existing, candidate Node + wantSameMachineFolds bool + }{ + { + name: "same cluster added twice with matching fingerprints", + existing: clusterNode("enacon-a", "enacon", "https://192.168.1.11:8006", "AA:AA:AA:AA"), + candidate: clusterNode("enacon-b", "enacon", "https://192.168.1.11:8006", "aaaaaaaa"), + wantSameMachineFolds: true, + }, + { + name: "same-named clusters without fingerprints", + existing: clusterNode("site-a", "cluster", "https://192.168.1.11:8006", ""), + candidate: clusterNode("site-b", "cluster", "https://192.168.1.11:8006", ""), + }, + { + name: "same-named clusters with different fingerprints", + existing: clusterNode("site-a", "cluster", "https://192.168.1.11:8006", "AA:AA"), + candidate: clusterNode("site-b", "cluster", "https://192.168.1.11:8006", "BB:BB"), + }, + { + name: "different clusters reusing a node name and address", + existing: clusterNode("site-a", "a", "https://192.168.1.11:8006", "AA:AA"), + candidate: clusterNode("site-b", "b", "https://192.168.1.11:8006", "AA:AA"), + }, + { + name: "standalone nodes of one name at different addresses", + existing: clusterNode("site-a", "", "https://10.0.0.5:8006", ""), + candidate: clusterNode("site-b", "", "https://10.0.1.5:8006", ""), + }, + { + name: "standalone connections to one address", + existing: clusterNode("by-ip", "", "https://10.0.0.5:8006", ""), + candidate: clusterNode("by-ip-again", "", "https://10.0.0.5:8006", ""), + wantSameMachineFolds: true, + }, + { + name: "standalone view of a cluster member's address", + existing: clusterNode("cluster-view", "enacon", "https://192.168.1.11:8006", ""), + candidate: clusterNode("standalone-view", "", "https://192.168.1.11:8006", ""), + wantSameMachineFolds: true, + }, + { + name: "same cluster by hostname with matching fingerprints", + existing: clusterNode("enacon-a", "enacon", "https://pve01.lan:8006", "AA:AA"), + candidate: clusterNode("enacon-b", "enacon", "https://pve01.lan:8006", "AA:AA"), + wantSameMachineFolds: true, + }, + { + name: "different clusters by hostname", + existing: clusterNode("site-a", "a", "https://pve01.lan:8006", ""), + candidate: clusterNode("site-b", "b", "https://pve01.lan:8006", ""), + }, + } { + t.Run(tc.name, func(t *testing.T) { + state := &State{Nodes: []Node{tc.existing}} + state.UpdateNodesForInstance(tc.candidate.Instance, []Node{tc.candidate}) + if folded := len(state.Nodes) == 1; folded != tc.wantSameMachineFolds { + t.Fatalf("state fold = %v (nodes %#v), want %v", folded, state.Nodes, tc.wantSameMachineFolds) + } + if got := NodeObservationsSameMachine(tc.existing, tc.candidate); got != tc.wantSameMachineFolds { + t.Fatalf("NodeObservationsSameMachine = %v, want %v (the state's fold)", got, tc.wantSameMachineFolds) + } + if got := NodeObservationsSameMachine(tc.candidate, tc.existing); got != tc.wantSameMachineFolds { + t.Fatalf("NodeObservationsSameMachine reversed = %v, want %v", got, tc.wantSameMachineFolds) + } + }) + } +} + +// HostAgentBridgesNodeViews must agree with the fold UpdateNodesForInstance lets +// a linked agent make between two standalone views of one machine. +func TestHostAgentBridgesNodeViewsMatchesStateFold(t *testing.T) { + standalone := func(instance, host string) Node { + return Node{ID: instance + "-minipc", NodeIdentity: instance + "-minipc", Name: "minipc", Instance: instance, Host: host, Status: "online"} + } + for _, tc := range []struct { + name string + addresses []string + existingFP, candidateFP string + wantBridge bool + }{ + {name: "agent reports both connection addresses", addresses: []string{"10.0.0.5/24", "10.0.1.5/24"}, wantBridge: true}, + {name: "agent reports only the linked view's address", addresses: []string{"10.0.0.5/24"}}, + {name: "matching fingerprints", addresses: []string{"10.0.0.5/24", "10.0.1.5/24"}, existingFP: "AA:AA", candidateFP: "aaaa", wantBridge: true}, + {name: "contradicting fingerprints", addresses: []string{"10.0.0.5/24", "10.0.1.5/24"}, existingFP: "AA:AA", candidateFP: "BB:BB"}, + } { + t.Run(tc.name, func(t *testing.T) { + existing := standalone("lan-a", "https://10.0.0.5:8006") + existing.LinkedAgentID = "agent-minipc" + existing.TLSFingerprint = tc.existingFP + host := Host{ + ID: "agent-minipc", Hostname: "minipc", LinkedNodeID: existing.ID, ReportIP: "10.0.0.5", + NetworkInterfaces: []HostNetworkInterface{{Name: "eth0", Addresses: tc.addresses}}, + } + candidate := standalone("lan-b", "https://10.0.1.5:8006") + candidate.TLSFingerprint = tc.candidateFP + + state := &State{Nodes: []Node{existing}, Hosts: []Host{host}} + state.UpdateNodesForInstance(candidate.Instance, []Node{candidate}) + if folded := len(state.Nodes) == 1; folded != tc.wantBridge { + t.Fatalf("state fold = %v (nodes %#v), want %v", folded, state.Nodes, tc.wantBridge) + } + if got := HostAgentBridgesNodeViews(host, existing, candidate); got != tc.wantBridge { + t.Fatalf("HostAgentBridgesNodeViews = %v, want %v (the state's fold)", got, tc.wantBridge) + } + }) + } +} diff --git a/internal/monitoring/host_agent_temps.go b/internal/monitoring/host_agent_temps.go index 8b3fc4d3e0..83ae970aac 100644 --- a/internal/monitoring/host_agent_temps.go +++ b/internal/monitoring/host_agent_temps.go @@ -3,6 +3,7 @@ package monitoring import ( "math" "regexp" + "slices" "sort" "strconv" "strings" @@ -19,7 +20,7 @@ import ( // duplicate hostname scenarios), then falls back to hostname matching. // Returns nil if no matching host agent is found or if no temperature data is available. func (m *Monitor) getHostAgentTemperature(nodeName string) *models.Temperature { - return m.getHostAgentTemperatureByID("", nodeName) + return m.getHostAgentTemperatureForNode(models.Node{Name: nodeName}) } func shouldSkipTemperatureSSHCollection(hostAgentTemp *models.Temperature) bool { @@ -77,19 +78,23 @@ func hasUsableSMARTTemperature(temp *models.Temperature) bool { return false } -// getHostAgentTemperatureByID looks for a matching host agent by node ID first, -// then falls back to hostname matching. This correctly handles clusters where -// multiple nodes may have the same hostname (e.g., "px1" on different IPs). -func (m *Monitor) getHostAgentTemperatureByID(nodeID, nodeName string) *models.Temperature { +// getHostAgentTemperatureForNode returns a polled Proxmox node's reading from +// its host agent (see hostAgentForNode), else from a sibling agent's cluster +// sensor cache. node is the poller's model of the node: its source ID, native +// name, connection and identity stamps. +func (m *Monitor) getHostAgentTemperatureForNode(node models.Node) *models.Temperature { readState := m.GetUnifiedReadStateOrSnapshot() if readState == nil { return nil } + hosts := readState.Hosts() + nodes := readState.Nodes() + slot := m.polledNodeSlot(hosts, nodes, node) - matchedHost := hostAgentForNode(readState.Hosts(), nodeID, nodeName) + matchedHost := hostAgentForNode(hosts, nodes, slot, node.Name) if matchedHost == nil { // No directly-linked host agent found — check cluster sensor cache - return m.getClusterSensorTemperature(nodeName) + return m.getClusterSensorTemperature(hosts, nodes, node, slot) } // Check if the host agent has temperature data. SMART-only reports are @@ -98,7 +103,7 @@ func (m *Monitor) getHostAgentTemperatureByID(nodeID, nodeName string) *models.T sensors := matchedHost.Sensors() if sensors == nil || (len(sensors.TemperatureCelsius) == 0 && len(sensors.SMART) == 0) { // Host agent exists but has no temperature data — try cluster cache - return m.getClusterSensorTemperature(nodeName) + return m.getClusterSensorTemperature(hosts, nodes, node, slot) } // An agent that stopped reporting keeps its last sensors in state. Those @@ -108,7 +113,7 @@ func (m *Monitor) getHostAgentTemperatureByID(nodeID, nodeName string) *models.T // every PVE poll, so read the agent source's sighting. agentStatus, ok := matchedHost.SourceStatus(unifiedresources.SourceAgent) if !ok || !hostAgentReportCurrent(agentStatus.LastSeen, matchedHost.IntervalSeconds(), time.Now()) { - return m.getClusterSensorTemperature(nodeName) + return m.getClusterSensorTemperature(hosts, nodes, node, slot) } // Convert host agent sensor data to Temperature model, stamped with the @@ -125,7 +130,7 @@ func (m *Monitor) getHostAgentTemperatureByID(nodeID, nodeName string) *models.T // reading of that age is dropped too; it is already older than the lease, and // the bound errs toward a gap rather than a stale value. A reading stamped // after that report keeps the ordinary carry window. -func (m *Monitor) carriedTemperatureOutlivesAgentLease(nodeID, nodeName string, temp *models.Temperature, now time.Time) bool { +func (m *Monitor) carriedTemperatureOutlivesAgentLease(node models.Node, temp *models.Temperature, now time.Time) bool { if temp == nil { return false } @@ -133,7 +138,9 @@ func (m *Monitor) carriedTemperatureOutlivesAgentLease(nodeID, nodeName string, if readState == nil { return false } - matchedHost := hostAgentForNode(readState.Hosts(), nodeID, nodeName) + hosts := readState.Hosts() + nodes := readState.Nodes() + matchedHost := hostAgentForNode(hosts, nodes, m.polledNodeSlot(hosts, nodes, node), node.Name) if matchedHost == nil { return false } @@ -145,37 +152,186 @@ func (m *Monitor) carriedTemperatureOutlivesAgentLease(nodeID, nodeName string, return !temp.LastUpdate.After(agentStatus.LastSeen) } -// hostAgentForNode returns the host agent for a Proxmox node: the agent linked -// to the node ID, else one whose hostname matches. This correctly handles -// clusters where multiple nodes may have the same hostname (e.g., "px1" on -// different IPs). -func hostAgentForNode(hosts []*unifiedresources.HostView, nodeID, nodeName string) *unifiedresources.HostView { - // First, try to find a host agent that is explicitly linked to this node - // via LinkedNodeID. This is the most reliable method and handles duplicate - // hostnames correctly. - if nodeID != "" { - for i := range hosts { - if hosts[i].LinkedNodeID() == nodeID { - log.Debug(). - Str("nodeID", nodeID). - Str("hostAgentID", hosts[i].ID()). - Str("hostname", hosts[i].Hostname()). - Msg("Matched host agent to node via LinkedNodeID") - return hosts[i] - } +// nodeSlot is the read-state identity of one polled Proxmox node: the source +// node IDs and connections that denote it. +type nodeSlot struct { + ids []string + instances []string +} + +func (s *nodeSlot) add(id, instance string) { + if id = strings.TrimSpace(id); id != "" && !s.hasID(id) { + s.ids = append(s.ids, id) + } + if instance = strings.TrimSpace(instance); instance != "" && !s.hasInstance(instance) { + s.instances = append(s.instances, instance) + } +} + +func (s nodeSlot) hasID(id string) bool { + return id != "" && slices.Contains(s.ids, id) +} + +func (s nodeSlot) hasInstance(instance string) bool { + return instance != "" && slices.Contains(s.instances, instance) +} + +// polledNodeSlot returns the read-state identity of a polled Proxmox node. A +// node the read state holds under its own source ID is that node alone. The +// state folds the views of one machine reached through two connections (a +// cluster added twice, or a multi-homed host added by each address) into one +// slot, kept under whichever view its merge preference keeps, so a polled view +// the read state does not hold also counts as the one same-named node proven to +// be that machine: by models.NodeObservationsSameMachine, or by +// models.HostAgentBridgesNodeViews for the agent linked to that node. Two or +// more such candidates are ambiguous and none is taken, as the state's alias +// resolution declines them. Agent links and cluster-sensor scope are matched +// against the slot, never against a same-named node of another machine. +func (m *Monitor) polledNodeSlot(hosts []*unifiedresources.HostView, nodes []*unifiedresources.NodeView, node models.Node) nodeSlot { + var slot nodeSlot + if strings.TrimSpace(node.ID) == "" { + return slot + } + for _, view := range nodes { + if view != nil && view.SourceID() == node.ID { + slot.add(node.ID, firstNonEmptyString(node.Instance, view.Instance())) + return slot + } + } + slot.add(node.ID, node.Instance) + + var partner *unifiedresources.NodeView + for _, view := range nodes { + if view == nil || !strings.EqualFold(view.NodeName(), node.Name) { + continue + } + if !m.nodeViewIsSameMachine(hosts, view, node) { + continue + } + if partner != nil { + return slot + } + partner = view + } + if partner != nil { + slot.add(partner.SourceID(), partner.Instance()) + } + return slot +} + +// nodeViewIsSameMachine reports whether a read-state Proxmox node and a polled +// node are one machine by the evidence the state folds node views on. +func (m *Monitor) nodeViewIsSameMachine(hosts []*unifiedresources.HostView, view *unifiedresources.NodeView, node models.Node) bool { + stateNode := m.stateNodeFromView(view) + if models.NodeObservationsSameMachine(stateNode, node) { + return true + } + for _, host := range hosts { + if host.LinkedNodeID() == stateNode.ID { + return models.HostAgentBridgesNodeViews(hostFromReadStateView(host), stateNode, node) + } + } + return false +} + +// nodeHasProvenViewInConnection reports whether a polled node has a view under +// another Proxmox connection that is the same machine: a cluster added through +// two connections lists the same members, each with its TLS fingerprint, so +// that connection's view of the node's name is the polled node. +func (m *Monitor) nodeHasProvenViewInConnection(node models.Node, instance string) bool { + instanceCfg := m.getInstanceConfig(instance) + if instanceCfg == nil || !instanceCfg.IsCluster { + return false + } + fingerprint := pveNodeTLSFingerprint(instanceCfg, node.Name) + if fingerprint == "" { + return false + } + return models.NodeObservationsSameMachine(models.Node{ + Name: node.Name, + Instance: instanceCfg.Name, + ClusterName: instanceCfg.ClusterName, + IsClusterMember: true, + TLSFingerprint: fingerprint, + }, node) +} + +// stateNodeFromView restores the identity a read-state Proxmox node carries in +// state, including the config-derived TLS fingerprint the registry does not +// keep, the way previousNodesForInstance does. +func (m *Monitor) stateNodeFromView(view *unifiedresources.NodeView) models.Node { + node := previousNodeFromView(view) + node.TLSFingerprint = pveNodeTLSFingerprint(m.getInstanceConfig(node.Instance), node.Name) + return node +} + +// hostAgentForNode returns the host agent for a polled Proxmox node: the agent +// linked to the node's slot, else the one unlinked agent whose hostname is the +// node's name. Several connections can each have a node with the same name +// (one "px1" per site), so the hostname fallback never takes an agent linked to +// another node or to a guest, and finds nothing when a Proxmox node outside the +// slot or a second unlinked agent has the name. Every link source (automatic, +// manual, restored from continuity) stores the Proxmox source node ID, so slot +// IDs compare with LinkedNodeID directly. +func hostAgentForNode(hosts []*unifiedresources.HostView, nodes []*unifiedresources.NodeView, slot nodeSlot, nodeName string) *unifiedresources.HostView { + for _, host := range hosts { + if slot.hasID(host.LinkedNodeID()) { + log.Debug(). + Str("nodeID", host.LinkedNodeID()). + Str("hostAgentID", host.ID()). + Str("hostname", host.Hostname()). + Msg("Matched host agent to node via LinkedNodeID") + return host } } - // Fallback: match by hostname if no linked host was found - // This maintains backwards compatibility for setups where linking hasn't occurred yet - nodeLower := strings.ToLower(strings.TrimSpace(nodeName)) - for i := range hosts { - hostnameLower := strings.ToLower(strings.TrimSpace(hosts[i].Hostname())) - if hostnameLower == nodeLower { - return hosts[i] + // Fallback for an agent that is not linked yet. + name := strings.ToLower(strings.TrimSpace(nodeName)) + if name == "" || otherProxmoxNodeHasName(nodes, slot, name) { + return nil + } + var match *unifiedresources.HostView + for _, host := range hosts { + if host.LinkedNodeID() != "" || host.LinkedVMID() != "" || host.LinkedContainerID() != "" { + continue + } + if strings.ToLower(strings.TrimSpace(host.Hostname())) != name { + continue + } + if match != nil { + return nil + } + match = host + } + return match +} + +// otherProxmoxNodeHasName reports whether a Proxmox node outside slot is named +// name (lowercase). With an empty slot every node of that name counts. +func otherProxmoxNodeHasName(nodes []*unifiedresources.NodeView, slot nodeSlot, name string) bool { + for _, node := range nodes { + if node == nil || slot.hasID(node.SourceID()) { + continue + } + if strings.ToLower(node.NodeName()) == name { + return true } } - return nil + return false +} + +// proxmoxNodeInstance returns the Proxmox connection a node belongs to, or "" +// when the node is not in the read state. +func proxmoxNodeInstance(nodes []*unifiedresources.NodeView, nodeID string) string { + if nodeID == "" { + return "" + } + for _, node := range nodes { + if node != nil && node.SourceID() == nodeID { + return node.Instance() + } + } + return "" } func convertUnifiedHostSensorsToTemperature(sensors *unifiedresources.HostSensorMeta, lastSeen time.Time) *models.Temperature { @@ -243,30 +399,57 @@ func cloneSMARTAttributesModel(src *models.SMARTAttributes) *models.SMARTAttribu return &dest } -// getClusterSensorTemperature looks up cached temperature data that was collected -// by a sibling agent in the same Proxmox cluster via SSH. Returns nil if no -// recent data is available. -func (m *Monitor) getClusterSensorTemperature(nodeName string) *models.Temperature { - if nodeName == "" { - return nil - } - - key := strings.ToLower(strings.TrimSpace(nodeName)) - - m.clusterSensorsMu.RLock() - entry, ok := m.clusterSensorsCache[key] - m.clusterSensorsMu.RUnlock() - - if !ok { +// getClusterSensorTemperature returns the reading a host agent on a sibling +// cluster node collected for this node over SSH. Siblings are reported by bare +// node name, which nodes of different connections can share, so a reading only +// serves the node when the node its reporting agent is linked to belongs to one +// of the slot's connections, or to a connection with a proven view of the node +// (a cluster added twice). Readings from an unlinked agent serve no node. +func (m *Monitor) getClusterSensorTemperature(hosts []*unifiedresources.HostView, nodes []*unifiedresources.NodeView, node models.Node, slot nodeSlot) *models.Temperature { + name := strings.ToLower(strings.TrimSpace(node.Name)) + if name == "" { return nil } // Reuse the same staleness threshold as direct host agents (2 minutes) - if !isHostAgentTemperatureRecent(entry.updatedAt) { + var candidates []clusterSensorsCacheEntry + m.clusterSensorsMu.RLock() + for _, entry := range m.clusterSensorsCache { + if entry.nodeName == name && isHostAgentTemperatureRecent(entry.updatedAt) { + candidates = append(candidates, entry) + } + } + m.clusterSensorsMu.RUnlock() + + var newest *clusterSensorsCacheEntry + for i := range candidates { + reporterInstance := proxmoxNodeInstance(nodes, linkedNodeIDForAgent(hosts, candidates[i].reporterID)) + if reporterInstance == "" || + (!slot.hasInstance(reporterInstance) && !m.nodeHasProvenViewInConnection(node, reporterInstance)) { + continue + } + if newest == nil || candidates[i].updatedAt.After(newest.updatedAt) { + newest = &candidates[i] + } + } + if newest == nil { return nil } + return convertHostSensorsToTemperature(newest.sensors, newest.updatedAt) +} - return convertHostSensorsToTemperature(entry.sensors, entry.updatedAt) +// linkedNodeIDForAgent returns the Proxmox source node ID the agent with the +// given agent ID (models.Host.ID) is linked to, or "". +func linkedNodeIDForAgent(hosts []*unifiedresources.HostView, agentID string) string { + if agentID == "" { + return "" + } + for _, host := range hosts { + if host.AgentID() == agentID { + return host.LinkedNodeID() + } + } + return "" } // convertHostSensorsToTemperature converts HostSensorSummary to the Temperature model. diff --git a/internal/monitoring/host_agent_temps_test.go b/internal/monitoring/host_agent_temps_test.go index 5885b47ea8..7d330a21e3 100644 --- a/internal/monitoring/host_agent_temps_test.go +++ b/internal/monitoring/host_agent_temps_test.go @@ -8,6 +8,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/models" unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" "github.com/stretchr/testify/assert" ) @@ -466,7 +467,7 @@ func TestGetHostAgentTemperature(t *testing.T) { } m.state.UpsertHost(host) - result := m.getHostAgentTemperatureByID("node-123", "different-name") + result := m.getHostAgentTemperatureForNode(models.Node{ID: "node-123", Name: "different-name"}) assert.NotNil(t, result) assert.Equal(t, 60.0, result.CPUPackage) }) @@ -484,7 +485,7 @@ func TestGetHostAgentTemperature(t *testing.T) { } m.state.UpsertHost(host) - result := m.getHostAgentTemperatureByID("node-smart-only", "different-name") + result := m.getHostAgentTemperatureForNode(models.Node{ID: "node-smart-only", Name: "different-name"}) assert.NotNil(t, result) assert.True(t, result.Available) assert.True(t, result.HasSMART) @@ -573,115 +574,142 @@ func TestConvertHostSensorsToTemperature_ExtraBranches(t *testing.T) { }) } -func TestGetClusterSensorTemperature(t *testing.T) { - m := &Monitor{ +// readStateMonitor returns a monitor whose unified read state is the registry +// projection of nodes and hosts, as the poller sees it. Nodes are linked back to +// the hosts linked to them, as the agent linker leaves them. +func readStateMonitor(nodes []models.Node, hosts []models.Host) *Monitor { + for i := range nodes { + for _, host := range hosts { + if host.LinkedNodeID == nodes[i].ID { + nodes[i].LinkedAgentID = host.ID + } + } + } + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + adapter.PopulateFromSnapshot(models.StateSnapshot{Nodes: nodes, Hosts: hosts}) + return &Monitor{ state: models.NewState(), + resourceStore: adapter, clusterSensorsCache: make(map[string]clusterSensorsCacheEntry), } +} + +func readStateTestNode(instance, name string) models.Node { + return models.Node{ID: instance + "-" + name, Instance: instance, Name: name, Status: "online", LastSeen: time.Now()} +} + +func reportingTestHost(id, hostname, linkedNodeID string, cpuPackage float64) models.Host { + host := models.Host{ID: id, Hostname: hostname, Status: "online", LinkedNodeID: linkedNodeID, IntervalSeconds: 30, LastSeen: time.Now()} + if cpuPackage > 0 { + host.Sensors = models.HostSensorSummary{TemperatureCelsius: map[string]float64{"cpu_package": cpuPackage}} + } + return host +} + +func clusterSensorReport(nodeName string, cpuPackage float64) []agentshost.ClusterNodeSensors { + return []agentshost.ClusterNodeSensors{{ + NodeName: nodeName, + Sensors: agentshost.Sensors{TemperatureCelsius: map[string]float64{"cpu_package": cpuPackage, "cpu_core_0": cpuPackage - 3}}, + }} +} + +func TestGetClusterSensorTemperature(t *testing.T) { + m := readStateMonitor( + []models.Node{readStateTestNode("pve1", "node1"), readStateTestNode("pve1", "node2"), readStateTestNode("pve1", "MyNode"), readStateTestNode("pve1", "stale-node")}, + []models.Host{reportingTestHost("agent-node1", "node1", "pve1-node1", 0), reportingTestHost("agent-unlinked", "elsewhere", "", 0)}, + ) + lookup := func(nodeID, nodeName string) *models.Temperature { + readState := m.GetUnifiedReadStateOrSnapshot() + polled := models.Node{ID: nodeID, Name: nodeName} + slot := m.polledNodeSlot(readState.Hosts(), readState.Nodes(), polled) + return m.getClusterSensorTemperature(readState.Hosts(), readState.Nodes(), polled, slot) + } t.Run("empty cache returns nil", func(t *testing.T) { - result := m.getClusterSensorTemperature("node1") - assert.Nil(t, result) + assert.Nil(t, lookup("pve1-node2", "node2")) }) t.Run("cached data returned", func(t *testing.T) { - m.clusterSensorsCache["node2"] = clusterSensorsCacheEntry{ - sensors: models.HostSensorSummary{ - TemperatureCelsius: map[string]float64{ - "cpu_package": 58.0, - "cpu_core_0": 55.0, - }, - }, - updatedAt: time.Now(), - } + m.applyClusterSensors("agent-node1", clusterSensorReport("node2", 58), time.Now()) - result := m.getClusterSensorTemperature("node2") - assert.NotNil(t, result) - assert.Equal(t, 58.0, result.CPUPackage) - assert.True(t, result.HasCPU) + result := lookup("pve1-node2", "node2") + if assert.NotNil(t, result) { + assert.Equal(t, 58.0, result.CPUPackage) + assert.True(t, result.HasCPU) + } }) t.Run("stale data returns nil", func(t *testing.T) { - m.clusterSensorsCache["stale-node"] = clusterSensorsCacheEntry{ - sensors: models.HostSensorSummary{ - TemperatureCelsius: map[string]float64{"cpu_package": 50.0}, - }, - updatedAt: time.Now().Add(-5 * time.Minute), // older than 2min threshold - } + m.applyClusterSensors("agent-node1", clusterSensorReport("stale-node", 50), time.Now().Add(-3*time.Minute)) // older than 2min threshold - result := m.getClusterSensorTemperature("stale-node") - assert.Nil(t, result) + assert.Nil(t, lookup("pve1-stale-node", "stale-node")) }) t.Run("case insensitive lookup", func(t *testing.T) { - m.clusterSensorsCache["mynode"] = clusterSensorsCacheEntry{ - sensors: models.HostSensorSummary{ - TemperatureCelsius: map[string]float64{"cpu_package": 60.0}, - }, - updatedAt: time.Now(), - } + m.applyClusterSensors("agent-node1", clusterSensorReport("mynode", 60), time.Now()) - result := m.getClusterSensorTemperature("MyNode") - assert.NotNil(t, result) - assert.Equal(t, 60.0, result.CPUPackage) + result := lookup("pve1-MyNode", "MyNode") + if assert.NotNil(t, result) { + assert.Equal(t, 60.0, result.CPUPackage) + } + }) + + t.Run("reading from an unlinked agent serves no node", func(t *testing.T) { + m.applyClusterSensors("agent-unlinked", clusterSensorReport("node1", 40), time.Now()) + + assert.Nil(t, lookup("pve1-node1", "node1")) + }) + + t.Run("node of another connection returns nil", func(t *testing.T) { + assert.Nil(t, lookup("", "node2")) + assert.Nil(t, lookup("pve9-node2", "node2")) + }) + + t.Run("node not yet in the read state is scoped by its own connection", func(t *testing.T) { + m.applyClusterSensors("agent-node1", clusterSensorReport("node9", 47), time.Now()) + readState := m.GetUnifiedReadStateOrSnapshot() + polled := readStateTestNode("pve1", "node9") + result := m.getClusterSensorTemperature(readState.Hosts(), readState.Nodes(), polled, m.polledNodeSlot(readState.Hosts(), readState.Nodes(), polled)) + if assert.NotNil(t, result) { + assert.Equal(t, 47.0, result.CPUPackage) + } }) t.Run("empty node name returns nil", func(t *testing.T) { - result := m.getClusterSensorTemperature("") - assert.Nil(t, result) + assert.Nil(t, lookup("pve1-node2", "")) }) } func TestGetHostAgentTemperatureByID_ClusterFallback(t *testing.T) { - m := &Monitor{ - state: models.NewState(), - clusterSensorsCache: make(map[string]clusterSensorsCacheEntry), - } + m := readStateMonitor( + []models.Node{readStateTestNode("pve1", "sibling"), readStateTestNode("pve1", "orphan-node")}, + []models.Host{reportingTestHost("agent-sibling", "sibling", "pve1-sibling", 0)}, + ) - // No host agent, but cluster cache has data - m.clusterSensorsCache["orphan-node"] = clusterSensorsCacheEntry{ - sensors: models.HostSensorSummary{ - TemperatureCelsius: map[string]float64{ - "cpu_package": 62.0, - }, - }, - updatedAt: time.Now(), - } + // No host agent on the node, but a sibling's agent collected its sensors + m.applyClusterSensors("agent-sibling", clusterSensorReport("orphan-node", 62), time.Now()) - result := m.getHostAgentTemperatureByID("", "orphan-node") - assert.NotNil(t, result, "should fall back to cluster sensor cache") - assert.Equal(t, 62.0, result.CPUPackage) + result := m.getHostAgentTemperatureForNode(models.Node{ID: "pve1-orphan-node", Name: "orphan-node"}) + if assert.NotNil(t, result, "should fall back to cluster sensor cache") { + assert.Equal(t, 62.0, result.CPUPackage) + } } func TestGetHostAgentTemperatureByID_LocalAgentTakesPriority(t *testing.T) { - m := &Monitor{ - state: models.NewState(), - clusterSensorsCache: make(map[string]clusterSensorsCacheEntry), - } - - // Both local agent and cluster cache have data for the same node - m.state.UpsertHost(models.Host{ - ID: "host-local", - Hostname: "shared-node", - Sensors: models.HostSensorSummary{ - TemperatureCelsius: map[string]float64{ - "cpu_package": 70.0, // local agent reports 70 - }, + m := readStateMonitor( + []models.Node{readStateTestNode("pve1", "sibling"), readStateTestNode("pve1", "shared-node")}, + []models.Host{ + reportingTestHost("agent-sibling", "sibling", "pve1-sibling", 0), + reportingTestHost("host-local", "shared-node", "pve1-shared-node", 70), // local agent reports 70 }, - LastSeen: time.Now(), - }) - m.clusterSensorsCache["shared-node"] = clusterSensorsCacheEntry{ - sensors: models.HostSensorSummary{ - TemperatureCelsius: map[string]float64{ - "cpu_package": 55.0, // cluster cache says 55 - }, - }, - updatedAt: time.Now(), - } + ) + // Both the local agent and the cluster cache have data for the same node + m.applyClusterSensors("agent-sibling", clusterSensorReport("shared-node", 55), time.Now()) // cluster cache says 55 - result := m.getHostAgentTemperatureByID("", "shared-node") - assert.NotNil(t, result) - assert.Equal(t, 70.0, result.CPUPackage, "local agent data should take priority over cluster cache") + result := m.getHostAgentTemperatureForNode(models.Node{ID: "pve1-shared-node", Name: "shared-node"}) + if assert.NotNil(t, result) { + assert.Equal(t, 70.0, result.CPUPackage, "local agent data should take priority over cluster cache") + } } // An agent that stops reporting keeps its last sensors in state. Past the @@ -689,45 +717,23 @@ func TestGetHostAgentTemperatureByID_LocalAgentTakesPriority(t *testing.T) { // linked node as a live reading; the cluster cache (with its own recency) is // still consulted. func TestGetHostAgentTemperatureByID_IgnoresAgentPastReportingLease(t *testing.T) { - m := &Monitor{ - state: models.NewState(), - clusterSensorsCache: make(map[string]clusterSensorsCacheEntry), - } - m.state.UpsertHost(models.Host{ - ID: "host-silent", - Hostname: "silent-node", - LinkedNodeID: "node-silent", - IntervalSeconds: 30, - Sensors: models.HostSensorSummary{ - TemperatureCelsius: map[string]float64{"cpu_package": 95.0}, - }, - LastSeen: time.Now().Add(-hostAgentHealthWindow(30) - time.Minute), - }) + nodes := []models.Node{readStateTestNode("pve1", "silent-node"), readStateTestNode("pve1", "sibling")} + sibling := reportingTestHost("agent-sibling", "sibling", "pve1-sibling", 0) + silent := reportingTestHost("host-silent", "silent-node", "pve1-silent-node", 95) + silent.LastSeen = time.Now().Add(-hostAgentHealthWindow(30) - time.Minute) + m := readStateMonitor(nodes, []models.Host{silent, sibling}) - assert.Nil(t, m.getHostAgentTemperatureByID("node-silent", "silent-node"), + assert.Nil(t, m.getHostAgentTemperatureForNode(models.Node{ID: "pve1-silent-node", Name: "silent-node"}), "a silent agent's retained sensors must not be presented as a current reading") - m.clusterSensorsCache["silent-node"] = clusterSensorsCacheEntry{ - sensors: models.HostSensorSummary{ - TemperatureCelsius: map[string]float64{"cpu_package": 58.0}, - }, - updatedAt: time.Now(), - } - if result := m.getHostAgentTemperatureByID("node-silent", "silent-node"); assert.NotNil(t, result) { + m.applyClusterSensors("agent-sibling", clusterSensorReport("silent-node", 58), time.Now()) + if result := m.getHostAgentTemperatureForNode(models.Node{ID: "pve1-silent-node", Name: "silent-node"}); assert.NotNil(t, result) { assert.Equal(t, 58.0, result.CPUPackage, "a recent cluster-cache reading still serves the node") } - m.state.UpsertHost(models.Host{ - ID: "host-silent", - Hostname: "silent-node", - LinkedNodeID: "node-silent", - IntervalSeconds: 30, - Sensors: models.HostSensorSummary{ - TemperatureCelsius: map[string]float64{"cpu_package": 71.0}, - }, - LastSeen: time.Now(), - }) - if result := m.getHostAgentTemperatureByID("node-silent", "silent-node"); assert.NotNil(t, result) { + reporting := reportingTestHost("host-silent", "silent-node", "pve1-silent-node", 71) + m.resourceStore.(*unifiedresources.MonitorAdapter).PopulateFromSnapshot(models.StateSnapshot{Nodes: nodes, Hosts: []models.Host{reporting, sibling}}) + if result := m.getHostAgentTemperatureForNode(models.Node{ID: "pve1-silent-node", Name: "silent-node"}); assert.NotNil(t, result) { assert.Equal(t, 71.0, result.CPUPackage, "a reporting agent's reading is used again") } } @@ -804,7 +810,7 @@ func TestGetHostAgentTemperatureByID_UsesUnifiedReadState(t *testing.T) { resourceStore: unifiedresources.NewMonitorAdapter(registry), } - result := m.getHostAgentTemperatureByID("node-readstate", "ignored") + result := m.getHostAgentTemperatureForNode(models.Node{ID: "node-readstate", Name: "ignored"}) assert.NotNil(t, result) assert.Equal(t, 71.0, result.CPUPackage) if assert.Len(t, result.SMART, 1) { diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 65df2b3a2e..167d0ce22b 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -1373,8 +1373,10 @@ func (m *Monitor) getRuntimeContext() context.Context { // clusterSensorsCacheEntry stores temperature data collected by a sibling agent via SSH. type clusterSensorsCacheEntry struct { - sensors models.HostSensorSummary - updatedAt time.Time + reporterID string // host agent that collected the reading over SSH + nodeName string // lowercase Proxmox node name the reading describes + sensors models.HostSensorSummary + updatedAt time.Time } type rrdMemCacheEntry struct { diff --git a/internal/monitoring/monitor_additional_test.go b/internal/monitoring/monitor_additional_test.go index 6a9b566dae..93a3e66fbe 100644 --- a/internal/monitoring/monitor_additional_test.go +++ b/internal/monitoring/monitor_additional_test.go @@ -1585,3 +1585,213 @@ func TestCarriedNodeTemperatureLeavesDiskReadingsOut(t *testing.T) { }) } } + +// Two Proxmox connections can each have a node with the same name (one "px1" +// per site). The poller's temperature lookup must never hand one site's node the +// host agent, or the agent's cluster-sibling SSH readings, of the other. +func TestNodeTemperatureStaysWithinSameNamedNodesSite(t *testing.T) { + instanceCfg := config.PVEInstance{Name: "siteB"} + poll := func(m *Monitor, instance, nodeName string) *models.Temperature { + m.config = &config.Config{TemperatureMonitoringEnabled: true} + polled := readStateTestNode(instance, nodeName) + m.collectNodeTemperatureData(context.Background(), instance, &instanceCfg, proxmox.Node{Node: nodeName}, &polled, nil, "online") + return polled.Temperature + } + sites := func() []models.Node { + return []models.Node{readStateTestNode("siteA", "px1"), readStateTestNode("siteA", "px2"), readStateTestNode("siteB", "px1")} + } + + t.Run("agent linked to the other site's node", func(t *testing.T) { + m := readStateMonitor(sites(), []models.Host{reportingTestHost("agent-a", "px1", "siteA-px1", 77)}) + if temp := poll(m, "siteA", "px1"); temp == nil || temp.CPUPackage != 77 { + t.Fatalf("siteA px1 temperature = %#v, want its linked agent's 77", temp) + } + if temp := poll(m, "siteB", "px1"); temp != nil { + t.Fatalf("siteB px1 took site A's agent reading: %#v", temp) + } + }) + + t.Run("unlinked agent whose hostname several nodes share", func(t *testing.T) { + m := readStateMonitor(sites(), []models.Host{reportingTestHost("agent-x", "px1", "", 66)}) + for _, instance := range []string{"siteA", "siteB"} { + if temp := poll(m, instance, "px1"); temp != nil { + t.Fatalf("%s px1 took an agent the linker could not place: %#v", instance, temp) + } + } + }) + + t.Run("unlinked agent whose hostname one node has", func(t *testing.T) { + m := readStateMonitor(sites(), []models.Host{reportingTestHost("agent-y", "px2", "", 61)}) + if temp := poll(m, "siteA", "px2"); temp == nil || temp.CPUPackage != 61 { + t.Fatalf("siteA px2 temperature = %#v, want the unambiguous agent's 61", temp) + } + }) + + t.Run("two unlinked agents with the node's hostname", func(t *testing.T) { + m := readStateMonitor(sites(), []models.Host{reportingTestHost("agent-y1", "px2", "", 61), reportingTestHost("agent-y2", "px2", "", 62)}) + if temp := poll(m, "siteA", "px2"); temp != nil { + t.Fatalf("siteA px2 took one of two candidate agents: %#v", temp) + } + }) + + t.Run("one cluster added through two connections", func(t *testing.T) { + // The state folds both views of pve01 into one slot, held under whichever + // view merged last, and the agent link follows the slot. Each view's poll + // must still take the agent's reading. + var instances []config.PVEInstance + for _, name := range []string{"enacon-a", "enacon-b"} { + instances = append(instances, config.PVEInstance{ + Name: name, IsCluster: true, ClusterName: "enacon", + ClusterEndpoints: []config.ClusterEndpoint{{NodeName: "pve01", Host: "https://192.168.1.11:8006", Fingerprint: "AA:AA"}}, + }) + } + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m := &Monitor{config: &config.Config{TemperatureMonitoringEnabled: true, PVEInstances: instances}, state: models.NewState(), resourceStore: adapter} + view := func(instanceCfg *config.PVEInstance) models.Node { + return models.Node{ + ID: instanceCfg.Name + "-pve01", NodeIdentity: instanceCfg.Name + "-pve01", Name: "pve01", Instance: instanceCfg.Name, + ClusterName: "enacon", IsClusterMember: true, Host: "https://192.168.1.11:8006", + TLSFingerprint: pveNodeTLSFingerprint(instanceCfg, "pve01"), Status: "online", LastSeen: time.Now(), + } + } + m.state.UpdateNodesForInstance("enacon-a", []models.Node{view(&instances[0])}) + m.state.UpsertHost(reportingTestHost("agent-pve01", "pve01", "", 64)) + if err := m.state.LinkHostAgentToNode("agent-pve01", "enacon-a-pve01"); err != nil { + t.Fatalf("LinkHostAgentToNode: %v", err) + } + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + + for round := 0; round < 3; round++ { + for i := range instances { + polled := view(&instances[i]) + m.collectNodeTemperatureData(context.Background(), instances[i].Name, &instances[i], proxmox.Node{Node: "pve01"}, &polled, nil, "online") + if polled.Temperature == nil || polled.Temperature.CPUPackage != 64 { + t.Fatalf("round %d %s temperature = %#v, want the folded node's agent reading 64", round, instances[i].Name, polled.Temperature) + } + m.state.UpdateNodesForInstance(instances[i].Name, []models.Node{polled}) + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + } + } + if nodes := m.state.GetSnapshot().Nodes; len(nodes) != 1 { + t.Fatalf("state holds %d nodes, want both views folded into one", len(nodes)) + } + }) + + t.Run("unclustered node whose address two sites' namesakes share", func(t *testing.T) { + // Both site nodes would fold with the polled view by address alone; the + // state declines an ambiguous alias, so the lookup must take neither. + nodes := sites() + for i := range nodes { + nodes[i].ClusterName = map[string]string{"siteA": "a", "siteB": "b"}[nodes[i].Instance] + nodes[i].Host = "https://10.0.0.5:8006" + } + m := readStateMonitor(nodes, []models.Host{reportingTestHost("agent-a", "px1", "siteA-px1", 77)}) + polled := readStateTestNode("siteC", "px1") + polled.Host = "https://10.0.0.5:8006" + m.config = &config.Config{TemperatureMonitoringEnabled: true} + m.collectNodeTemperatureData(context.Background(), "siteC", &config.PVEInstance{Name: "siteC"}, proxmox.Node{Node: "px1"}, &polled, nil, "online") + if polled.Temperature != nil { + t.Fatalf("siteC px1 took a namesake's agent reading: %#v", polled.Temperature) + } + }) + + t.Run("multi-homed host added through two standalone connections", func(t *testing.T) { + // The agent reports both connection addresses, so the state folds the two + // views into one slot through it. Each view's poll must take its reading. + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m := &Monitor{config: &config.Config{TemperatureMonitoringEnabled: true}, state: models.NewState(), resourceStore: adapter} + views := map[string]string{"lan-a": "https://10.0.0.5:8006", "lan-b": "https://10.0.1.5:8006"} + view := func(instance string) models.Node { + polled := readStateTestNode(instance, "minipc") + polled.Host = views[instance] + return polled + } + m.state.UpdateNodesForInstance("lan-a", []models.Node{view("lan-a")}) + agent := reportingTestHost("agent-minipc", "minipc", "", 64) + agent.ReportIP = "10.0.0.5" + agent.NetworkInterfaces = []models.HostNetworkInterface{{Name: "eth0", Addresses: []string{"10.0.0.5/24"}}, {Name: "eth1", Addresses: []string{"10.0.1.5/24"}}} + m.state.UpsertHost(agent) + if err := m.state.LinkHostAgentToNode("agent-minipc", "lan-a-minipc"); err != nil { + t.Fatalf("LinkHostAgentToNode: %v", err) + } + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + + for round := 0; round < 3; round++ { + for _, instance := range []string{"lan-a", "lan-b"} { + polled := view(instance) + m.collectNodeTemperatureData(context.Background(), instance, &config.PVEInstance{Name: instance}, proxmox.Node{Node: "minipc"}, &polled, nil, "online") + if polled.Temperature == nil || polled.Temperature.CPUPackage != 64 { + t.Fatalf("round %d %s temperature = %#v, want the folded host's agent reading 64", round, instance, polled.Temperature) + } + m.state.UpdateNodesForInstance(instance, []models.Node{polled}) + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + } + } + if nodes := m.state.GetSnapshot().Nodes; len(nodes) != 1 { + t.Fatalf("state holds %d nodes, want both views folded into one", len(nodes)) + } + }) + + t.Run("namesake's agent reporting the polled node's address", func(t *testing.T) { + // Site A's px1 agent also reports 10.0.1.5, the address site B's px1 + // uses. The fingerprints contradict, so site B's node is another machine. + siteA := readStateTestNode("siteA", "px1") + siteA.Host = "https://10.0.0.5:8006" + agent := reportingTestHost("agent-a", "px1", "siteA-px1", 77) + agent.ReportIP = "10.0.0.5" + agent.NetworkInterfaces = []models.HostNetworkInterface{{Name: "eth0", Addresses: []string{"10.0.0.5/24", "10.0.1.5/24"}}} + m := readStateMonitor([]models.Node{siteA}, []models.Host{agent}) + m.config = &config.Config{TemperatureMonitoringEnabled: true, PVEInstances: []config.PVEInstance{ + {Name: "siteA", Host: "https://10.0.0.5:8006", Fingerprint: "AA:AA"}, + {Name: "siteB", Host: "https://10.0.1.5:8006", Fingerprint: "BB:BB"}, + }} + polled := readStateTestNode("siteB", "px1") + polled.Host = "https://10.0.1.5:8006" + polled.TLSFingerprint = pveNodeTLSFingerprint(&m.config.PVEInstances[1], "px1") + m.collectNodeTemperatureData(context.Background(), "siteB", &m.config.PVEInstances[1], proxmox.Node{Node: "px1"}, &polled, nil, "online") + if polled.Temperature != nil { + t.Fatalf("siteB px1 took site A's agent reading: %#v", polled.Temperature) + } + }) + + t.Run("sibling reading across the two connections of one cluster", func(t *testing.T) { + // A cluster added twice can keep px1 under one connection's view and the + // reporting px2 under the other's; both connections list the same + // members with the same fingerprints, so the reading is px1's. + var instances []config.PVEInstance + for _, name := range []string{"enacon-a", "enacon-b"} { + instances = append(instances, config.PVEInstance{ + Name: name, IsCluster: true, ClusterName: "enacon", + ClusterEndpoints: []config.ClusterEndpoint{{NodeName: "px1", Fingerprint: "11:11"}, {NodeName: "px2", Fingerprint: "22:22"}}, + }) + } + member := func(instanceCfg *config.PVEInstance, name string) models.Node { + node := readStateTestNode(instanceCfg.Name, name) + node.ClusterName, node.IsClusterMember = "enacon", true + node.TLSFingerprint = pveNodeTLSFingerprint(instanceCfg, name) + return node + } + m := readStateMonitor( + []models.Node{member(&instances[0], "px1"), member(&instances[1], "px2")}, + []models.Host{reportingTestHost("agent-px2", "px2", "enacon-b-px2", 50)}, + ) + m.applyClusterSensors("agent-px2", clusterSensorReport("px1", 88), time.Now()) + m.config = &config.Config{TemperatureMonitoringEnabled: true, PVEInstances: instances} + polled := member(&instances[0], "px1") + m.collectNodeTemperatureData(context.Background(), "enacon-a", &instances[0], proxmox.Node{Node: "px1"}, &polled, nil, "online") + if polled.Temperature == nil || polled.Temperature.CPUPackage != 88 { + t.Fatalf("px1 temperature = %#v, want the sibling reading 88", polled.Temperature) + } + }) + + t.Run("cluster sibling reading from the other site", func(t *testing.T) { + m := readStateMonitor(sites(), []models.Host{reportingTestHost("agent-a2", "px2", "siteA-px2", 50)}) + m.applyClusterSensors("agent-a2", clusterSensorReport("px1", 88), time.Now()) + if temp := poll(m, "siteA", "px1"); temp == nil || temp.CPUPackage != 88 { + t.Fatalf("siteA px1 temperature = %#v, want its cluster sibling's 88", temp) + } + if temp := poll(m, "siteB", "px1"); temp != nil { + t.Fatalf("siteB px1 took site A's cluster sibling reading: %#v", temp) + } + }) +} diff --git a/internal/monitoring/monitor_agents.go b/internal/monitoring/monitor_agents.go index 5fcf15d549..376fe3c101 100644 --- a/internal/monitoring/monitor_agents.go +++ b/internal/monitoring/monitor_agents.go @@ -3781,7 +3781,7 @@ func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config. } // Store cluster peer sensor data if present and evict stale entries - m.applyClusterSensors(report.ClusterSensors, observedAt) + m.applyClusterSensors(host.ID, report.ClusterSensors, observedAt) // Availability results are ingested only once the host identity is // committed, because ownership is checked against that host ID. m.ApplyProbeAvailabilityResults(host.ID, probeAvailabilityResultsFromReport(report.AvailabilityResults)) @@ -4351,10 +4351,11 @@ func (m *Monitor) proxmoxPhysicalDiskMatchesForLinkedNode(linkedNodeID string) [ return matches } -// applyClusterSensors stores temperature data collected from Proxmox cluster -// siblings via SSH. Each entry is keyed by lowercase node name so that -// getHostAgentTemperatureByID can use it as a fallback. -func (m *Monitor) applyClusterSensors(entries []agentshost.ClusterNodeSensors, reportTime time.Time) { +// applyClusterSensors stores temperature data a host agent collected from its +// Proxmox cluster siblings via SSH, so that getHostAgentTemperatureForNode can use +// it as a fallback. Entries are keyed by reporting agent and lowercase node +// name: agents in different clusters can each report a sibling of the same name. +func (m *Monitor) applyClusterSensors(reporterID string, entries []agentshost.ClusterNodeSensors, reportTime time.Time) { // Fast path: nothing to add and cache is empty — skip lock if len(entries) == 0 { m.clusterSensorsMu.RLock() @@ -4377,7 +4378,9 @@ func (m *Monitor) applyClusterSensors(entries []agentshost.ClusterNodeSensors, r continue } - m.clusterSensorsCache[nodeName] = clusterSensorsCacheEntry{ + m.clusterSensorsCache[reporterID+"\x00"+nodeName] = clusterSensorsCacheEntry{ + reporterID: reporterID, + nodeName: nodeName, sensors: models.HostSensorSummary{ TemperatureCelsius: cloneStringFloatMap(entry.Sensors.TemperatureCelsius), FanRPM: cloneStringFloatMap(entry.Sensors.FanRPM), diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go index 3ef3414d7d..8fc230e0b6 100644 --- a/internal/monitoring/monitor_host_agents_test.go +++ b/internal/monitoring/monitor_host_agents_test.go @@ -972,6 +972,39 @@ func TestApplyHostReportRefreshesUnifiedReadStateWithoutBroadcast(t *testing.T) t.Fatal("accepted host report did not refresh the canonical headless read state") } +// An agent reports its Proxmox cluster siblings' sensors by bare node name, +// which another connection's node can share. Ingest records the reporting +// agent, so the readings serve only nodes of that agent's own connection. +func TestApplyHostReportScopesClusterSensorsToReportersConnection(t *testing.T) { + monitor := newTestMonitor(t) + monitor.clusterSensorsCache = make(map[string]clusterSensorsCacheEntry) + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + monitor.SetResourceStore(adapter) + monitor.state.UpdateNodesForInstance("siteA", []models.Node{readStateTestNode("siteA", "px1"), readStateTestNode("siteA", "px2")}) + monitor.state.UpdateNodesForInstance("siteB", []models.Node{readStateTestNode("siteB", "px1")}) + adapter.PopulateFromSnapshot(monitor.state.GetSnapshot()) + + host, err := monitor.ApplyHostReport(agentshost.Report{ + Agent: agentshost.AgentInfo{ID: "agent-px2", IntervalSeconds: 30}, + Host: agentshost.HostInfo{ID: "machine-px2", Hostname: "px2", Platform: "linux"}, + ClusterSensors: clusterSensorReport("px1", 88), + Timestamp: time.Now().UTC(), + }, nil) + if err != nil { + t.Fatalf("ApplyHostReport: %v", err) + } + if host.LinkedNodeID != "siteA-px2" { + t.Fatalf("reporting agent linked to %q, want siteA-px2", host.LinkedNodeID) + } + + if temp := monitor.getHostAgentTemperatureForNode(readStateTestNode("siteA", "px1")); temp == nil || temp.CPUPackage != 88 { + t.Fatalf("siteA px1 temperature = %#v, want the sibling reading 88", temp) + } + if temp := monitor.getHostAgentTemperatureForNode(readStateTestNode("siteB", "px1")); temp != nil { + t.Fatalf("siteB px1 took site A's cluster sibling reading: %#v", temp) + } +} + func TestApplyHostReportProjectsAndPreservesLibvirtInventory(t *testing.T) { monitor := newTestMonitor(t) adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) diff --git a/internal/monitoring/monitor_polling_node_helpers.go b/internal/monitoring/monitor_polling_node_helpers.go index e8ef1e849a..dd5e9abba9 100644 --- a/internal/monitoring/monitor_polling_node_helpers.go +++ b/internal/monitoring/monitor_polling_node_helpers.go @@ -164,9 +164,9 @@ func (m *Monitor) collectNodeTemperatureData( if effectiveStatus == "online" && tempMonitoringEnabled { // First, check if there's a matching host agent with temperature data. // Host agent temperatures are preferred because they don't require SSH access. - // Use getHostAgentTemperatureByID with the unique node ID to correctly handle - // duplicate hostname scenarios (e.g., two "px1" nodes on different IPs). - hostAgentTemp := m.getHostAgentTemperatureByID(modelNode.ID, node.Node) + // Match by the polled node's identity, not its name, so two connections' + // same-named nodes (e.g., "px1" at two sites) never share an agent. + hostAgentTemp := m.getHostAgentTemperatureForNode(*modelNode) if hostAgentTemp != nil { log.Debug(). Str("node", node.Node). @@ -331,7 +331,7 @@ func (m *Monitor) collectNodeTemperatureData( } } - if m.carriedTemperatureOutlivesAgentLease(modelNode.ID, node.Node, prevTemp, time.Now()) { + if m.carriedTemperatureOutlivesAgentLease(*modelNode, prevTemp, time.Now()) { // The reading came from a host agent that has stopped reporting. // Its reporting lease already rode out transient gaps, so // carrying it would outlive that lease. diff --git a/internal/monitoring/monitor_pve.go b/internal/monitoring/monitor_pve.go index 7f5405f896..8b03fac7e0 100644 --- a/internal/monitoring/monitor_pve.go +++ b/internal/monitoring/monitor_pve.go @@ -785,7 +785,7 @@ func (m *Monitor) boundCarriedNodeTemperature(node *models.Node, now time.Time) return } if now.Sub(node.Temperature.LastUpdate) > m.nodeTemperatureCarryWindow() || - m.carriedTemperatureOutlivesAgentLease(node.ID, node.Name, node.Temperature, now) { + m.carriedTemperatureOutlivesAgentLease(*node, node.Temperature, now) { node.Temperature = nil return } From 41b94b29a86173e8b7c07609524ab3596dc33fa9 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:47:17 +0100 Subject: [PATCH 23/53] Keep a merged disk's temperature under its own row's state The registry merges a physical disk's rows (Proxmox, host agent SMART, Unraid inventory), choosing the temperature by source preference and merging field states separately. When two rows carry different temperatures the shown value could sit under the other row's "available" state: a silent agent's retained reading carried over a Proxmox row that collected its own reading now, in either ingest order. Risk, history and the UI then treated a frozen reading as current. pairPhysicalDiskTemperatureState runs at the end of each merge step: when the merged state says collected but the row the shown value came from says it was not, the merged state becomes that row's own. It never changes a value and never grants availability, so it cannot revive a withdrawn reading or override a resumed agent's fresh one; rows with the same value keep the merged state. This is exact for the two rows of a step. The merged disk keeps one state per field, so with three or more rows a withdrawal an earlier step replaced is invisible to a later one; closing that needs each reading's own state tracked across merges. Preferring another source's current value instead would need an ordering between rows that the merge does not record. With this in place the Unraid-native row no longer needs to withhold an "available" state to avoid masking a SMART row, so it now carries the same derived state as a SMART row borrowing its reading. --- .../internal/subsystems/unified-resources.md | 28 +++-- .../physical_disk_roundtrip_test.go | 7 +- .../unifiedresources/adapter_coverage_test.go | 20 ++- internal/unifiedresources/adapters.go | 6 +- internal/unifiedresources/registry.go | 34 +++++ .../registry_merge_policy_test.go | 117 ++++++++++++++++++ 6 files changed, 183 insertions(+), 29 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index f90e5ef236..48de7a7a75 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -3576,20 +3576,30 @@ state the Proxmox row still carries from the same source past its reporting lease must not stay "collected" through the Proxmox row's copy of the agent's own state, including when its host is down and no disk poll refreshes that copy. +Each merge step also stops the shown temperature borrowing another row's +availability (`pairPhysicalDiskTemperatureState`). The value is still chosen +by source preference and the states merged as above, so when the two rows of a +step carry different temperatures the shown value could sit under the other +row's `available` state, for example a silent agent's retained reading carried +over a Proxmox row that collected its own reading now. When the merged state +says collected but the row the shown value came from says it was not, the +merged state becomes that row's own. The rule never changes a value and never +grants availability; rows with the same value keep the merged state. It sees +only the two rows of one step: the merged disk keeps one state per field, so +with three or more rows for one disk a withdrawal an earlier step replaced is +not visible to a later step, and the earlier step's merged state stands in for +its row. Tracking each reading's own state across merges would close that. Unraid array-inventory rows carry no per-field provenance, so the adapter -derives the state of a temperature taken from one: `unavailable` from `unraid` +derives the state of a positive temperature taken from one (a row without a +temperature claims no state): `unavailable` from `unraid` with "host agent stopped reporting" once `State.ExpireHostTelemetry` has marked the host offline, or with "disk is reported spun down" for a spun-down disk, and otherwise `available`. The value is kept as last-known context; it is collected again only when a reporting host sends a positive reading for a disk -that is no longer spun down. A SMART row that falls back to -the inventory reading because it has none of its own takes that state with it. -The Unraid-native disk row records only the withdrawing states, never -`available`: the registry chooses a merged disk's temperature and its state -separately, and an `available` claim from the provenance-less row would -outrank the SMART row's explicit state for the same disk when the SMART -reading is the one shown. A withdrawn inventory temperature also stops -counting as heat in the disk's risk assessment. +that is no longer spun down. The Unraid-native disk row carries that state, and +so does a SMART row that falls back to the inventory reading because it has +none of its own. The adapter's risk assessment of an Unraid row no longer counts +a withdrawn inventory temperature as heat. Cross-source correlation compares normalized serial and WWN values across fields without truncation, allowing a PVE bare-hex array-volume serial to join diff --git a/internal/monitoring/physical_disk_roundtrip_test.go b/internal/monitoring/physical_disk_roundtrip_test.go index 0c07448d64..38c45b8963 100644 --- a/internal/monitoring/physical_disk_roundtrip_test.go +++ b/internal/monitoring/physical_disk_roundtrip_test.go @@ -687,11 +687,10 @@ func TestSilentUnraidAgentDiskTemperatureIsRetainedButNotCollected(t *testing.T) return got } - // A reporting agent's readings are collected. The inventory-only disk - // keeps no state of its own (it counts as collected); the SMART rows that - // borrow the inventory reading take its provenance. + // A reporting agent's inventory readings are collected, including where + // a SMART row borrows them. collected := map[string]diskinventory.FieldStatus{ - "UNRAID-ONLY1": {}, + "UNRAID-ONLY1": diskinventory.Available("unraid"), "UNRAID-SMART2": diskinventory.Available("unraid"), "UNRAID-SAS3": diskinventory.Available("unraid"), } diff --git a/internal/unifiedresources/adapter_coverage_test.go b/internal/unifiedresources/adapter_coverage_test.go index 154ca7e4aa..8953fb4a49 100644 --- a/internal/unifiedresources/adapter_coverage_test.go +++ b/internal/unifiedresources/adapter_coverage_test.go @@ -1049,10 +1049,9 @@ func TestMetricPercent(t *testing.T) { // Unraid inventory temperatures have no agent provenance, so the adapter // derives their state from the host's reporting lease and the disk's spin -// state. A native row records only a state that withdraws its reading, a SMART -// row that borrows the inventory reading takes the inventory's state, and a -// SMART row with its own reading keeps the agent's state. A withdrawn reading -// never counts as heat in the disk's risk. +// state. The native row and a SMART row that borrows the inventory reading +// both carry it, and a SMART row with its own reading keeps the agent's state. +// A withdrawn reading never counts as heat in the disk's risk. func TestUnraidDiskTemperatureFollowsTheHostLease(t *testing.T) { smartWithout := models.HostDiskSMART{Device: "/dev/sdc", Serial: "UNRAID-1", Collection: &diskinventory.CollectionStatus{ Serial: diskinventory.Available("smartctl"), @@ -1081,7 +1080,7 @@ func TestUnraidDiskTemperatureFollowsTheHostLease(t *testing.T) { native diskinventory.FieldStatus borrowed diskinventory.FieldStatus }{ - {"reporting", "online", false, diskinventory.FieldStatus{}, diskinventory.Available("unraid")}, + {"reporting", "online", false, diskinventory.Available("unraid"), diskinventory.Available("unraid")}, {"silent agent", "offline", false, stopped, stopped}, {"spun down", "online", true, spunDown, spunDown}, } { @@ -1131,10 +1130,8 @@ func TestUnraidDiskTemperatureFollowsTheHostLease(t *testing.T) { } // The registry merges an Unraid inventory row and the agent's SMART row for the -// same disk, choosing the temperature and the collection state separately. A -// reporting host's inventory row must not lend the merged disk an "available" -// state when the reading it shows is the SMART row's, and that row says it was -// not collected. +// same disk. When they disagree, the inventory's "available" never lends +// itself to the SMART row's uncollected reading. func TestUnraidInventoryRowDoesNotMaskTheSMARTRowState(t *testing.T) { state := models.NewState() state.UpsertHost(models.Host{ @@ -1158,7 +1155,8 @@ func TestUnraidInventoryRowDoesNotMaskTheSMARTRowState(t *testing.T) { if len(disks) != 1 { t.Fatalf("physical disks = %d, want the inventory and SMART rows merged into one", len(disks)) } - if got := disks[0]; got.Temperature != 40 || diskinventory.TemperatureCollected(got.Temperature, got.Collection) { - t.Fatalf("merged disk presents the SMART row's uncollected reading as collected: temperature=%d collection=%+v", got.Temperature, got.Collection) + if got := disks[0]; got.Temperature != 40 || got.Collection == nil || + got.Collection.Temperature != diskinventory.Unavailable("smartctl", "disk is in standby") { + t.Fatalf("merged disk presents the SMART row's uncollected reading under another state: temperature=%d collection=%+v", got.Temperature, got.Collection) } } diff --git a/internal/unifiedresources/adapters.go b/internal/unifiedresources/adapters.go index 5196aab935..da65047d35 100644 --- a/internal/unifiedresources/adapters.go +++ b/internal/unifiedresources/adapters.go @@ -806,12 +806,8 @@ func resourceFromHostUnraidPhysicalDisk(host models.Host, disk models.HostUnraid health := unraidPhysicalDiskHealth(disk) assessment := assessUnraidPhysicalDisk(host, disk) sizeBytes := unraidDiskSizeBytes(host, disk) - // The row records only a state that withdraws its reading. A positive - // claim from this provenance-less row would outrank another source's - // explicit state for the same disk when the registry merges them, even - // where that source's temperature is the one shown. var collection *diskinventory.CollectionStatus - if status := unraidDiskTemperatureStatus(host, disk); status.State == diskinventory.FieldUnavailable { + if status := unraidDiskTemperatureStatus(host, disk); status.State != "" { collection = &diskinventory.CollectionStatus{Temperature: status} } resource := Resource{ diff --git a/internal/unifiedresources/registry.go b/internal/unifiedresources/registry.go index cb2f10a48a..fc932ff604 100644 --- a/internal/unifiedresources/registry.go +++ b/internal/unifiedresources/registry.go @@ -3732,6 +3732,7 @@ func (rr *ResourceRegistry) mergeInto(existing *Resource, incoming Resource, sou incoming.PhysicalDisk.Collection, ) } + pairPhysicalDiskTemperatureState(existing.PhysicalDisk, previous, incoming.PhysicalDisk) } if existing.PhysicalDisk != nil && (mergedPhysicalDisk || len(incoming.Incidents) > 0) { existing.PhysicalDisk.Risk = physicalDiskRiskFromMeta(existing.PhysicalDisk, existing.Incidents) @@ -4006,6 +4007,39 @@ func mergeTrueNASData(existing *TrueNASData, incoming *TrueNASData) *TrueNASData return &merged } +// pairPhysicalDiskTemperatureState stops one merge step presenting the shown +// temperature under the other row's availability. The merge rules choose the +// value by source preference and merge field states separately, so when the +// two rows carry different temperatures the shown value can sit under the +// other row's "available" state: a silent agent's retained reading beside a +// Proxmox row that collected its own reading now, for example. When the +// merged state says collected but the row the shown value came from says it +// was not, the merged state becomes that row's own. Values never change, and a +// state is only ever withdrawn here, never granted. previous may already be an +// earlier step's merge, whose single state per field stands in for its rows. +func pairPhysicalDiskTemperatureState(merged, previous, incoming *PhysicalDiskMeta) { + if merged == nil || previous == nil || incoming == nil || previous.Temperature == incoming.Temperature || + !diskinventory.TemperatureCollected(merged.Temperature, merged.Collection) { + return + } + var shown *PhysicalDiskMeta + switch merged.Temperature { + case incoming.Temperature: + shown = incoming + case previous.Temperature: + shown = previous + default: + return + } + if diskinventory.TemperatureCollected(shown.Temperature, shown.Collection) { + return + } + if merged.Collection == nil { + merged.Collection = &diskinventory.CollectionStatus{} + } + merged.Collection.Temperature = shown.Collection.Temperature +} + func mergePhysicalDiskData(existing *PhysicalDiskMeta, incoming *PhysicalDiskMeta) *PhysicalDiskMeta { if existing == nil { return incoming diff --git a/internal/unifiedresources/registry_merge_policy_test.go b/internal/unifiedresources/registry_merge_policy_test.go index 83c8fda61a..ac3b15ec22 100644 --- a/internal/unifiedresources/registry_merge_policy_test.go +++ b/internal/unifiedresources/registry_merge_policy_test.go @@ -634,3 +634,120 @@ func TestPhysicalDiskMergeFollowsAgentWithdrawalOfItsOwnReadings(t *testing.T) { }) } } + +// A merged disk must never present its temperature under a state the +// reading's own row does not claim. A silent agent's retained reading stays as +// last-known context under the agent's own state, whatever the Proxmox row +// collected or copied; a resumed agent's fresh reading stays collected over a +// Proxmox copy of its earlier withdrawal; a standby withdrawal keeps the +// pre-sleep reading as last-known. +func TestPhysicalDiskMergePairsTemperatureWithItsCollectionState(t *testing.T) { + disk := func(temperature int, status diskinventory.FieldStatus) Resource { + return Resource{ + Type: ResourceTypePhysicalDisk, Name: "WDC", Status: StatusOnline, + PhysicalDisk: &PhysicalDiskMeta{ + DevPath: "/dev/sdb", Serial: "WD-PAIR1", Temperature: temperature, + Collection: &diskinventory.CollectionStatus{Temperature: status}, + }, + } + } + // The Proxmox row can also carry the SMART attributes copied from the + // agent's earlier report, which changes whose value the merge shows. + withSMART := func(r Resource) Resource { + powerOnHours := int64(1000) + r.PhysicalDisk.SMART = &SMARTMeta{PowerOnHours: &powerOnHours} + return r + } + identity := ResourceIdentity{MachineID: "WD-PAIR1", Hostnames: []string{"node1"}} + retained := diskinventory.Unavailable("host_agent", "host agent stopped reporting") + nodeSMART := diskinventory.Available("proxmox_node_smart") + smartctl := diskinventory.Available("smartctl") + withdrawn := diskinventory.Unavailable("smartctl", "host agent stopped reporting") + standby := diskinventory.Unavailable("smartctl", "disk is in standby") + noProxmoxTemp := diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature") + + for _, tc := range []struct { + name string + agent, proxmox Resource + // wantTemperature 0 accepts either row's value: source preference, + // not this rule, decides which one is shown. + wantTemperature int + wantState diskinventory.FieldStatus + }{ + {"silent agent, Proxmox collected", disk(72, retained), disk(40, nodeSMART), 72, retained}, + {"silent agent, no Proxmox reading", disk(72, retained), disk(0, noProxmoxTemp), 72, retained}, + {"reporting agent, Proxmox collected", disk(41, smartctl), disk(40, nodeSMART), 41, smartctl}, + {"silent agent, Proxmox copy of its earlier reading", disk(72, withdrawn), disk(41, smartctl), 72, withdrawn}, + {"silent agent, Proxmox copy with SMART attributes", disk(72, withdrawn), withSMART(disk(41, smartctl)), 0, withdrawn}, + {"resumed agent, Proxmox copy of its withdrawal", disk(50, smartctl), disk(41, withdrawn), 50, smartctl}, + {"standby agent, Proxmox copy of its pre-sleep reading", disk(0, standby), disk(41, smartctl), 41, standby}, + } { + for _, proxmoxFirst := range []bool{true, false} { + registry := NewRegistry(nil) + ingestProxmox := func() { registry.ingest(SourceProxmox, "pve1-node1-sdb", tc.proxmox, identity) } + ingestAgent := func() { registry.ingest(SourceAgent, "agent-1-sdb", tc.agent, identity) } + if proxmoxFirst { + ingestProxmox() + ingestAgent() + } else { + ingestAgent() + ingestProxmox() + } + disks := registry.ListByType(ResourceTypePhysicalDisk) + if len(disks) != 1 || disks[0].PhysicalDisk == nil { + t.Fatalf("%s (proxmox first=%v): expected one merged disk, got %+v", tc.name, proxmoxFirst, disks) + } + got := disks[0].PhysicalDisk + var state diskinventory.FieldStatus + if got.Collection != nil { + state = got.Collection.Temperature + } + valueOK := got.Temperature == tc.wantTemperature || + (tc.wantTemperature == 0 && (got.Temperature == tc.agent.PhysicalDisk.Temperature || got.Temperature == tc.proxmox.PhysicalDisk.Temperature)) + if !valueOK || state != tc.wantState { + t.Errorf("%s (proxmox first=%v): temperature=%d state=%+v, want %d with %+v", + tc.name, proxmoxFirst, got.Temperature, state, tc.wantTemperature, tc.wantState) + } + } + } +} + +// Three rows for one disk with different values, none collected now: the +// agent's SMART row withdrew its reading, the Unraid inventory row is past the +// same lease, and the Proxmox row still carries a copy of the agent's earlier +// "available" state. Whatever the ingest order, the merged disk must not +// present a reading as collected. +func TestPhysicalDiskMergeNeverPresentsAWithdrawnTemperatureAsCollected(t *testing.T) { + disk := func(temperature int, status diskinventory.FieldStatus) Resource { + return Resource{ + Type: ResourceTypePhysicalDisk, Name: "WDC", Status: StatusOnline, + PhysicalDisk: &PhysicalDiskMeta{ + DevPath: "/dev/sdb", Serial: "WD-THREE1", Temperature: temperature, + Collection: &diskinventory.CollectionStatus{Temperature: status}, + }, + } + } + identity := ResourceIdentity{MachineID: "WD-THREE1", Hostnames: []string{"tower"}} + rows := []struct { + source DataSource + sourceID string + resource Resource + }{ + {SourceAgent, "agent-1-sdb", disk(72, diskinventory.Unavailable("smartctl", "host agent stopped reporting"))}, + {SourceAgent, "agent-1-unraid-sdb", disk(37, diskinventory.Unavailable("unraid", "host agent stopped reporting"))}, + {SourceProxmox, "pve1-node1-sdb", disk(41, diskinventory.Available("smartctl"))}, + } + for _, order := range [][]int{{0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}} { + registry := NewRegistry(nil) + for _, i := range order { + registry.ingest(rows[i].source, rows[i].sourceID, rows[i].resource, identity) + } + disks := registry.ListByType(ResourceTypePhysicalDisk) + if len(disks) != 1 || disks[0].PhysicalDisk == nil { + t.Fatalf("order %v: expected one merged disk, got %d", order, len(disks)) + } + if got := disks[0].PhysicalDisk; diskinventory.TemperatureCollected(got.Temperature, got.Collection) { + t.Errorf("order %v: withdrawn temperature presented as collected: temperature=%d collection=%+v", order, got.Temperature, got.Collection) + } + } +} From f4c104621580024ea79b321a61c8c9b5879aa4a9 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:07:06 +0100 Subject: [PATCH 24/53] Record Docker host, service and sub-resource alert history Alert lifecycle events resolve to canonical resources since Proxmox and agent references gained durable resolution, but Docker host alerts ("docker:"), Swarm service alerts ("docker:/service/") and sub-resource alerts such as ZFS pool and device health ("/zfs-pool:") or host filesystem usage ("agent:/disk:var") were still journaled under references no resource answers to. Resource facets, timelines, resource intelligence and Assistant context read history by canonical ID, so they never saw them. Docker host and service references now resolve through the registry's own Docker source identities. A service reference must carry the service's real ID in the host's Swarm cluster, from any manager that reports it; services without an ID are referenced by name and never bind. Container names and shortened IDs still bind nothing and are no longer retried against later inventory. Sub-resource references bind to the owner whose durable ID they carry: the storage for ZFS pools and devices, the Unraid array storage, and the host for filesystem, SMART disk, disk temperature, RAID and custom-sensor alerts. Mount and kernel device labels can name a different disk after a reboot, so they never bind a physical disk. Any other shape stays unbound. On a mock stack none of the Docker host, Swarm service, ZFS and agent disk alert rows were readable through their owner's timeline API before this change; all of them are after it, including rows journaled earlier. The mock's shortened container ID rows stay unbound. --- .../v6/internal/subsystems/monitoring.md | 14 + .../internal/subsystems/unified-resources.md | 46 +++- .../monitoring/monitor_alert_handling_test.go | 61 +++++ internal/unifiedresources/history_identity.go | 196 ++++++++++++-- .../unifiedresources/history_identity_test.go | 243 ++++++++++++++++++ internal/unifiedresources/monitor_adapter.go | 26 +- 6 files changed, 546 insertions(+), 40 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 01286836ea..e1b8d4a5e7 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -1052,6 +1052,20 @@ replay stays idempotent. The real alert-manager callback path is covered by `TestProxmoxAlertTimelineUsesCanonicalHistoryIdentity` in `internal/monitoring/monitor_alert_handling_test.go`. +Docker host and Swarm service alert lifecycle events, and sub-resource alert +events (ZFS pools and devices, host filesystems, disks, RAID arrays, sensors +and the Unraid array), pass through the same writer, which binds them to the +Docker host, Swarm service, storage or host named in the unified-resources +history identity clause. Container names and shortened container IDs still +bind nothing. `CheckDockerHost` gives a container without an ID its host's +reference, so that container's events join the host's history. A producer +that changes one of these reference shapes must change +`dockerHostHistoryReference` or `historySubResourceOwner` with it. +`TestOwnerAlertTimelinesUseCanonicalHistoryIdentity` in +`internal/monitoring/monitor_alert_handling_test.go` pins the Docker host, +Swarm service, ZFS pool and device, and host filesystem shapes against the +real alert manager. + TrueNAS native alert projection preserves the trimmed, uppercase provider level in ResourceIncident.NativeSeverity. INFO and NOTICE retain the same canonical monitor risk; consumers must not lose their distinct actionability when projecting provider evidence. Native CRITICAL, ALERT, and EMERGENCY all project to canonical critical severity; EMERGENCY must not be discarded as unknown or make a still-active condition appear recovered. WARNING remains warning, and INFO and NOTICE remain informational at this projection boundary. Verification: `TestIncidentProjectionPreservesNativeSeverity` in `internal/truenas/provider_pool_health_contract_test.go` covers all seven native levels and case/whitespace normalization. `TestTrueNASNativeSeverityDispatch` in `internal/alerts/truenas_native_dispatch_test.go` verifies downstream INFO suppression, NOTICE preservation, notification severity, duplicate-poll retention, and confirmed recovery callback identity. The TrueNAS lifecycle tests in `internal/alerts/unified_incidents_test.go` require repeated EMERGENCY evidence to interrupt recovery confirmation. These are fixture-based projection and manager checks, not appliance ingestion or external notification-provider receipt proof. diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 8b80254b40..1b1e4d58d9 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -5846,10 +5846,12 @@ and storage IDs), a node-scoped Proxmox guest reference after migration, or `ResolveReferenceID` used by alert policy, so a resource named like a system reference cannot capture its events. A reference two resources answer to binds to neither: its event keeps its own reference and creates or changes no -binding, though an earlier binding still covers that reference on read. A +binding, though an earlier binding still covers that reference on read. Like +any unbound reference it is retried, and once a later generation resolves it +to one resource it binds there, replacing an earlier binding. A reference no resource answers to follows its retained binding after inventory removal, or keeps its -own history. Docker references never take this path. Alert journal references +own history. Docker container references never take this path. Alert journal references without a binding (read once per process, plus any written before inventory named their resource) are retried on each published registry generation for `legacyHistoryBindWindow`. Bound rows keep their recorded resource ID, so a @@ -5861,6 +5863,46 @@ Assistant and Patrol resource contexts read it from the store. Proof: `TestHistoryIdentityMonitorAdapterResolvesProxmoxAlertReferences` and `TestHistoryIdentityBindsLegacyAlertRowsFromRegistryGenerations` in `internal/unifiedresources/history_identity_test.go`. +Docker host alerts (`docker:`) and Swarm service alerts +(`docker:/service/`) resolve only through the registry's +Docker source identities: the host ID is the Docker host's source ID, and a +service reference names the service with that exact ID in the host's Swarm +cluster, whichever manager reported it. A hostname in the host position is +only ever looked up as a Docker host source ID. A service without an ID is +referenced by its normalized name in the same place, so while its cluster has +such a service every service reference there is a conflict. Alert references +do not record whether they came from an ID, so a row journaled by an ID-less +service can still join another service's history if a later generation or +event binds the same reference; Swarm always reports service IDs, so that needs a malformed +report naming a service exactly like another service's ID. Hostless +`docker-service:` references, container names and shortened container +IDs never bind and are never retried against later inventory. A container +without an ID alerts under its host's reference (`alerts.DockerResourceID`), so +its events join that host's history, never another container's. +Sub-resource alert references name no resource of their own and bind to the +owner whose durable ID they carry (`historySubResourceOwner`): ZFS pool and +device alerts (`/zfs-pool:[/device:]`) to the +storage, the Unraid array alert (`agent:/storage:unraid-array`) to +the array storage with that agent source ID, and host filesystem, SMART disk, +disk temperature, RAID and custom-sensor alerts (`agent:/disk:`, +`/disk_temp:`, `/raid:`, `/custom:`) to the host. The owner reference must +resolve to exactly one resource of the expected type; an ambiguous or +incompatible owner is a conflict, so the event keeps its own reference, +follows no retained binding and is retried. Pool, mount and kernel device labels are names, +and a device name can denote a different disk after a reboot, so these alerts +never bind a physical disk. Any other sub-resource shape stays unbound until +its producer and this list change together. The Docker, storage and platform +host-row drawers do not request per-resource history, so these bindings reach +the facets, timeline and intelligence APIs and Assistant resource context, not +a drawer. Patrol's scoped change feed matches the stored resource ID without +expanding bindings, so it includes events recorded under the owner's +canonical ID but not rows journaled before their binding. +Proof: `TestHistoryIdentityMonitorAdapterResolvesDockerHostAndServiceReferences` +and `TestHistoryIdentityMonitorAdapterResolvesSubResourceReferences` in +`internal/unifiedresources/history_identity_test.go`, and +`TestOwnerAlertTimelinesUseCanonicalHistoryIdentity` in +`internal/monitoring/monitor_alert_handling_test.go` against the real alert +producers. That same shared timeline vocabulary now includes the `activity` change kind for provider-read breadcrumbs such as VMware tasks and events, plus the `vmware_adapter` source-adapter token for canonical provenance drill-down. diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go index bb65eb663a..a8b7454b6c 100644 --- a/internal/monitoring/monitor_alert_handling_test.go +++ b/internal/monitoring/monitor_alert_handling_test.go @@ -150,6 +150,67 @@ func TestProxmoxAlertTimelineUsesCanonicalHistoryIdentity(t *testing.T) { require.Equal(t, canonical["pve1"], changes[0].ResourceID) } +// Docker host, Swarm service and sub-resource alerts (a ZFS pool and device, +// a host filesystem) carry references the registry has no resource for. Their +// lifecycle must reach the owner's canonical history, so the producers' ref +// shapes are pinned here against the real alert manager. +func TestOwnerAlertTimelinesUseCanonicalHistoryIdentity(t *testing.T) { + store, err := unifiedresources.NewSQLiteResourceStore(t.TempDir(), "default") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + manager := alerts.NewManagerWithDataDir(t.TempDir()) + t.Cleanup(manager.Stop) + config := manager.GetConfig() + config.Enabled = true + config.ActivationState = alerts.ActivationPending + config.TimeThresholds = map[string]int{"all": 0} + config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 0}} + config.SuppressionWindow = 0 + manager.UpdateConfig(config) + now := time.Now() + dockerHost := models.DockerHost{ID: "dh-7f3a", Hostname: "tower", DisplayName: "tower", Status: "online", LastSeen: now, + Swarm: &models.DockerSwarmInfo{NodeID: "node-a", NodeRole: "manager", LocalState: "active", ControlAvailable: true, ClusterID: "swarm-1"}, + Services: []models.DockerService{{ID: "x7k2m9q4w1e8r5t3y6u0i2o4p", Name: "web", DesiredTasks: 2, RunningTasks: 0}}} + storage := models.Storage{ID: "lab-pve1-local-zfs", Name: "local-zfs", Node: "pve1", Instance: "lab", Type: "zfspool", Status: "available", Total: 100, Used: 10, Usage: 10, Enabled: true, Active: true, LastSeen: now, + ZFSPool: &models.ZFSPool{Name: "rpool", State: "DEGRADED", Status: "Degraded", Devices: []models.ZFSDevice{{Name: "sda2", Type: "disk", State: "FAULTED"}}}} + agentHost := models.Host{ID: "host-nas", Hostname: "nas", DisplayName: "nas", MachineID: "fedcba9876543210", Status: "online", LastSeen: now, + Disks: []models.Disk{{Mountpoint: "/var", Device: "/dev/sdb1", Type: "ext4", Total: 100, Used: 99, Free: 1, Usage: 99}}} + registry := unifiedresources.NewRegistry(store) + registry.IngestSnapshot(models.StateSnapshot{DockerHosts: []models.DockerHost{dockerHost}, Storage: []models.Storage{storage}, Hosts: []models.Host{agentHost}}) + monitor := &Monitor{alertManager: manager, resourceStore: unifiedresources.NewMonitorAdapter(registry)} + manager.SubscribeLifecycleCallback(monitor.handleAlertLifecycleEvent) + manager.CheckDockerHost(dockerHost) + manager.CheckStorage(storage) + manager.CheckHost(agentHost) + for range 3 { + manager.HandleDockerHostOffline(dockerHost) + } + + owner := map[string]string{} + for _, resource := range registry.List() { + owner[string(resource.Type)+"/"+resource.Name] = resource.ID + } + filters := unifiedresources.ResourceChangeFilters{Kinds: []unifiedresources.ChangeKind{unifiedresources.ChangeAlertFired}} + want := map[string]string{ + "docker-host-offline": owner["agent/tower"], + "docker-service-health": owner["docker-service/web"], + "zfs-pool-state": owner["storage/local-zfs"], + "zfs-device": owner["storage/local-zfs"], + "disk": owner["agent/nas"], + } + for alertType, canonicalID := range want { + require.NotEmpty(t, canonicalID, alertType) + changes, err := store.GetRecentChangesFiltered(canonicalID, time.Time{}, 10, filters) + require.NoError(t, err) + found := false + for _, change := range changes { + require.Equal(t, canonicalID, change.ResourceID) + found = found || change.Metadata["alert_type"] == alertType + } + require.True(t, found, "%s alert missing from %s history: %+v", alertType, canonicalID, changes) + } +} + func TestMonitor_HandleAlertFired_Extra(t *testing.T) { // 1. Alert is nil m1 := &Monitor{} diff --git a/internal/unifiedresources/history_identity.go b/internal/unifiedresources/history_identity.go index 9dc69d886e..61ddc1ce1b 100644 --- a/internal/unifiedresources/history_identity.go +++ b/internal/unifiedresources/history_identity.go @@ -32,35 +32,180 @@ func legacyDockerHistoryIdentity(ref string) (sourceID, canonicalID string, ok b return sourceID, SourceSpecificID(ResourceTypeAppContainer, SourceDocker, sourceID), true } -// isDockerHistoryReference marks Docker alert source references, which only a -// complete container ID binds (legacyDockerHistoryIdentity). A container name -// or shortened ID must never join history through general reference matching. +// isDockerHistoryReference marks Docker alert source references, including the +// hostless "docker-service:" form. They never join history through +// general reference matching: only a Docker host, a Swarm service ID +// (dockerHostHistoryReference) or a complete container ID +// (legacyDockerHistoryIdentity) binds. func isDockerHistoryReference(ref string) bool { - return strings.HasPrefix(strings.TrimSpace(ref), "docker:") + ref = strings.TrimSpace(ref) + return strings.HasPrefix(ref, "docker:") || strings.HasPrefix(ref, "docker-service:") } -// resolveHistoryReference resolves a non-Docker alert reference to the -// resource whose history it joins. Only durable identities qualify: the -// canonical ID, a retired era, a source ID (Proxmox node, guest and storage -// IDs), a node-scoped Proxmox guest reference, or an agent alert reference -// ("agent:"). Names and hostnames never bind history, so a resource -// named like a system reference cannot capture its events. claimed reports -// that inventory answers to the reference, possibly ambiguously; an ambiguous -// reference resolves to nothing and must not follow a retained binding. +// isDockerNameHistoryReference marks Docker alert references that are not host +// or service references: container references, which only a complete +// container ID binds, and hostless service names. A name or shortened ID must +// never join history, not even through a retained binding. +func isDockerNameHistoryReference(ref string) bool { + _, _, ok := dockerHostHistoryReference(ref) + return isDockerHistoryReference(ref) && !ok +} + +// dockerHostHistoryReference parses the Docker host alert reference +// ("docker:", alerts.DockerResourceID without a container) and the +// Swarm service alert reference ("docker:/service/", +// alerts.DockerServiceResourceID). serviceID is empty for a host reference. +// "docker:unknown" is the shared fallback for alerts without a host. +func dockerHostHistoryReference(ref string) (hostID, serviceID string, ok bool) { + rest, found := strings.CutPrefix(strings.TrimSpace(ref), "docker:") + if !found { + return "", "", false + } + hostID, serviceID, isService := strings.Cut(rest, "/service/") + if hostID == "" || hostID == "unknown" || strings.TrimSpace(hostID) != hostID || strings.Contains(hostID, "/") { + return "", "", false + } + if isService && (serviceID == "" || strings.TrimSpace(serviceID) != serviceID || strings.Contains(serviceID, "/")) { + return "", "", false + } + return hostID, serviceID, true +} + +// dockerHistoryOwnerLocked resolves a Docker host or Swarm service alert +// reference through the registry's own Docker source identities. The host ID +// is the Docker host's source ID. A service reference names the service with +// that ID in the host's Swarm cluster, which every manager of the cluster +// reports under its own host ID. A service without an ID is referenced by its +// normalized name in the same place, so while the cluster has such a service +// every service reference there is a conflict. +func (rr *ResourceRegistry) dockerHistoryOwnerLocked(ref string) (resourceID string, conflict bool) { + hostID, serviceID, ok := dockerHostHistoryReference(ref) + if !ok { + return "", false + } + hostResourceID := rr.bySource[SourceDocker][hostID] + host := rr.resources[hostResourceID] + if host == nil || CanonicalResourceType(host.Type) != ResourceTypeAgent { + return "", false + } + if serviceID == "" { + return hostResourceID, false + } + if host.Docker == nil { + return "", false + } + cluster := dockerSwarmClusterKeyFromMeta(host.Docker.Swarm) + if cluster == "" { + return "", false + } + for _, other := range rr.resources { + if CanonicalResourceType(other.Type) == ResourceTypeDockerService && other.Docker != nil && + strings.TrimSpace(other.Docker.ServiceID) == "" && dockerSwarmClusterKeyFromMeta(other.Docker.Swarm) == cluster { + return "", true + } + } + serviceResourceID := rr.bySource[SourceDocker][normalizeSourceID(cluster+":service:"+serviceID)] + service := rr.resources[serviceResourceID] + if service == nil || CanonicalResourceType(service.Type) != ResourceTypeDockerService || + service.Docker == nil || strings.TrimSpace(service.Docker.ServiceID) != serviceID { + return "", false + } + return serviceResourceID, false +} + +// historySubResourceOwner names the owner of a sub-resource alert reference, +// one that names no resource of its own, and the type that owner must have. +// Only these producer shapes qualify, each carrying its owner's durable +// identity: +// - "/zfs-pool:[/device:]" (ZFS pool state, +// errors and device health) belongs to the storage. +// - "agent:/storage:" (Unraid array) is the agent source ID +// "/storage:" of the host's array storage. +// - "agent:/disk:", ".../disk_temp:", +// ".../raid:" and ".../custom:" belong to the host. +// +// Pool, mount and kernel device labels are names: a device name can belong to +// a different disk after a reboot, so these never bind a physical disk. +func historySubResourceOwner(ref string) (ownerRef string, ownerType ResourceType, ok bool) { + if rest, found := strings.CutPrefix(ref, "agent:"); found { + hostID, child, _ := strings.Cut(rest, "/") + kind, label, _ := strings.Cut(child, ":") + if hostID == "" || label == "" { + return "", "", false + } + switch kind { + case "storage": + return hostID + "/" + child, ResourceTypeStorage, true + case "disk", "disk_temp", "raid", "custom": + return "agent:" + hostID, ResourceTypeAgent, true + } + return "", "", false + } + if i := strings.LastIndex(ref, "/zfs-pool:"); i > 0 { + pool, device, hasDevice := strings.Cut(ref[i+len("/zfs-pool:"):], "/device:") + if pool == "" || strings.Contains(pool, "/") || (hasDevice && (device == "" || strings.Contains(device, "/"))) { + return "", "", false + } + return ref[:i], ResourceTypeStorage, true + } + return "", "", false +} + +// resolveHistoryReference resolves an alert reference to the resource whose +// history it joins. Only durable identities qualify: the canonical ID, a +// retired era, a source ID (Proxmox node, guest and storage IDs), a +// node-scoped Proxmox guest reference, an agent alert reference +// ("agent:"), a Docker host or Swarm service ID, or the owner of a +// sub-resource reference (historySubResourceOwner). Names and hostnames never +// bind history, so a resource named like a system reference cannot capture +// its events. claimed reports that inventory answers to the reference, +// possibly ambiguously; an ambiguous or conflicting reference resolves to +// nothing and must not follow a retained binding. func (rr *ResourceRegistry) resolveHistoryReference(ref string) (resourceID string, claimed bool) { ref = CanonicalResourceID(ref) - if rr == nil || ref == "" || isDockerHistoryReference(ref) { + if rr == nil || ref == "" { return "", false } rr.mu.RLock() defer rr.mu.RUnlock() + if isDockerHistoryReference(ref) { + resourceID, conflict := rr.dockerHistoryOwnerLocked(ref) + return resourceID, resourceID != "" || conflict + } + matches := rr.durableHistoryMatchesLocked(ref) + if ownerRef, ownerType, ok := historySubResourceOwner(ref); ok && len(matches) == 0 { + // The owner must resolve uniquely and have the expected type. An owner + // reference that names an ambiguous or incompatible resource is a + // conflict: the event keeps its own reference. + owners := rr.durableHistoryMatchesLocked(ownerRef) + for id := range owners { + if len(owners) != 1 || CanonicalResourceType(rr.resources[id].Type) != ownerType { + return "", true + } + matches[id] = struct{}{} + } + } + if len(matches) != 1 { + return "", len(matches) > 1 + } + for id := range matches { + resourceID = id + } + return resourceID, true +} + +// durableHistoryMatchesLocked lists the resources that answer to a reference +// by durable identity. +func (rr *ResourceRegistry) durableHistoryMatchesLocked(ref string) map[string]struct{} { + matches := make(map[string]struct{}, 1) if rr.resources[ref] != nil { - return ref, true + matches[ref] = struct{}{} + return matches } if id := rr.supersededResourceIDLocked(ref); rr.resources[id] != nil { - return id, true + matches[id] = struct{}{} + return matches } - matches := make(map[string]struct{}, 1) for _, mapping := range rr.bySource { if id := mapping[ref]; rr.resources[id] != nil { matches[id] = struct{}{} @@ -78,13 +223,7 @@ func (rr *ResourceRegistry) resolveHistoryReference(ref string) (resourceID stri } } } - if len(matches) != 1 { - return "", len(matches) > 1 - } - for id := range matches { - resourceID = id - } - return resourceID, true + return matches } // resourceHistoryIdentityWriter binds a source reference to the canonical @@ -187,11 +326,12 @@ func (b *legacyHistoryBackfill) add(ref string, now time.Time) { } // unboundHistoryReferences lists alert journal references without a binding. -// Docker references are excluded: the store migration binds full container -// IDs, and names or shortened IDs must never bind. +// Docker container references and hostless service names are excluded: the +// store migration binds full container IDs, and names or shortened IDs must +// never bind. func (s *SQLiteResourceStore) unboundHistoryReferences() ([]string, error) { rows, err := s.db.Query(`SELECT DISTINCT canonical_id FROM resource_changes - WHERE kind GLOB 'alert_*' AND canonical_id NOT GLOB 'docker:*' + WHERE kind GLOB 'alert_*' AND canonical_id NOT IN (SELECT source_id FROM resource_history_aliases) ORDER BY canonical_id`) if err != nil { @@ -204,7 +344,9 @@ func (s *SQLiteResourceStore) unboundHistoryReferences() ([]string, error) { if err := rows.Scan(&ref); err != nil { return nil, err } - refs = append(refs, ref) + if !isDockerNameHistoryReference(ref) { + refs = append(refs, ref) + } } return refs, rows.Err() } diff --git a/internal/unifiedresources/history_identity_test.go b/internal/unifiedresources/history_identity_test.go index 935e260fb4..4744e08503 100644 --- a/internal/unifiedresources/history_identity_test.go +++ b/internal/unifiedresources/history_identity_test.go @@ -333,3 +333,246 @@ func TestHistoryIdentityBindsLegacyAlertRowsFromRegistryGenerations(t *testing.T require.NoError(t, err) require.Equal(t, []string{"lab:pve1:999"}, refs) } + +func requireHistoryReferenceBindings(t *testing.T, store *SQLiteResourceStore, bound map[string]string, unbound []string) { + t.Helper() + byResource := make(map[string][]string) + for ref, canonicalID := range bound { + byResource[canonicalID] = append(byResource[canonicalID], "fired-"+ref) + id, found, err := store.ResolveHistorySourceIdentity(ref) + require.NoError(t, err) + require.True(t, found, ref) + require.Equal(t, canonicalID, id, ref) + } + for canonicalID, want := range byResource { + got, err := store.GetRecentChangesFiltered(canonicalID, time.Time{}, 50, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertFired}}) + require.NoError(t, err) + ids := make([]string, 0, len(got)) + for _, change := range got { + ids = append(ids, change.ID) + // A row journaled before its binding keeps its recorded reference. + require.Contains(t, []string{canonicalID, strings.TrimPrefix(change.ID, "fired-")}, change.ResourceID, change.ID) + } + require.ElementsMatch(t, want, ids, canonicalID) + } + for _, ref := range unbound { + _, found, err := store.ResolveHistorySourceIdentity(ref) + require.NoError(t, err) + require.False(t, found, ref) + got, err := store.GetRecentChanges(ref, time.Time{}, 10) + require.NoError(t, err) + require.Len(t, got, 1, ref) + require.Equal(t, ref, got[0].ResourceID, ref) + } +} + +// Docker host alerts reference the host's source ID ("docker:") and +// Swarm service alerts the service ID ("docker:/service/") from +// every manager that reports the service. Both join the canonical host and +// service history, also for a host merged into its agent's machine. Hostnames, +// service names, container names and shortened container IDs never bind, not +// even to the history of what they name, and a cluster with a service that +// lacks an ID, whose alerts carry its normalized name, binds no service. +func TestHistoryIdentityMonitorAdapterResolvesDockerHostAndServiceReferences(t *testing.T) { + store, err := NewSQLiteResourceStore(t.TempDir(), "default") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + now := time.Now().UTC().Truncate(time.Second) + container := strings.Repeat("f", 64) + serviceID := "x7k2m9q4w1e8r5t3y6u0i2o4p" + otherServiceID := "q1w2e3r4t5y6u7i8o9p0a1s2d" + manager := func(id, hostname, cluster string, services ...models.DockerService) models.DockerHost { + return models.DockerHost{ID: id, Hostname: hostname, Status: "online", LastSeen: now, Services: services, + Swarm: &models.DockerSwarmInfo{NodeID: id + "-node", NodeRole: "manager", LocalState: "active", ControlAvailable: true, ClusterID: cluster}} + } + web := models.DockerService{ID: serviceID, Name: "web"} + // hostA runs on a machine whose agent also reports, so the two merge. + hostA := manager("dh-7f3a", "tower", "swarm-1", web) + hostA.AgentID, hostA.MachineID = "host-tower", "0123456789abcdef" + hostA.Containers = []models.DockerContainer{{ID: container, Name: "worker", State: "running"}} + hostB := manager("dh-91c0", "rack", "swarm-1", web) + // In swarm-2 a service without an ID is named like another service's ID. + hostC := manager("dh-c4d2", "edge", "swarm-2", models.DockerService{ID: otherServiceID, Name: "api"}, models.DockerService{Name: strings.ToUpper(otherServiceID)}) + // Rows journaled before this process started are bound by a later generation. + legacy := []string{"docker:dh-91c0", "docker:dh-91c0/service/" + serviceID, "docker:dh-7f3a/worker"} + for i, ref := range legacy { + require.NoError(t, store.RecordChange(ResourceChange{ID: "fired-" + ref, ResourceID: ref, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Duration(i) * time.Second)})) + } + adapter := NewMonitorAdapter(NewRegistry(store)) + adapter.PopulateFromSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ID: "host-tower", Hostname: "tower", MachineID: "0123456789abcdef", Status: "online", LastSeen: now}}, + DockerHosts: []models.DockerHost{hostA, hostB, hostC}, + }) + registry := adapter.currentRegistry() + hostAID := registry.sourceResourceID(SourceDocker, "dh-7f3a") + hostBID := registry.sourceResourceID(SourceDocker, "dh-91c0") + serviceResourceID := historyIdentityResourceID(t, registry, ResourceTypeDockerService, "web") + containerID := registry.sourceResourceID(SourceDocker, "dh-7f3a/container/"+container) + require.Equal(t, registry.sourceResourceID(SourceAgent, "host-tower"), hostAID, "the Docker host merges into its agent's machine") + require.NotEqual(t, hostAID, hostBID) + require.NotEmpty(t, containerID) + + bound := map[string]string{ + "docker:dh-91c0": hostBID, + "docker:dh-91c0/service/" + serviceID: serviceResourceID, + "docker:dh-7f3a": hostAID, + "docker:dh-7f3a/service/" + serviceID: serviceResourceID, + } + unbound := []string{ + "docker:tower", // hostname + "docker:dh-7f3a/service/web", // service name + "docker:dh-c4d2/service/" + otherServiceID, // its cluster has a service without an ID + "docker-service:web", // hostless service name + "docker:dh-7f3a/worker", // container name + "docker:dh-7f3a/" + container[:12], // shortened container ID + "docker:unknown", + } + for i, ref := range append([]string{"docker:dh-7f3a", "docker:dh-7f3a/service/" + serviceID}, unbound...) { + if ref == "docker:dh-7f3a/worker" { + continue // journaled above + } + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "fired-" + ref, ResourceID: ref, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Minute + time.Duration(i)*time.Second)})) + } + requireHistoryReferenceBindings(t, store, bound, unbound) + // Names and shortened IDs are never retried against later inventory. + require.Contains(t, adapter.legacyHistory.pending, "docker:tower") + for _, ref := range []string{"docker:dh-7f3a/worker", "docker:dh-7f3a/" + container[:12], "docker-service:web"} { + require.NotContains(t, adapter.legacyHistory.pending, ref) + } + got, err := store.GetRecentChangesFiltered(containerID, time.Time{}, 10, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertFired}}) + require.NoError(t, err) + require.Empty(t, got, "a container name or short ID never joins the container's history") + // While its cluster has a service without an ID, a service reference is a + // conflict and must not follow a binding recorded earlier. + collision := "docker:dh-c4d2/service/" + otherServiceID + apiID := historyIdentityResourceID(t, registry, ResourceTypeDockerService, "api") + require.NoError(t, store.RecordChangeWithSourceIdentity(ResourceChange{ID: "earlier-api", ResourceID: apiID, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now}, collision)) + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "conflict-api", ResourceID: collision, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(2 * time.Minute)})) + // The same holds once the service with that ID has left inventory. + hostC.Services = hostC.Services[1:] + adapter.PopulateFromSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ID: "host-tower", Hostname: "tower", MachineID: "0123456789abcdef", Status: "online", LastSeen: now}}, + DockerHosts: []models.DockerHost{hostA, hostB, hostC}, + }) + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "conflict-api-removed", ResourceID: collision, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(3 * time.Minute)})) + for _, id := range []string{"conflict-api", "conflict-api-removed"} { + var recordedAs string + require.NoError(t, store.db.QueryRow(`SELECT canonical_id FROM resource_changes WHERE id = ?`, id).Scan(&recordedAs)) + require.Equal(t, collision, recordedAs, id) + } + + // After the service leaves inventory its alerts follow the retained binding; + // a container name still binds nothing. + removed := NewMonitorAdapter(NewRegistry(store)) + for _, ref := range []string{"docker:dh-91c0/service/" + serviceID, "docker:dh-7f3a/worker"} { + require.NoError(t, removed.RecordChange(ResourceChange{ID: "resolved-" + ref, ResourceID: ref, Kind: ChangeAlertResolved, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Hour)})) + } + got, err = store.GetRecentChangesFiltered(serviceResourceID, time.Time{}, 10, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertFired, ChangeAlertResolved}}) + require.NoError(t, err) + require.Len(t, got, 3) + require.Equal(t, "resolved-docker:dh-91c0/service/"+serviceID, got[0].ID) + require.Equal(t, serviceResourceID, got[0].ResourceID) + got, err = store.GetRecentChanges("docker:dh-7f3a/worker", time.Time{}, 10) + require.NoError(t, err) + require.Len(t, got, 2) + require.Equal(t, "docker:dh-7f3a/worker", got[0].ResourceID) +} + +// Sub-resource alerts (ZFS pools and devices, host filesystems, disks, RAID +// arrays, sensors, the Unraid array) name no resource of their own. They join +// the history of the owner their reference carries a durable ID for: the +// storage, the host, or the Unraid array storage. Device labels never bind a +// physical disk, and an owner of the wrong type or a name binds nothing. +func TestHistoryIdentityMonitorAdapterResolvesSubResourceReferences(t *testing.T) { + store, err := NewSQLiteResourceStore(t.TempDir(), "default") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + now := time.Now().UTC().Truncate(time.Second) + snapshot := proxmoxHistoryIdentitySnapshot(now) + snapshot.Storage = []models.Storage{{ID: "lab-pve1-local-zfs", Name: "local-zfs", Node: "pve1", Instance: "lab", Type: "zfspool", Status: "available", Total: 100, Used: 10, LastSeen: now}} + snapshot.Hosts[0].Sensors.SMART = []models.HostDiskSMART{{Device: "sda", Serial: "S3Z1NB0K", Model: "Samsung SSD", Health: "PASSED", Temperature: 34}} + snapshot.Hosts = append(snapshot.Hosts, models.Host{ID: "host-nas", Hostname: "nas", MachineID: "fedcba9876543210", Status: "online", LastSeen: now, + Unraid: &models.HostUnraidStorage{ArrayStarted: true, ArrayState: "STARTED", Disks: []models.HostUnraidDisk{{Name: "disk1", Device: "sdb", Role: "data", Status: "DISK_OK", Serial: "WD-1"}}}}) + adapter := NewMonitorAdapter(NewRegistry(store)) + adapter.PopulateFromSnapshot(snapshot) + registry := adapter.currentRegistry() + nodeID := historyIdentityResourceID(t, registry, ResourceTypeAgent, "pve1") + storageID := registry.sourceResourceID(SourceProxmox, "lab-pve1-local-zfs") + unraidID := registry.sourceResourceID(SourceAgent, "host-nas/storage:unraid-array") + diskID := registry.sourceResourceID(SourceAgent, HostSMARTDiskSourceID(snapshot.Hosts[0], snapshot.Hosts[0].Sensors.SMART[0])) + require.NotEmpty(t, storageID) + require.NotEmpty(t, unraidID) + require.NotEmpty(t, diskID) + + bound := map[string]string{ + "lab-pve1-local-zfs/zfs-pool:local-zfs": storageID, + "lab-pve1-local-zfs/zfs-pool:local-zfs/device:sda2": storageID, + "agent:host-pve1/disk:var": nodeID, + "agent:host-pve1/disk:sda": nodeID, + "agent:host-pve1/disk_temp:sda": nodeID, + "agent:host-pve1/raid:md0": nodeID, + "agent:host-pve1/custom:fan1": nodeID, + "agent:host-nas/storage:unraid-array": unraidID, + } + unbound := []string{ + "lab-pve1/zfs-pool:local-zfs", // the owner part names a node, not a storage + "local-zfs/zfs-pool:local-zfs", // a storage name, not its ID + "agent:pve1/disk:var", // a hostname, not the host ID + "agent:host-pve1/service:sshd", // not a sub-resource shape any producer emits + "agent:host-pve1/disk:", + } + refs := append([]string{}, unbound...) + for ref := range bound { + refs = append(refs, ref) + } + for i, ref := range refs { + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "fired-" + ref, ResourceID: ref, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Duration(i) * time.Second)})) + } + requireHistoryReferenceBindings(t, store, bound, unbound) + got, err := store.GetRecentChangesFiltered(diskID, time.Time{}, 10, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertFired}}) + require.NoError(t, err) + require.Empty(t, got, "a kernel device label never binds a physical disk") + + // Recovery after the storage left inventory follows the retained binding. + pool := "lab-pve1-local-zfs/zfs-pool:local-zfs" + removed := NewMonitorAdapter(NewRegistry(store)) + require.NoError(t, removed.RecordChange(ResourceChange{ID: "resolved-" + pool, ResourceID: pool, Kind: ChangeAlertResolved, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Hour)})) + got, err = store.GetRecentChangesFiltered(storageID, time.Time{}, 10, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertResolved}}) + require.NoError(t, err) + require.Len(t, got, 1) + require.Equal(t, storageID, got[0].ResourceID) + + // An owner reference that two resources answer to, or that names a resource + // of another type, binds nothing and must not follow the retained binding. + vmID := historyIdentityResourceID(t, registry, ResourceTypeVM, "web") + for _, conflict := range []struct { + name string + source DataSource + }{{"ambiguous", SourcePBS}, {"wrong-type", SourceProxmox}} { + registry.mu.Lock() + delete(registry.bySource[SourcePBS], "lab-pve1-local-zfs") + if registry.bySource[conflict.source] == nil { + registry.bySource[conflict.source] = make(map[string]string) + } + registry.bySource[conflict.source]["lab-pve1-local-zfs"] = vmID + registry.mu.Unlock() + id := conflict.name + "-" + pool + require.NoError(t, adapter.RecordChange(ResourceChange{ID: id, ResourceID: pool, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(2 * time.Hour)})) + var recordedAs string + require.NoError(t, store.db.QueryRow(`SELECT canonical_id FROM resource_changes WHERE id = ?`, id).Scan(&recordedAs)) + require.Equal(t, pool, recordedAs, conflict.name) + bound, _, err := store.ResolveHistorySourceIdentity(pool) + require.NoError(t, err) + require.Equal(t, storageID, bound, conflict.name) + } + // A conflicting reference without a binding is retried, like one found at + // startup, and binds once a generation names its owner unambiguously. + device := "lab-pve1-local-zfs/zfs-pool:local-zfs/device:sdc2" + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "fired-" + device, ResourceID: device, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(3 * time.Hour)})) + require.Contains(t, adapter.legacyHistory.pending, device) + adapter.PopulateFromSnapshot(snapshot) + deviceBinding, found, err := store.ResolveHistorySourceIdentity(device) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, storageID, deviceBinding) +} diff --git a/internal/unifiedresources/monitor_adapter.go b/internal/unifiedresources/monitor_adapter.go index 94795237b4..26a39e0958 100644 --- a/internal/unifiedresources/monitor_adapter.go +++ b/internal/unifiedresources/monitor_adapter.go @@ -198,7 +198,8 @@ func (a *MonitorAdapter) currentRegistry() *ResourceRegistry { // RecordChange forwards canonical resource-history events into the underlying // resource store when monitoring has a durable store attached. Alert lifecycle // events carry source-native references (Proxmox node and guest IDs, agent -// IDs); they resolve here so history reads by canonical ID include them. +// IDs, Docker host and Swarm service IDs, sub-resource references); they +// resolve here so history reads by canonical ID include them. func (a *MonitorAdapter) RecordChange(change ResourceChange) error { registry := a.currentRegistry() if registry == nil || registry.store == nil { @@ -229,16 +230,19 @@ func (a *MonitorAdapter) RecordChange(change ResourceChange) error { } } else if resolvedID, claimed := registry.resolveHistoryReference(sourceRef); resolvedID != "" { change.ResourceID = resolvedID - } else if hasHistory && !claimed && !isDockerHistoryReference(sourceRef) { - // No resource answers to it: keep the binding an earlier event recorded. - id, found, err := history.ResolveHistorySourceIdentity(CanonicalResourceID(sourceRef)) - if err != nil { - return err - } - if found { - change.ResourceID = id - } else { - unbound = true + } else if hasHistory && !isDockerNameHistoryReference(sourceRef) { + // A later generation may name the resource or settle a conflict. + unbound = true + if !claimed { + // No resource answers to it: keep the binding an earlier event recorded. + id, found, err := history.ResolveHistorySourceIdentity(CanonicalResourceID(sourceRef)) + if err != nil { + return err + } + if found { + change.ResourceID = id + unbound = false + } } } if hasHistory && CanonicalResourceID(sourceRef) != CanonicalResourceID(change.ResourceID) { From 5becf5953676d14ccdd99d3a289255ae1281bf7e Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:46:05 +0100 Subject: [PATCH 25/53] Remove the unreachable host metric cells from grouped workload rows Both production mounts of the Workloads surface, the Proxmox and vSphere overviews, set compactGroupHeaders unconditionally. WorkloadPanel's per-column node cells (CPU, memory, disk and I/O bars or sparklines, uptime, version, temperature) and NodeGroupHeader's column mode and inline host facts therefore never rendered. They were left over from the retired standalone Workloads page, whose group rows had no hosts table beside them. Drop the option, the dead renderers and the NodeGroupHeader props that only served them. Grouped host rows keep exactly the shape both pages already render: one spanning cell with status, linked name, cluster and agent badges, while host stats stay in each page's own hosts table. Tests that only pinned the option are removed. The NodeGroupHeader test now covers the spanning-cell shape and fails against the old default facts, and the PVE version formatter guardrail moves to proxmoxPageModel, the remaining version renderer. --- .../subsystems/frontend-primitives.md | 20 +- .../subsystems/performance-and-scalability.md | 30 +- .../internal/subsystems/storage-recovery.md | 5 + .../internal/subsystems/unified-resources.md | 5 +- .../8ada2c9fe2438ea7.json | 49 +++ .../components/Workloads/WorkloadPanel.tsx | 304 ------------------ .../components/Workloads/WorkloadsSurface.tsx | 1 - .../components/Workloads/WorkloadsTable.tsx | 4 - ...loadsSurface.performance.contract.test.tsx | 26 -- .../components/Workloads/useWorkloadsState.ts | 7 - .../src/components/shared/NodeGroupHeader.tsx | 71 +--- .../SharedPrimitives.guardrails.test.ts | 7 +- .../shared/__tests__/NodeGroupHeader.test.tsx | 39 +-- .../features/proxmox/ProxmoxPageSurface.tsx | 2 - .../ProxmoxPageSurface.contract.test.tsx | 10 + .../src/features/vmware/VmwarePageSurface.tsx | 2 - .../temperatureDisplayGuardrails.test.ts | 7 +- 17 files changed, 120 insertions(+), 469 deletions(-) create mode 100644 frontend-modern/browser-verification/8ada2c9fe2438ea7.json diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 85c281c011..6c07a49441 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -2911,9 +2911,9 @@ Agent`), with the plain-language source phrase available through accessible `pveVersion` or a Pulse Agent report whose OS identity resolves to Unraid or Proxmox VE. They must omit the version rather than showing unrelated collector OS versions, such as Debian 12, beside an API-backed PVE badge. - Shared row primitives that render Proxmox node identity, including - `frontend-modern/src/components/shared/NodeGroupHeader.tsx`, must route raw - PVE manager payloads through + Surfaces that render Proxmox node versions, including the Proxmox page + model at `frontend-modern/src/features/proxmox/proxmoxPageModel.ts`, must + route raw PVE manager payloads through `frontend-modern/src/utils/proxmoxVersion.ts` rather than inlining page-local parsing or falling back to unrelated agent OS versions. System title metadata must apply the same identity rule: once the primary @@ -4395,12 +4395,14 @@ The shared table chrome now allows `TableCardHeader` to expose a right-aligned action slot, currently used by the Workloads/Proxmox metric display control. That slot belongs to the table header band and must not reintroduce nested cards or page-local toolbar wrappers inside `TableCard`. Proxmox host grouping -also extends the shared `NodeGroupHeader` row pattern: host metrics may align -with workload table columns, but the shared primitive owns the header/table -shell boundary rather than platform pages copying their own card headers. -Compact PVE version text in that header must come from the shared Proxmox -version formatter so raw `pve-manager/...` payloads and platform-page host -version cells stay consistent. +also extends the shared `NodeGroupHeader` row pattern, which owns the +header/table shell boundary rather than platform pages copying their own card +headers. `frontend-modern/src/components/shared/NodeGroupHeader.tsx` renders +node identity only (status dot, linked name, cluster and agent badges); in +table-row mode that identity sits in one cell spanning the table. It carries no +per-column cell renderer and no inline +version, temperature, or uptime facts: host metrics and versions belong to the +platform page's host table, so a group row must not duplicate them. Mobile navigation now recognizes `proxmox` as a first-class platform tab in the shared priority model so app-shell ordering remains centralized. diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 87517e7399..5231c21704 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -1948,22 +1948,20 @@ query persistence once per resource, or run on the steady-state alert hot path; ancestor lookup remains bounded by canonical hierarchy depth for each active alert actually evaluated. -The embedded WorkloadsSurface exposes a `compactGroupHeaders` prop on -`frontend-modern/src/components/Workloads/useWorkloadsState.ts` that -platform pages owning their own hosts table (Proxmox overview today) set -to strip per-host metric cells out of the `NodeGroupHeader` rows in -grouped mode. The flag is threaded through -`frontend-modern/src/components/Workloads/WorkloadsSurface.tsx`, -`frontend-modern/src/components/Workloads/WorkloadsTable.tsx`, and -`frontend-modern/src/components/Workloads/WorkloadPanel.tsx`; when set, -`WorkloadPanel` calls `NodeGroupHeader` without `columns` / -`renderColumnCell` so the existing colspan layout renders just the -status dot, linked node name, cluster badge, and agent badge — the -duplicate CPU / Memory / Disk / uptime / temperature / version stats -the top-of-page hosts table already owns are dropped. New embedded -platform-page consumers that render their own hosts summary must set -this flag; standalone Workloads surfaces (no top hosts table) keep the -original verbose group rows by default. +Grouped host rows in the embedded WorkloadsSurface are identity-only +dividers. `frontend-modern/src/components/Workloads/WorkloadPanel.tsx` +renders each grouped node through `NodeGroupHeader` as one cell spanning +the table: status dot, linked node name, cluster badge, and agent badge. +Per-host CPU, memory, disk, network and disk I/O, uptime, temperature, +and version stats belong to the platform page's own hosts table (the +Proxmox nodes table today), so the workloads table must not render them +per column or inline in the group row. The former `compactGroupHeaders` +option and the per-column node metric cells it switched off were removed +because both remaining production mounts (Proxmox and vSphere overview) +set it unconditionally, so the non-compact path no longer rendered +anywhere. A new embedded consumer that needs host +stats beside its workloads must render its own hosts table rather than +re-growing metric cells in the shared group row. WorkloadsSurface stays monitoring-first. It must not render a persistent aggregate banner just because running VMs or system containers lack an diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index cd2ec5bf9e..fb2158b9ac 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -970,6 +970,11 @@ the page supplies it through the Workloads `tableTitle` slot and the shared states. This heading alignment changes neither workload backup scope nor the storage/recovery evidence carried by the adjacent Backup column and Backups tab. +The guest table's grouped node rows are identity-only dividers. Node stats and +the node drawer stay with the Proxmox nodes table above it, so `ProxmoxPageSurface` +passes `groupNodeDrawerMode: 'disabled'` and no host-metric option to the +embedded Workloads state. Those dividers carry no backup or recovery evidence; +the per-guest Backup column remains the overview's protection signal. The overview's node-row action selects the shared workload node scope and reveals the guest section. The title names the selected node and reports the filtered guest count, including zero. Activating the selected node again or diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index f3c5938c4c..d505a37b64 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -2973,7 +2973,10 @@ canonical WebSocket snapshot, including the public `vmware-vsphere` alias for the raw `vmware` source, rather than degrading a source-scoped page to periodic REST refreshes. VMware Overview passes that same source-scoped snapshot into the embedded Workloads state, so hosts and VMs share one inventory generation -and one explicit refresh path. +and one explicit refresh path. Beyond disabling the grouped host drawer, it +passes no option that adds host metrics to grouped rows: the shared Workloads +group row carries host identity only, and per-host stats stay in the page's +own hosts table. Its page-owned workload toolbar consumes the complete shared `getWorkloadsMetricFilterProps` binding, so vSphere VMs expose the same Bars, Trends, Details, History, range, and first-use discovery contract as every diff --git a/frontend-modern/browser-verification/8ada2c9fe2438ea7.json b/frontend-modern/browser-verification/8ada2c9fe2438ea7.json new file mode 100644 index 0000000000..4f8e022c8f --- /dev/null +++ b/frontend-modern/browser-verification/8ada2c9fe2438ea7.json @@ -0,0 +1,49 @@ +{ + "version": 1, + "base_sha": "c3adf0c7b9fa7f4283f7fd832a806659b009b51b", + "verified_at": "2026-10-06T21:45:33Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Workloads/WorkloadPanel.tsx", + "frontend-modern/src/components/Workloads/WorkloadsSurface.tsx", + "frontend-modern/src/components/Workloads/WorkloadsTable.tsx", + "frontend-modern/src/components/Workloads/useWorkloadsState.ts", + "frontend-modern/src/components/shared/NodeGroupHeader.tsx", + "frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx", + "frontend-modern/src/features/vmware/VmwarePageSurface.tsx" + ], + "content_sha256": { + "frontend-modern/src/components/Workloads/WorkloadPanel.tsx": "67c2c51ffceb96893f968dc141dbce54f0affe66fb389ffeeba8c8c94988968c", + "frontend-modern/src/components/Workloads/WorkloadsSurface.tsx": "e49032445f566b4e6a4329314f4ada41ed6f7652ba52e9e3fc413047a49f7878", + "frontend-modern/src/components/Workloads/WorkloadsTable.tsx": "a340532f9e3e0e28d83e9a0fb369c8be973acb0dea7b321fd646345ff924db38", + "frontend-modern/src/components/Workloads/useWorkloadsState.ts": "a29034c5149b7323e86bc2fe795d9b18db805b1ad32e32013fb71547256d61f5", + "frontend-modern/src/components/shared/NodeGroupHeader.tsx": "e435c1e6ab04b85a022459cf19a67645bc8339f15ef986ffcb5b42ba27ce667f", + "frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx": "80c860be3bee37251c16d21044074267e48991b9cb39b87d335cb2155066ec96", + "frontend-modern/src/features/vmware/VmwarePageSurface.tsx": "6c74063a298a259ce5f60485174529e9bffbe7293ae0710f53dc95d6f3c11265" + }, + "routes": ["/proxmox/overview", "/vmware"], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Production vite build of this tree (rebased on c3adf0c7b9) embedded in a private DEMO_MODE mock backend (50 PVE nodes, 929 guests, 7 ESXi hosts), headless Chromium via Playwright; desktop 1440x900 and phone 390x844.", + "Proxmox Overview guest table, Grouped layout: every grouped node row (8 desktop, 2 phone in the rendered window) is one td whose colspan equals the guest table's visible header count (8 desktop, 5 phone), with status dot, linked node name, cluster badge and Agent badge only; no data-workload-col cells, no PVE version, temperature or uptime text.", + "vSphere Overview VM table: the 7 host group rows render as one spanning cell (colspan 6 desktop, 5 phone) with host name and cluster badge, unchanged shape.", + "Bars and Trends metric modes both inspected: Trends adds four sparklines per guest row (Proxmox 140 to 560 images, vSphere 18 to 72) while each grouped host row keeps exactly its one status-dot image.", + "Screenshots of grouped rows in both modes and of open guest drawers inspected at both viewports; no page errors.", + "This is the shape both pages already rendered on main, where compactGroupHeaders was always set; it is not a check of host drawers, which both pages disable." + ], + "interactions": [ + "Opened the View popover (desktop) and the Filters popover (phone) and toggled Bars to Trends and back.", + "Clicked a grouped host row on each page: no inline node drawer row appears and nothing else changes.", + "Clicked the first guest row under a group on each page: its inline guest drawer opens.", + "Scrolled the first grouped host row to the viewport centre before each capture." + ] +} diff --git a/frontend-modern/src/components/Workloads/WorkloadPanel.tsx b/frontend-modern/src/components/Workloads/WorkloadPanel.tsx index 9caf56f762..de8a3cab1d 100644 --- a/frontend-modern/src/components/Workloads/WorkloadPanel.tsx +++ b/frontend-modern/src/components/Workloads/WorkloadPanel.tsx @@ -1,10 +1,6 @@ import { createMemo, For, Show } from 'solid-js'; import { ComponentErrorBoundary } from '@/components/ErrorBoundary'; -import { EnhancedCPUBar } from '@/components/Workloads/EnhancedCPUBar'; -import { MetricMiniSparkline } from '@/components/Workloads/MetricMiniSparkline'; -import { StackedDiskBar } from '@/components/Workloads/StackedDiskBar'; -import { StackedMemoryBar } from '@/components/Workloads/StackedMemoryBar'; import { GROUPED_TABLE_ROW_BADGE_CLASS, getGroupedTableRowCellClass, @@ -16,12 +12,9 @@ import { NodeGroupHeader } from '@/components/shared/NodeGroupHeader'; import { createSummaryInteractiveRowPreviewHandlers } from '@/components/shared/summaryInteractionA11y'; import { buildSummaryDisclosureControlsId } from '@/components/shared/summaryInteractionA11y'; import { TableBody, TableCell, TableRow } from '@/components/shared/Table'; -import type { Node } from '@/types/api'; import { getAlertsForResource, getAlertStyles } from '@/utils/alerts'; import { guestOverrideIdCandidates } from '@/features/alerts/guestOverrideIdentity'; -import { formatSpeed, formatUptime } from '@/utils/format'; import { isNodeOnline } from '@/utils/status'; -import { formatTemperature, getCpuTemperature, getTemperatureTextClass } from '@/utils/temperature'; import { getCanonicalWorkloadId, getWorkloadMetadataIdCandidates } from '@/utils/workloads'; import { resolveSummaryGroupMemberInteractionState, @@ -34,14 +27,12 @@ import { NodeDrawer } from './NodeDrawer'; import { buildWorkloadSummaryGroupScope } from './workloadSelectors'; import { getWorkloadGuestMetadataRecord } from './workloadGuestMetadataRecord'; import type { WorkloadsState } from './useWorkloadsState'; -import type { WorkloadTableMetric } from './workloadMetricHistoryModel'; type WorkloadPanelProps = Pick< WorkloadsState, | 'activeAlerts' | 'alertsEnabled' | 'bottomSpacerHeight' - | 'compactGroupHeaders' | 'getGroupLabel' | 'getNodeTemperatureThresholds' | 'groupedGuests' @@ -58,7 +49,6 @@ type WorkloadPanelProps = Pick< | 'focusedSummaryWorkloadGroupScope' | 'focusedSummaryWorkloadGroupId' | 'hoveredSummaryWorkloadGroupScope' - | 'isMobile' | 'nestedWorkloadContextByGuestId' | 'nodeByInstance' | 'search' @@ -80,289 +70,9 @@ type WorkloadPanelProps = Pick< | 'workloadTableLayoutMode' | 'workloadTableVisibleColumnIds' | 'workloadColumnWidths' - | 'workloadTableVisibleColumns' >; -const GROUP_NODE_METRIC_CELL_CLASS = 'px-1.5 sm:px-2 py-0.5 align-middle'; -const GROUP_NODE_NAME_CELL_CLASS = getGroupedTableRowCellClass( - 'py-1! pl-2! sm:pl-3! pr-1.5! sm:pr-2!', -); - -const getGroupNodeColumnCellClass = (_columnId: string, isNameColumn: boolean): string => { - if (isNameColumn) return GROUP_NODE_NAME_CELL_CLASS; - return GROUP_NODE_METRIC_CELL_CLASS; -}; - -const normalizeMetricPercent = (value: number | null | undefined): number => { - if (typeof value !== 'number' || !Number.isFinite(value)) return 0; - return value <= 1 ? Math.max(0, value * 100) : Math.max(0, value); -}; - -const formatMetricPercent = (value: number | null | undefined): string => - `${Math.round(normalizeMetricPercent(value))}%`; - -const getUsedPercent = (used?: number | null, total?: number | null): number => { - if (typeof used !== 'number' || typeof total !== 'number' || total <= 0) return 0; - return normalizeMetricPercent((used / total) * 100); -}; - -const getGroupNodeMetricKey = (node: Node): string => node.linkedAgentId || node.id || node.name; - -const getGroupNodePveVersion = (node: Node): string => { - const version = (node.pveVersion || '').trim(); - if (!version || version.toLowerCase() === 'unknown') return ''; - return ( - version.match(/pve-manager\/([^/\s]+)/i)?.[1] || version.match(/\d+(?:\.\d+)+/)?.[0] || version - ); -}; - -const renderGroupNodeEmptyCell = () => ( -
- -
-); - export function WorkloadPanel(props: WorkloadPanelProps) { - const renderGroupNodeSparkline = ( - node: Node, - metric: WorkloadTableMetric, - valueLabel: string, - title: string, - unit = '%', - valueLabelMode: 'inline' | 'tooltip' | 'hidden' = 'inline', - formatValue?: (value: number) => string, - ) => ( - - ); - - const renderGroupNodeRatePair = ( - leftLabel: string, - leftClass: string, - leftValue: number | null | undefined, - rightLabel: string, - rightClass: string, - rightValue: number | null | undefined, - ) => { - const left = Math.max(0, leftValue ?? 0); - const right = Math.max(0, rightValue ?? 0); - if (left <= 0 && right <= 0) return renderGroupNodeEmptyCell(); - - return ( -
- {leftLabel} - - {formatSpeed(left)} - - {rightLabel} - - {formatSpeed(right)} - -
- ); - }; - - const renderGroupNodeInfoCell = (node: Node) => { - const version = getGroupNodePveVersion(node); - const temperature = getCpuTemperature(node.temperature); - if (!version && temperature === null) return renderGroupNodeEmptyCell(); - - return ( -
- - PVE {version} - - - - {formatTemperature(temperature)} - - -
- ); - }; - - const renderGroupNodeColumnCell = (columnId: string, node: Node) => { - switch (columnId) { - case 'type': - return ( -
- - PVE - -
- ); - case 'info': - case 'vmid': - return renderGroupNodeInfoCell(node); - case 'cpu': - if (props.workloadMetricDisplayMode() === 'sparklines') { - return renderGroupNodeSparkline( - node, - 'cpu', - formatMetricPercent(node.cpu), - `${node.name} CPU history`, - '%', - 'inline', - formatMetricPercent, - ); - } - return ( -
- -
- ); - case 'memory': - if (props.workloadMetricDisplayMode() === 'sparklines') { - return renderGroupNodeSparkline( - node, - 'memory', - node.memory?.usageUnavailable - ? 'N/A' - : formatMetricPercent(getUsedPercent(node.memory?.used, node.memory?.total)), - `${node.name} memory history`, - '%', - 'inline', - formatMetricPercent, - ); - } - return ( -
- -
- ); - case 'disk': - if (props.workloadMetricDisplayMode() === 'sparklines') { - return renderGroupNodeSparkline( - node, - 'disk', - formatMetricPercent(node.disk?.usage), - `${node.name} disk usage history`, - '%', - 'inline', - formatMetricPercent, - ); - } - return ( -
- -
- ); - case 'uptime': - return ( -
- 0} fallback={renderGroupNodeEmptyCell()}> - 0 && node.uptime < 3600 ? 'text-orange-500' : 'text-muted' - }`} - > - {formatUptime(node.uptime, props.isMobile())} - - -
- ); - case 'node': - case 'context': - return ( -
- - {node.name} - -
- ); - case 'os': - return ( -
- PVE -
- ); - case 'netIo': - if (props.workloadMetricDisplayMode() === 'sparklines') { - return renderGroupNodeSparkline( - node, - 'netIo', - `${formatSpeed(node.networkIn ?? 0)} / ${formatSpeed(node.networkOut ?? 0)}`, - `${node.name} network I/O history`, - 'B/s', - 'tooltip', - formatSpeed, - ); - } - return renderGroupNodeRatePair( - '↓', - 'text-emerald-500', - node.networkIn, - '↑', - 'text-orange-400', - node.networkOut, - ); - case 'diskIo': - if (props.workloadMetricDisplayMode() === 'sparklines') { - return renderGroupNodeSparkline( - node, - 'diskIo', - `${formatSpeed(node.diskRead ?? 0)} / ${formatSpeed(node.diskWrite ?? 0)}`, - `${node.name} disk I/O history`, - 'B/s', - 'tooltip', - formatSpeed, - ); - } - return renderGroupNodeRatePair( - 'R', - 'font-mono text-blue-500', - node.diskRead, - 'W', - 'font-mono text-amber-500', - node.diskWrite, - ); - default: - return renderGroupNodeEmptyCell(); - } - }; - return ( 0}> @@ -485,22 +195,8 @@ export function WorkloadPanel(props: WorkloadPanelProps) { > td>div]:flex-nowrap' diff --git a/frontend-modern/src/components/Workloads/WorkloadsSurface.tsx b/frontend-modern/src/components/Workloads/WorkloadsSurface.tsx index 0f413620b6..fe11021d39 100644 --- a/frontend-modern/src/components/Workloads/WorkloadsSurface.tsx +++ b/frontend-modern/src/components/Workloads/WorkloadsSurface.tsx @@ -192,7 +192,6 @@ export function WorkloadsSurface(props: WorkloadsSurfaceComponentProps) { activeAlerts={state.activeAlerts} alertsEnabled={state.alertsEnabled} bottomSpacerHeight={state.bottomSpacerHeight} - compactGroupHeaders={state.compactGroupHeaders} getGroupLabel={state.getGroupLabel} getNodeTemperatureThresholds={state.getNodeTemperatureThresholds} groupedGuests={state.groupedGuests} diff --git a/frontend-modern/src/components/Workloads/WorkloadsTable.tsx b/frontend-modern/src/components/Workloads/WorkloadsTable.tsx index 2f25d5598b..5193c87d1c 100644 --- a/frontend-modern/src/components/Workloads/WorkloadsTable.tsx +++ b/frontend-modern/src/components/Workloads/WorkloadsTable.tsx @@ -17,7 +17,6 @@ type WorkloadsTableProps = Pick< | 'activeAlerts' | 'alertsEnabled' | 'bottomSpacerHeight' - | 'compactGroupHeaders' | 'getGroupLabel' | 'getNodeTemperatureThresholds' | 'groupedGuests' @@ -133,7 +132,6 @@ export function WorkloadsTable(props: WorkloadsTableProps) { activeAlerts={props.activeAlerts} alertsEnabled={props.alertsEnabled} bottomSpacerHeight={props.bottomSpacerHeight} - compactGroupHeaders={props.compactGroupHeaders} getGroupLabel={props.getGroupLabel} getNodeTemperatureThresholds={props.getNodeTemperatureThresholds} groupedGuests={props.groupedGuests} @@ -150,7 +148,6 @@ export function WorkloadsTable(props: WorkloadsTableProps) { focusedSummaryWorkloadGroupScope={props.focusedSummaryWorkloadGroupScope} focusedSummaryWorkloadGroupId={props.focusedSummaryWorkloadGroupId} hoveredSummaryWorkloadGroupScope={props.hoveredSummaryWorkloadGroupScope} - isMobile={props.isMobile} nestedWorkloadContextByGuestId={props.nestedWorkloadContextByGuestId} nodeByInstance={props.nodeByInstance} search={props.search} @@ -171,7 +168,6 @@ export function WorkloadsTable(props: WorkloadsTableProps) { workloadMetricHistory={props.workloadMetricHistory} workloadTableLayoutMode={props.workloadTableLayoutMode} workloadTableVisibleColumnIds={props.workloadTableVisibleColumnIds} - workloadTableVisibleColumns={props.workloadTableVisibleColumns} workloadColumnWidths={props.workloadColumnWidths} /> diff --git a/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx b/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx index d2aa75fff9..7a272f8b5c 100644 --- a/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx @@ -454,32 +454,6 @@ describe('Workloads platform-page embed contract', () => { expect(nodesTableSource).toContain('isSparklineMode'); }); - it('exposes compactGroupHeaders so platform pages can strip duplicate host stats from group rows', async () => { - const stateSource = (await import('../useWorkloadsState.ts?raw')).default; - expect(stateSource).toContain('compactGroupHeaders?: boolean;'); - expect(stateSource).toContain('compactGroupHeaders: () => props.compactGroupHeaders === true,'); - - const tableSource = (await import('../WorkloadsTable.tsx?raw')).default; - expect(tableSource).toContain(`| 'compactGroupHeaders'`); - expect(tableSource).toContain('compactGroupHeaders={props.compactGroupHeaders}'); - - const panelSource = (await import('../WorkloadPanel.tsx?raw')).default; - expect(panelSource).toContain(`| 'compactGroupHeaders'`); - // When the flag is set the panel falls back to NodeGroupHeader's - // colspan layout: no per-column metric cells, no inline node facts. - expect(panelSource).toContain( - 'props.compactGroupHeaders() ? undefined : props.workloadTableVisibleColumns()', - ); - expect(panelSource).toContain( - 'props.compactGroupHeaders() ? undefined : renderGroupNodeColumnCell', - ); - expect(panelSource).toContain('!props.compactGroupHeaders() &&'); - - const proxmoxSource = (await import('../../../features/proxmox/ProxmoxPageSurface.tsx?raw')) - .default; - expect(proxmoxSource).toContain('compactGroupHeaders'); - }); - it('lets platform pages move grouped host drawers to a dedicated host table owner', async () => { const stateSource = (await import('../useWorkloadsState.ts?raw')).default; expect(stateSource).toContain("groupNodeDrawerMode?: 'inline' | 'disabled';"); diff --git a/frontend-modern/src/components/Workloads/useWorkloadsState.ts b/frontend-modern/src/components/Workloads/useWorkloadsState.ts index e041d9d20c..195099789b 100644 --- a/frontend-modern/src/components/Workloads/useWorkloadsState.ts +++ b/frontend-modern/src/components/Workloads/useWorkloadsState.ts @@ -105,12 +105,6 @@ export interface WorkloadsSurfaceProps { }>; resourceSnapshotRefetch?: () => Promise; statusModeStorageScope?: string; - // Platform pages that render their own hosts table above the embedded - // workloads surface (e.g. Proxmox overview) own the per-host CPU / Memory - // / Disk / Temperature / uptime / version stats. Setting - // `compactGroupHeaders` strips those stats from the NodeGroupHeader rows - // in grouped mode so the section dividers don't duplicate the info. - compactGroupHeaders?: boolean; // Default Workloads behavior owns grouped host row drawers inline. Platform // pages with a dedicated host table can disable that drawer so host details // open from the host-owned table instead of the embedded guest table. @@ -728,7 +722,6 @@ export function useWorkloadsState(props: WorkloadsSurfaceProps) { workloadsNoInventoryState, ws, groupingMode, - compactGroupHeaders: () => props.compactGroupHeaders === true, groupNodeDrawerMode: () => props.groupNodeDrawerMode ?? 'inline', groupLabelBadges, } as const; diff --git a/frontend-modern/src/components/shared/NodeGroupHeader.tsx b/frontend-modern/src/components/shared/NodeGroupHeader.tsx index 0c3b9feb69..2dfa8db0fc 100644 --- a/frontend-modern/src/components/shared/NodeGroupHeader.tsx +++ b/frontend-modern/src/components/shared/NodeGroupHeader.tsx @@ -1,12 +1,8 @@ -import { Component, For, Show, type JSX } from 'solid-js'; +import { Component, Show, type JSX } from 'solid-js'; import type { Node } from '@/types/api'; -import type { MetricDisplayThresholds } from '@/utils/metricThresholds'; import { getNodeDisplayName, getNodeExternalUrl, hasAlternateDisplayName } from '@/utils/nodes'; import { StatusDot } from '@/components/shared/StatusDot'; import { getNodeStatusIndicator } from '@/utils/status'; -import { formatUptime } from '@/utils/format'; -import { formatTemperature, getCpuTemperature, getTemperatureTextClass } from '@/utils/temperature'; -import { formatProxmoxVersion } from '@/utils/proxmoxVersion'; import { GROUPED_TABLE_ROW_BADGE_CLASS, getGroupedTableRowCellClass, @@ -17,13 +13,8 @@ import { ResourceNameWithWebInterfaceLink } from './WebInterfaceLink'; interface NodeGroupHeaderProps { node: Node; colspan?: number; - columns?: Array<{ id: string }>; - columnCellClass?: (columnId: string, isNameColumn: boolean) => string; leadingAction?: JSX.Element; - renderColumnCell?: (columnId: string, node: Node) => JSX.Element; renderAs?: 'tr' | 'div'; - showFactsInName?: boolean; - temperatureThresholds?: MetricDisplayThresholds | null; trClass?: string; trProps?: JSX.HTMLAttributes & Partial>; @@ -35,12 +26,6 @@ export const NodeGroupHeader: Component = (props) => { const nodeUrl = () => getNodeExternalUrl(props.node); const displayName = () => getNodeDisplayName(props.node); const showActualName = () => hasAlternateDisplayName(props.node); - const pveVersion = () => formatProxmoxVersion(props.node.pveVersion); - const cpuTemperature = () => getCpuTemperature(props.node.temperature); - const hasNodeFacts = () => - Boolean(pveVersion()) || - cpuTemperature() !== null || - (typeof props.node.uptime === 'number' && props.node.uptime > 0); const InnerContent = () => (
= (props) => { Agent - - - -
); @@ -118,38 +84,9 @@ export const NodeGroupHeader: Component = (props) => { } > - 0 && Boolean(props.renderColumnCell)} - fallback={ - - - - } - > - - {(column) => { - const isNameColumn = () => column.id === 'name'; - return ( - - - - - - ); - }} - - + + + ); diff --git a/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts b/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts index 679ab96c40..7697db0332 100644 --- a/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts +++ b/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts @@ -216,6 +216,7 @@ import proxmoxCoverageTableSource from '@/features/proxmox/ProxmoxCoverageTable. import proxmoxMailGatewayTableSource from '@/features/proxmox/ProxmoxMailGatewayTable.tsx?raw'; import proxmoxNodesTableSource from '@/features/proxmox/ProxmoxNodesTable.tsx?raw'; import proxmoxHostTableModelSource from '@/features/proxmox/proxmoxHostTableModel.ts?raw'; +import proxmoxPageModelSource from '@/features/proxmox/proxmoxPageModel.ts?raw'; import proxmoxRecoverableTableSource from '@/features/proxmox/ProxmoxRecoverableTable.tsx?raw'; import proxmoxReplicationTableSource from '@/features/proxmox/ProxmoxReplicationTable.tsx?raw'; import vsphereHostsTableSource from '@/features/vmware/VsphereHostsTable.tsx?raw'; @@ -1406,9 +1407,9 @@ describe('shared primitive guardrails', () => { }); it('routes Proxmox node version presentation through the shared formatter', () => { - expect(nodeGroupHeaderSource).toContain("from '@/utils/proxmoxVersion'"); - expect(nodeGroupHeaderSource).toContain('formatProxmoxVersion(props.node.pveVersion)'); - expect(nodeGroupHeaderSource).not.toContain('pve-manager\\/'); + expect(proxmoxPageModelSource).toContain("from '@/utils/proxmoxVersion'"); + expect(proxmoxPageModelSource).toContain('formatProxmoxVersion(resource.proxmox?.pveVersion)'); + expect(proxmoxPageModelSource).not.toContain('pve-manager\\/'); expect(proxmoxVersionSource).toContain('formatProxmoxVersion'); expect(proxmoxVersionSource).toContain('pve-manager\\/'); diff --git a/frontend-modern/src/components/shared/__tests__/NodeGroupHeader.test.tsx b/frontend-modern/src/components/shared/__tests__/NodeGroupHeader.test.tsx index 88f23a0606..f31f9a4ea3 100644 --- a/frontend-modern/src/components/shared/__tests__/NodeGroupHeader.test.tsx +++ b/frontend-modern/src/components/shared/__tests__/NodeGroupHeader.test.tsx @@ -1,9 +1,8 @@ import { render, screen } from '@solidjs/testing-library'; -import { beforeEach, describe, expect, it } from 'vitest'; +import { describe, expect, it } from 'vitest'; import { NodeGroupHeader } from '@/components/shared/NodeGroupHeader'; import type { Node } from '@/types/api'; -import { temperatureStore } from '@/utils/temperature'; const makeNode = (overrides: Partial = {}): Node => ({ id: 'cluster-a-pve1', @@ -34,18 +33,11 @@ const makeNode = (overrides: Partial = {}): Node => ({ }); describe('NodeGroupHeader', () => { - beforeEach(() => { - temperatureStore.setUnit('celsius'); - }); - - it('renders compact infrastructure facts in the grouped workload header', () => { + it('renders node identity in the grouped workload header', () => { render(() => ); expect(screen.getByText('pve1')).toBeInTheDocument(); expect(screen.getByText('homelab')).toBeInTheDocument(); - expect(screen.getByText('PVE 9.1.9')).toBeInTheDocument(); - expect(screen.getByText('62°C')).toBeInTheDocument(); - expect(screen.getByText('2h')).toBeInTheDocument(); const link = screen.getByRole('link', { name: 'Open web interface for pve1' }); expect(link).toHaveAttribute('href', 'https://pve1:8006'); @@ -53,20 +45,21 @@ describe('NodeGroupHeader', () => { expect(link).toHaveAttribute('rel', 'noopener noreferrer'); }); - it('colors grouped header temperatures from configured thresholds', () => { - render(() => ( - + it('spans the table in one cell and leaves host stats to the platform host table', () => { + const { container } = render(() => ( + + + + +
)); - expect(screen.getByText('76°C')).toHaveClass('text-green-600'); + const cells = container.querySelectorAll('tr > td'); + expect(cells).toHaveLength(1); + expect(cells[0]).toHaveAttribute('colspan', '9'); + expect(cells[0]).toHaveTextContent('pve1'); + expect(screen.queryByText('PVE 9.1.9')).not.toBeInTheDocument(); + expect(screen.queryByText(/°[CF]/)).not.toBeInTheDocument(); + expect(screen.queryByText('2h')).not.toBeInTheDocument(); }); }); diff --git a/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx b/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx index 0cf4d4c4ba..e34418e333 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx @@ -495,7 +495,6 @@ function ProxmoxOverview(props: ProxmoxOverviewProps) { // surface, so label it with the platform vocabulary instead of 'Info'. columnLabelOverrides: { info: 'ID' }, statusModeStorageScope: PROXMOX_WORKLOAD_STATUS_STORAGE_SCOPE, - compactGroupHeaders: true, groupNodeDrawerMode: 'disabled', metricDisplayMode: props.metricDisplayMode, onMetricDisplayModeChange: props.setMetricDisplayMode, @@ -638,7 +637,6 @@ function ProxmoxOverview(props: ProxmoxOverviewProps) { forcedPlatform={PROXMOX_PLATFORM_FILTER} excludedWorkloadTypes={PROXMOX_WORKLOAD_EXCLUDED_TYPES} showNestedExcludedWorkloads - compactGroupHeaders groupNodeDrawerMode="disabled" suppressFilterToolbar emptyStateTitle="No Proxmox workloads" diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx b/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx index 0979b3a487..198c6b273c 100644 --- a/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx +++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx @@ -214,6 +214,16 @@ describe('ProxmoxPageSurface contract', () => { ); }); + it('leaves host details to the nodes table instead of the embedded guest table', () => { + setResources([makeResource({ id: 'vm-1', type: 'vm' })]); + + renderSurface(); + + expect(mockWorkloadsOptions).toHaveBeenCalledWith( + expect.objectContaining({ groupNodeDrawerMode: 'disabled' }), + ); + }); + afterEach(() => { cleanup(); vi.clearAllMocks(); diff --git a/frontend-modern/src/features/vmware/VmwarePageSurface.tsx b/frontend-modern/src/features/vmware/VmwarePageSurface.tsx index 0e8d015897..1a34ea8be5 100644 --- a/frontend-modern/src/features/vmware/VmwarePageSurface.tsx +++ b/frontend-modern/src/features/vmware/VmwarePageSurface.tsx @@ -474,7 +474,6 @@ function VmwareOverview(props: VmwareOverviewProps) { statusModeStorageScope: VMWARE_WORKLOAD_STATUS_STORAGE_SCOPE, columnVisibilityStorageScope: VMWARE_WORKLOAD_COLUMN_VISIBILITY_SCOPE, additionalDefaultHiddenColumnIds: [...VMWARE_WORKLOAD_DEFAULT_HIDDEN_COLUMN_IDS], - compactGroupHeaders: true, inventorySourcesQuery: props.inventorySourcesQuery, groupNodeDrawerMode: 'disabled', metricDisplayMode: props.metricDisplayMode, @@ -545,7 +544,6 @@ function VmwareOverview(props: VmwareOverviewProps) { useWorkloads forcedPlatform={VMWARE_PLATFORM_FILTER} forcedViewMode="vm" - compactGroupHeaders groupNodeDrawerMode="disabled" suppressFilterToolbar emptyStateTitle="No vSphere VMs" diff --git a/frontend-modern/src/utils/__tests__/temperatureDisplayGuardrails.test.ts b/frontend-modern/src/utils/__tests__/temperatureDisplayGuardrails.test.ts index 3f8e8898e8..37c1175902 100644 --- a/frontend-modern/src/utils/__tests__/temperatureDisplayGuardrails.test.ts +++ b/frontend-modern/src/utils/__tests__/temperatureDisplayGuardrails.test.ts @@ -2,15 +2,15 @@ import { describe, expect, it } from 'vitest'; import nodeDrawerOverviewSource from '@/components/Workloads/NodeDrawerOverview.tsx?raw'; import workloadPanelSource from '@/components/Workloads/WorkloadPanel.tsx?raw'; -import nodeGroupHeaderSource from '@/components/shared/NodeGroupHeader.tsx?raw'; import temperatureGaugeSource from '@/components/shared/TemperatureGauge.tsx?raw'; import dockerHostDrawerOverviewSource from '@/features/docker/DockerHostDrawerOverview.tsx?raw'; import agentsMachinesTableSource from '@/features/standalone/AgentsMachinesTable.tsx?raw'; describe('temperature display guardrails', () => { it('keeps colored temperature displays threshold-aware', () => { - expect(nodeGroupHeaderSource).toContain('temperatureThresholds'); - expect(workloadPanelSource).toContain('getNodeTemperatureThresholds(node)'); + expect(workloadPanelSource).toContain( + 'temperatureThresholds={props.getNodeTemperatureThresholds(node()!)}', + ); expect(nodeDrawerOverviewSource).toContain('getTemperatureTextClass(value, thresholds)'); expect(dockerHostDrawerOverviewSource).toContain('temperatureThresholds()'); expect(agentsMachinesTableSource).toContain('getAgentMachineTemperatureMetric'); @@ -19,7 +19,6 @@ describe('temperature display guardrails', () => { }); it('does not reintroduce the previous local temperature severity calls', () => { - expect(nodeGroupHeaderSource).not.toContain('getTemperatureTextClass(cpuTemperature())'); expect(workloadPanelSource).not.toContain('getTemperatureTextClass(temperature)}'); expect(nodeDrawerOverviewSource).not.toContain('getTemperatureTextClass(value)'); expect(dockerHostDrawerOverviewSource).not.toContain('getTemperatureTextClass(temperature)'); From 94b7870855120b2c62fce19102903601b9fc4bfc Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:47:48 +0100 Subject: [PATCH 26/53] Call a disk that is only hot Running Hot, not Replace Now The Physical Disks Health verdict mapped any critical disk risk to "Replace Now". Disk risk also grades temperature (temperature_high at >=70C critical, >=60C warning), so a SMART-passed disk with good life running at 72C told the user to buy a new disk when the fix is airflow. Alerts already keep heat apart: the SMART health and wearout alerts skip temperature_high and raise disk heat as a Temperature metric. The verdict now splits risk reasons into heat and everything else. Heat alone reads "Running Hot" (phone "Hot"), red at the critical tier and amber at warning, with the temperature reason still as its title and drawer summary. FAILED health, any other critical reason, and a critical level no reason explains keep "Replace Now". The more severe class names the verdict and failure evidence wins a tie, so merged reasons that list heat first no longer hide pending sectors. Sorting and the health filter follow the same verdict, so hot disks stay in "Needs attention" at their temperature tier. A reason's code and severity count even when it has no display text. A critical level that no reason explains shows the generic summary rather than a weaker reason's text. Thresholds are unchanged. The tier still comes from the server's flat 60/70C rule, not the per-type alert thresholds that colour the Temp cell. Reconciling those is separate work. --- .../subsystems/frontend-primitives.md | 5 + .../internal/subsystems/storage-recovery.md | 20 ++ .../ad11a59541280777.json | 44 +++ .../Storage/__tests__/DiskList.test.tsx | 105 +++++++ .../__tests__/diskPresentation.test.ts | 290 ++++++++++++++++++ .../storageBackups/diskPresentation.ts | 178 +++++++++-- 6 files changed, 610 insertions(+), 32 deletions(-) create mode 100644 frontend-modern/browser-verification/ad11a59541280777.json diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index b741322660..6fb37dbe06 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -412,6 +412,11 @@ disk header and tooltips. Structured ZFS scan activity supplies a compact badge only during reported rebuild activity; its complete provider summary remains available. Stable disk History catalog/organisation/access ownership is unchanged. +The physical-disk verdict words and their phone forms are `Needs Attention` +(`Attention`), `Running Hot` (`Hot`) and `Replace Now` (`Replace`), all from +`getPhysicalDiskHealthCompactLabel` in `diskPresentation.ts`. A hot disk keeps +the same label at either tier; its tone carries the severity. + Controller expansions carry the supplied kind-specific targets, absolute times, duration and cluster/namespace when the phone row omits those columns. Proxmox coverage distinguishes independent backups from guest-local snapshots; By date diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index cd2ec5bf9e..85d3b643e8 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -6727,6 +6727,26 @@ disk/pool fixture `frontend-modern/browser-tests/history-touch.cjs` and mounted `HistoryChart.test.tsx` regressions verify presentation/input behaviour only, not live appliance collection or recovery success. +### Physical-disk heat is not a replacement verdict + +The Physical Disks Health verdict names the action it asks for. Risk whose only +reason is `temperature_high` reads `Running Hot` (compact `Hot`), red at the +critical tier and amber at warning, with the temperature reason as its summary. +`Replace Now` stays reserved for FAILED health, any other critical reason, and +a critical level that no listed reason explains. `Needs Attention` keeps the +remaining warning evidence. When heat and other evidence both apply, the more +severe class names the verdict and failure evidence wins a tie, whatever order +merged reasons arrive in. Sorting follows the verdict (replace, critical heat, +attention, warning heat), and the health filter places a hot disk at its +temperature tier, so `Needs attention` still lists it. This matches alerting, +which raises disk heat as a Temperature metric and never as disk health. +Thresholds are unchanged: the tier still comes from the server's disk risk +(flat 60/70C), not the per-type alert thresholds that colour the Temp column. +Reconciling those is separate work. `frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts` +and `frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx` cover +the split, merged reason order, the unexplained critical level, sorting and +filtering. + ### Pool-to-physical-disk ownership in Storage details `storagePoolDetailPresentation.ts` must scope inferred ZFS device and UnRAID diff --git a/frontend-modern/browser-verification/ad11a59541280777.json b/frontend-modern/browser-verification/ad11a59541280777.json new file mode 100644 index 0000000000..f0678d07dd --- /dev/null +++ b/frontend-modern/browser-verification/ad11a59541280777.json @@ -0,0 +1,44 @@ +{ + "version": 1, + "base_sha": "8259a74c9d0503df630b09c12a2c1be906fa3d78", + "verified_at": "2026-10-06T22:20:00Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/features/storageBackups/diskPresentation.ts" + ], + "content_sha256": { + "frontend-modern/src/features/storageBackups/diskPresentation.ts": "87469b0d62a31a1657a94a5f645041ad34987cc5af55ca238c6cbf49b0c068ee" + }, + "routes": [ + "/proxmox/storage?tab=disks", + "/proxmox/storage?tab=disks&status=attention", + "/proxmox/storage?tab=disks&status=critical", + "/proxmox/storage?tab=disks&status=warning" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack (8 nodes) whose generator was overlaid locally, never committed, to pin readings from a control file on pve3: Crucial MX500 2TB /dev/nvme2 (SMART PASSED) at 72C, WD Blue SN570 /dev/sd1 at 63C, and Crucial MX500 2TB /dev/nvme0 at 72C with 2 pending sectors. The API returned risk {critical: [temperature_high]}, {warning: [temperature_high]} and {critical: [pending_sectors, temperature_high]} respectively; the backend binary was rebuilt from base 8259a74c9d (includes #2611)", + "Same-session control with the pre-change diskPresentation.ts (3ac5664d95, whose verdict code is unchanged on base 8259a74c9d) swapped in under the running Vite: the 72C temperature-only disk read 'Replace Now' (phone 'Replace') titled 'Disk temperature is 72C' in the row and in its drawer, and the 63C disk read 'Needs Attention' and sorted above the 7%-life disk", + "After the change at 1440 (full layout): rows sort Replace Now (pending sectors, title 'Pending sectors detected (2)', rgb(185,28,28)), Running Hot (title 'Disk temperature is 72C', rgb(185,28,28)), Needs Attention (970 EVO, 'SSD life remaining is 7%', rgb(180,83,9)), Running Hot (title 'Disk temperature is 63C', rgb(180,83,9)), then Healthy/Unknown disks unchanged", + "After the change at 390 (narrow layout): the same rows show the compact words Replace, Hot, Attention, Hot with the same tones and titles", + "Disk drawer header for the 72C temperature-only disk reads 'Running Hot Disk temperature is 72C' at both widths", + "Health filter from the URL at both widths: status=attention lists the two Replace/Hot critical rows plus Attention and the amber Hot row; status=critical lists only Replace Now and the red Running Hot; status=warning lists Needs Attention and the amber Running Hot", + "Known and unchanged: the 63C NVMe reads Running Hot in amber beside a green Temp cell, because the risk tier is the server's flat 60C while the Temp cell uses the NVMe alert threshold; the same disk read Needs Attention before this change. Threshold reconciliation is separate work", + "Amended content (empty-summary reasons keep their code and severity, an unexplained critical level no longer borrows a weaker reason's text): full pass repeated at 1440 and 390 with identical rows, compact words, titles, colours, drawer header and status=attention/critical/warning results" + ], + "interactions": [ + "Loaded the Physical Disks tab unfiltered at 1440x900 and 390x844 and read every row's visible health word, its title, computed colour and the Temp cell", + "Opened the inline drawer for the 72C temperature-only disk by clicking its row at both widths and read the health header", + "Loaded status=attention, status=critical and status=warning at both widths and read the listed rows", + "Swapped the pre-change diskPresentation.ts in under the running Vite, repeated the reads, restored the change and confirmed it byte-identical with cmp; after rebasing onto 8259a74c9d repeated the full after-change pass with a rebuilt backend" + ] +} diff --git a/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx b/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx index 73b9cd330d..887fe416aa 100644 --- a/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx +++ b/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx @@ -343,6 +343,111 @@ describe('DiskList', () => { ); }); + it('says a disk that is only hot is running hot, not due for replacement', () => { + const view = renderDiskList({ + disks: [ + buildDisk('sda', 'pve3', { + model: 'Crucial MX500 2TB', + diskType: 'ssd', + wearout: 95, + temperature: 72, + risk: { + level: 'critical', + reasons: [ + { + code: 'temperature_high', + severity: 'critical', + summary: 'Disk temperature is 72C', + }, + ], + }, + }), + buildDisk('sdb', 'pve3', { + temperature: 72, + smart: { pendingSectors: 2 }, + risk: { + level: 'critical', + reasons: [ + { + code: 'temperature_high', + severity: 'critical', + summary: 'Disk temperature is 72C', + }, + { + code: 'pending_sectors', + severity: 'critical', + summary: 'Pending sectors detected (2)', + }, + ], + }, + }), + ], + nodes: [buildNode('node-pve3', 'pve3')], + selectedNode: null, + searchTerm: '', + }); + + const rows = Array.from(view.container.querySelectorAll('[data-row-id]')); + // The disk that needs replacing sorts above the one that needs cooling. + expect(rows.map((row) => row.getAttribute('data-row-id'))).toEqual(['sdb', 'sda']); + const hotRow = within(rows[1] as HTMLElement); + expect(hotRow.queryByText('Replace Now')).not.toBeInTheDocument(); + expect(hotRow.getByText('Running Hot')).toHaveClass( + 'platform-table-label-full', + 'text-red-700', + ); + expect(hotRow.getByText('Running Hot')).toHaveAttribute('title', 'Disk temperature is 72C'); + expect(hotRow.getByText('Hot')).toHaveClass('platform-table-label-compact', 'text-red-700'); + const failingRow = within(rows[0] as HTMLElement); + expect(failingRow.getByText('Replace Now')).toHaveAttribute( + 'title', + 'Pending sectors detected (2)', + ); + }); + + it('follows a disk from warm to critically hot and back to healthy', async () => { + const heat = (severity: 'warning' | 'critical', celsius: number) => ({ + level: severity, + reasons: [{ code: 'temperature_high', severity, summary: `Disk temperature is ${celsius}C` }], + }); + const [disks, setDisks] = createStore({ + items: [buildDisk('sda', 'pve3', { temperature: 63, risk: heat('warning', 63) })], + }); + const [healthFilter, setHealthFilter] = createSignal<'all' | 'critical'>('all'); + const view = render(() => ( + {}} + /> + )); + const row = () => view.container.querySelector('[data-row-id="sda"]') as HTMLElement | null; + expect(within(row()!).getByText('Running Hot')).toHaveClass('text-amber-700'); + expect(within(row()!).getByText('Hot')).toHaveClass('text-amber-700'); + setHealthFilter('critical'); + await waitFor(() => expect(row()).toBeNull()); + + setDisks( + 'items', + reconcile([buildDisk('sda', 'pve3', { temperature: 72, risk: heat('critical', 72) })]), + ); + await waitFor(() => expect(row()).not.toBeNull()); + expect(within(row()!).getByText('Running Hot')).toHaveClass('text-red-700'); + expect(within(row()!).getByText('Running Hot')).toHaveAttribute( + 'title', + 'Disk temperature is 72C', + ); + + setHealthFilter('all'); + setDisks('items', reconcile([buildDisk('sda', 'pve3', { temperature: 41 })])); + await waitFor(() => expect(within(row()!).getByText('Healthy')).toBeInTheDocument()); + expect(within(row()!).queryByText('Running Hot')).not.toBeInTheDocument(); + }); + it('renders SSD life and falls back to the Proxmox usage string for Belongs', () => { renderDiskList({ disks: [ diff --git a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts index a18d7ccdfe..04ac75dc46 100644 --- a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts +++ b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts @@ -268,10 +268,300 @@ describe('diskPresentation', () => { it('shortens only the health words that cannot fit a phone health column', () => { expect(getPhysicalDiskHealthCompactLabel('Needs Attention')).toBe('Attention'); expect(getPhysicalDiskHealthCompactLabel('Replace Now')).toBe('Replace'); + expect(getPhysicalDiskHealthCompactLabel('Running Hot')).toBe('Hot'); expect(getPhysicalDiskHealthCompactLabel('Healthy')).toBe('Healthy'); expect(getPhysicalDiskHealthCompactLabel('Unknown')).toBe('Unknown'); }); + describe('heat versus replacement evidence', () => { + const buildRiskDisk = ( + id: string, + risk: { level: string; reasons: { code: string; severity: string; summary: string }[] }, + overrides: Record = {}, + ): Resource => + ({ + id, + name: id, + type: 'physical_disk', + status: 'warning', + physicalDisk: { + devPath: `/dev/${id}`, + model: 'Crucial MX500 2TB', + diskType: 'sata', + health: 'PASSED', + wearout: 95, + temperature: 72, + risk, + ...overrides, + }, + identity: { hostname: 'pve3' }, + canonicalIdentity: { hostname: 'pve3' }, + platformType: 'proxmox-pve', + }) as unknown as Resource; + const hot = (severity: string, celsius = 72) => ({ + code: 'temperature_high', + severity, + summary: `Disk temperature is ${celsius}C`, + }); + + it('calls a disk that is only hot Running Hot at the tier of its reading', () => { + const criticalDisk = buildRiskDisk('sda', { level: 'critical', reasons: [hot('critical')] }); + const criticalData = extractPhysicalDiskPresentationData(criticalDisk); + expect(criticalData.riskReasonDetails).toEqual([hot('critical')]); + expect(getPhysicalDiskHealthStatus(criticalData)).toEqual({ + label: 'Running Hot', + summary: 'Disk temperature is 72C', + tone: 'text-red-700 dark:text-red-300', + }); + expect(getPhysicalDiskNormalizedHealth(criticalDisk, criticalData)).toBe('critical'); + + const warmDisk = buildRiskDisk( + 'sdb', + { level: 'warning', reasons: [hot('warning', 63)] }, + { temperature: 63 }, + ); + const warmData = extractPhysicalDiskPresentationData(warmDisk); + expect(getPhysicalDiskHealthStatus(warmData)).toEqual({ + label: 'Running Hot', + summary: 'Disk temperature is 63C', + tone: 'text-amber-700 dark:text-amber-300', + }); + expect(getPhysicalDiskNormalizedHealth(warmDisk, warmData)).toBe('warning'); + }); + + it('keeps Replace Now for failure evidence even when heat is listed first', () => { + // Merged risks keep insertion order, so heat can precede the reason + // that actually calls for a new disk. + const disk = buildRiskDisk('sda', { + level: 'critical', + reasons: [ + hot('critical'), + { + code: 'pending_sectors', + severity: 'critical', + summary: 'Pending sectors detected (2)', + }, + ], + }); + const data = extractPhysicalDiskPresentationData(disk); + expect(getPhysicalDiskHealthStatus(data)).toEqual({ + label: 'Replace Now', + summary: 'Pending sectors detected (2)', + tone: 'text-red-700 dark:text-red-300', + }); + + const failed = extractPhysicalDiskPresentationData( + buildRiskDisk( + 'sdb', + { + level: 'critical', + reasons: [ + hot('critical'), + { + code: 'health_status', + severity: 'critical', + summary: 'Disk reports health status FAILED', + }, + ], + }, + { health: 'FAILED' }, + ), + ); + expect(getPhysicalDiskHealthStatus(failed).label).toBe('Replace Now'); + expect(getPhysicalDiskHealthStatus(failed).summary).toBe('Disk reports health status FAILED'); + }); + + it('lets the more severe of heat and wear evidence name the verdict', () => { + const hotAndWorn = extractPhysicalDiskPresentationData( + buildRiskDisk('sda', { + level: 'critical', + reasons: [ + hot('critical'), + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + ], + }), + ); + expect(getPhysicalDiskHealthStatus(hotAndWorn).label).toBe('Running Hot'); + expect(getPhysicalDiskHealthStatus(hotAndWorn).summary).toBe('Disk temperature is 72C'); + + const warmAndWorn = extractPhysicalDiskPresentationData( + buildRiskDisk( + 'sdb', + { + level: 'warning', + reasons: [ + hot('warning', 63), + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + ], + }, + { temperature: 63 }, + ), + ); + expect(getPhysicalDiskHealthStatus(warmAndWorn)).toEqual({ + label: 'Needs Attention', + summary: 'SSD life remaining is 8%', + tone: 'text-amber-700 dark:text-amber-300', + }); + }); + + it('keeps a critical level that no listed reason explains as Replace Now', () => { + const data = extractPhysicalDiskPresentationData( + buildRiskDisk('sda', { level: 'critical', reasons: [hot('warning', 63)] }), + ); + expect(getPhysicalDiskHealthStatus(data)).toEqual({ + label: 'Replace Now', + summary: 'Disk health has degraded to a critical state.', + tone: 'text-red-700 dark:text-red-300', + }); + }); + + it('decides the verdict from codes and severities even without display text', () => { + const pendingWithoutText = extractPhysicalDiskPresentationData( + buildRiskDisk('sda', { + level: 'critical', + reasons: [ + hot('critical'), + { code: 'pending_sectors', severity: 'critical', summary: '' }, + ], + }), + ); + expect(pendingWithoutText.riskReasons).toEqual(['Disk temperature is 72C']); + expect(getPhysicalDiskHealthStatus(pendingWithoutText)).toEqual({ + label: 'Replace Now', + summary: 'Disk health has degraded to a critical state.', + tone: 'text-red-700 dark:text-red-300', + }); + + const heatWithoutText = extractPhysicalDiskPresentationData( + buildRiskDisk('sdb', { + level: 'critical', + reasons: [{ code: 'temperature_high', severity: 'critical', summary: '' }], + }), + ); + expect(getPhysicalDiskHealthStatus(heatWithoutText).label).toBe('Running Hot'); + }); + + it('does not let a weaker reason stand in for an unexplained critical level', () => { + const data = extractPhysicalDiskPresentationData( + buildRiskDisk('sda', { + level: 'critical', + reasons: [ + { code: 'crc_errors', severity: 'monitor', summary: 'UDMA CRC errors detected (2)' }, + hot('warning', 63), + ], + }), + ); + expect(getPhysicalDiskHealthStatus(data)).toEqual({ + label: 'Replace Now', + summary: 'Disk health has degraded to a critical state.', + tone: 'text-red-700 dark:text-red-300', + }); + }); + + it('reads severities case-insensitively and in any order within a class', () => { + const shouting = extractPhysicalDiskPresentationData( + buildRiskDisk('sda', { + level: 'CRITICAL', + reasons: [{ ...hot('critical'), severity: ' Critical ' }], + }), + ); + expect(getPhysicalDiskHealthStatus(shouting).label).toBe('Running Hot'); + expect(getPhysicalDiskHealthStatus(shouting).tone).toBe('text-red-700 dark:text-red-300'); + + const unsorted = extractPhysicalDiskPresentationData( + buildRiskDisk('sdb', { + level: 'critical', + reasons: [ + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + { + code: 'pending_sectors', + severity: 'critical', + summary: 'Pending sectors detected (2)', + }, + ], + }), + ); + expect(getPhysicalDiskHealthStatus(unsorted).summary).toBe('Pending sectors detected (2)'); + }); + + it('lets SMART counters and low life outrank warning heat', () => { + const warmAndLow = extractPhysicalDiskPresentationData( + buildRiskDisk( + 'sda', + { level: 'warning', reasons: [hot('warning', 63)] }, + { diskType: 'ssd', wearout: 8, temperature: 63 }, + ), + ); + expect(getPhysicalDiskHealthStatus(warmAndLow)).toEqual({ + label: 'Needs Attention', + summary: 'SSD life is running low.', + tone: 'text-amber-700 dark:text-amber-300', + }); + + const warmAndReallocating = extractPhysicalDiskPresentationData( + buildRiskDisk( + 'sdb', + { level: 'warning', reasons: [hot('warning', 63)] }, + { temperature: 63, smart: { reallocatedSectors: 3 } }, + ), + ); + expect(getPhysicalDiskHealthStatus(warmAndReallocating).label).toBe('Needs Attention'); + }); + + it('sorts a FAILED disk with no risk payload ahead of a critically hot one', () => { + const failedDisk = buildRiskDisk( + 'sdb', + { level: 'healthy', reasons: [] }, + { risk: undefined, health: 'FAILED', temperature: 40 }, + ); + const hotDisk = buildRiskDisk('sda', { level: 'critical', reasons: [hot('critical')] }); + const failedData = extractPhysicalDiskPresentationData(failedDisk); + const hotData = extractPhysicalDiskPresentationData(hotDisk); + expect(getPhysicalDiskHealthStatus(failedData).label).toBe('Replace Now'); + expect( + comparePhysicalDiskPresentation(failedDisk, failedData, hotDisk, hotData), + ).toBeLessThan(0); + }); + + it('sorts and filters hot disks by the action their verdict asks for', () => { + const warm = buildRiskDisk('sdc', { level: 'warning', reasons: [hot('warning', 63)] }); + const worn = buildRiskDisk('sdd', { + level: 'warning', + reasons: [ + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + ], + }); + const hotDisk = buildRiskDisk('sda', { level: 'critical', reasons: [hot('critical')] }); + const failing = buildRiskDisk('sdb', { + level: 'critical', + reasons: [ + { code: 'media_errors', severity: 'critical', summary: 'Media errors detected (4)' }, + ], + }); + const healthy = buildRiskDisk( + 'sde', + { level: 'healthy', reasons: [] }, + { risk: undefined, temperature: 38 }, + ); + const disks = [healthy, warm, worn, hotDisk, failing]; + const dataMap = buildPhysicalDiskPresentationDataMap(disks); + const sorted = (healthFilter: 'all' | 'attention' | 'critical' | 'warning') => + filterAndSortPhysicalDisks(disks, { + selectedNode: null, + healthFilter, + searchTerm: '', + getDiskData: (disk) => dataMap.get(disk.id)!, + matchesNode: () => true, + }).map((disk) => disk.id); + + // Device order alone would put each hot disk first. + expect(sorted('all')).toEqual(['sdb', 'sda', 'sdd', 'sdc', 'sde']); + expect(sorted('attention')).toEqual(['sdb', 'sda', 'sdd', 'sdc']); + expect(sorted('critical')).toEqual(['sdb', 'sda']); + expect(sorted('warning')).toEqual(['sdd', 'sdc']); + }); + }); + it('detects SMART warnings from counters', () => { expect( hasPhysicalDiskSmartWarning({ diff --git a/frontend-modern/src/features/storageBackups/diskPresentation.ts b/frontend-modern/src/features/storageBackups/diskPresentation.ts index 2499be5969..3248b29bf6 100644 --- a/frontend-modern/src/features/storageBackups/diskPresentation.ts +++ b/frontend-modern/src/features/storageBackups/diskPresentation.ts @@ -3,6 +3,7 @@ import type { PhysicalDiskCollectionStatus, PhysicalDiskFieldStatus, Resource, + ResourceStorageRiskReason, } from '@/types/resource'; import { getPlatformTableWeightedColumnWidthStyle } from '@/features/platformPage/sharedPlatformPage'; import { @@ -53,6 +54,8 @@ export interface PhysicalDiskPresentationData { health: string; riskLevel?: string; riskReasons: string[]; + /** The same reasons with their codes and severities, as the API sent them. */ + riskReasonDetails?: ResourceStorageRiskReason[]; wearout: number; storageRole?: string; storageGroup?: string; @@ -198,6 +201,7 @@ export const getPhysicalDiskCellPaddingClass = (layout: PhysicalDiskTableLayoutM export const getPhysicalDiskHealthCompactLabel = (label: string): string => { if (label === 'Needs Attention') return 'Attention'; if (label === 'Replace Now') return 'Replace'; + if (label === 'Running Hot') return 'Hot'; return label; }; @@ -402,14 +406,21 @@ export function extractPhysicalDiskPresentationData( ): PhysicalDiskPresentationData { const pd = resource.physicalDisk || ((resource.platformData as any)?.physicalDisk ?? {}); const diskNode = getPhysicalDiskNodeIdentity(resource); - const riskReasons = Array.isArray(pd.risk?.reasons) + // A reason's code and severity decide the verdict even when it carries no + // display text, so only the summary list drops empty summaries. + const riskReasonDetails: ResourceStorageRiskReason[] = Array.isArray(pd.risk?.reasons) ? pd.risk.reasons - .map((reason: { summary?: unknown }) => reason?.summary) - .filter( - (summary: unknown): summary is string => - typeof summary === 'string' && summary.length > 0, - ) + .filter((reason: unknown) => reason !== null && typeof reason === 'object') + .map((reason: { code?: unknown; severity?: unknown; summary?: unknown }) => ({ + code: typeof reason.code === 'string' ? reason.code : '', + severity: typeof reason.severity === 'string' ? reason.severity : '', + summary: typeof reason.summary === 'string' ? reason.summary : '', + })) + .filter((reason: ResourceStorageRiskReason) => reason.code || reason.summary) : []; + const riskReasons = riskReasonDetails + .map((reason) => reason.summary) + .filter((summary) => summary.length > 0); return { node: diskNode.node, @@ -438,6 +449,7 @@ export function extractPhysicalDiskPresentationData( collection: pd.collection, riskLevel: pd.risk?.level, riskReasons, + riskReasonDetails, smartAttributes: pd.smart ? { powerOnHours: pd.smart.powerOnHours, @@ -534,9 +546,24 @@ export function buildPhysicalDiskPresentationDataMap( return map; } +// Disks sort in the order their verdicts ask for action: replacement first, +// then a disk running critically hot, then other warnings, then a warm disk. +const getPhysicalDiskVerdictPriority = (disk: PhysicalDiskPresentationData): number => { + const verdict = getPhysicalDiskHealthVerdict(disk); + switch (verdict.kind) { + case 'replace': + return 400; + case 'hot': + return verdict.critical ? 300 : 100; + case 'attention': + return 200; + default: + return 0; + } +}; + const getPhysicalDiskPriority = (disk: PhysicalDiskPresentationData): number => - (disk.riskLevel === 'critical' ? 300 : disk.riskLevel === 'warning' ? 200 : 0) + - (hasPhysicalDiskSmartWarning(disk) ? 50 : 0); + getPhysicalDiskVerdictPriority(disk) + (hasPhysicalDiskSmartWarning(disk) ? 50 : 0); export function matchesPhysicalDiskSearch( resource: Resource, @@ -685,31 +712,116 @@ export function hasPhysicalDiskSmartWarning(disk: PhysicalDiskPresentationData): ); } +// Heat is the one disk risk whose fix is cooling, not a replacement disk. +// Alerts raise it as a temperature metric rather than as disk health, so the +// verdict names it apart from the evidence that does mean replacing the disk. +const PHYSICAL_DISK_TEMPERATURE_RISK_CODE = 'temperature_high'; + +const PHYSICAL_DISK_RISK_SEVERITY_RANK: Record = { + monitor: 1, + warning: 2, + critical: 3, +}; + +const getPhysicalDiskRiskSeverityRank = (severity: string | undefined): number => + PHYSICAL_DISK_RISK_SEVERITY_RANK[(severity || '').trim().toLowerCase()] ?? 0; + +interface PhysicalDiskRiskEvidence { + rank: number; + summary?: string; +} + +// Splits the disk's risk into heat and everything else, keeping the most +// severe reason of each. A risk level that none of its reasons explains, or +// one sent without reason codes, stays with the disk-health evidence. +function splitPhysicalDiskRisk(disk: PhysicalDiskPresentationData): { + health: PhysicalDiskRiskEvidence; + heat: PhysicalDiskRiskEvidence; +} { + const health: PhysicalDiskRiskEvidence = { rank: 0 }; + const heat: PhysicalDiskRiskEvidence = { rank: 0 }; + const details = + disk.riskReasonDetails ?? + disk.riskReasons.map((summary) => ({ code: '', severity: disk.riskLevel || '', summary })); + for (const reason of details) { + const evidence = reason.code === PHYSICAL_DISK_TEMPERATURE_RISK_CODE ? heat : health; + const rank = getPhysicalDiskRiskSeverityRank(reason.severity); + if (rank > evidence.rank) { + evidence.rank = rank; + evidence.summary = reason.summary || undefined; + } else if (rank === evidence.rank && !evidence.summary) { + evidence.summary = reason.summary || undefined; + } + } + const levelRank = getPhysicalDiskRiskSeverityRank(disk.riskLevel); + if (levelRank > Math.max(health.rank, heat.rank)) { + // A reason with a known, lower severity does not explain this level, so + // its text must not stand in as the verdict's reason. + if (health.rank > 0) health.summary = undefined; + health.rank = levelRank; + } + return { health, heat }; +} + +type PhysicalDiskHealthVerdict = + | { kind: 'replace'; summary: string } + | { kind: 'hot'; critical: boolean; summary: string } + | { kind: 'attention'; summary: string } + | { kind: 'clear' }; + +function getPhysicalDiskHealthVerdict( + disk: PhysicalDiskPresentationData, +): PhysicalDiskHealthVerdict { + const { health, heat } = splitPhysicalDiskRisk(disk); + const critical = PHYSICAL_DISK_RISK_SEVERITY_RANK.critical; + const warning = PHYSICAL_DISK_RISK_SEVERITY_RANK.warning; + const lowLife = isPhysicalDiskWearoutReported(disk) && disk.wearout < 10; + + if (normalizePhysicalDiskHealth(disk.health) === 'FAILED' || health.rank >= critical) { + return { + kind: 'replace', + summary: health.summary || 'Disk health has degraded to a critical state.', + }; + } + if (heat.rank >= critical) { + return { kind: 'hot', critical: true, summary: heat.summary || 'Disk temperature is high.' }; + } + if (health.rank >= warning || hasPhysicalDiskSmartWarning(disk) || lowLife) { + return { + kind: 'attention', + summary: + health.summary || + (lowLife ? 'SSD life is running low.' : 'SMART counters indicate elevated risk.'), + }; + } + if (heat.rank >= warning) { + return { kind: 'hot', critical: false, summary: heat.summary || 'Disk temperature is high.' }; + } + return { kind: 'clear' }; +} + +const PHYSICAL_DISK_CRITICAL_TONE = 'text-red-700 dark:text-red-300'; +const PHYSICAL_DISK_WARNING_TONE = 'text-amber-700 dark:text-amber-300'; + export function getPhysicalDiskHealthStatus( disk: PhysicalDiskPresentationData, ): DiskHealthStatusPresentation { - const normalizedHealth = normalizePhysicalDiskHealth(disk.health); - const criticalRisk = (disk.riskLevel || '').trim().toLowerCase() === 'critical'; - const warningRisk = (disk.riskLevel || '').trim().toLowerCase() === 'warning'; - const smartWarning = hasPhysicalDiskSmartWarning(disk); - const lowLife = isPhysicalDiskWearoutReported(disk) && disk.wearout < 10; + const verdict = getPhysicalDiskHealthVerdict(disk); - if (normalizedHealth === 'FAILED' || criticalRisk) { + if (verdict.kind === 'replace') { + return { label: 'Replace Now', summary: verdict.summary, tone: PHYSICAL_DISK_CRITICAL_TONE }; + } + + if (verdict.kind === 'hot') { return { - label: 'Replace Now', - summary: disk.riskReasons[0] || 'Disk health has degraded to a critical state.', - tone: 'text-red-700 dark:text-red-300', + label: 'Running Hot', + summary: verdict.summary, + tone: verdict.critical ? PHYSICAL_DISK_CRITICAL_TONE : PHYSICAL_DISK_WARNING_TONE, }; } - if (warningRisk || smartWarning || lowLife) { - return { - label: 'Needs Attention', - summary: - disk.riskReasons[0] || - (lowLife ? 'SSD life is running low.' : 'SMART counters indicate elevated risk.'), - tone: 'text-amber-700 dark:text-amber-300', - }; + if (verdict.kind === 'attention') { + return { label: 'Needs Attention', summary: verdict.summary, tone: PHYSICAL_DISK_WARNING_TONE }; } if (isUnraidPhysicalDisk(disk) && !hasUnraidPhysicalDiskFaultSignal(disk)) { @@ -720,7 +832,7 @@ export function getPhysicalDiskHealthStatus( }; } - const isHealthy = PHYSICAL_DISK_HEALTHY_STATES.has(normalizedHealth); + const isHealthy = PHYSICAL_DISK_HEALTHY_STATES.has(normalizePhysicalDiskHealth(disk.health)); return { label: isHealthy ? 'Healthy' : 'Unknown', summary: isHealthy ? 'No active disk-health issues.' : 'Health state is not reported.', @@ -732,13 +844,15 @@ export function getPhysicalDiskNormalizedHealth( resource: Resource, disk: PhysicalDiskPresentationData, ): NormalizedHealth { - const status = getPhysicalDiskHealthStatus(disk).label; - if (status === 'Online') return 'healthy'; + const verdict = getPhysicalDiskHealthVerdict(disk); + const clearLabel = verdict.kind === 'clear' ? getPhysicalDiskHealthStatus(disk).label : null; + if (clearLabel === 'Online') return 'healthy'; if (resource.status === 'offline') return 'offline'; - if (status === 'Replace Now') return 'critical'; - if (status === 'Needs Attention') return 'warning'; - if (status === 'Healthy') return 'healthy'; - return 'unknown'; + if (verdict.kind === 'replace') return 'critical'; + // A disk running hot sits in the health filter at its temperature tier. + if (verdict.kind === 'hot') return verdict.critical ? 'critical' : 'warning'; + if (verdict.kind === 'attention') return 'warning'; + return clearLabel === 'Healthy' ? 'healthy' : 'unknown'; } export function matchesPhysicalDiskHealthFilter( From 23fdfbc493ebd8376c56233742fd151b1de02d4e Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:40:10 +0100 Subject: [PATCH 27/53] Keep the cleared history hint box so hovered guests stay put The first-use "Hover a guest to preview history" hint clears on the first populated row preview, while the pointer is over a guest row. It used to unmount at that moment. On widths where the hint wraps the filter action row onto its own line (1024-1440px with a typical estate), the toolbar then shrank by 36px, the table slid up under the pointer, and the history lens opened on the next guest instead of the one being hovered. Laptop-width users hit this on their very first use of the feature. WorkloadsFilter now keeps the cleared hint mounted but invisible while row hover stays in bars + history mode, so clearing it never changes toolbar geometry at any width. Leaving that mode or remounting the filter releases the box, and the next visit gets the compact one-line toolbar. --- .../subsystems/performance-and-scalability.md | 9 +++ .../758bbf731f948f33.json | 56 +++++++++++++++++++ .../components/Workloads/WorkloadsFilter.tsx | 20 ++++++- .../__tests__/WorkloadsFilter.test.tsx | 34 +++++++++++ 4 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 frontend-modern/browser-verification/758bbf731f948f33.json diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 87517e7399..6b5a240faf 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -1931,6 +1931,15 @@ mode, row-hover mode, history range, and first-use hint on the same state owner for the generic surface and the Proxmox and vSphere platform compositions; provider pages must not recreate a partial metric prop list that can enable prefetching without exposing the matching control or discovery state. +That first-use hint clears while the pointer is over a guest row, because the +first populated preview marks it seen. Clearing it must not unmount it or +change the toolbar height: wherever the hint wraps the action row onto its own +line, removing it moved the table up under the pointer and the history lens +retargeted the next guest. `WorkloadsFilter.tsx` therefore keeps the cleared +hint mounted but `invisible` wherever the desktop History controls render, +for as long as row hover stays in bars + history mode. Leaving that mode or +remounting the filter releases the box. `WorkloadsFilter.test.tsx` pins the +retained node and its release on a switch to Trends. The alert-bridge patrol-trigger callback wired in `internal/api/router.go` now short-circuits before queuing a scoped patrol when a firing alert does not meet diff --git a/frontend-modern/browser-verification/758bbf731f948f33.json b/frontend-modern/browser-verification/758bbf731f948f33.json new file mode 100644 index 0000000000..0aa4ba26b4 --- /dev/null +++ b/frontend-modern/browser-verification/758bbf731f948f33.json @@ -0,0 +1,56 @@ +{ + "version": 1, + "base_sha": "0fb5ccb62b2c8b07637c09547cc7e397e82925a7", + "verified_at": "2026-10-06T21:51:39Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Workloads/WorkloadsFilter.tsx" + ], + "content_sha256": { + "frontend-modern/src/components/Workloads/WorkloadsFilter.tsx": "382e16198535227445f6cf823a1390502bcb24409ed2f12e19e028fa325793bf" + }, + "routes": [ + "/proxmox", + "/vmware" + ], + "viewports": [ + { + "width": 1440, + "height": 1000 + }, + { + "width": 1280, + "height": 1000 + }, + { + "width": 1600, + "height": 1000 + }, + { + "width": 768, + "height": 1000 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack (backend + Vite on 7797/5307) built from main 0fb5ccb62b (including the Net and Disk I/O row history lens) plus this change, re-run after rebasing from 3ac5664d95, compared with the same stack on unchanged main; large mock estate (50 nodes, 10 VMs + 8 LXCs each) so the guest list matches the reported repro", + "Fresh browser context so the first-use 'Hover a guest to preview history' hint is unseen; bars display mode with history row hover (the defaults)", + "Before, /proxmox at 1440 and 1280 with the page unscrolled: the hint wraps the filter action row onto its own line (filter bar 138px); after hovering artifact-cache-310 (675) the hint unmounted, the bar shrank to 102px, the row moved up 36px, pointer events went enter 675, leave 675, enter 679, and the history lens ended on auth-service-314 (679)", + "Before, /proxmox at 1600: hint inline on one row, bar 102px both ways, no shift", + "After, /proxmox at 1024, 1280, 1366, 1440, 1600 and 1920, both unscrolled and with a mid-list row scrolled to the centre: row y shift 0, filter bar height unchanged (138 or 102), a single pointerenter on the hovered row, and data-history-lens-active only on the hovered guest; the hint is still mounted with visibility hidden", + "After, /vmware at 1024 (wrapped, 138px) and 1440 (inline): row shift 0 and the lens stays on the hovered VM", + "After, 768: the hint span is display none so it has no box either way; bar 174px unchanged and the lens stays on the hovered row. 390: the mobile toolbar never renders the hint (54px before and after reload)", + "After the first preview, a reload shows no hint and the compact one-line toolbar (102px at 1440)", + "After the first preview, switching View > Metrics to Trends removes the hidden span instead of leaving a gap, and switching back to Bars does not bring it back" + ], + "interactions": [ + "Moved the real mouse onto a guest row's name cell (clear of the expand chevron) and waited for the populated history preview, recording the row bounding box, filter bar height, scroll position, pointerenter/pointerleave events and [data-history-lens-active] rows before and after", + "Repeated the hover with the page unscrolled and with the target row scrolled to the viewport centre", + "Reloaded the page after the first preview to check the next-visit toolbar", + "Opened View and switched Metrics between Trends and Bars after the hint had cleared", + "Captured screenshots of the toolbar and first rows before and after hover on main and on this change" + ] +} diff --git a/frontend-modern/src/components/Workloads/WorkloadsFilter.tsx b/frontend-modern/src/components/Workloads/WorkloadsFilter.tsx index 75814e225a..34d226b784 100644 --- a/frontend-modern/src/components/Workloads/WorkloadsFilter.tsx +++ b/frontend-modern/src/components/Workloads/WorkloadsFilter.tsx @@ -294,6 +294,20 @@ export const WorkloadsFilter: Component = (props) => { buildWorkloadSearchSuggestions(props.searchSuggestionWorkloads?.() ?? []), ); + // The history hint clears on the first populated row preview, while the + // pointer is over the table. Once shown, it keeps its box for as long as row + // hover previews history: removing it could unwrap the action row and slide + // the hovered guest out from under the pointer, retargeting the lens to the + // next row. Leaving bars + history hover, or remounting, releases the box. + const historyHintSlotReserved = createMemo( + (reserved) => + props.metricHistoryHintVisible?.() === true || + (reserved && + props.metricDisplayMode?.() === 'bars' && + (props.metricHoverMode?.() ?? 'history') === 'history'), + false, + ); + return ( = (props) => { } >
- + diff --git a/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx b/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx index d2aa75fff9..4fa12f172d 100644 --- a/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx @@ -1667,6 +1667,12 @@ describe('Workloads performance contract', () => { expect(guestDrawerManageSource).toContain('ResourceOperatorStateSection'); expect(guestDrawerOverviewSource).toContain('buildWorkloadsDiskPresentation'); expect(guestDrawerOverviewSource).toContain('Filesystems'); + // Linked physical disks load with an open agent guest's drawer, so the + // disk presentation module stays out of the WorkloadsSurface chunk. + expect(guestDrawerOverviewSource).toContain("import('./GuestPhysicalDisks')"); + expect(guestDrawerOverviewSource).not.toMatch( + /^import (?!type )[^;]*from '\.\/GuestPhysicalDisks';/m, + ); // The shared DetailSectionTable gives the earlier of the last two sections // the wider span, so the longer filesystem list must be declared before the // short tag list for mount points to get the readable panel width. From 18f0fbd826741519c63ee1909fc78b88543050eb Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:17:11 +0100 Subject: [PATCH 30/53] Keep PVE disk alert references out of resource history PVE disk health and wearout alerts reference the disk by instance, node and device path. Operator mutes resolve that reference to the disk at the path (#2112), but resource history must not bind it: a replacement disk in the same slot, or a reboot that reorders devices, takes over the path, and disk alerts are evaluated before their poll's disks reach the registry. A history binding belongs to the reference string, so it would carry a replaced disk's journaled alerts into its successor's history and make the Alerts history Resource action show the successor's incidents for its predecessor's alert. A binding guarded by each alert's recorded disk serial was built and measured on a mock estate. It bound correctly while the disk stayed at its path, but rejected events still joined whichever disk the reference was bound to, and the serial check cannot be atomic with journal writes. Record the decision in the unified-resources contract and pin it with a test, so binding these alerts later goes through ownership per row rather than a per-reference binding. --- .../internal/subsystems/unified-resources.md | 29 +++++++++++++ .../unifiedresources/history_identity_test.go | 43 +++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 3815c3ddf8..086635b28a 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -5831,6 +5831,35 @@ and `TestHistoryIdentityMonitorAdapterResolvesSubResourceReferences` in `TestOwnerAlertTimelinesUseCanonicalHistoryIdentity` in `internal/monitoring/monitor_alert_handling_test.go` against the real alert producers. +PVE disk health and wearout alerts (`ProxmoxPhysicalDiskAlertResourceID`, +`::disk:`) stay unbound, although +`ResolveReferenceID` resolves the same reference to the physical disk at that +path for operator mutes (#2112). A device path is not hardware identity: a +replacement disk in the same slot, or a reboot that reorders devices, takes +over the path, and disk alerts are evaluated before their poll's disks reach +the registry, so a lifecycle event can resolve against a generation that still +places another disk there. A binding belongs to the reference string and +moves to its latest target, carrying every row journaled under the reference +and every read of it. Bound, the path would put rows a replaced disk journaled +into its successor's history, and the Alerts history Resource action, which +reads incidents by the alert's own reference, would show the successor's +incidents for its predecessor's alert. Checking the serial each disk alert +records (`disk_serial`) narrows this but does not close it: an event rejected +for naming other hardware is still journaled under the reference and joins the +disk the reference is bound to, the check and the journal write are not +atomic, and an atomic check would still leave one binding owning every row +under the path. A disk's history therefore omits its own SMART health and +wearout alerts; they stay under the alert's reference, where incident +timelines by alert identifier and start time find them. The Resource action +reads the whole reference, so it lists every disk that has held the path, and +alert identifiers derive from the path too. Binding them needs ownership per +row, not per reference: each event written under the disk its recorded +hardware identity names, rows with missing, unusable or ambiguous identity +left under the reference rather than inferred from the current path, and a +decided read for the Resource action. `proxmoxDiskAlertMetadata` records the +serial but not the WWN, so WWN ownership also needs a producer change. Proof: +`TestHistoryIdentityLeavesProxmoxDiskAlertReferencesUnbound` in +`internal/unifiedresources/history_identity_test.go`. That same shared timeline vocabulary now includes the `activity` change kind for provider-read breadcrumbs such as VMware tasks and events, plus the `vmware_adapter` source-adapter token for canonical provenance drill-down. diff --git a/internal/unifiedresources/history_identity_test.go b/internal/unifiedresources/history_identity_test.go index 4744e08503..3ac088b73d 100644 --- a/internal/unifiedresources/history_identity_test.go +++ b/internal/unifiedresources/history_identity_test.go @@ -576,3 +576,46 @@ func TestHistoryIdentityMonitorAdapterResolvesSubResourceReferences(t *testing.T require.True(t, found) require.Equal(t, storageID, deviceBinding) } + +// PVE disk health and wearout alerts reference the disk by instance, node and +// device path. Operator mutes resolve that reference to the disk at the path +// (#2112), but history never binds it: the path passes to a replacement disk +// in the same slot, and a binding would carry every row journaled under the +// reference, and every read of the reference, to whichever disk held the path +// last. +func TestHistoryIdentityLeavesProxmoxDiskAlertReferencesUnbound(t *testing.T) { + store, err := NewSQLiteResourceStore(t.TempDir(), "default") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + now := time.Now().UTC().Truncate(time.Second) + ref := ProxmoxPhysicalDiskAlertResourceID("lab", "pve1", "/dev/sda") + event := func(id string, at time.Time) ResourceChange { + return ResourceChange{ID: id, ResourceID: ref, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: at, + Metadata: map[string]any{"alert_type": "disk-health", "disk_serial": "ZA1A2B3C"}} + } + // Journaled before this process started, then raised live. + require.NoError(t, store.RecordChange(event("legacy", now.Add(-time.Hour)))) + snapshot := proxmoxHistoryIdentitySnapshot(now) + snapshot.PhysicalDisks = []models.PhysicalDisk{{ID: ProxmoxPhysicalDiskSourceID("lab", "pve1", "/dev/sda", "", ""), Node: "pve1", Instance: "lab", + DevPath: "/dev/sda", Model: "Seagate ST2000DM008", Serial: "ZA1A2B3C", Type: "hdd", Health: "FAILED", Wearout: -1, LastChecked: now}} + adapter := NewMonitorAdapter(NewRegistry(store)) + adapter.PopulateFromSnapshot(snapshot) + registry := adapter.currentRegistry() + diskID, ok := registry.ResolveReferenceID(ref) + require.True(t, ok, "operator mutes resolve the reference to the disk") + require.Equal(t, ResourceTypePhysicalDisk, registry.resources[diskID].Type) + require.NoError(t, adapter.RecordChange(event("live", now))) + + _, found, err := store.ResolveHistorySourceIdentity(ref) + require.NoError(t, err) + require.False(t, found) + got, err := store.GetRecentChanges(ref, time.Time{}, 10) + require.NoError(t, err) + require.Len(t, got, 2) + for _, change := range got { + require.Equal(t, ref, change.ResourceID, change.ID) + } + got, err = store.GetRecentChangesFiltered(diskID, time.Time{}, 10, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertFired}}) + require.NoError(t, err) + require.Empty(t, got) +} From 723f3165a1292268634b6054ab93d3015805d990 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:23:18 +0100 Subject: [PATCH 31/53] Show a silent agent's retained disk temperatures as last known MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a host agent passes its reporting lease, its collected SMART temperatures keep their values with collection.temperature unavailable (#2593), and #2603 taught the Storage disk surfaces to show such a value as last known. The frontend HostDiskSMART type never carried `collection`, so the agent's own surfaces kept presenting the retained readings as current: - The Machines temperature cell fell back to the hottest SMART reading and coloured it against the disk thresholds. On a live mock stack a silent, SMART-only agent read 52°C in threshold green. - Its tooltip, the machine drawer's Thermals rows and the guest drawer's Physical Disks card listed the retained disks as plain readings. The row does not make this obvious. By the time the lease expires the registry has marked a standalone agent's source stale, which outranks offline, so a silent agent reaches the table as `warning` with `agent.stale`, not `offline`. The offline metric fallback never blanks its cells, and the temperature cell has no such gate. Now `HostDiskSMART.collection` is typed and every reader decides through `isPhysicalDiskTemperatureCurrent`: - A retained disk reading never stands for the machine while another disk is current. When no non-standby disk is current and the machine has no positive direct or temperatureCelsius reading, the cell shows the hottest retained non-standby value muted with a dotted underline and screen-reader "last known", without threshold colour. - Tooltip and Thermals rows read "(last known)", the Thermals row with the collection reason as its title. A disk with a zero or negative reading no longer shows a temperature row there. - The guest Physical Disks card uses the Storage table's last-known treatment. The temperature helpers move to diskTemperaturePresentation.ts, which diskPresentation.ts re-exports. On the earlier base, importing diskPresentation from the Machines page and the drawer mapper folded the whole Storage presenter into the shared chunk (+4.9 kB gzip) and renamed its tracked budget entry, failing the bundle check. With the small module, Standalone grows 0.22 kB, the shared chunk 0.20 kB, and Workloads is flat against a build of main. The guest card still loads diskPresentation for extraction and health. Direct and temperatureCelsius readings carry no collection state and still lead the cell. A stale agent's CPU, memory, disk and sensor values still render as current. That is a row-wide decision, left for a separate change. --- .../v6/internal/subsystems/api-contracts.md | 9 + .../subsystems/frontend-primitives.md | 10 ++ .../subsystems/performance-and-scalability.md | 7 + .../internal/subsystems/storage-recovery.md | 15 ++ .../internal/subsystems/unified-resources.md | 14 ++ .../47c60098fc12bf96.json | 54 ++++++ .../__tests__/resourceDetailMappers.test.ts | 85 ++++++++- .../Infrastructure/resourceDetailMappers.ts | 11 +- .../Workloads/GuestPhysicalDisks.tsx | 21 ++- .../__tests__/GuestPhysicalDisks.test.tsx | 96 +++++++++- .../standalone/AgentsMachinesTable.tsx | 28 ++- .../__tests__/AgentsMachinesTable.test.tsx | 148 ++++++++++++++++ .../__tests__/agentMachineTableModel.test.ts | 164 ++++++++++++++++++ .../standalone/agentMachineTableModel.ts | 66 +++++-- .../__tests__/diskPresentation.test.ts | 20 +++ .../storageBackups/diskPresentation.ts | 53 +----- .../diskTemperaturePresentation.ts | 53 ++++++ frontend-modern/src/types/api.ts | 3 + 18 files changed, 787 insertions(+), 70 deletions(-) create mode 100644 frontend-modern/browser-verification/47c60098fc12bf96.json create mode 100644 frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index ae4efc4a57..ecae82c5c7 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -5378,6 +5378,15 @@ Backend API payloads and `frontend-modern/src/types/api.ts` must preserve that optional map without making it a required compatibility field, and clients must keep it as descriptive host telemetry rather than a temperature metric, resource identity, alert metric, or storage/recovery signal. +Host sensor SMART rows (`smart[]`) also carry the optional per-field +`collection` status (`PhysicalDiskCollectionStatus`). Backend API payloads and +`frontend-modern/src/types/api.ts` (`HostDiskSMART.collection`) must preserve it +on the agent facet. When a host agent passes its reporting lease, a SMART +temperature that was current (`available`, or present without provenance) keeps +its value and becomes `unavailable`. A state that was already not available is +kept. Clients must read that state through the shared +`isPhysicalDiskTemperatureCurrent` decision before treating the value as a +current reading. Agent resource-context sections expose host package posture only as bounded operational facts: package manager, pending count, inventory state, inspection freshness, and reboot-required state. Raw package identifiers, versions, and diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 85c281c011..f585ee45a4 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -7803,6 +7803,16 @@ never through the threshold colour classes, so a retained reading cannot look hot or healthy. Table, drawer and pool surfaces share that one decision in `frontend-modern/src/features/storageBackups/diskPresentation.ts` rather than each re-reading `collection.temperature`. +That decision and its class now live in the small +`frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts` +module, which `diskPresentation.ts` re-exports, so the Machines table and the +machine drawer's Thermals rows apply the same treatment without pulling the +Storage presenter into their chunks. Those surfaces and the guest drawer's +Physical Disks card import it directly. +When the cell's value is a retained host-agent SMART temperature, it renders +muted with a dotted underline and screen-reader "last known", as does a +retained value in the guest card. Tooltip and Thermals rows say "(last known)", +and the Thermals row carries the collection reason as its title. The focused browser proofs are `frontend-modern/src/features/patrol/__tests__/patrolRunAcceptance.test.ts`, diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 87517e7399..a7f0088b89 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -2044,6 +2044,13 @@ render those sensor values already present on the selected resource payload, but it must not add host powercap reads, sensor-specific history reads, per-row polling, browser-side command assumptions, or table-wide aggregation work. +Host-agent SMART disk rows in `resourceDetailMappers.ts` mark a retained +temperature "(last known)", with its collection reason as the title, from the +payload's own `collection.temperature` state, through `getPhysicalDiskTemperaturePresentation` +in `frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts`. +That module stays separate from `diskPresentation.ts` so the drawer and +Machines chunks take the decision without the Storage presenter, and the rows +add no reads, polling or aggregation work. The Proxmox node drawer overview should follow the existing guest drawer compact detail-section pattern and expose node-specific context such as platform, kernel, hardware, raw capacity, telemetry, and thermal facts rather than diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index cd2ec5bf9e..2397cd4bb5 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -3814,6 +3814,21 @@ or alert evaluation. Verification: `diskPresentation.test.ts`, `diskDetailPresentation.test.ts`, `storagePoolDetailPresentation.test.ts`, `DiskList.test.tsx` (a row moving between current, retained and legacy states) and `DiskDetail.test.tsx`. +The same decision reaches the disks that only a host agent's own SMART rows +describe. `HostDiskSMART.collection` now crosses into the frontend type, and the +helpers live in `frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts` +(re-exported by `diskPresentation.ts`, one implementation) so the Machines +table and the machine drawer can use them without loading the Storage +presenter. The guest drawer's card already loads it for extraction and health. +A silent agent's retained SMART temperature never becomes the Machines row's +value while another disk has a current reading. When the machine has no +positive direct or `temperatureCelsius` reading and no non-standby disk is +current, the cell shows the hottest retained non-standby value as last known, +without the per-disk-type threshold colour. The guest drawer's Physical Disks card marks a +retained value the way the Physical Disks table does. Still presentation only. +Verification: `diskPresentation.test.ts` (one implementation behind both import +paths), `agentMachineTableModel.test.ts`, `AgentsMachinesTable.test.tsx`, +`resourceDetailMappers.test.ts` and `GuestPhysicalDisks.test.tsx`. Direct SATA, SAS, and NVMe device fallback identities remain compatible. Controller-member fallback identities add controller/target scope only where diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 79650fb3d1..d14cbc296c 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -2269,6 +2269,20 @@ and temperature facts must also compose the frontend-primitives `InfoCardKeyValueRow`. Mobile rows retain their condensed endpoint layout; desktop rows use the shared fixed label track so labels and values remain visually adjacent instead of spanning the full drawer width. +A standalone host agent past its reporting lease reaches the Machines table as +`warning` with `agent.stale`, not `offline`: by the time the lease expires the +registry has marked its only source stale, which outranks offline. The offline +metric fallback therefore does not blank its cells, and the temperature cell +has no such gate. For SMART disk temperatures the provenance travels on +`agent.sensors.smart[].collection`: the Machines temperature cell +(`AgentsMachinesTable.tsx`, `agentMachineTableModel.ts`), its tooltip and the +drawer's Thermals rows (`resourceDetailMappers.ts`) read it through +`isPhysicalDiskTemperatureCurrent`, so a retained disk reading never stands for +the machine while another disk has a current one. A positive direct +`temperature` or `temperatureCelsius` reading carries no collection state and +still leads the cell. Without one, and with no current non-standby disk, the +hottest retained non-standby disk reading shows as last known without +threshold colour. The same boundary applies to availability facts, resource change-history metadata, Docker/PBS/PMG service facts, nested PMG queue/mail breakdowns, and Docker container-update management facts. It also covers action-history facts, diff --git a/frontend-modern/browser-verification/47c60098fc12bf96.json b/frontend-modern/browser-verification/47c60098fc12bf96.json new file mode 100644 index 0000000000..eb0d451e5a --- /dev/null +++ b/frontend-modern/browser-verification/47c60098fc12bf96.json @@ -0,0 +1,54 @@ +{ + "version": 1, + "base_sha": "dcb69cfb020193d05d9e4e668d19b820a0c46d31", + "verified_at": "2026-10-06T22:07:56Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts", + "frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx", + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx", + "frontend-modern/src/features/standalone/agentMachineTableModel.ts", + "frontend-modern/src/features/storageBackups/diskPresentation.ts", + "frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts", + "frontend-modern/src/types/api.ts" + ], + "content_sha256": { + "frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts": "6cdf797534d6ecba8f8007561195f9245899ff881a225f83c2c6c6293b557f8c", + "frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx": "98e9dc568360b5b6c9552433207e0e2fa78b91ba17da53fa513fb42484373ff5", + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx": "a1dfacc661118c77fea482d26ffbc6d1c6aca0d351d77744f7a2dbbd4770ef6f", + "frontend-modern/src/features/standalone/agentMachineTableModel.ts": "1f9be08ed7100a330142eab8900b04474360785e75c1b95a8a42d4bc35fa7233", + "frontend-modern/src/features/storageBackups/diskPresentation.ts": "debb0d30e95b84d010d5b7478d61ce5fa9487c0bfb43b3eb812e16b4a75acf2e", + "frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts": "c62e4d1e74b9278ef34d93e8101c3d41ab67b17885b6810116a2f96bd112301c", + "frontend-modern/src/types/api.ts": "bb2712595745499cc51ed5c6e54ea24998e00d42ce45c8715b1d6c3eed6afc3e" + }, + "routes": [ + "/standalone", + "/proxmox?q=analytics-batch-01" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack built from this tree (base dcb69cfb02) plus a local-only go build -overlay of internal/mock/generator.go: three standalone Linux agents get SMART rows; Apollo-114 (CPU sensors) and Meridian-731 (SMART only) are stamped as State.ExpireHostTelemetry does (status offline, temperatures kept, collection.temperature unavailable 'host agent stopped reporting', source kept), Europa-405 keeps reporting. /api/resources returned agent.sensors.smart[].collection end to end, status warning and agent.stale true for the silent agents.", + "Same-session control with the HEAD runtime files under the same Vite: Meridian-731's Temp cell rendered 52°C in threshold green, the tooltip and the drawer Thermals card listed the retained disks as plain current readings, and the guest Physical Disks card showed plain muted 52°C/41°C with no last-known cue.", + "After: Meridian-731's Temp cell renders '52°C' muted (slate-600 light, slate-400 dark) with a dotted underline, no threshold colour and screen-reader ', last known'; its tooltip lists both disks '(last known)' muted. Apollo-114 keeps its CPU package value (no collection state) while its disk tooltip rows read '(last known)'. Europa-405 is unchanged.", + "Machine drawer (Platform details > Machine details > Thermals) at desktop and phone: 'Disk nvme0n1 52°C (last known)' and 'Disk sda 41°C (last known)' with title 'Last known reading, not current: host agent stopped reporting'; cpu.package unchanged.", + "Guest drawer Physical Disks card at desktop and phone: Playwright routes add the agent version a VM-resident agent would carry to analytics-batch-01 and answer its physical_disk child query with Meridian-731's real registry disks (collection state untouched) re-parented to the VM, because mock mode never links an agent inside a VM. Both disks render muted, dotted, with the reason title and ', last known'.", + "Light and dark schemes on the Machines page. Bundle check: Standalone +0.22 kB, shared chunk +0.20 kB, WorkloadsSurface and InfrastructureWorkspace flat versus a same-machine main build, no tracked chunk renamed.", + "Not covered: a real host agent going silent on hardware, an agent genuinely running inside a VM, or stale-row CPU/memory/disk cells (flagged separately)." + ], + "interactions": [ + "Enabled the Temp column from View > column picker on /standalone and read every row's cell text, class, computed colour and text decoration.", + "Hovered the Temp cell of the SMART-only and the CPU-sensor silent agents and read the tooltip rows and their muted state.", + "Opened the Apollo-114 drawer by row click (desktop) and CPU cell tap (phone), expanded Platform details and the Machine Show details, and read the Thermals rows with their titles.", + "Opened the analytics-batch-01 guest drawer from the Proxmox guest table at desktop and phone and read the Physical Disks card summaries, data-temperature-reading, titles and decoration.", + "Swapped the runtime files to HEAD under the running Vite for the same-session control, then restored and re-ran every pass on the final tree." + ] +} diff --git a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts index 0df9376977..5596569845 100644 --- a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts +++ b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts @@ -10,7 +10,7 @@ import { } from '@/components/Infrastructure/resourceDetailMappers'; import resourceDetailMappersSource from '@/components/Infrastructure/resourceDetailMappers.ts?raw'; import resourceDetailDiscoveryModelSource from '@/components/Infrastructure/resourceDetailDiscoveryModel.ts?raw'; -import type { Resource } from '@/types/resource'; +import type { PhysicalDiskCollectionStatus, Resource } from '@/types/resource'; const createHybridHostResource = (): Resource => ({ @@ -203,6 +203,89 @@ describe('resourceDetailMappers', () => { }, ]); }); + + it('marks a silent agent disk temperature as last known and drops disks without one', () => { + const rows = buildTemperatureRows({ + smart: [ + { + device: 'sda', + temperature: 71, + // A host agent past its reporting lease keeps its last reading. + collection: { + temperature: { + state: 'unavailable', + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }, + }, + { + device: 'sdb', + temperature: 38, + collection: { temperature: { state: 'available', source: 'smartctl' } }, + }, + { + device: 'sdc', + temperature: 0, + collection: { + temperature: { + state: 'unsupported', + source: 'smartctl', + reason: 'device did not expose a temperature reading', + }, + }, + }, + ], + }); + + expect(rows).toEqual([ + { + label: 'Disk sda', + value: '71°C (last known)', + valueTitle: 'Last known reading, not current: host agent stopped reporting', + }, + { label: 'Disk sdb', value: '38°C', valueTitle: '38.0°C' }, + ]); + }); + + it('lists a SMART row only for a positive reading in any state, never a standby disk', () => { + const states: (PhysicalDiskCollectionStatus | undefined)[] = [ + undefined, + { temperature: { state: 'available', source: 'smartctl' } }, + { temperature: { state: 'unavailable', source: 'smartctl' } }, + ]; + for (const collection of states) { + for (const temperature of [0, -3]) { + expect( + buildTemperatureRows({ smart: [{ device: 'sda', temperature, collection }] }), + ).toEqual([]); + } + } + + expect( + buildTemperatureRows({ + smart: [ + { + device: 'sda', + temperature: 44, + standby: true, + collection: { temperature: { state: 'unavailable', source: 'smartctl' } }, + }, + { + device: 'sdb', + temperature: 39, + collection: { temperature: { state: 'missing', source: 'smartctl' } }, + }, + ], + }), + ).toEqual([ + { + label: 'Disk sdb', + value: '39°C (last known)', + valueTitle: 'Last known reading, not current', + }, + ]); + }); }); describe('buildCustomSensorRows', () => { diff --git a/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts b/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts index 3a77a7aded..b6b94a7ea1 100644 --- a/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts +++ b/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts @@ -15,6 +15,7 @@ import type { } from '@/types/api'; import type { Resource, ResourceMetric, ResourceVMwareMeta } from '@/types/resource'; import { formatTemperature } from '@/utils/temperature'; +import { getPhysicalDiskTemperaturePresentation } from '@/features/storageBackups/diskTemperaturePresentation'; import { formatBytes } from '@/utils/format'; import { getActionableAgentIdFromResource } from '@/utils/agentResources'; import { @@ -502,13 +503,17 @@ export const buildTemperatureRows = (sensors?: HostSensorSummary) => { const smart = sensors?.smart; if (smart) { smart - .filter((disk) => !disk.standby && Number.isFinite(disk.temperature)) + .filter((disk) => !disk.standby) .sort((a, b) => a.device.localeCompare(b.device)) .forEach((disk) => { + const temperature = getPhysicalDiskTemperaturePresentation(disk); + if (!temperature) return; + // A retained reading, such as a silent agent's, says so in the value + // because this card has no other cue that it is not current. rows.push({ label: `Disk ${disk.device}`, - value: formatTemperature(disk.temperature), - valueTitle: `${disk.temperature.toFixed(1)}°C`, + value: temperature.current ? temperature.label : `${temperature.label} (last known)`, + valueTitle: temperature.title ?? `${disk.temperature.toFixed(1)}°C`, }); }); } diff --git a/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx b/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx index a9c75d3db7..ceeaa26a50 100644 --- a/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx +++ b/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx @@ -5,9 +5,12 @@ import { extractPhysicalDiskPresentationData, getPhysicalDiskHealthStatus, } from '@/features/storageBackups/diskPresentation'; +import { + PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS, + getPhysicalDiskTemperaturePresentation, +} from '@/features/storageBackups/diskTemperaturePresentation'; import type { Resource } from '@/types/resource'; import { formatBytes } from '@/utils/format'; -import { formatTemperature } from '@/utils/temperature'; // SMART history and chart controls are only needed after a disk row is opened. // Keep their Storage detail module out of the initial Workloads surface. @@ -19,6 +22,7 @@ const GuestPhysicalDiskRow: Component<{ disk: Resource }> = (props) => { const [expanded, setExpanded] = createSignal(false); const data = () => extractPhysicalDiskPresentationData(props.disk); const health = () => getPhysicalDiskHealthStatus(data()); + const temperature = () => getPhysicalDiskTemperaturePresentation(data()); const label = () => data().model || data().devPath || props.disk.name; return ( @@ -35,8 +39,19 @@ const GuestPhysicalDiskRow: Component<{ disk: Resource }> = (props) => { {health().label} - 0}> - {formatTemperature(data().temperature)} + + {(reading) => ( + + {reading().label} + + , last known + + + )} 0}> {formatBytes(data().size)} diff --git a/frontend-modern/src/components/Workloads/__tests__/GuestPhysicalDisks.test.tsx b/frontend-modern/src/components/Workloads/__tests__/GuestPhysicalDisks.test.tsx index 0fa8f2c11a..0b316595db 100644 --- a/frontend-modern/src/components/Workloads/__tests__/GuestPhysicalDisks.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/GuestPhysicalDisks.test.tsx @@ -1,10 +1,13 @@ import { cleanup, fireEvent, render, screen, within } from '@solidjs/testing-library'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { createSignal } from 'solid-js'; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import type { Resource } from '@/types/resource'; import { GuestPhysicalDisks } from '../GuestPhysicalDisks'; const queryState = vi.hoisted(() => ({ resources: [] as Resource[], + // Set to a signal accessor to drive live resource updates. + accessor: undefined as (() => Resource[]) | undefined, error: undefined as unknown, query: '', enabled: false, @@ -14,7 +17,10 @@ vi.mock('@/hooks/useUnifiedResources', () => ({ useUnifiedResources: (options: { query: string; enabled: () => boolean }) => { queryState.query = options.query; queryState.enabled = options.enabled(); - return { resources: () => queryState.resources, error: () => queryState.error }; + return { + resources: () => (queryState.accessor ? queryState.accessor() : queryState.resources), + error: () => queryState.error, + }; }, })); @@ -41,8 +47,15 @@ const disk: Resource = { }; describe('GuestPhysicalDisks', () => { + // The first expansion lazy-loads the Storage disk detail; a cold transform of + // that module can outlast findByText's default wait on a loaded machine. + beforeAll(async () => { + await import('@/components/Storage/DiskDetail'); + }, 30_000); + beforeEach(() => { queryState.resources = []; + queryState.accessor = undefined; queryState.error = undefined; queryState.query = ''; queryState.enabled = false; @@ -84,6 +97,85 @@ describe('GuestPhysicalDisks', () => { expect(screen.queryByTestId('guest-physical-disks')).toBeNull(); }); + it('marks a retained disk temperature as last known', () => { + queryState.resources = [ + disk, + { + ...disk, + id: 'disk-101-b', + displayName: 'Guest NVMe', + physicalDisk: { + ...disk.physicalDisk!, + devPath: '/dev/nvme0', + model: 'Guest NVMe', + temperature: 61, + collection: { + temperature: { + state: 'unavailable', + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }, + }, + }, + ]; + render(() => ); + + const [currentRow, retainedRow] = screen.getAllByTestId('guest-physical-disk'); + const current = within(currentRow).getByText('45°C'); + expect(current).toHaveAttribute('data-temperature-reading', 'current'); + expect(current).not.toHaveAttribute('title'); + + const retained = retainedRow.querySelector('[data-temperature-reading="last-known"]'); + expect(retained).not.toBeNull(); + expect(retained).toHaveTextContent('61°C, last known'); + expect(retained).toHaveClass('underline', 'decoration-dotted', 'text-muted'); + expect(retained).toHaveAttribute( + 'title', + 'Last known reading, not current: host agent stopped reporting', + ); + }); + + it('follows a disk between current and last known as its collection state changes', () => { + const withReading = ( + temperature: number, + collection?: NonNullable['collection'], + ): Resource => ({ + ...disk, + physicalDisk: { ...disk.physicalDisk!, temperature, collection }, + }); + const [resources, setResources] = createSignal([disk]); + queryState.accessor = resources; + render(() => ); + const reading = () => + screen.getByTestId('guest-physical-disk').querySelector('[data-temperature-reading]'); + + expect(reading()).toHaveAttribute('data-temperature-reading', 'current'); + + setResources([ + withReading(61, { + temperature: { + state: 'unavailable', + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }), + ]); + expect(reading()).toHaveAttribute('data-temperature-reading', 'last-known'); + expect(reading()).toHaveTextContent('61°C, last known'); + expect(reading()).toHaveClass('decoration-dotted'); + + setResources([withReading(47, { temperature: { state: 'available', source: 'smartctl' } })]); + expect(reading()).toHaveAttribute('data-temperature-reading', 'current'); + expect(reading()).toHaveTextContent('47°C'); + expect(reading()).not.toHaveTextContent('last known'); + expect(reading()).not.toHaveAttribute('title'); + expect(reading()).not.toHaveClass('decoration-dotted'); + + setResources([withReading(0)]); + expect(reading()).toBeNull(); + }); + it('makes a failed child query visible rather than silently implying no disks', () => { queryState.error = new Error('unavailable'); render(() => ); diff --git a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx index 7e0f98124e..2609a62fa7 100644 --- a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx +++ b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx @@ -57,6 +57,7 @@ import { type PlatformResourceStatusFilter, withPlatformStatusCounts, } from '@/features/platformPage/sharedPlatformPage'; +import { PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS } from '@/features/storageBackups/diskTemperaturePresentation'; import { useColumnVisibility } from '@/hooks/useColumnVisibility'; import type { Disk } from '@/types/api'; import type { Resource, ResourceAvailabilityMeta } from '@/types/resource'; @@ -76,6 +77,7 @@ import { import { asTrimmedString } from '@/utils/stringUtils'; import { getPreferredResourceIP } from '@/utils/resourceIdentity'; import { useRelativeTimeNow } from '@/utils/relativeTimeClock'; +import { formatTemperature } from '@/utils/temperature'; import { RESOURCE_METADATA_CHANGED_EVENT, type ResourceMetadataChangedDetail, @@ -101,6 +103,7 @@ import { getAgentMachineTemperatureTitle, getAgentMachineThermalPressurePresentation, getNextAgentMachineSortState, + isAgentMachineTemperatureLastKnown, sortAgentMachines, type AgentMachineColumn, type AgentMachineColumnId, @@ -183,6 +186,8 @@ const AgentMachineMetricTooltip: Component<{ const AgentMachineTemperatureCell: Component<{ celsius: number | undefined; + /** The value is a retained reading, not a current one. */ + lastKnown?: boolean; metric: AgentMachineTemperatureMetric; sections: AgentMachineTemperatureDetailSection[]; thresholds?: MetricDisplayThresholds | null; @@ -222,7 +227,26 @@ const AgentMachineTemperatureCell: Component<{ } > {(value) => ( - + + {formatTemperature(value())} + , last known + + } + > + + )} } @@ -1565,6 +1589,7 @@ export const AgentsMachinesTable: Component<{ const raidArrays = () => getAgentMachineRaidArrayDetails(machine); const raidSummary = () => getAgentMachineRaidSummary(machine); const temperature = () => getAgentMachineTemperatureCelsius(machine); + const temperatureLastKnown = () => isAgentMachineTemperatureLastKnown(machine); const temperatureMetric = () => getAgentMachineTemperatureMetric(machine); const temperatureThresholds = () => temperatureMetric() === 'diskTemperature' @@ -1791,6 +1816,7 @@ export const AgentsMachinesTable: Component<{ > { expect(screen.getByText('1400 RPM')).toBeInTheDocument(); }); + it('shows a silent agent disk temperature as last known, never as a current reading', async () => { + // A host agent past its reporting lease is stale and keeps its last SMART + // temperatures with a non-available collection state. + const stoppedReporting = { + temperature: { + state: 'unavailable' as const, + source: 'smartctl', + reason: 'host agent stopped reporting', + }, + }; + const { container } = render(() => ( + ({ ...disk, collection: stoppedReporting })), + }, + }, + }), + resource({ + id: 'mixed-host', + name: 'Mixed Host', + agent: { + sensors: { + smart: [ + { + device: '/dev/sda', + model: 'Retained HDD', + temperature: 69, + collection: stoppedReporting, + }, + { device: '/dev/nvme0', model: 'Fast SSD', temperature: 44 }, + ], + }, + }, + }), + ]} + emptyIcon={emptyIcon} + emptyTitle="No machines" + emptyDescription="Install Pulse Agent." + /> + )); + + await openMachineColumnPicker(); + await fireEvent.click(screen.getByLabelText('Temp')); + + const triggerFor = (id: string) => + container.querySelector( + `[data-agents-machine-row="${id}"] [data-agent-machine-temperature-trigger="true"]`, + ) as HTMLElement | null; + + // Nothing is collected now, so the cell keeps the hottest retained value + // without a threshold colour and says it is last known. + const silent = triggerFor('silent-nas'); + expect(silent).not.toBeNull(); + if (!silent) return; + const retained = silent.querySelector('[data-temperature-reading="last-known"]'); + expect(retained).not.toBeNull(); + expect(retained).toHaveTextContent('71°C, last known'); + expect(retained).toHaveClass('underline', 'decoration-dotted', 'text-muted'); + expect(retained?.className).not.toMatch(/text-(green|yellow|red)-/); + + await fireEvent.mouseEnter(silent); + const tooltip = await waitFor(() => { + const element = document.querySelector('[data-agent-machine-temperature-tooltip="true"]'); + expect(element).not.toBeNull(); + return element as HTMLElement; + }); + expect(within(tooltip).getByText('71°C (last known)')).toHaveClass('text-muted'); + expect(within(tooltip).getByText('66°C (last known)')).toHaveClass('text-muted'); + await fireEvent.mouseLeave(silent); + + // A disk collected now leads the cell even when a retained one is hotter. + const mixed = triggerFor('mixed-host'); + expect(mixed).not.toBeNull(); + if (!mixed) return; + expect(mixed).toHaveTextContent('44°C'); + expect(mixed.querySelector('[data-temperature-reading="last-known"]')).toBeNull(); + expect(within(mixed).queryByText('69°C')).toBeNull(); + }); + + it('moves a machine temperature between current and last known as its disk reports change', async () => { + // Fresh objects per update, like parsed API payloads: the store reconciles + // into the objects it was given, so a reused fixture would be rewritten. + const collected = () => ({ temperature: { state: 'available' as const, source: 'smartctl' } }); + const stoppedReporting = () => ({ + temperature: { + state: 'unavailable' as const, + source: 'smartctl', + reason: 'host agent stopped reporting', + }, + }); + const nas = ( + temperature: number, + collection?: ReturnType | ReturnType, + ) => + resource({ + id: 'nas', + name: 'NAS', + agent: { + sensors: { smart: [{ device: '/dev/sda', type: 'sata', temperature, collection }] }, + }, + }); + const [resources, setResources] = createSignal([nas(52, collected())]); + const { container } = render(() => ( + + )); + + await openMachineColumnPicker(); + await fireEvent.click(screen.getByLabelText('Temp')); + const cell = () => + container.querySelector( + '[data-agents-machine-row="nas"] [data-agent-machine-temperature-trigger="true"]', + ) as HTMLElement; + const lastKnown = () => cell().querySelector('[data-temperature-reading="last-known"]'); + + expect(cell()).toHaveTextContent('52°C'); + expect(lastKnown()).toBeNull(); + + setResources([nas(52, stoppedReporting())]); + await waitFor(() => expect(lastKnown()).not.toBeNull()); + expect(cell()).toHaveTextContent('52°C, last known'); + + setResources([nas(47, collected())]); + await waitFor(() => expect(lastKnown()).toBeNull()); + expect(cell()).toHaveTextContent('47°C'); + expect(cell().querySelector('.decoration-dotted')).toBeNull(); + expect(cell()).not.toHaveTextContent('last known'); + + setResources([nas(0)]); + await waitFor(() => expect(cell()).toHaveTextContent('—')); + expect(lastKnown()).toBeNull(); + }); + it('shows thermal pressure when macOS reports no Celsius temperature', async () => { render(() => ( { expect(getAgentMachineTemperatureMetric(machine)).toBe('diskTemperature'); }); + it('shows a retained SMART temperature only as last known', () => { + // A host agent past its reporting lease keeps its last SMART temperature + // with a non-available collection state. + const stoppedReporting = { + temperature: { + state: 'unavailable' as const, + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }; + const mixed = resource({ + agent: { + sensors: { + smart: [ + { + device: '/dev/sda', + model: 'Retained HDD', + type: 'sas', + temperature: 71, + collection: stoppedReporting, + }, + { + device: '/dev/nvme0', + model: 'Fast SSD', + type: 'nvme', + temperature: 44, + collection: { temperature: { state: 'available' as const, source: 'smartctl' } }, + }, + // Agents that predate collection state report no provenance. + { device: '/dev/sdb', temperature: 39 }, + { device: '/dev/sdc', model: 'Cold Standby', temperature: 0, standby: true }, + ], + }, + }, + }); + + // A disk collected now outranks a hotter retained one. + expect(getAgentMachineTemperatureCelsius(mixed)).toBe(44); + expect(isAgentMachineTemperatureLastKnown(mixed)).toBe(false); + expect(getAgentMachineTemperatureMetric(mixed)).toBe('diskTemperature'); + expect(getAgentMachineHottestSmartDiskType(mixed)).toBe('nvme'); + expect(getAgentMachineTemperatureTitle(mixed).split('\n')).toEqual([ + 'Disk Temperatures', + 'Disk /dev/nvme0 Fast SSD: 44°C', + 'Disk /dev/sdb: 39°C', + 'Disk /dev/sda Retained HDD: 71°C (last known)', + 'Disk /dev/sdc Cold Standby: standby', + ]); + expect(getAgentMachineTemperatureDetailSections(mixed)[0].rows[2]).toEqual({ + label: 'Disk /dev/sda Retained HDD', + value: '71°C (last known)', + muted: true, + }); + + // With nothing collected now, the cell keeps the hottest retained value + // and marks it last known. + const onlyRetained = resource({ + agent: { + sensors: { + smart: [ + { device: '/dev/sda', type: 'sata', temperature: 71, collection: stoppedReporting }, + { device: '/dev/nvme0', type: 'nvme', temperature: 58, collection: stoppedReporting }, + ], + }, + }, + }); + expect(getAgentMachineTemperatureCelsius(onlyRetained)).toBe(71); + expect(isAgentMachineTemperatureLastKnown(onlyRetained)).toBe(true); + expect(getAgentMachineTemperatureMetric(onlyRetained)).toBe('diskTemperature'); + expect(getAgentMachineHottestSmartDiskType(onlyRetained)).toBe('sata'); + expect(getAgentMachineTemperatureTitle(onlyRetained)).toBe( + 'Disk Temperatures\nDisk /dev/sda: 71°C (last known)\nDisk /dev/nvme0: 58°C (last known)', + ); + + // CPU sensors carry no collection state, so they still lead the cell. + const withSensors = resource({ + agent: { + sensors: { + temperatureCelsius: { 'cpu.package': 49 }, + smart: [{ device: '/dev/sda', temperature: 71, collection: stoppedReporting }], + }, + }, + }); + expect(getAgentMachineTemperatureCelsius(withSensors)).toBe(49); + expect(isAgentMachineTemperatureLastKnown(withSensors)).toBe(false); + expect(getAgentMachineTemperatureMetric(withSensors)).toBe('temperature'); + }); + + it('offers only positive, non-standby SMART readings to the cell, behind direct and CPU sensors', () => { + const stoppedReporting = { + temperature: { + state: 'unavailable' as const, + source: 'smartctl', + reason: 'host agent stopped reporting', + }, + }; + const retainedDisk = (temperature: number) => ({ + device: '/dev/sda', + temperature, + collection: stoppedReporting, + }); + + // A disk in standby is never a temperature candidate, retained or not. + const standbyOnly = resource({ + agent: { sensors: { smart: [{ ...retainedDisk(52), standby: true }] } }, + }); + expect(getAgentMachineTemperatureCelsius(standbyOnly)).toBeUndefined(); + expect(isAgentMachineTemperatureLastKnown(standbyOnly)).toBe(false); + expect(getAgentMachineTemperatureTitle(standbyOnly)).toBe( + 'Disk Temperatures\nDisk /dev/sda: standby', + ); + + // A direct machine temperature leads a retained disk. + const direct = resource({ + temperature: 47, + agent: { sensors: { smart: [retainedDisk(71)] } }, + }); + expect(getAgentMachineTemperatureCelsius(direct)).toBe(47); + expect(isAgentMachineTemperatureLastKnown(direct)).toBe(false); + expect(getAgentMachineTemperatureMetric(direct)).toBe('temperature'); + + // Additional sensors never lead the cell, so the retained disk is shown. + const additionalOnly = resource({ + agent: { sensors: { additional: { vrm: 49 }, smart: [retainedDisk(71)] } }, + }); + expect(getAgentMachineTemperatureCelsius(additionalOnly)).toBe(71); + expect(isAgentMachineTemperatureLastKnown(additionalOnly)).toBe(true); + + // Sorting follows the value the cell shows: a hotter retained disk does not + // lift a machine whose shown reading is a cooler current one. + const machines = [ + resource({ + id: 'retained', + name: 'Retained', + agent: { sensors: { smart: [retainedDisk(71)] } }, + }), + resource({ + id: 'mixed', + name: 'Mixed', + agent: { + sensors: { smart: [retainedDisk(80), { device: '/dev/sdb', temperature: 35 }] }, + }, + }), + resource({ + id: 'current', + name: 'Current', + agent: { sensors: { smart: [{ device: '/dev/sda', temperature: 50 }] } }, + }), + ]; + const sortedIds = (direction: 'asc' | 'desc') => + sortAgentMachines( + machines, + 'temp', + direction, + () => '', + () => '', + ).map((machine) => machine.id); + expect(sortedIds('desc')).toEqual(['retained', 'current', 'mixed']); + expect(sortedIds('asc')).toEqual(['mixed', 'current', 'retained']); + }); + it('prefers direct and sensor temperatures over SMART fallback temperatures', () => { const direct = resource({ temperature: 55, diff --git a/frontend-modern/src/features/standalone/agentMachineTableModel.ts b/frontend-modern/src/features/standalone/agentMachineTableModel.ts index 5fe81f91f9..661e02c86c 100644 --- a/frontend-modern/src/features/standalone/agentMachineTableModel.ts +++ b/frontend-modern/src/features/standalone/agentMachineTableModel.ts @@ -2,6 +2,7 @@ import type { ColumnDef } from '@/hooks/useColumnVisibility'; import type { HostDiskIO, HostRAIDArray, HostRAIDDevice } from '@/types/api'; import type { Resource } from '@/types/resource'; import { getPlatformTableFiniteMetric } from '@/features/platformPage/sharedPlatformPage'; +import { isPhysicalDiskTemperatureCurrent } from '@/features/storageBackups/diskTemperaturePresentation'; import { formatBytes, normalizeDiskArray } from '@/utils/format'; import { asTrimmedString } from '@/utils/stringUtils'; @@ -155,6 +156,9 @@ type TemperatureReading = { type SmartTemperatureReading = TemperatureReading & { standby?: boolean; diskType?: string; + // False for a retained value, such as the last temperature of a host agent + // that stopped reporting. + current: boolean; }; export type AgentMachineTemperatureDetailRow = { @@ -257,13 +261,21 @@ const getSmartTemperatureReadings = (machine: Resource): SmartTemperatureReading value: temperature ?? 0, standby: disk.standby, diskType: asTrimmedString(disk.type), + current: isPhysicalDiskTemperatureCurrent(disk.collection), }); return readings; }, []); -const getActiveSmartTemperatureReadings = (machine: Resource): TemperatureReading[] => +// Only a temperature collected now can stand for the machine or pick its +// threshold. A retained one stays in the detail rows, marked last known. +const getActiveSmartTemperatureReadings = (machine: Resource): SmartTemperatureReading[] => getSmartTemperatureReadings(machine).filter( - (reading): reading is TemperatureReading => !reading.standby && reading.value > 0, + (reading) => !reading.standby && reading.value > 0 && reading.current, + ); + +const getLastKnownSmartTemperatureReadings = (machine: Resource): SmartTemperatureReading[] => + getSmartTemperatureReadings(machine).filter( + (reading) => !reading.standby && reading.value > 0 && !reading.current, ); const byHighestTemperature = (left: TemperatureReading, right: TemperatureReading): number => @@ -536,22 +548,38 @@ export const getAgentMachineDiskIODetails = (machine: Resource): AgentMachineDis }, []); }; +const hasMachineSensorTemperature = (machine: Resource): boolean => + positiveTemperature(machine.temperature) !== undefined || + getSensorTemperatureReadings(machine).length > 0; + +// The SMART readings the cell falls back to: the ones collected now, or the +// retained ones when no disk has a current reading. +const getCellSmartTemperatureReadings = (machine: Resource): SmartTemperatureReading[] => { + const active = getActiveSmartTemperatureReadings(machine); + return active.length > 0 ? active : getLastKnownSmartTemperatureReadings(machine); +}; + export const getAgentMachineTemperatureCelsius = (machine: Resource): number | undefined => { const direct = positiveTemperature(machine.temperature); if (direct !== undefined) return direct; return ( maxTemperatureReading(getSensorTemperatureReadings(machine)) ?? - maxTemperatureReading(getActiveSmartTemperatureReadings(machine)) + maxTemperatureReading(getCellSmartTemperatureReadings(machine)) ); }; -// Disk type of the hottest active SMART reading — the disk whose temperature -// the cell displays when it falls back to the diskTemperature metric. +// True when the cell's value is a retained SMART reading, such as the last +// disk temperature of a host agent that stopped reporting. +export const isAgentMachineTemperatureLastKnown = (machine: Resource): boolean => + !hasMachineSensorTemperature(machine) && + getActiveSmartTemperatureReadings(machine).length === 0 && + getLastKnownSmartTemperatureReadings(machine).length > 0; + +// Disk type of the hottest SMART reading the cell displays when it falls back +// to the diskTemperature metric. export const getAgentMachineHottestSmartDiskType = (machine: Resource): string | undefined => { - const readings = getSmartTemperatureReadings(machine).filter( - (reading) => !reading.standby && reading.value > 0, - ); + const readings = getCellSmartTemperatureReadings(machine); if (readings.length === 0) return undefined; return readings.reduce((worst, reading) => (reading.value > worst.value ? reading : worst)) .diskType; @@ -560,13 +588,8 @@ export const getAgentMachineHottestSmartDiskType = (machine: Resource): string | export const getAgentMachineTemperatureMetric = ( machine: Resource, ): AgentMachineTemperatureMetric => { - if (positiveTemperature(machine.temperature) !== undefined) return 'temperature'; - if (maxTemperatureReading(getSensorTemperatureReadings(machine)) !== undefined) { - return 'temperature'; - } - if (maxTemperatureReading(getActiveSmartTemperatureReadings(machine)) !== undefined) { - return 'diskTemperature'; - } + if (hasMachineSensorTemperature(machine)) return 'temperature'; + if (getCellSmartTemperatureReadings(machine).length > 0) return 'diskTemperature'; return 'temperature'; }; @@ -585,6 +608,13 @@ export const getAgentMachineTemperatureDetailSections = ( label: `Disk ${reading.label}`, value: formatTemperatureValue(reading), })); + const lastKnownSmartReadings = getLastKnownSmartTemperatureReadings(machine) + .sort(byHighestTemperature) + .map((reading) => ({ + label: `Disk ${reading.label}`, + value: `${formatTemperatureValue(reading)} (last known)`, + muted: true, + })); const standbySmartReadings = getSmartTemperatureReadings(machine) .filter((reading) => reading.standby) .sort(byLabel) @@ -609,7 +639,11 @@ export const getAgentMachineTemperatureDetailSections = ( ...section('Temperatures', capTemperatureRows(sensorReadings)), ...section( 'Disk Temperatures', - capTemperatureRows([...activeSmartReadings, ...standbySmartReadings]), + capTemperatureRows([ + ...activeSmartReadings, + ...lastKnownSmartReadings, + ...standbySmartReadings, + ]), ), ...section('Fan Speeds', capTemperatureRows(fanReadings)), ...section('Other Sensors', capTemperatureRows(additionalReadings)), diff --git a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts index a18d7ccdfe..c3c13377bd 100644 --- a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts +++ b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts @@ -18,6 +18,7 @@ import { PHYSICAL_DISK_SOURCE_BADGE_CLASS, PHYSICAL_DISK_TABLE_CLASS, PHYSICAL_DISK_TABLE_ROW_HOVER_CLASS, + PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS, getPhysicalDiskEmptyStatePresentation, getPhysicalDiskCellPaddingClass, getPhysicalDiskColumnWidthStyle, @@ -27,6 +28,7 @@ import { getPhysicalDiskHealthStatus, getPhysicalDiskHealthSummary, getPhysicalDiskHostLabel, + getPhysicalDiskLastKnownTemperatureTitle, getPhysicalDiskLifeLabel, getPhysicalDiskLifeTextClass, getPhysicalDiskNormalizedHealth, @@ -48,6 +50,7 @@ import { normalizePhysicalDiskFacetFilter, type PhysicalDiskPresentationData, } from '@/features/storageBackups/diskPresentation'; +import * as diskTemperaturePresentation from '@/features/storageBackups/diskTemperaturePresentation'; function makeDiskData( overrides: Partial = {}, @@ -131,6 +134,23 @@ describe('diskPresentation', () => { } }); + it('serves the same temperature decision the Machines table and drawers import directly', () => { + // Tables outside Storage import the small module so they do not pull this + // one into their chunks. Both paths must stay one implementation. + expect(isPhysicalDiskTemperatureCurrent).toBe( + diskTemperaturePresentation.isPhysicalDiskTemperatureCurrent, + ); + expect(getPhysicalDiskTemperaturePresentation).toBe( + diskTemperaturePresentation.getPhysicalDiskTemperaturePresentation, + ); + expect(getPhysicalDiskLastKnownTemperatureTitle).toBe( + diskTemperaturePresentation.getPhysicalDiskLastKnownTemperatureTitle, + ); + expect(PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS).toBe( + diskTemperaturePresentation.PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS, + ); + }); + it('distinguishes unsupported, unavailable, and unexpectedly missing disk evidence', () => { expect( getPhysicalDiskFieldStatusMessage('Disk I/O', { diff --git a/frontend-modern/src/features/storageBackups/diskPresentation.ts b/frontend-modern/src/features/storageBackups/diskPresentation.ts index 2499be5969..fe1ad5ad4e 100644 --- a/frontend-modern/src/features/storageBackups/diskPresentation.ts +++ b/frontend-modern/src/features/storageBackups/diskPresentation.ts @@ -14,7 +14,6 @@ import { getAllFilterOptionLabel } from '@/components/shared/filterOptionPresent import { getPhysicalDiskNodeIdentity } from '@/components/Storage/diskResourceUtils'; import { getInfrastructureSettingsLocationLabel } from '@/utils/infrastructureSettingsPresentation'; import { normalizeStorageSourceKey, storageSourceMatchesFilter } from '@/utils/storageSources'; -import { formatTemperature } from '@/utils/temperature'; import type { NormalizedHealth, StorageHealthFilter } from './models'; import { matchesStorageNodeTerms, parseStorageSearchQuery } from './storageSearchQuery'; @@ -254,11 +253,6 @@ export const PHYSICAL_DISK_HEALTH_WRAP_CLASS = // that would compete with it for the track. export const PHYSICAL_DISK_HEALTH_LABEL_CLASS = 'shrink-0 text-[11px] font-semibold'; export const PHYSICAL_DISK_TEMPERATURE_CLASS = 'text-[11px] font-medium'; -// A retained reading takes no threshold colour, so it cannot read as a disk -// running hot or cool right now. The dotted underline marks the value as one -// its title explains. -export const PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS = - 'text-muted underline decoration-dotted underline-offset-2 cursor-help'; export const PHYSICAL_DISK_SIZE_VALUE_CLASS = 'text-[11px] text-base-content'; const titleize = (value: string | undefined | null): string => @@ -473,44 +467,15 @@ export function getPhysicalDiskFieldStatusMessage( } } -// Normalization may keep the last known temperature when the current -// observation is not available, such as a disk in standby or a host agent that -// stopped reporting. The value is a current reading only when its collection -// state is available, or when the source predates collection state. -export function isPhysicalDiskTemperatureCurrent( - collection: PhysicalDiskCollectionStatus | null | undefined, -): boolean { - const state = collection?.temperature?.state; - return !state || state === 'available'; -} - -export function getPhysicalDiskLastKnownTemperatureTitle( - status: PhysicalDiskFieldStatus | null | undefined, -): string { - const reason = status?.reason?.trim(); - return reason ? `Last known reading, not current: ${reason}` : 'Last known reading, not current'; -} - -export interface PhysicalDiskTemperaturePresentation { - label: string; - current: boolean; - /** Set only for a last-known reading: why the value is not current. */ - title?: string; -} - -export function getPhysicalDiskTemperaturePresentation( - disk: Pick, -): PhysicalDiskTemperaturePresentation | null { - if (!Number.isFinite(disk.temperature) || disk.temperature <= 0) return null; - const current = isPhysicalDiskTemperatureCurrent(disk.collection); - return { - label: formatTemperature(disk.temperature), - current, - title: current - ? undefined - : getPhysicalDiskLastKnownTemperatureTitle(disk.collection?.temperature), - }; -} +// Whether a disk temperature is a current reading lives in its own module so +// tables outside the Storage page can use it without this one. +export { + PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS, + getPhysicalDiskLastKnownTemperatureTitle, + getPhysicalDiskTemperaturePresentation, + isPhysicalDiskTemperatureCurrent, + type PhysicalDiskTemperaturePresentation, +} from './diskTemperaturePresentation'; export function getPhysicalDiskCollectionMessages(disk: PhysicalDiskPresentationData): string[] { const collection = disk.collection; diff --git a/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts b/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts new file mode 100644 index 0000000000..acb63c612a --- /dev/null +++ b/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts @@ -0,0 +1,53 @@ +import type { PhysicalDiskCollectionStatus, PhysicalDiskFieldStatus } from '@/types/resource'; +import { formatTemperature } from '@/utils/temperature'; + +// The canonical decision on whether a disk temperature is a current reading. +// It lives apart from diskPresentation so the Machines table, the machine +// drawer and the guest drawer can use it without pulling the Storage page +// presentation module into their chunks. + +// A retained reading takes no threshold colour, so it cannot read as a disk +// running hot or cool right now. The dotted underline marks the value as one +// its title explains. +export const PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS = + 'text-muted underline decoration-dotted underline-offset-2 cursor-help'; + +// Normalization may keep the last known temperature when the current +// observation is not available, such as a disk in standby or a host agent that +// stopped reporting. The value is a current reading only when its collection +// state is available, or when the source predates collection state. +export function isPhysicalDiskTemperatureCurrent( + collection: PhysicalDiskCollectionStatus | null | undefined, +): boolean { + const state = collection?.temperature?.state; + return !state || state === 'available'; +} + +export function getPhysicalDiskLastKnownTemperatureTitle( + status: PhysicalDiskFieldStatus | null | undefined, +): string { + const reason = status?.reason?.trim(); + return reason ? `Last known reading, not current: ${reason}` : 'Last known reading, not current'; +} + +export interface PhysicalDiskTemperaturePresentation { + label: string; + current: boolean; + /** Set only for a last-known reading: why the value is not current. */ + title?: string; +} + +export function getPhysicalDiskTemperaturePresentation(disk: { + temperature: number; + collection?: PhysicalDiskCollectionStatus | null; +}): PhysicalDiskTemperaturePresentation | null { + if (!Number.isFinite(disk.temperature) || disk.temperature <= 0) return null; + const current = isPhysicalDiskTemperatureCurrent(disk.collection); + return { + label: formatTemperature(disk.temperature), + current, + title: current + ? undefined + : getPhysicalDiskLastKnownTemperatureTitle(disk.collection?.temperature), + }; +} diff --git a/frontend-modern/src/types/api.ts b/frontend-modern/src/types/api.ts index 5b15015d7a..eaa1bfcc3f 100644 --- a/frontend-modern/src/types/api.ts +++ b/frontend-modern/src/types/api.ts @@ -626,6 +626,9 @@ export interface HostDiskSMART { temperature: number; // Temperature in Celsius health?: string; // PASSED, FAILED, UNKNOWN standby?: boolean; // True if disk was in standby + // A non-available temperature state marks the value as last known, such as + // a host agent that stopped reporting. + collection?: import('./resource').PhysicalDiskCollectionStatus; } export interface HostRAIDArray { From 38954b0b714a0a52917ba9a5f64404ee553b306a Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:23:17 +0100 Subject: [PATCH 32/53] Give Docker containers and services without IDs their own alert refs A container reported without an ID alerted under its Docker host's own reference (docker:). Its alerts showed as host alerts, joined the host's resource history, were never cleared when the container left the report, and made the host's Patrol scope alias ambiguous. A Swarm service without an ID put its normalized name in the service-ID slot, so a name equal to another service's ID shared that service's alerts and history, and the history resolver had to treat the whole cluster as a conflict. Both now alert under their name, marked in the ID position (docker:/name:, docker:/service/name:). Docker and Swarm IDs never contain a colon, so these cannot equal a host or ID reference, and one with neither an ID nor a name raises no alert. The policy switches, cleanup, Patrol aliases and the history resolver use the same builders and parse the same shapes, so the resolver no longer needs the cluster-wide conflict. A pre-upgrade container alert left on the host's reference is cleared on the host's next report, and a pending image update recorded there keeps its age. Canonical state IDs were split at the first "::", but Docker hosts that collide are disambiguated as "::". Every container alert of such a host was rewritten to the reference of the host named , so it joined that host's history and skipped its own host's cleanup. State IDs now split before the alert's recorded spec ID, or after its own resource ID, so alerts saved that way also recover on restore. --- .../v6/internal/subsystems/ai-runtime.md | 14 + .../v6/internal/subsystems/alerts.md | 39 ++ .../v6/internal/subsystems/monitoring.md | 9 +- .../internal/subsystems/unified-resources.md | 25 +- internal/ai/patrol_run_test.go | 19 +- internal/ai/patrol_state.go | 33 +- internal/ai/patrol_state_scope_alias_test.go | 11 +- internal/alerts/alerts_test.go | 373 +++++++++++++++--- internal/alerts/canonical_identity.go | 45 ++- .../cleanup_observation_retention_test.go | 2 +- internal/alerts/config_runtime.go | 40 +- internal/alerts/docker.go | 101 +++-- internal/alerts/docker_override_migration.go | 2 +- internal/alerts/unified_eval_test.go | 2 +- internal/alerts/update_alerts_test.go | 12 +- .../monitoring/monitor_alert_handling_test.go | 28 +- internal/unifiedresources/history_identity.go | 54 ++- .../unifiedresources/history_identity_test.go | 44 ++- 18 files changed, 646 insertions(+), 207 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 56b2a47c66..ad2bb34f87 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -6508,6 +6508,20 @@ Patrol scope payloads. API-backed TrueNAS systems may still keep `truenas` platform metadata and separate run-history coverage counts, but AI resource type fields must normalize to canonical `agent` once they cross the governed runtime boundary. +Patrol scope resolution registers the alert subsystem's Docker references as +aliases, built by the same `internal/alerts` builders rather than formatted +locally: `DockerHostResourceID` on the Docker host record, and on each +container record `dockerAlertScopeAlias`, which passes the container's name so +a container reported without an ID answers to its marked name reference +(`docker:/name:`) and never to its host's reference. +`dockerServiceAlertScopeAliases` registers service references, including +`docker:/service/name:` for a service without an ID, on the +owning host. The legacy snapshot record of a container without an ID has no ID +to scope by, so only the unified read state can scope its alerts. Proof: +`TestResolvePatrolScopeResolvesDockerAlertResourceIDs` in +`internal/ai/patrol_run_test.go` and +`TestDockerServiceAlertScopeAliasesDeriveIDFromName` in +`internal/ai/patrol_state_scope_alias_test.go`. The same governed-context rule also applies to the main unified AI resource overview: infrastructure, workload, alert-label, and top-consumer summaries must not leak raw resource names, cluster labels, IP addresses, or unresolved diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index c80daddd90..a600448ef0 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -1477,6 +1477,45 @@ short-ID, unified-hash, and slash-tail forms as trailing lookup candidates). Container override work must not reintroduce a runtime-container-ID persistence key. +Docker alert references come only from the builders in +`internal/alerts/docker.go`: a host's own alert `docker:` +(`DockerHostResourceID`, used only by its connectivity alert), a container +`docker:/` (`DockerContainerResourceID`), and a Swarm +service `docker:/service/` (`DockerServiceResourceID`). +A container or service reported without an ID alerts under its name, marked in +the ID position (`docker:/name:`, +`docker:/service/name:`). Docker and Swarm IDs never +contain a colon, so a name reference never takes its host's reference or +another service's ID reference, and one with neither an ID nor a name raises +no alert. Report cleanup, host removal and the host's offline alert firing +clear children by the `docker:/` prefix; the container and service +policy switches classify by the first path segment after the host +(`dockerAlertResourcePath`), so a container name containing `/service/` stays +a container; and Patrol's scope aliases and resource history parse the same +forms. Service overrides key on the same reference, so an override written for +a service without an ID under its old unmarked name no longer applies. A +reference can contain the canonical state separator (a Docker host +disambiguated as `::`), so `splitCanonicalStateID` splits a +state ID before the alert's recorded spec ID, or else after its resource ID, +and only without either at the first `::`. Splitting at the first `::` gave +such a host's alerts the reference of the host named ``; an alert saved +that way recovers its reference on restore because its spec ID is intact. +Before name references existed, a container without an ID alerted under its +host's reference. Only the connectivity alert belongs there, so the host's +next report, or its offline alert firing, clears any such alert +left active whose recorded host matches and whose container ID is empty. A +pending image update recorded there keeps its age under the container's new +reference when that first report still shows the update pending; a first +report with missing or failed update evidence clears it, and the delay starts +again on the next positive report. Proof: +`TestDockerAlertsWithoutIDsUseTheirOwnReferences`, +`TestDockerHostIDWithSeparatorKeepsItsAlertReferences`, +`TestDockerAlertsRestoredWithCutShortReferencesRecover`, +`TestDockerContainerWithoutIDKeepsPendingUpdateAgeAcrossUpgrade` and +`TestDockerContainerResourceID` in `internal/alerts/alerts_test.go`, and +`TestOwnerAlertTimelinesUseCanonicalHistoryIdentity` in +`internal/monitoring/monitor_alert_handling_test.go`. + Backup orphan evaluation is also inventory-scoped. The alerts runtime may evaluate recovery rollups for backup age, but unresolved Proxmox PVE backup subjects must not be treated as orphaned until monitoring has supplied the diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index a912bfce62..dc954d645f 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -1065,14 +1065,15 @@ events (ZFS pools and devices, host filesystems, disks, RAID arrays, sensors and the Unraid array), pass through the same writer, which binds them to the Docker host, Swarm service, storage or host named in the unified-resources history identity clause. Container names and shortened container IDs still -bind nothing. `CheckDockerHost` gives a container without an ID its host's -reference, so that container's events join the host's history. A producer +bind nothing, and neither does the marked name a container or Swarm service +reported without an ID alerts under, so its events join neither its host's +history nor that of a service whose ID it is named like. A producer that changes one of these reference shapes must change `dockerHostHistoryReference` or `historySubResourceOwner` with it. `TestOwnerAlertTimelinesUseCanonicalHistoryIdentity` in `internal/monitoring/monitor_alert_handling_test.go` pins the Docker host, -Swarm service, ZFS pool and device, and host filesystem shapes against the -real alert manager. +Swarm service, ID-less container and service, ZFS pool and device, and host +filesystem shapes against the real alert manager. TrueNAS native alert projection preserves the trimmed, uppercase provider level in ResourceIncident.NativeSeverity. INFO and NOTICE retain the same canonical monitor risk; consumers must not lose their distinct actionability when projecting provider evidence. Native CRITICAL, ALERT, and EMERGENCY all project to canonical critical severity; EMERGENCY must not be discarded as unknown or make a still-active condition appear recovered. WARNING remains warning, and INFO and NOTICE remain informational at this projection boundary. diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 862a3f245a..41a4a33a4b 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -5809,17 +5809,20 @@ Docker host alerts (`docker:`) and Swarm service alerts Docker source identities: the host ID is the Docker host's source ID, and a service reference names the service with that exact ID in the host's Swarm cluster, whichever manager reported it. A hostname in the host position is -only ever looked up as a Docker host source ID. A service without an ID is -referenced by its normalized name in the same place, so while its cluster has -such a service every service reference there is a conflict. Alert references -do not record whether they came from an ID, so a row journaled by an ID-less -service can still join another service's history if a later generation or -event binds the same reference; Swarm always reports service IDs, so that needs a malformed -report naming a service exactly like another service's ID. Hostless -`docker-service:` references, container names and shortened container -IDs never bind and are never retried against later inventory. A container -without an ID alerts under its host's reference (`alerts.DockerResourceID`), so -its events join that host's history, never another container's. +only ever looked up as a Docker host source ID. The registry keys a service +without an ID by its raw name, so the service entry must also carry exactly +the referenced ID. A container or service reported without an ID alerts under +its name, marked in the ID position (`docker:/name:` and +`docker:/service/name:`, from `alerts.DockerContainerResourceID` +and `alerts.DockerServiceResourceID`). Docker and Swarm IDs never contain a +colon, so such a reference never equals its host's reference or another +service's ID reference, and `dockerHostHistoryReference` treats a service slot +with a colon as a name. These name references, hostless `docker-service:` +references, container names and shortened container IDs never bind and are +never retried against later inventory. Rows journaled before name references +existed keep the reference they were written under: an ID-less container's +rows stay in its host's history, and an ID-less service's rows bind only to a +service in that cluster whose ID equals its normalized name. Sub-resource alert references name no resource of their own and bind to the owner whose durable ID they carry (`historySubResourceOwner`): ZFS pool and device alerts (`/zfs-pool:[/device:]`) to the diff --git a/internal/ai/patrol_run_test.go b/internal/ai/patrol_run_test.go index e45bd0c944..de708b56b0 100644 --- a/internal/ai/patrol_run_test.go +++ b/internal/ai/patrol_run_test.go @@ -149,12 +149,18 @@ func TestResolvePatrolScopeCanonicalIDOutranksAliasCollision(t *testing.T) { } func TestResolvePatrolScopeResolvesDockerAlertResourceIDs(t *testing.T) { + // A container reported without an ID alerts under its own name reference, + // so the host's reference stays unambiguous. snapshot := models.StateSnapshot{DockerHosts: []models.DockerHost{{ ID: "nas", Hostname: "nas", - Containers: []models.DockerContainer{{ID: "abc123def456", Name: "dockhand-hawser-updater-nas", State: "exited"}}, + Containers: []models.DockerContainer{ + {ID: "abc123def456", Name: "dockhand-hawser-updater-nas", State: "exited"}, + {Name: "legacy-sidecar", State: "exited"}, + }, }}} - containerAlertID := alerts.DockerResourceID("nas", "abc123def456") - hostAlertID := alerts.DockerResourceID("nas", "") + containerAlertID := alerts.DockerContainerResourceID("nas", "abc123def456", "dockhand-hawser-updater-nas") + namedContainerAlertID := alerts.DockerContainerResourceID("nas", "", "legacy-sidecar") + hostAlertID := alerts.DockerHostResourceID("nas") snapshotState := newPatrolRuntimeState(snapshot) registry := unifiedresources.NewRegistry(nil) @@ -165,7 +171,12 @@ func TestResolvePatrolScopeResolvesDockerAlertResourceIDs(t *testing.T) { for _, tc := range []struct { requested string resourceType string - }{{containerAlertID, "app-container"}, {hostAlertID, "docker-host"}} { + }{{containerAlertID, "app-container"}, {namedContainerAlertID, "app-container"}, {hostAlertID, "docker-host"}} { + if tc.requested == namedContainerAlertID && state.readState == nil { + // The legacy snapshot record of a container without an ID has + // no ID to scope by; only the unified read state can. + continue + } resolved, resolution := resolvePatrolScopeState(state, PatrolScope{ResourceIDs: []string{tc.requested}}) if len(resolution.UnmatchedResourceIDs) != 0 || len(resolution.AmbiguousResourceIDs) != 0 { t.Fatalf("alert resource ID %q did not resolve exactly: %+v", tc.requested, resolution) diff --git a/internal/ai/patrol_state.go b/internal/ai/patrol_state.go index 822728ba0a..9b33d909c2 100644 --- a/internal/ai/patrol_state.go +++ b/internal/ai/patrol_state.go @@ -9,32 +9,37 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" ) -// dockerAlertScopeAlias mirrors the alert subsystem's docker resource IDs -// ("docker:" and "docker:/") so an alert's resource ID -// resolves to the patrol runtime record for that host or container (#1699). -// Returns "" without a host part rather than the shared "docker:unknown" -// fallback, which would alias unrelated records together. -func dockerAlertScopeAlias(hostID, containerID string) string { +// dockerAlertScopeAlias mirrors the alert subsystem's Docker container +// resource IDs ("docker:/", or "docker:/name:" +// for a container without an ID) so an alert's resource ID resolves to the +// patrol runtime record for that container (#1699). A host's own records use +// alerts.DockerHostResourceID. Returns "" without a host part rather than a +// hostless form, which would alias unrelated records together. +func dockerAlertScopeAlias(hostID, containerID, containerName string) string { if strings.TrimSpace(hostID) == "" { return "" } - return alerts.DockerResourceID(hostID, containerID) + return alerts.DockerContainerResourceID(hostID, containerID, containerName) } // dockerServiceAlertScopeAliases mirrors the alert subsystem's Swarm service -// resource IDs ("docker:/service/") onto the owning Docker +// resource IDs ("docker:/service/", or +// "docker:/service/name:" without an ID) onto the owning Docker // host record. Patrol has no service-level analysis, and a service's tasks run // as containers on the host, so the host is the smallest unit Patrol can // investigate for a service alert (#1699). Hosts without an ID are skipped for // the same anti-aliasing reason as dockerAlertScopeAlias: the host-less -// "docker-service:" fallback could collide across hosts. +// "docker-service:" fallback could collide across hosts. A service with +// neither an ID nor a name raises no alert, so it has no alias. func dockerServiceAlertScopeAliases(hostID string, services []models.DockerService) []string { if strings.TrimSpace(hostID) == "" || len(services) == 0 { return nil } aliases := make([]string, 0, len(services)) for _, service := range services { - aliases = append(aliases, alerts.DockerServiceResourceID(hostID, service.ID, service.Name)) + if alias := alerts.DockerServiceResourceID(hostID, service.ID, service.Name); alias != "" { + aliases = append(aliases, alias) + } } return aliases } @@ -246,14 +251,14 @@ func patrolVisitRuntimeResources(s patrolRuntimeState, visit func(patrolRuntimeR } } for _, dh := range rs.DockerHosts() { - aliases := append([]string{dh.Name(), dh.Hostname(), dockerAlertScopeAlias(dh.HostSourceID(), "")}, + aliases := append([]string{dh.Name(), dh.Hostname(), alerts.DockerHostResourceID(dh.HostSourceID())}, dockerServiceAlertScopeAliases(dh.HostSourceID(), dh.Services())...) if !emit(patrolRuntimeResourceDockerHost, []string{dh.ID(), dh.HostSourceID()}, aliases...) { return } } for _, dc := range rs.DockerContainers() { - if !emit(patrolRuntimeResourceDockerItem, []string{dc.ID(), dc.ContainerID()}, dc.Name(), dockerAlertScopeAlias(dc.HostSourceID(), dc.ContainerID())) { + if !emit(patrolRuntimeResourceDockerItem, []string{dc.ID(), dc.ContainerID()}, dc.Name(), dockerAlertScopeAlias(dc.HostSourceID(), dc.ContainerID(), dc.Name())) { return } } @@ -299,13 +304,13 @@ func patrolVisitRuntimeResources(s patrolRuntimeState, visit func(patrolRuntimeR } } for _, dh := range s.DockerHosts { - aliases := append([]string{dh.DisplayName, dh.CustomDisplayName, dh.Hostname, dockerAlertScopeAlias(dh.ID, "")}, + aliases := append([]string{dh.DisplayName, dh.CustomDisplayName, dh.Hostname, alerts.DockerHostResourceID(dh.ID)}, dockerServiceAlertScopeAliases(dh.ID, dh.Services)...) if !emit(patrolRuntimeResourceDockerHost, []string{dh.ID}, aliases...) { return } for _, dc := range dh.Containers { - if !emit(patrolRuntimeResourceDockerItem, []string{dc.ID}, dc.Name, dockerAlertScopeAlias(dh.ID, dc.ID)) { + if !emit(patrolRuntimeResourceDockerItem, []string{dc.ID}, dc.Name, dockerAlertScopeAlias(dh.ID, dc.ID, dc.Name)) { return } } diff --git a/internal/ai/patrol_state_scope_alias_test.go b/internal/ai/patrol_state_scope_alias_test.go index 0889178589..c0d9e88b09 100644 --- a/internal/ai/patrol_state_scope_alias_test.go +++ b/internal/ai/patrol_state_scope_alias_test.go @@ -78,16 +78,17 @@ func TestDockerServiceAlertScopeAliasesWithoutServices(t *testing.T) { } // TestDockerServiceAlertScopeAliasesDeriveIDFromName covers services that carry -// no ID, where the canonical builder derives one from the service name. The -// alias has to follow that derivation rather than formatting its own string. +// no ID, where the canonical builder references the marked service name. The +// alias has to follow that derivation rather than formatting its own string, +// and a service with neither an ID nor a name raises no alert, so no alias. func TestDockerServiceAlertScopeAliasesDeriveIDFromName(t *testing.T) { - got := dockerServiceAlertScopeAliases("host-1", []models.DockerService{{Name: "Web API"}}) + got := dockerServiceAlertScopeAliases("host-1", []models.DockerService{{Name: "Web API"}, {Name: " "}}) want := alerts.DockerServiceResourceID("host-1", "", "Web API") if len(got) != 1 || got[0] != want { t.Fatalf("aliases = %v, want [%q]", got, want) } - if got[0] != "docker:host-1/service/web-api" { - t.Errorf("alias = %q, want %q", got[0], "docker:host-1/service/web-api") + if got[0] != "docker:host-1/service/name:web-api" { + t.Errorf("alias = %q, want %q", got[0], "docker:host-1/service/name:web-api") } } diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go index f95e034cbf..677d046d37 100644 --- a/internal/alerts/alerts_test.go +++ b/internal/alerts/alerts_test.go @@ -2720,7 +2720,7 @@ func TestCheckDockerHostIgnoresContainersByPrefix(t *testing.T) { Containers: []models.DockerContainer{container}, } - resourceID := DockerResourceID(host.ID, container.ID) + resourceID := DockerContainerResourceID(host.ID, container.ID, "") alertID := fmt.Sprintf("docker-container-state-%s", resourceID) // Run twice to satisfy the confirmation threshold when not ignored @@ -2759,7 +2759,7 @@ func TestCheckDockerHostContainerCPUUsesCapacityNormalizedPercent(t *testing.T) CPUPercent: 240, }}, } - alertID := canonicalMetricStateID(DockerResourceID(host.ID, "container-1"), "cpu") + alertID := canonicalMetricStateID(DockerContainerResourceID(host.ID, "container-1", ""), "cpu") m.CheckDockerHost(host) m.mu.RLock() @@ -3058,7 +3058,7 @@ func TestDockerContainerStateUsesDockerDefaults(t *testing.T) { m.CheckDockerHost(host) m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, container.ID) + resourceID := DockerContainerResourceID(host.ID, container.ID, "") alertID := fmt.Sprintf("docker-container-state-%s", resourceID) alert, exists := testLookupActiveAlert(t, m, alertID) if !exists { @@ -3091,7 +3091,7 @@ func TestDockerContainerStateRespectsDisableDefault(t *testing.T) { m.CheckDockerHost(host) m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, container.ID) + resourceID := DockerContainerResourceID(host.ID, container.ID, "") alertID := fmt.Sprintf("docker-container-state-%s", resourceID) if testHasActiveAlert(t, m, alertID) { t.Fatalf("did not expect docker container state alert when defaults disable connectivity") @@ -3121,7 +3121,7 @@ func TestDockerContainerMemoryLimitHysteresis(t *testing.T) { m.CheckDockerHost(hostHigh) - resourceID := DockerResourceID(hostID, containerID) + resourceID := DockerContainerResourceID(hostID, containerID, "") alertID := fmt.Sprintf("docker-container-memory-limit-%s", resourceID) alert, exists := testLookupActiveAlert(t, m, alertID) if !exists { @@ -3190,7 +3190,7 @@ func TestDockerContainerDiskUsageAlert(t *testing.T) { m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, host.Containers[0].ID) + resourceID := DockerContainerResourceID(host.ID, host.Containers[0].ID, "") alertID := canonicalMetricStateID(resourceID, "disk") alert, exists := testLookupActiveAlert(t, m, alertID) if !exists { @@ -3782,7 +3782,7 @@ func TestCheckDockerHostIgnoredPrefixClearsExistingAlerts(t *testing.T) { Hostname: "docker-host.local", Containers: []models.DockerContainer{container}, } - resourceID := DockerResourceID(host.ID, container.ID) + resourceID := DockerContainerResourceID(host.ID, container.ID, "") stateAlertID := fmt.Sprintf("docker-container-state-%s", resourceID) healthAlertID := fmt.Sprintf("docker-container-health-%s", resourceID) restartAlertID := fmt.Sprintf("docker-container-restart-loop-%s", resourceID) @@ -3954,20 +3954,25 @@ func TestDockerContainerDisplayName(t *testing.T) { } } -func TestDockerResourceID(t *testing.T) { +func TestDockerContainerResourceID(t *testing.T) { // t.Parallel() tests := []struct { - name string - hostID string - containerID string - want string + name string + hostID string + containerID string + containerName string + want string }{ - {name: "both ids present", hostID: "host1", containerID: "abc", want: "docker:host1/abc"}, + {name: "both ids present", hostID: "host1", containerID: "abc", containerName: "web", want: "docker:host1/abc"}, {name: "trims ids", hostID: " host1 ", containerID: " abc ", want: "docker:host1/abc"}, {name: "missing host id", hostID: "", containerID: "abc", want: "docker:container/abc"}, - {name: "missing container id", hostID: "host1", containerID: "", want: "docker:host1"}, - {name: "both missing", hostID: "", containerID: "", want: "docker:unknown"}, + // A container without an ID never takes its host's reference. + {name: "missing container id uses the marked name", hostID: "host1", containerID: "", containerName: " /web ", want: "docker:host1/name:web"}, + {name: "a name never equals an ID reference", hostID: "host1", containerID: "", containerName: "abc", want: "docker:host1/name:abc"}, + {name: "no id and no name", hostID: "host1", containerID: "", containerName: " / ", want: ""}, + {name: "a name without a host identifies nothing", hostID: "", containerID: "", containerName: "web", want: ""}, + {name: "all missing", want: ""}, } for _, tc := range tests { @@ -3975,11 +3980,255 @@ func TestDockerResourceID(t *testing.T) { t.Run(tc.name, func(t *testing.T) { // t.Parallel() - if got := DockerResourceID(tc.hostID, tc.containerID); got != tc.want { - t.Fatalf("DockerResourceID(%q, %q) = %q, want %q", tc.hostID, tc.containerID, got, tc.want) + if got := DockerContainerResourceID(tc.hostID, tc.containerID, tc.containerName); got != tc.want { + t.Fatalf("DockerContainerResourceID(%q, %q, %q) = %q, want %q", tc.hostID, tc.containerID, tc.containerName, got, tc.want) } }) } + if got := DockerHostResourceID(" host1 "); got != "docker:host1" { + t.Fatalf("DockerHostResourceID = %q, want docker:host1", got) + } + if got := DockerHostResourceID(" "); got != "" { + t.Fatalf("DockerHostResourceID without a host = %q, want empty", got) + } +} + +// Containers and Swarm services reported without an ID alert under their own +// marked name, never under the host's reference or another service's ID +// reference, so cleanup, removal and the policy switches reach them like any +// other container or service. +func TestDockerAlertsWithoutIDsUseTheirOwnReferences(t *testing.T) { + m := newTestManager(t) + config := m.GetConfig() + config.Enabled = true + config.TimeThresholds = map[string]int{"all": 0} + config.SuppressionWindow = 0 + m.UpdateConfig(config) + + containerID := strings.Repeat("c", 64) + serviceID := "x7k2m9q4w1e8r5t3y6u0i2o4p" + host := models.DockerHost{ID: "h1", Hostname: "tower", DisplayName: "tower", Status: "online", LastSeen: time.Now(), + Containers: []models.DockerContainer{ + {ID: containerID, Name: "web", State: "running", Health: "unhealthy"}, + {Name: "/sidecar", State: "running", Health: "unhealthy"}, + // Docker forbids slashes in names, but a report can carry anything. + {Name: "jobs/service/runner", State: "running", Health: "unhealthy"}, + {State: "running", Health: "unhealthy"}, // no identity at all + }, + Services: []models.DockerService{ + {ID: serviceID, Name: "api", DesiredTasks: 2}, + {Name: strings.ToUpper(serviceID), DesiredTasks: 2}, + {DesiredTasks: 2}, + }} + refs := func() map[string]bool { + got := map[string]bool{} + for _, alert := range m.GetActiveAlerts() { + got[alert.ResourceID] = true + } + return got + } + var firedMu sync.Mutex + fired := map[string]bool{} + m.SubscribeLifecycleCallback(func(event LifecycleEvent) { + if event.Type == eventlog.TypeFired && event.Alert != nil { + firedMu.Lock() + fired[event.Alert.ResourceID] = true + firedMu.Unlock() + } + }) + m.CheckDockerHost(host) + want := map[string]bool{ + "docker:h1/" + containerID: true, + "docker:h1/name:sidecar": true, + "docker:h1/name:jobs/service/runner": true, + "docker:h1/service/" + serviceID: true, + "docker:h1/service/name:" + serviceID: true, + } + if got := refs(); !reflect.DeepEqual(got, want) { + t.Fatalf("alert references = %v, want %v", got, want) + } + firedMu.Lock() + if !reflect.DeepEqual(fired, want) { + t.Fatalf("fired alert references = %v, want %v", fired, want) + } + firedMu.Unlock() + + // A container that leaves the report clears under its own reference. + reported := host + reported.Containers = host.Containers[:1] + m.CheckDockerHost(reported) + if got := refs(); got["docker:h1/name:sidecar"] || got["docker:h1/name:jobs/service/runner"] { + t.Fatalf("alerts of removed containers without an ID survived the report: %v", got) + } + + // Before this reference existed such a container alerted under its host's. + // No producer refreshes those, so the next report clears them. + legacy := func() { + m.mu.Lock() + m.activeAlerts["legacy"] = &Alert{ID: "legacy", Type: "docker-container-health", ResourceID: "docker:h1", StartTime: time.Now(), LastSeen: time.Now(), + Metadata: map[string]interface{}{"hostId": "h1", "containerId": "", "containerName": "sidecar"}} + m.mu.Unlock() + } + legacy() + m.CheckDockerHost(host) + if refs()["docker:h1"] { + t.Fatal("a pre-upgrade container alert on the host's reference survived the host's report") + } + // Going offline clears it too, and keeps the host's own alert. + legacy() + for range 3 { + m.HandleDockerHostOffline(host) + } + offline := 0 + for _, alert := range m.GetActiveAlerts() { + if alert.ResourceID == "docker:h1" { + if alert.Type != "docker-host-offline" { + t.Fatalf("a pre-upgrade container alert on the host's reference survived going offline: %+v", alert) + } + offline++ + } + } + if offline != 1 { + t.Fatalf("host offline alerts = %d, want 1", offline) + } + + // The policy switches classify the name references by kind. + m.CheckDockerHost(host) + config = m.GetConfig() + config.DisableAllDockerContainers = true + m.UpdateConfig(config) + if got := refs(); len(got) != 2 || !got["docker:h1/service/"+serviceID] || !got["docker:h1/service/name:"+serviceID] { + t.Fatalf("after disabling containers, alert references = %v", got) + } + config.DisableAllDockerServices = true + m.UpdateConfig(config) + if got := refs(); len(got) != 0 { + t.Fatalf("after disabling services, alert references = %v", got) + } +} + +// A Docker host disambiguated as "::" carries the canonical +// state separator in its ID. Its alerts keep their full reference, so they +// never take the reference of the host named "", and a restored alert +// whose reference was cut short there is not that host's to clear. +func TestDockerHostIDWithSeparatorKeepsItsAlertReferences(t *testing.T) { + m := newTestManager(t) + config := m.GetConfig() + config.Enabled = true + config.TimeThresholds = map[string]int{"all": 0} + m.UpdateConfig(config) + containerID := strings.Repeat("d", 64) + report := func(hostID string) models.DockerHost { + return models.DockerHost{ID: hostID, Hostname: "clone", LastSeen: time.Now(), + Containers: []models.DockerContainer{{ID: containerID, Name: "web", State: "running", Health: "unhealthy"}}} + } + m.CheckDockerHost(report("base::suffix")) + m.CheckDockerHost(report("base")) + got := map[string]string{} + for _, alert := range m.GetActiveAlerts() { + got[alert.Metadata["hostId"].(string)] = alert.ResourceID + } + want := map[string]string{"base": "docker:base/" + containerID, "base::suffix": "docker:base::suffix/" + containerID} + if !reflect.DeepEqual(got, want) { + t.Fatalf("alert references by host = %v, want %v", got, want) + } + + restored := &Alert{ID: "docker:base::suffix/x::docker:base::suffix/x-health", Type: "docker-container-health", ResourceID: "docker:base", + StartTime: time.Now(), LastSeen: time.Now(), Metadata: map[string]interface{}{"hostId": "base::suffix", "containerId": "x"}} + m.mu.Lock() + m.activeAlerts[restored.ID] = restored + m.mu.Unlock() + m.CheckDockerHost(report("base")) + m.mu.RLock() + _, kept := m.activeAlerts[restored.ID] + m.mu.RUnlock() + if !kept { + t.Fatal("host base cleared another host's restored alert as its own pre-upgrade container alert") + } +} + +// Alerts saved before state IDs split after their own spec carry a resource +// ID cut short at the first "::" of a host ID like "::". They +// recover their reference on restore, so the owning host's report clears them +// and carries the age of a pending update. +func TestDockerAlertsRestoredWithCutShortReferencesRecover(t *testing.T) { + m := newTestManager(t) + config := m.GetConfig() + config.Enabled = true + config.TimeThresholds = map[string]int{"all": 0} + config.DockerDefaults.UpdateAlertDelayHours = 24 + m.UpdateConfig(config) + hostRef := "docker:base::suffix" + containerRef := hostRef + "/" + strings.Repeat("e", 64) + startedAt := time.Now().Add(-72 * time.Hour).Truncate(time.Second) + saved := []*Alert{ + {ID: buildCanonicalStateID(hostRef, hostRef+"-image-update"), CanonicalSpecID: hostRef + "-image-update", Type: "docker-container-update", + ResourceID: "docker:base", Level: AlertLevelWarning, StartTime: startedAt, LastSeen: time.Now(), + Metadata: map[string]interface{}{"hostId": "base::suffix", "containerId": "", "containerName": "sidecar"}}, + {ID: buildCanonicalStateID(containerRef, containerRef+"-health"), CanonicalSpecID: containerRef + "-health", Type: "docker-container-health", + ResourceID: "docker:base", Level: AlertLevelCritical, StartTime: startedAt, LastSeen: time.Now(), + Metadata: map[string]interface{}{"hostId": "base::suffix", "containerId": strings.Repeat("e", 64), "containerName": "gone"}}, + } + data, err := json.Marshal(saved) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(m.getAlertsDir(), alertsDirPerm); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(m.getAlertsDir(), "active-alerts.json"), data, alertsFilePerm); err != nil { + t.Fatal(err) + } + if err := m.LoadActiveAlerts(); err != nil { + t.Fatalf("LoadActiveAlerts() error = %v", err) + } + restored := map[string]bool{} + for _, alert := range m.GetActiveAlerts() { + restored[alert.ResourceID] = true + } + if want := map[string]bool{hostRef: true, containerRef: true}; !reflect.DeepEqual(restored, want) { + t.Fatalf("restored references = %v, want %v", restored, want) + } + + m.CheckDockerHost(models.DockerHost{ID: "base::suffix", Hostname: "clone", LastSeen: time.Now(), Containers: []models.DockerContainer{{ + Name: "sidecar", State: "running", + UpdateStatus: &models.DockerContainerUpdateStatus{UpdateAvailable: true, CurrentDigest: "sha256:a", LatestDigest: "sha256:b", LastChecked: time.Now()}, + }}}) + active := m.GetActiveAlerts() + if len(active) != 1 || active[0].ResourceID != hostRef+"/name:sidecar" || !active[0].StartTime.Equal(startedAt) { + t.Fatalf("active alerts = %+v, want only the update on %s/name:sidecar started %s", active, hostRef, startedAt) + } +} + +// Before a container without an ID had a reference of its own, its pending +// image update was recorded under its host's reference. The first report +// after the upgrade keeps that age instead of restarting the delay. +func TestDockerContainerWithoutIDKeepsPendingUpdateAgeAcrossUpgrade(t *testing.T) { + m := newTestManager(t) + config := m.GetConfig() + config.Enabled = true + config.DockerDefaults.UpdateAlertDelayHours = 24 + m.UpdateConfig(config) + startedAt := time.Now().Add(-72 * time.Hour).Truncate(time.Second) + legacyID := buildCanonicalStateID("docker:h1", "docker:h1-image-update") + m.mu.Lock() + m.activeAlerts[legacyID] = &Alert{ID: legacyID, Type: "docker-container-update", ResourceID: "docker:h1", StartTime: startedAt, LastSeen: time.Now(), + Metadata: map[string]interface{}{"hostId": "h1", "containerId": "", "containerName": "sidecar"}} + m.mu.Unlock() + + m.CheckDockerHost(models.DockerHost{ID: "h1", Hostname: "tower", LastSeen: time.Now(), Containers: []models.DockerContainer{{ + Name: "sidecar", State: "running", Image: "nginx:latest", + UpdateStatus: &models.DockerContainerUpdateStatus{UpdateAvailable: true, CurrentDigest: "sha256:a", LatestDigest: "sha256:b", LastChecked: time.Now()}, + }}}) + var updates []Alert + for _, alert := range m.GetActiveAlerts() { + if alert.Type == "docker-container-update" { + updates = append(updates, alert) + } + } + if len(updates) != 1 || updates[0].ResourceID != "docker:h1/name:sidecar" || !updates[0].StartTime.Equal(startedAt) { + t.Fatalf("update alerts = %+v, want one on docker:h1/name:sidecar started %s", updates, startedAt) + } } func TestHasKnownFirmwareBug(t *testing.T) { @@ -6127,81 +6376,89 @@ func TestDockerServiceResourceID(t *testing.T) { expected: "docker-service:svc-123", }, { - name: "derives ID from service name when ID empty", + name: "marks the derived name when ID empty", hostID: "host-1", serviceID: "", serviceName: "My Service", - expected: "docker:host-1/service/my-service", + expected: "docker:host-1/service/name:my-service", }, { name: "special chars in name replaced with dash", hostID: "host-1", serviceID: "", serviceName: "my/service:v1.0", - expected: "docker:host-1/service/my-service-v1-0", + expected: "docker:host-1/service/name:my-service-v1-0", }, { name: "backslash and colon replaced", hostID: "host-1", serviceID: "", serviceName: "path\\to:service", - expected: "docker:host-1/service/path-to-service", + expected: "docker:host-1/service/name:path-to-service", }, { name: "preserves alphanumeric and underscore", hostID: "host-1", serviceID: "", serviceName: "my_service_123", - expected: "docker:host-1/service/my_service_123", + expected: "docker:host-1/service/name:my_service_123", }, { name: "preserves hyphens", hostID: "host-1", serviceID: "", serviceName: "my-service-name", - expected: "docker:host-1/service/my-service-name", + expected: "docker:host-1/service/name:my-service-name", }, { name: "trims leading/trailing dashes and underscores", hostID: "host-1", serviceID: "", serviceName: "---my-service___", - expected: "docker:host-1/service/my-service", + expected: "docker:host-1/service/name:my-service", }, { name: "truncates long derived ID to 32 chars", hostID: "host-1", serviceID: "", serviceName: "this-is-a-very-long-service-name-that-exceeds-the-limit", - expected: "docker:host-1/service/this-is-a-very-long-service-name", + expected: "docker:host-1/service/name:this-is-a-very-long-service-name", }, { - name: "uses 'service' when name is all special chars", + name: "no identity when name is all special chars", hostID: "host-1", serviceID: "", serviceName: "!!!@@@###", - expected: "docker:host-1/service/service", + expected: "", }, { - name: "uses 'service' when both ID and name empty", + name: "no identity when both ID and name empty", hostID: "host-1", serviceID: "", serviceName: "", - expected: "docker:host-1/service/service", + expected: "", }, { - name: "uses 'service' when both ID and name whitespace", + name: "no identity when both ID and name whitespace", hostID: "host-1", serviceID: " ", serviceName: " ", - expected: "docker:host-1/service/service", + expected: "", }, { name: "no host and derived name", hostID: "", serviceID: "", serviceName: "webserver", - expected: "docker-service:webserver", + expected: "docker-service:name:webserver", + }, + { + // A service named like another service's Swarm ID keeps its own reference. + name: "a name never equals an ID reference", + hostID: "host-1", + serviceID: "", + serviceName: "X7K2M9Q4W1E8R5T3Y6U0I2O4P", + expected: "docker:host-1/service/name:x7k2m9q4w1e8r5t3y6u0i2o4p", }, } @@ -13154,7 +13411,7 @@ func TestDockerContainerHealthAlert(t *testing.T) { m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, host.Containers[0].ID) + resourceID := DockerContainerResourceID(host.ID, host.Containers[0].ID, "") alertID := fmt.Sprintf("docker-container-health-%s", resourceID) if testHasActiveAlert(t, m, alertID) { t.Fatal("expected no health alert for healthy container") @@ -13181,7 +13438,7 @@ func TestDockerContainerHealthAlert(t *testing.T) { m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, host.Containers[0].ID) + resourceID := DockerContainerResourceID(host.ID, host.Containers[0].ID, "") alertID := fmt.Sprintf("docker-container-health-%s", resourceID) if testHasActiveAlert(t, m, alertID) { t.Fatal("expected no health alert for container with empty health") @@ -13208,7 +13465,7 @@ func TestDockerContainerHealthAlert(t *testing.T) { m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, host.Containers[0].ID) + resourceID := DockerContainerResourceID(host.ID, host.Containers[0].ID, "") alertID := fmt.Sprintf("docker-container-health-%s", resourceID) if testHasActiveAlert(t, m, alertID) { t.Fatal("expected no health alert for container with none health") @@ -13235,7 +13492,7 @@ func TestDockerContainerHealthAlert(t *testing.T) { m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, host.Containers[0].ID) + resourceID := DockerContainerResourceID(host.ID, host.Containers[0].ID, "") alertID := fmt.Sprintf("docker-container-health-%s", resourceID) if testHasActiveAlert(t, m, alertID) { t.Fatal("expected no health alert for starting container") @@ -13262,7 +13519,7 @@ func TestDockerContainerHealthAlert(t *testing.T) { m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, host.Containers[0].ID) + resourceID := DockerContainerResourceID(host.ID, host.Containers[0].ID, "") alertID := fmt.Sprintf("docker-container-health-%s", resourceID) alert, exists := testLookupActiveAlert(t, m, alertID) if !exists { @@ -13302,7 +13559,7 @@ func TestDockerContainerHealthAlert(t *testing.T) { m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, host.Containers[0].ID) + resourceID := DockerContainerResourceID(host.ID, host.Containers[0].ID, "") alertID := fmt.Sprintf("docker-container-health-%s", resourceID) alert, exists := testLookupActiveAlert(t, m, alertID) if !exists { @@ -13333,7 +13590,7 @@ func TestDockerContainerHealthAlert(t *testing.T) { m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, host.Containers[0].ID) + resourceID := DockerContainerResourceID(host.ID, host.Containers[0].ID, "") alertID := fmt.Sprintf("docker-container-health-%s", resourceID) if !testHasActiveAlert(t, m, alertID) { t.Fatal("expected running unhealthy container to raise a health alert") @@ -13376,7 +13633,7 @@ func TestDockerContainerHealthAlert(t *testing.T) { m.CheckDockerHost(hostUnhealthy) - resourceID := DockerResourceID(hostID, containerID) + resourceID := DockerContainerResourceID(hostID, containerID, "") alertID := fmt.Sprintf("docker-container-health-%s", resourceID) if !testHasActiveAlert(t, m, alertID) { t.Fatal("expected health alert to be raised") @@ -13427,7 +13684,7 @@ func TestDockerContainerOOMKillAlert(t *testing.T) { m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, host.Containers[0].ID) + resourceID := DockerContainerResourceID(host.ID, host.Containers[0].ID, "") alertID := fmt.Sprintf("docker-container-oom-%s", resourceID) if testHasActiveAlert(t, m, alertID) { t.Fatal("expected no OOM alert for running container") @@ -13454,7 +13711,7 @@ func TestDockerContainerOOMKillAlert(t *testing.T) { m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, host.Containers[0].ID) + resourceID := DockerContainerResourceID(host.ID, host.Containers[0].ID, "") alertID := fmt.Sprintf("docker-container-oom-%s", resourceID) if testHasActiveAlert(t, m, alertID) { t.Fatal("expected no OOM alert for container with exit code 1") @@ -13478,7 +13735,7 @@ func TestDockerContainerOOMKillAlert(t *testing.T) { m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, host.Containers[0].ID) + resourceID := DockerContainerResourceID(host.ID, host.Containers[0].ID, "") if testHasActiveAlert(t, m, fmt.Sprintf("docker-container-oom-%s", resourceID)) { t.Fatal("exit code 137 alone must not create an OOM alert") } @@ -13508,7 +13765,7 @@ func TestDockerContainerOOMKillAlert(t *testing.T) { m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, host.Containers[0].ID) + resourceID := DockerContainerResourceID(host.ID, host.Containers[0].ID, "") alertID := fmt.Sprintf("docker-container-oom-%s", resourceID) alert, exists := testLookupActiveAlert(t, m, alertID) if !exists { @@ -13552,7 +13809,7 @@ func TestDockerContainerOOMKillAlert(t *testing.T) { m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, host.Containers[0].ID) + resourceID := DockerContainerResourceID(host.ID, host.Containers[0].ID, "") alertID := fmt.Sprintf("docker-container-oom-%s", resourceID) alert, exists := testLookupActiveAlert(t, m, alertID) if !exists { @@ -13588,7 +13845,7 @@ func TestDockerContainerOOMKillAlert(t *testing.T) { // First check - should create alert m.CheckDockerHost(host) - resourceID := DockerResourceID(hostID, containerID) + resourceID := DockerContainerResourceID(hostID, containerID, "") alertID := fmt.Sprintf("docker-container-oom-%s", resourceID) alert1, exists := testLookupActiveAlert(t, m, alertID) if !exists { @@ -13633,7 +13890,7 @@ func TestDockerContainerOOMKillAlert(t *testing.T) { m.CheckDockerHost(hostOOM) - resourceID := DockerResourceID(hostID, containerID) + resourceID := DockerContainerResourceID(hostID, containerID, "") alertID := fmt.Sprintf("docker-container-oom-%s", resourceID) if !testHasActiveAlert(t, m, alertID) { t.Fatal("expected OOM alert to be raised") @@ -13687,7 +13944,7 @@ func TestDockerContainerOOMKillAlert(t *testing.T) { m.CheckDockerHost(hostOOM) - resourceID := DockerResourceID(hostID, containerID) + resourceID := DockerContainerResourceID(hostID, containerID, "") alertID := fmt.Sprintf("docker-container-oom-%s", resourceID) if !testHasActiveAlert(t, m, alertID) { t.Fatal("expected OOM alert to be raised") @@ -13738,7 +13995,7 @@ func TestDockerContainerRestartLoopAlert(t *testing.T) { m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, host.Containers[0].ID) + resourceID := DockerContainerResourceID(host.ID, host.Containers[0].ID, "") alertID := fmt.Sprintf("docker-container-restart-loop-%s", resourceID) if testHasActiveAlert(t, m, alertID) { t.Fatal("expected no restart loop alert on first check (just initializes tracking)") @@ -13786,7 +14043,7 @@ func TestDockerContainerRestartLoopAlert(t *testing.T) { // Third check - still same restart count m.CheckDockerHost(host) - resourceID := DockerResourceID(hostID, containerID) + resourceID := DockerContainerResourceID(hostID, containerID, "") alertID := fmt.Sprintf("docker-container-restart-loop-%s", resourceID) if testHasActiveAlert(t, m, alertID) { t.Fatal("expected no restart loop alert for stable container") @@ -13827,7 +14084,7 @@ func TestDockerContainerRestartLoopAlert(t *testing.T) { host.Containers[0].RestartCount = 3 m.CheckDockerHost(host) - resourceID := DockerResourceID(hostID, containerID) + resourceID := DockerContainerResourceID(hostID, containerID, "") alertID := fmt.Sprintf("docker-container-restart-loop-%s", resourceID) if testHasActiveAlert(t, m, alertID) { t.Fatal("expected no restart loop alert when restarts <= threshold") @@ -13873,7 +14130,7 @@ func TestDockerContainerRestartLoopAlert(t *testing.T) { host.Containers[0].RestartCount = 4 m.CheckDockerHost(host) - resourceID := DockerResourceID(hostID, containerID) + resourceID := DockerContainerResourceID(hostID, containerID, "") alertID := fmt.Sprintf("docker-container-restart-loop-%s", resourceID) alert, exists := testLookupActiveAlert(t, m, alertID) if !exists { @@ -13909,7 +14166,7 @@ func TestDockerContainerRestartLoopAlert(t *testing.T) { hostID := "host-restart-5" containerID := "container-5" - resourceID := DockerResourceID(hostID, containerID) + resourceID := DockerContainerResourceID(hostID, containerID, "") // Manually set up a restart loop state m.mu.Lock() @@ -13989,7 +14246,7 @@ func TestDockerContainerRestartLoopAlert(t *testing.T) { // First check - initializes m.CheckDockerHost(host) - resourceID := DockerResourceID(hostID, containerID) + resourceID := DockerContainerResourceID(hostID, containerID, "") alertID := fmt.Sprintf("docker-container-restart-loop-%s", resourceID) // Restart 1 @@ -14042,7 +14299,7 @@ func TestDockerContainerRestartLoopAlert(t *testing.T) { host.Containers[0].RestartCount = 5 m.CheckDockerHost(host) - resourceID := DockerResourceID(hostID, containerID) + resourceID := DockerContainerResourceID(hostID, containerID, "") alertID := fmt.Sprintf("docker-container-restart-loop-%s", resourceID) alert1, exists := testLookupActiveAlert(t, m, alertID) if !exists { @@ -20293,7 +20550,7 @@ func TestDockerContainerOverrideSurvivesContainerRecreate(t *testing.T) { } } stateAlertID := func(hostID, containerID string) string { - return fmt.Sprintf("docker-container-state-%s", DockerResourceID(hostID, containerID)) + return fmt.Sprintf("docker-container-state-%s", DockerContainerResourceID(hostID, containerID, "")) } // Subtests keep the two managers in separate lifetimes: newTestManager @@ -20345,13 +20602,13 @@ func TestDockerContainerOverrideLegacyIDKeyStillHonoured(t *testing.T) { } m.mu.Lock() - m.config.Overrides[DockerResourceID(host.ID, container.ID)] = ThresholdConfig{Disabled: true} + m.config.Overrides[DockerContainerResourceID(host.ID, container.ID, "")] = ThresholdConfig{Disabled: true} m.mu.Unlock() m.CheckDockerHost(host) m.CheckDockerHost(host) - alertID := fmt.Sprintf("docker-container-state-%s", DockerResourceID(host.ID, container.ID)) + alertID := fmt.Sprintf("docker-container-state-%s", DockerContainerResourceID(host.ID, container.ID, "")) if testHasActiveAlert(t, m, alertID) { t.Fatalf("expected legacy ID-keyed override to still disable container alerts") } diff --git a/internal/alerts/canonical_identity.go b/internal/alerts/canonical_identity.go index e02961a42c..e6365029d8 100644 --- a/internal/alerts/canonical_identity.go +++ b/internal/alerts/canonical_identity.go @@ -16,6 +16,27 @@ func buildCanonicalStateID(resourceID, specID string) string { return resourceID + canonicalStateSeparator + specID } +// splitCanonicalStateID splits a canonical state ID into its resource and +// spec parts. A resource ID can itself contain the separator (a Docker host +// disambiguated as "::"), so an ID that ends with the alert's +// spec ID splits before it, and one that starts with the alert's resource ID +// splits after it. Only without either does it split at the first separator. +// The spec comes first: an alert restored from before this rule can carry a +// resource ID that was cut short at the first separator. +func splitCanonicalStateID(id, resourceID, specID string) (resource, spec string, ok bool) { + if specID = strings.TrimSpace(specID); specID != "" { + if prefix, found := strings.CutSuffix(id, canonicalStateSeparator+specID); found && prefix != "" { + return prefix, specID, true + } + } + if resourceID = strings.TrimSpace(resourceID); resourceID != "" { + if rest, found := strings.CutPrefix(id, resourceID+canonicalStateSeparator); found && rest != "" { + return resourceID, rest, true + } + } + return strings.Cut(id, canonicalStateSeparator) +} + func canonicalMetricSpecID(resourceID, metric string) string { if resourceID == "" || metric == "" { return "" @@ -149,17 +170,24 @@ func deriveCanonicalIdentity(alert *Alert) canonicalIdentity { } } +// canonicalSpecIDHint is the spec ID an alert already records, on the alert +// or in its metadata. +func canonicalSpecIDHint(alert *Alert) string { + if alert.CanonicalSpecID != "" { + return alert.CanonicalSpecID + } + specID, _ := alert.Metadata["canonicalSpecID"].(string) + return specID +} + func inferCanonicalResourceIDFromLegacyAlert(alert *Alert) string { if alert == nil { return "" } id := alert.ID - if strings.Contains(id, canonicalStateSeparator) { - parts := strings.SplitN(id, canonicalStateSeparator, 2) - if len(parts) == 2 { - return parts[0] - } + if resource, _, ok := splitCanonicalStateID(id, alert.ResourceID, canonicalSpecIDHint(alert)); ok { + return resource } switch { case strings.HasPrefix(id, "host-offline-"): @@ -230,11 +258,8 @@ func inferCanonicalSpecIDFromLegacyAlert(alert *Alert) string { return "" } id := alert.ID - if strings.Contains(id, canonicalStateSeparator) { - parts := strings.SplitN(id, canonicalStateSeparator, 2) - if len(parts) == 2 { - return parts[1] - } + if _, spec, ok := splitCanonicalStateID(id, alert.ResourceID, canonicalSpecIDHint(alert)); ok { + return spec } resourceID := inferCanonicalResourceIDFromLegacyAlert(alert) switch { diff --git a/internal/alerts/cleanup_observation_retention_test.go b/internal/alerts/cleanup_observation_retention_test.go index 9670acc63a..5a4081e1ed 100644 --- a/internal/alerts/cleanup_observation_retention_test.go +++ b/internal/alerts/cleanup_observation_retention_test.go @@ -48,7 +48,7 @@ func TestCleanupContinuingDockerUpdateKeepsOccurrence(t *testing.T) { LastChecked: time.Now().Add(-6 * time.Hour), } container := models.DockerContainer{ID: "retention-container", Name: "web", Image: "mongo:7", UpdateStatus: pending} - resourceID := DockerResourceID(host.ID, container.ID) + resourceID := DockerContainerResourceID(host.ID, container.ID, "") trackingKey := dockerUpdateTrackingKey(host, container) first := time.Now().Add(-25 * time.Hour) m.mu.Lock() diff --git a/internal/alerts/config_runtime.go b/internal/alerts/config_runtime.go index 308d134046..3b6259e71e 100644 --- a/internal/alerts/config_runtime.go +++ b/internal/alerts/config_runtime.go @@ -284,41 +284,39 @@ func (m *Manager) applyGlobalOfflineSettingsLocked() { } } -// dockerAlertResourcePath returns the portion of the alert's resource ID after -// the "docker:" scheme, matching the IDs built by DockerResourceID and -// DockerServiceResourceID. Canonical alerts are stored under +// dockerAlertResourcePath splits the alert's resource ID after the "docker:" +// scheme into the host and the child path, matching the IDs built by +// DockerHostResourceID (""), DockerContainerResourceID +// ("/" or "/name:") and +// DockerServiceResourceID ("/service/" or +// "/service/name:"). child is empty for a host's own alert. A +// name can contain anything, so callers classify by the first segment of +// child, never by searching the whole path. Canonical alerts are stored under // "::" state IDs, so the legacy "docker-container-" / // "docker-service-" alert-ID prefixes never match them. -func dockerAlertResourcePath(alert *Alert) (string, bool) { +func dockerAlertResourcePath(alert *Alert) (host, child string, ok bool) { if alert == nil { - return "", false + return "", "", false } - resourceID := strings.TrimSpace(alert.ResourceID) - if !strings.HasPrefix(resourceID, "docker:") { - return "", false + path, found := strings.CutPrefix(strings.TrimSpace(alert.ResourceID), "docker:") + if !found { + return "", "", false } - return strings.TrimPrefix(resourceID, "docker:"), true + host, child, _ = strings.Cut(path, "/") + return host, child, true } func isDockerContainerAlert(alert *Alert) bool { - path, ok := dockerAlertResourcePath(alert) - if !ok { - return false - } - // "hostID/containerID" has a path separator; host-level IDs ("hostID") do - // not, and services use "hostID/service/serviceID". - return strings.Contains(path, "/") && !strings.Contains(path, "/service/") + _, child, ok := dockerAlertResourcePath(alert) + return ok && child != "" && !strings.HasPrefix(child, "service/") } func isDockerServiceAlert(alert *Alert) bool { if alert != nil && strings.HasPrefix(strings.TrimSpace(alert.ResourceID), "docker-service:") { return true } - path, ok := dockerAlertResourcePath(alert) - if !ok { - return false - } - return strings.Contains(path, "/service/") + _, child, ok := dockerAlertResourcePath(alert) + return ok && strings.HasPrefix(child, "service/") } func alertPrimaryResourceType(alert *Alert) string { diff --git a/internal/alerts/docker.go b/internal/alerts/docker.go index 773d3c645e..3eaa0d370a 100644 --- a/internal/alerts/docker.go +++ b/internal/alerts/docker.go @@ -80,22 +80,43 @@ func (m *Manager) lookupDockerContainerOverrideNoLock(hostID, containerName, leg return ThresholdConfig{}, false } -// DockerResourceID builds a stable identifier for Docker container alerts. -// Patrol scope resolution registers these forms as known aliases so an -// alert's resource ID resolves to the collected container or host. -func DockerResourceID(hostID, containerID string) string { +// dockerNameReferencePrefix marks the name of a Docker container or Swarm +// service that has no ID in the slot of an alert reference that otherwise +// holds the ID. Docker and Swarm IDs never contain a colon, so a name +// reference can never equal an ID reference or the host's own reference. +const dockerNameReferencePrefix = "name:" + +// DockerHostResourceID builds the reference of a Docker host's own alerts +// ("docker:"). Only the host's connectivity alert uses it. Returns "" +// without a host ID. +func DockerHostResourceID(hostID string) string { hostID = strings.TrimSpace(hostID) - containerID = strings.TrimSpace(containerID) - if containerID == "" { - if hostID == "" { - return "docker:unknown" + if hostID == "" { + return "" + } + return "docker:" + hostID +} + +// DockerContainerResourceID builds the reference of a Docker container's +// alerts: "docker:/". A container reported without an +// ID is referenced by its name, "docker:/name:", so it never +// shares its host's reference or another container's. Returns "" for a +// container with neither, which has no identity to alert under. Patrol scope +// resolution registers the same forms as aliases of the collected container. +func DockerContainerResourceID(hostID, containerID, containerName string) string { + hostID = strings.TrimSpace(hostID) + ref := strings.TrimSpace(containerID) + if ref == "" { + name := strings.TrimLeft(strings.TrimSpace(containerName), "/") + if hostID == "" || name == "" { + return "" } - return fmt.Sprintf("docker:%s", hostID) + ref = dockerNameReferencePrefix + name } if hostID == "" { - return fmt.Sprintf("docker:container/%s", containerID) + return fmt.Sprintf("docker:container/%s", ref) } - return fmt.Sprintf("docker:%s/%s", hostID, containerID) + return fmt.Sprintf("docker:%s/%s", hostID, ref) } func normalizeDockerUpdateTrackingPart(part string) string { @@ -164,17 +185,18 @@ func dockerServiceDisplayName(service models.DockerService) string { return serviceID } -// DockerServiceResourceID builds a stable identifier for Docker Swarm service -// alerts. Patrol scope resolution registers this form as a known alias on the -// owning Docker host record so a service alert's resource ID resolves there. +// DockerServiceResourceID builds the reference of a Docker Swarm service's +// alerts: "docker:/service/". A service reported without +// an ID is referenced by its normalized name, "docker:/service/name:", +// so it never shares another service's ID reference. Returns "" for a service +// with neither, which has no identity to alert under. Patrol scope resolution +// registers the same forms as aliases on the owning Docker host record so a +// service alert's resource ID resolves there. func DockerServiceResourceID(hostID, serviceID, serviceName string) string { hostID = strings.TrimSpace(hostID) normalizedServiceID := strings.TrimSpace(serviceID) if normalizedServiceID == "" { name := strings.TrimSpace(serviceName) - if name == "" { - name = "service" - } builder := strings.Builder{} for _, r := range strings.ToLower(name) { switch { @@ -190,11 +212,12 @@ func DockerServiceResourceID(hostID, serviceID, serviceName string) string { } normalizedServiceID = strings.Trim(builder.String(), "-_") if normalizedServiceID == "" { - normalizedServiceID = "service" + return "" } if len(normalizedServiceID) > 32 { normalizedServiceID = normalizedServiceID[:32] } + normalizedServiceID = dockerNameReferencePrefix + normalizedServiceID } if hostID == "" { return fmt.Sprintf("docker-service:%s", normalizedServiceID) @@ -272,8 +295,12 @@ func (m *Manager) CheckDockerHost(host models.DockerHost) { seen := make(map[string]struct{}, len(host.Containers)+len(host.Services)) seenUpdateTracking := make(map[string]struct{}, len(host.Containers)) for _, container := range host.Containers { + resourceID := DockerContainerResourceID(host.ID, container.ID, container.Name) + if resourceID == "" { + // Neither an ID nor a name: nothing identifies its alerts. + continue + } containerName := dockerContainerDisplayName(container) - resourceID := DockerResourceID(host.ID, container.ID) updateTrackingKey := dockerUpdateTrackingKey(host, container) if matchesDockerIgnoredPrefix(containerName, container.ID, ignoredPrefixes) { @@ -301,6 +328,9 @@ func (m *Manager) CheckDockerHost(host models.DockerHost) { for _, service := range host.Services { resourceID := DockerServiceResourceID(host.ID, service.ID, service.Name) + if resourceID == "" { + continue + } seen[resourceID] = struct{}{} m.evaluateDockerService(host, service, resourceID) } @@ -656,7 +686,7 @@ func (m *Manager) HandleDockerHostOnline(host models.DockerHost) { return } - resourceID := fmt.Sprintf("docker:%s", strings.TrimSpace(host.ID)) + resourceID := DockerHostResourceID(host.ID) alertID := canonicalConnectivityStateID(resourceID) m.mu.Lock() @@ -697,7 +727,7 @@ func (m *Manager) HandleDockerHostOffline(host models.DockerHost) { _, disableDockerHostsOffline := m.alertPolicyTypeSwitchesNoLock("docker-host") m.mu.RUnlock() - resourceID := fmt.Sprintf("docker:%s", strings.TrimSpace(host.ID)) + resourceID := DockerHostResourceID(host.ID) alertID := canonicalConnectivityStateID(resourceID) instanceName := dockerInstanceName(host) nodeName := strings.TrimSpace(host.Hostname) @@ -1400,6 +1430,16 @@ func (m *Manager) checkDockerContainerImageUpdate(host models.DockerHost, contai active.Type == "docker-container-update" && active.ResourceID == resourceID && !active.StartTime.IsZero() { firstSeen, exists = active.StartTime, true + } else if strings.TrimSpace(container.ID) == "" { + // A container without an ID used to record its pending update under + // its host's reference. Keep that age; the host's report cleanup then + // clears the old occurrence. + legacyRef := DockerHostResourceID(host.ID) + if legacy, ok := m.getActiveAlertNoLock(buildCanonicalStateID(legacyRef, legacyRef+"-image-update")); ok && + isLegacyDockerHostContainerAlert(legacy, host) && legacy.Type == "docker-container-update" && + alertMetadataString(legacy, "containerName") == containerName && !legacy.StartTime.IsZero() { + firstSeen, exists = legacy.StartTime, true + } } if !exists { firstSeen = time.Now() @@ -1480,6 +1520,19 @@ func (m *Manager) cleanupDockerContainerAlerts(host models.DockerHost, seen map[ m.cleanupDockerContainerAlertsWithTracking(host, seen, nil) } +// isLegacyDockerHostContainerAlert reports an alert that a container without +// an ID on this host raised under the host's own reference, before such +// containers had a reference of their own. Only the host's connectivity alert +// belongs there, so no producer refreshes these. The recorded host and empty +// container ID must match too: an alert restored with a resource ID cut short +// at a "::" in another host's ID ("::") can carry this host's +// reference. +func isLegacyDockerHostContainerAlert(alert *Alert, host models.DockerHost) bool { + hostRef := DockerHostResourceID(host.ID) + return alert != nil && hostRef != "" && alert.ResourceID == hostRef && alert.Type != "docker-host-offline" && + alertMetadataString(alert, "hostId") == strings.TrimSpace(host.ID) && alertMetadataString(alert, "containerId") == "" +} + func (m *Manager) cleanupDockerContainerAlertsWithTracking(host models.DockerHost, seen map[string]struct{}, seenUpdateTracking map[string]struct{}) { prefix := fmt.Sprintf("docker:%s/", strings.TrimSpace(host.ID)) updateTrackingPrefix := dockerUpdateTrackingHostPrefix(host) @@ -1488,6 +1541,10 @@ func (m *Manager) cleanupDockerContainerAlertsWithTracking(host models.DockerHos toClear := make([]string, 0) for storageKey, alert := range m.activeAlerts { alertID := effectiveAlertID(alert, storageKey) + if isLegacyDockerHostContainerAlert(alert, host) { + toClear = append(toClear, alertID) + continue + } if !strings.HasPrefix(alert.ResourceID, prefix) { continue } @@ -1540,7 +1597,7 @@ func (m *Manager) clearDockerHostContainerAlerts(host models.DockerHost) { toClear := make([]string, 0) for storageKey, alert := range m.activeAlerts { alertID := effectiveAlertID(alert, storageKey) - if strings.HasPrefix(alert.ResourceID, prefix) { + if strings.HasPrefix(alert.ResourceID, prefix) || isLegacyDockerHostContainerAlert(alert, host) { toClear = append(toClear, alertID) } } diff --git a/internal/alerts/docker_override_migration.go b/internal/alerts/docker_override_migration.go index e9b32b838d..9599c87d1e 100644 --- a/internal/alerts/docker_override_migration.go +++ b/internal/alerts/docker_override_migration.go @@ -75,7 +75,7 @@ func MigrateDockerContainerOverrideKeys(config *AlertConfig, resources []unified legacyKey := "" if containerID := strings.TrimSpace(resource.Docker.ContainerID); containerID != "" { - legacyKey = DockerResourceID(hostID, containerID) + legacyKey = DockerContainerResourceID(hostID, containerID, "") live[legacyKey] = struct{}{} } if legacyKey != "" && legacyKey != stableKey { diff --git a/internal/alerts/unified_eval_test.go b/internal/alerts/unified_eval_test.go index e5c732cee1..bdb0b7080a 100644 --- a/internal/alerts/unified_eval_test.go +++ b/internal/alerts/unified_eval_test.go @@ -1287,7 +1287,7 @@ func TestCheckDockerContainerStateAnnotatesCanonicalSpecMetadata(t *testing.T) { m.CheckDockerHost(host) m.CheckDockerHost(host) - resourceID := DockerResourceID(host.ID, "container-1") + resourceID := DockerContainerResourceID(host.ID, "container-1", "") alert := activeAlert(t, m, "docker-container-state-"+resourceID) if got := alert.Metadata["canonicalAlertKind"]; got != "discrete-state" { t.Fatalf("canonicalAlertKind = %v, want discrete-state", got) diff --git a/internal/alerts/update_alerts_test.go b/internal/alerts/update_alerts_test.go index 83093943cd..644bb3133a 100644 --- a/internal/alerts/update_alerts_test.go +++ b/internal/alerts/update_alerts_test.go @@ -469,7 +469,7 @@ func TestCheckDockerContainerImageUpdatePreservesDelayAcrossHostIDChange(t *test newHost.ID = "docker-host-new" newHost.DisplayName = "New Docker Host" - oldResourceID := DockerResourceID(oldHost.ID, container.ID) + oldResourceID := DockerContainerResourceID(oldHost.ID, container.ID, "") m.checkDockerContainerImageUpdate(oldHost, container, oldResourceID, "web", "docker-instance", "docker.local") firstSeen := time.Now().Add(-25 * time.Hour) @@ -479,7 +479,7 @@ func TestCheckDockerContainerImageUpdatePreservesDelayAcrossHostIDChange(t *test m.dockerUpdateFirstSeenByIdentity[trackingKey] = firstSeen m.mu.Unlock() - newResourceID := DockerResourceID(newHost.ID, container.ID) + newResourceID := DockerContainerResourceID(newHost.ID, container.ID, "") m.checkDockerContainerImageUpdate(newHost, container, newResourceID, "web", "docker-instance", "docker.local") alertID := "docker-container-update-" + newResourceID @@ -639,7 +639,7 @@ func TestUpdateConfigClearsDockerContainerUpdateAlertsWhenDisabled(t *testing.T) Name: "/frontend", Image: "nginx:latest", } - resourceID := DockerResourceID(host.ID, container.ID) + resourceID := DockerContainerResourceID(host.ID, container.ID, "") firstSeen := time.Now().Add(-48 * time.Hour) _, canonicalAlertID, trackingKey := seedDockerUpdateAlert(t, m, host, container, resourceID, firstSeen) @@ -677,7 +677,7 @@ func TestUpdateConfigKeepsDockerContainerUpdateAlertsWhenStillEnabled(t *testing Name: "/api", Image: "ghcr.io/example/api:latest", } - resourceID := DockerResourceID(host.ID, container.ID) + resourceID := DockerContainerResourceID(host.ID, container.ID, "") firstSeen := time.Now().Add(-30 * time.Hour) _, canonicalAlertID, trackingKey := seedDockerUpdateAlert(t, m, host, container, resourceID, firstSeen) @@ -723,7 +723,7 @@ func TestEvaluateDockerContainerClearsUpdateAlertWhenOverrideDisabled(t *testing Name: "/worker", Image: "ghcr.io/example/worker:latest", } - resourceID := DockerResourceID(host.ID, container.ID) + resourceID := DockerContainerResourceID(host.ID, container.ID, "") firstSeen := time.Now().Add(-36 * time.Hour) alertID, canonicalAlertID, trackingKey := seedDockerUpdateAlert(t, m, host, container, resourceID, firstSeen) @@ -884,7 +884,7 @@ func TestDockerUpdateRestartRestoresActivePendingAge(t *testing.T) { } container := models.DockerContainer{ID: "container", Name: "web", Image: "mongo:7", State: "running", UpdateStatus: pending} host := models.DockerHost{ID: "host", AgentID: "agent", Hostname: "host", Containers: []models.DockerContainer{container}} - resourceID := DockerResourceID(host.ID, container.ID) + resourceID := DockerContainerResourceID(host.ID, container.ID, "") canonicalID := buildCanonicalStateID(resourceID, resourceID+"-image-update") trackingKey := dockerUpdateTrackingKey(host, container) first := time.Now().Add(-time.Duration(delay+2) * time.Hour) diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go index a8b7454b6c..3a5b27f43f 100644 --- a/internal/monitoring/monitor_alert_handling_test.go +++ b/internal/monitoring/monitor_alert_handling_test.go @@ -153,7 +153,10 @@ func TestProxmoxAlertTimelineUsesCanonicalHistoryIdentity(t *testing.T) { // Docker host, Swarm service and sub-resource alerts (a ZFS pool and device, // a host filesystem) carry references the registry has no resource for. Their // lifecycle must reach the owner's canonical history, so the producers' ref -// shapes are pinned here against the real alert manager. +// shapes are pinned here against the real alert manager. A container or +// service reported without an ID alerts under its own name reference, so it +// joins neither its host's history nor that of the service whose ID it is +// named like. func TestOwnerAlertTimelinesUseCanonicalHistoryIdentity(t *testing.T) { store, err := unifiedresources.NewSQLiteResourceStore(t.TempDir(), "default") require.NoError(t, err) @@ -169,8 +172,10 @@ func TestOwnerAlertTimelinesUseCanonicalHistoryIdentity(t *testing.T) { manager.UpdateConfig(config) now := time.Now() dockerHost := models.DockerHost{ID: "dh-7f3a", Hostname: "tower", DisplayName: "tower", Status: "online", LastSeen: now, - Swarm: &models.DockerSwarmInfo{NodeID: "node-a", NodeRole: "manager", LocalState: "active", ControlAvailable: true, ClusterID: "swarm-1"}, - Services: []models.DockerService{{ID: "x7k2m9q4w1e8r5t3y6u0i2o4p", Name: "web", DesiredTasks: 2, RunningTasks: 0}}} + Swarm: &models.DockerSwarmInfo{NodeID: "node-a", NodeRole: "manager", LocalState: "active", ControlAvailable: true, ClusterID: "swarm-1"}, + Containers: []models.DockerContainer{{Name: "sidecar", State: "running", Health: "unhealthy"}}, + Services: []models.DockerService{{ID: "x7k2m9q4w1e8r5t3y6u0i2o4p", Name: "web", DesiredTasks: 2, RunningTasks: 0}, + {Name: "X7K2M9Q4W1E8R5T3Y6U0I2O4P", DesiredTasks: 2, RunningTasks: 0}}} storage := models.Storage{ID: "lab-pve1-local-zfs", Name: "local-zfs", Node: "pve1", Instance: "lab", Type: "zfspool", Status: "available", Total: 100, Used: 10, Usage: 10, Enabled: true, Active: true, LastSeen: now, ZFSPool: &models.ZFSPool{Name: "rpool", State: "DEGRADED", Status: "Degraded", Devices: []models.ZFSDevice{{Name: "sda2", Type: "disk", State: "FAULTED"}}}} agentHost := models.Host{ID: "host-nas", Hostname: "nas", DisplayName: "nas", MachineID: "fedcba9876543210", Status: "online", LastSeen: now, @@ -209,6 +214,23 @@ func TestOwnerAlertTimelinesUseCanonicalHistoryIdentity(t *testing.T) { } require.True(t, found, "%s alert missing from %s history: %+v", alertType, canonicalID, changes) } + alertTypes := func(ref string) []any { + changes, err := store.GetRecentChangesFiltered(ref, time.Time{}, 10, filters) + require.NoError(t, err) + types := make([]any, 0, len(changes)) + for _, change := range changes { + types = append(types, change.Metadata["alert_type"]) + } + return types + } + require.ElementsMatch(t, []any{"docker-host-offline"}, alertTypes(owner["agent/tower"])) + require.ElementsMatch(t, []any{"docker-service-health"}, alertTypes(owner["docker-service/web"])) + require.ElementsMatch(t, []any{"docker-container-health"}, alertTypes("docker:dh-7f3a/name:sidecar")) + require.ElementsMatch(t, []any{"docker-service-health"}, alertTypes("docker:dh-7f3a/service/name:x7k2m9q4w1e8r5t3y6u0i2o4p")) + for _, name := range []string{"app-container/sidecar", "docker-service/X7K2M9Q4W1E8R5T3Y6U0I2O4P"} { + require.NotEmpty(t, owner[name], name) + require.Empty(t, alertTypes(owner[name]), "a name binds nothing: %s", name) + } } func TestMonitor_HandleAlertFired_Extra(t *testing.T) { diff --git a/internal/unifiedresources/history_identity.go b/internal/unifiedresources/history_identity.go index 61ddc1ce1b..c6d71b126b 100644 --- a/internal/unifiedresources/history_identity.go +++ b/internal/unifiedresources/history_identity.go @@ -26,7 +26,7 @@ func legacyDockerHistoryIdentity(ref string) (sourceID, canonicalID string, ok b return "", "", false } sourceID = host + "/container/" + container - if host == "container" { // DockerResourceID's explicit hostless form. + if host == "container" { // DockerContainerResourceID's explicit hostless form. sourceID = container } return sourceID, SourceSpecificID(ResourceTypeAppContainer, SourceDocker, sourceID), true @@ -43,19 +43,23 @@ func isDockerHistoryReference(ref string) bool { } // isDockerNameHistoryReference marks Docker alert references that are not host -// or service references: container references, which only a complete -// container ID binds, and hostless service names. A name or shortened ID must -// never join history, not even through a retained binding. +// or service ID references: container references, which only a complete +// container ID binds, the names of containers and services reported without an +// ID, and hostless service names. A name or shortened ID must never join +// history, not even through a retained binding. func isDockerNameHistoryReference(ref string) bool { _, _, ok := dockerHostHistoryReference(ref) return isDockerHistoryReference(ref) && !ok } // dockerHostHistoryReference parses the Docker host alert reference -// ("docker:", alerts.DockerResourceID without a container) and the -// Swarm service alert reference ("docker:/service/", +// ("docker:", alerts.DockerHostResourceID) and the Swarm service +// alert reference ("docker:/service/", // alerts.DockerServiceResourceID). serviceID is empty for a host reference. -// "docker:unknown" is the shared fallback for alerts without a host. +// A service reported without an ID alerts under "name:" in the ID +// position, and Swarm IDs never contain a colon, so that is a name, not a +// service reference. "docker:unknown" was the old builder's fallback for +// alerts without a host. func dockerHostHistoryReference(ref string) (hostID, serviceID string, ok bool) { rest, found := strings.CutPrefix(strings.TrimSpace(ref), "docker:") if !found { @@ -65,7 +69,7 @@ func dockerHostHistoryReference(ref string) (hostID, serviceID string, ok bool) if hostID == "" || hostID == "unknown" || strings.TrimSpace(hostID) != hostID || strings.Contains(hostID, "/") { return "", "", false } - if isService && (serviceID == "" || strings.TrimSpace(serviceID) != serviceID || strings.Contains(serviceID, "/")) { + if isService && (serviceID == "" || strings.TrimSpace(serviceID) != serviceID || strings.ContainsAny(serviceID, "/:")) { return "", "", false } return hostID, serviceID, true @@ -74,43 +78,37 @@ func dockerHostHistoryReference(ref string) (hostID, serviceID string, ok bool) // dockerHistoryOwnerLocked resolves a Docker host or Swarm service alert // reference through the registry's own Docker source identities. The host ID // is the Docker host's source ID. A service reference names the service with -// that ID in the host's Swarm cluster, which every manager of the cluster -// reports under its own host ID. A service without an ID is referenced by its -// normalized name in the same place, so while the cluster has such a service -// every service reference there is a conflict. -func (rr *ResourceRegistry) dockerHistoryOwnerLocked(ref string) (resourceID string, conflict bool) { +// that exact ID in the host's Swarm cluster, which every manager of the +// cluster reports under its own host ID. +func (rr *ResourceRegistry) dockerHistoryOwnerLocked(ref string) string { hostID, serviceID, ok := dockerHostHistoryReference(ref) if !ok { - return "", false + return "" } hostResourceID := rr.bySource[SourceDocker][hostID] host := rr.resources[hostResourceID] if host == nil || CanonicalResourceType(host.Type) != ResourceTypeAgent { - return "", false + return "" } if serviceID == "" { - return hostResourceID, false + return hostResourceID } if host.Docker == nil { - return "", false + return "" } cluster := dockerSwarmClusterKeyFromMeta(host.Docker.Swarm) if cluster == "" { - return "", false - } - for _, other := range rr.resources { - if CanonicalResourceType(other.Type) == ResourceTypeDockerService && other.Docker != nil && - strings.TrimSpace(other.Docker.ServiceID) == "" && dockerSwarmClusterKeyFromMeta(other.Docker.Swarm) == cluster { - return "", true - } + return "" } + // The registry keys a service without an ID by its raw name, so the entry + // must carry exactly this ID. serviceResourceID := rr.bySource[SourceDocker][normalizeSourceID(cluster+":service:"+serviceID)] service := rr.resources[serviceResourceID] if service == nil || CanonicalResourceType(service.Type) != ResourceTypeDockerService || service.Docker == nil || strings.TrimSpace(service.Docker.ServiceID) != serviceID { - return "", false + return "" } - return serviceResourceID, false + return serviceResourceID } // historySubResourceOwner names the owner of a sub-resource alert reference, @@ -169,8 +167,8 @@ func (rr *ResourceRegistry) resolveHistoryReference(ref string) (resourceID stri rr.mu.RLock() defer rr.mu.RUnlock() if isDockerHistoryReference(ref) { - resourceID, conflict := rr.dockerHistoryOwnerLocked(ref) - return resourceID, resourceID != "" || conflict + resourceID = rr.dockerHistoryOwnerLocked(ref) + return resourceID, resourceID != "" } matches := rr.durableHistoryMatchesLocked(ref) if ownerRef, ownerType, ok := historySubResourceOwner(ref); ok && len(matches) == 0 { diff --git a/internal/unifiedresources/history_identity_test.go b/internal/unifiedresources/history_identity_test.go index 4744e08503..c6e183f117 100644 --- a/internal/unifiedresources/history_identity_test.go +++ b/internal/unifiedresources/history_identity_test.go @@ -391,6 +391,8 @@ func TestHistoryIdentityMonitorAdapterResolvesDockerHostAndServiceReferences(t * hostA.AgentID, hostA.MachineID = "host-tower", "0123456789abcdef" hostA.Containers = []models.DockerContainer{{ID: container, Name: "worker", State: "running"}} hostB := manager("dh-91c0", "rack", "swarm-1", web) + // A container without an ID, named like its host's ID. + hostB.Containers = []models.DockerContainer{{Name: "dh-91c0", State: "running"}} // In swarm-2 a service without an ID is named like another service's ID. hostC := manager("dh-c4d2", "edge", "swarm-2", models.DockerService{ID: otherServiceID, Name: "api"}, models.DockerService{Name: strings.ToUpper(otherServiceID)}) // Rows journaled before this process started are bound by a later generation. @@ -407,6 +409,7 @@ func TestHistoryIdentityMonitorAdapterResolvesDockerHostAndServiceReferences(t * hostAID := registry.sourceResourceID(SourceDocker, "dh-7f3a") hostBID := registry.sourceResourceID(SourceDocker, "dh-91c0") serviceResourceID := historyIdentityResourceID(t, registry, ResourceTypeDockerService, "web") + apiID := historyIdentityResourceID(t, registry, ResourceTypeDockerService, "api") containerID := registry.sourceResourceID(SourceDocker, "dh-7f3a/container/"+container) require.Equal(t, registry.sourceResourceID(SourceAgent, "host-tower"), hostAID, "the Docker host merges into its agent's machine") require.NotEqual(t, hostAID, hostBID) @@ -417,17 +420,22 @@ func TestHistoryIdentityMonitorAdapterResolvesDockerHostAndServiceReferences(t * "docker:dh-91c0/service/" + serviceID: serviceResourceID, "docker:dh-7f3a": hostAID, "docker:dh-7f3a/service/" + serviceID: serviceResourceID, + // A service without an ID in the same cluster no longer shares this + // reference, so it binds. + "docker:dh-c4d2/service/" + otherServiceID: apiID, } unbound := []string{ - "docker:tower", // hostname - "docker:dh-7f3a/service/web", // service name - "docker:dh-c4d2/service/" + otherServiceID, // its cluster has a service without an ID - "docker-service:web", // hostless service name - "docker:dh-7f3a/worker", // container name - "docker:dh-7f3a/" + container[:12], // shortened container ID + "docker:tower", // hostname + "docker:dh-7f3a/service/web", // service name + "docker:dh-c4d2/service/name:" + otherServiceID, // service without an ID + "docker-service:name:web", // hostless service name + "docker:dh-7f3a/worker", // container name + "docker:dh-7f3a/" + container[:12], // shortened container ID + "docker:dh-91c0/name:dh-91c0", // container without an ID "docker:unknown", } - for i, ref := range append([]string{"docker:dh-7f3a", "docker:dh-7f3a/service/" + serviceID}, unbound...) { + ids := []string{"docker:dh-7f3a", "docker:dh-7f3a/service/" + serviceID, "docker:dh-c4d2/service/" + otherServiceID} + for i, ref := range append(ids, unbound...) { if ref == "docker:dh-7f3a/worker" { continue // journaled above } @@ -436,29 +444,29 @@ func TestHistoryIdentityMonitorAdapterResolvesDockerHostAndServiceReferences(t * requireHistoryReferenceBindings(t, store, bound, unbound) // Names and shortened IDs are never retried against later inventory. require.Contains(t, adapter.legacyHistory.pending, "docker:tower") - for _, ref := range []string{"docker:dh-7f3a/worker", "docker:dh-7f3a/" + container[:12], "docker-service:web"} { + for _, ref := range []string{"docker:dh-7f3a/worker", "docker:dh-7f3a/" + container[:12], "docker-service:name:web", + "docker:dh-91c0/name:dh-91c0", "docker:dh-c4d2/service/name:" + otherServiceID} { require.NotContains(t, adapter.legacyHistory.pending, ref) } got, err := store.GetRecentChangesFiltered(containerID, time.Time{}, 10, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertFired}}) require.NoError(t, err) require.Empty(t, got, "a container name or short ID never joins the container's history") - // While its cluster has a service without an ID, a service reference is a - // conflict and must not follow a binding recorded earlier. - collision := "docker:dh-c4d2/service/" + otherServiceID - apiID := historyIdentityResourceID(t, registry, ResourceTypeDockerService, "api") - require.NoError(t, store.RecordChangeWithSourceIdentity(ResourceChange{ID: "earlier-api", ResourceID: apiID, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now}, collision)) - require.NoError(t, adapter.RecordChange(ResourceChange{ID: "conflict-api", ResourceID: collision, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(2 * time.Minute)})) - // The same holds once the service with that ID has left inventory. + // The service's ID reference follows its binding once the service has left + // inventory, while the service without an ID is still there; the name + // reference stays its own. hostC.Services = hostC.Services[1:] adapter.PopulateFromSnapshot(models.StateSnapshot{ Hosts: []models.Host{{ID: "host-tower", Hostname: "tower", MachineID: "0123456789abcdef", Status: "online", LastSeen: now}}, DockerHosts: []models.DockerHost{hostA, hostB, hostC}, }) - require.NoError(t, adapter.RecordChange(ResourceChange{ID: "conflict-api-removed", ResourceID: collision, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(3 * time.Minute)})) - for _, id := range []string{"conflict-api", "conflict-api-removed"} { + nameRef := "docker:dh-c4d2/service/name:" + otherServiceID + for id, ref := range map[string]string{"api-removed": "docker:dh-c4d2/service/" + otherServiceID, "name-removed": nameRef} { + require.NoError(t, adapter.RecordChange(ResourceChange{ID: id, ResourceID: ref, Kind: ChangeAlertResolved, SourceType: SourceHeuristic, ObservedAt: now.Add(3 * time.Minute)})) + } + for id, want := range map[string]string{"api-removed": apiID, "name-removed": nameRef} { var recordedAs string require.NoError(t, store.db.QueryRow(`SELECT canonical_id FROM resource_changes WHERE id = ?`, id).Scan(&recordedAs)) - require.Equal(t, collision, recordedAs, id) + require.Equal(t, want, recordedAs, id) } // After the service leaves inventory its alerts follow the retained binding; From 1c101b024d7adb43cbcb615272b0558b1352fd3e Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:26:46 +0100 Subject: [PATCH 33/53] Replace stored alerts with each payload instead of merging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The websocket store wrote every incoming active alert with setActiveAlerts(id, alert). Solid's path setter shallow-merges into an existing object, so a field the server stops sending kept its old value in the browser. After a backend restart, threshold alerts are restored without metricStatus until their next evaluation, but an open tab kept the pre-restart live reading and the node drawer and node table went on saying "78°C now" for a reading the server no longer reports. An unacknowledge from another session left ackTime and ackUser behind the same way, as did a dropped nodeDisplayName or metadata. Active and recently resolved alerts are now written with reconcile, so omitted keys are removed on the snapshot, keyed-delta and REST recovery commits alike. reconcile diffs nested values in place, so an unchanged alert's live status no longer re-notifies its readers on every frame, which the shallow merge did by swapping in a fresh object each time. The pending acknowledgement hold is unchanged: it still skips payloads that contradict an unconfirmed local acknowledge or unacknowledge. updateAlert now stores a copy of each local update. Because payloads reconcile the stored alert in place, a snooze's optimistic record, which shares nested objects with the record the caller keeps for rollback, would otherwise have that rollback record rewritten by the next payload. --- .../v6/internal/subsystems/alerts.md | 12 + .../subsystems/performance-and-scalability.md | 8 +- .../3b5ee6cfe34d3e8c.json | 40 +++ .../__tests__/websocket-unified.test.ts | 321 +++++++++++++++++- frontend-modern/src/stores/websocket.ts | 20 +- 5 files changed, 395 insertions(+), 6 deletions(-) create mode 100644 frontend-modern/browser-verification/3b5ee6cfe34d3e8c.json diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index c80daddd90..c7e098c5ac 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -787,6 +787,18 @@ that fast path entirely: active-alert and resolved-alert stores keep their own commit path, resource `alerts` facets are not in the fast-path allow-list, and a row whose patch touches alert-relevant structure always takes the full canonical merge. +That commit path replaces each stored alert with the server's payload for its +id instead of merging into it. A field the server omits is removed from the +store, whether the payload came from a socket snapshot, a keyed delta, or REST +recovery: a threshold alert restored after a restart carries no `metricStatus` +until its next evaluation, so the previous live reading must not keep reading +as "now"; an unacknowledge from another session clears `ackTime` and +`ackUser`; a dropped `nodeDisplayName` or `metadata` disappears. Resolved +alerts follow the same rule. The pending acknowledgement hold still skips a +payload that contradicts an unconfirmed local acknowledge or unacknowledge, +and the confirming payload then replaces the optimistic copy. Because a +replacement reconciles the stored alert in place, local optimistic updates are +stored as copies, so it never rewrites a record a caller keeps for rollback. Operational evidence and lifecycle identity are typed through `internal/operationaltrust`. Evidence envelopes distinguish completeness, confidence, permissions, freshness, correlation, and bounded provider detail. diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index c7ea65f894..693d316073 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -673,7 +673,13 @@ the store with reference-stable untouched items so the per-tick deep walk of the whole projection is gone. Active alerts travel the same keyed transport (`activeAlertsDelta`, replacing the ~37KB whole-array re-ship whenever any alert changed) while their application stays immediate per the alerts -subsystem boundary. These projection baselines are socket-owned: an oversized +subsystem boundary. Each committed alert is reconciled in place against its +copy in the keyed active-alert store, so a reader of an unchanged alert there, +or of an unchanged nested value such as its live `metricStatus` or `metadata`, +is not notified although every commit carries freshly parsed objects; omitted +fields are deleted rather than shallow-merged. The `state.activeAlerts` array +is still replaced on every alert commit. These projection baselines are +socket-owned: an oversized frame invalidates resources, connected infrastructure, and active alerts together, while REST recovery hydrates display state only. A later keyed delta without a socket baseline requests the throttled recovery path instead of diff --git a/frontend-modern/browser-verification/3b5ee6cfe34d3e8c.json b/frontend-modern/browser-verification/3b5ee6cfe34d3e8c.json new file mode 100644 index 0000000000..ffc8d4beb4 --- /dev/null +++ b/frontend-modern/browser-verification/3b5ee6cfe34d3e8c.json @@ -0,0 +1,40 @@ +{ + "version": 1, + "base_sha": "8424fd65057af75e1ea3c1fc9fe29528d7c0d68e", + "verified_at": "2026-10-06T21:24:22Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/stores/websocket.ts" + ], + "content_sha256": { + "frontend-modern/src/stores/websocket.ts": "1be5c0f23d3dbaf85a32f7b8b3906c2cb25b20bb02412c31c200ffbad9c6c2d5" + }, + "routes": [ + "/proxmox/overview", + "/alerts/overview" + ], + "viewports": [ + { + "width": 1280, + "height": 900 + }, + { + "width": 375, + "height": 812 + } + ], + "states": [ + "Private mock stack (backend + Vite from this tree) with a local-only overlay pinning node standalone2's evaluated CPU temperature and letting mock mode restore persisted alerts across restarts", + "standalone2 temperature alert latched below trigger (breach 95, live 78): node drawer Needs attention reads 'Temperature 78\u00b0C now, back under the 80\u00b0C alert level' and the node table temperature cell title carries the same summary", + "Backend restarted with the reading missing: /api/alerts/active returns the restored alert without metricStatus; the open tab reconnects and the drawer and cell title fall back to 'Node temperature at 95.0\u00b0C' within seconds", + "Reading returns: the next keyed delta adds metricStatus and the drawer reads '78\u00b0C now' again", + "Control on the unchanged store, same run: after the restart the open tab kept 'Temperature 78\u00b0C now' in the drawer and cell title for the full two-minute window", + "Alerts overview card snoozed ('Snoozed until ...', Resume shown), resumed, and acknowledged from the UI with no page errors" + ], + "interactions": [ + "Expanded the standalone2 row details in the Proxmox nodes table (desktop) and after Show all nodes (narrow)", + "Restarted the private backend while the tab stayed open, then changed the pinned reading back to 78", + "Clicked Snooze and the first preset, then Resume, then Acknowledge on the alert card", + "Acknowledged and unacknowledged through the API from another session with the tab open (card left and returned within 2s)" + ] +} diff --git a/frontend-modern/src/stores/__tests__/websocket-unified.test.ts b/frontend-modern/src/stores/__tests__/websocket-unified.test.ts index c630e3535e..2b94a4ea8c 100644 --- a/frontend-modern/src/stores/__tests__/websocket-unified.test.ts +++ b/frontend-modern/src/stores/__tests__/websocket-unified.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { createRoot } from 'solid-js'; +import { createComputed, createRoot } from 'solid-js'; // Mirrors MAX_INBOUND_WEBSOCKET_MESSAGE_BYTES in the store under test. const MAX_INBOUND_WEBSOCKET_MESSAGE_BYTES = 32 * 1024 * 1024; @@ -1291,6 +1291,325 @@ describe('websocket store unified resource contract', () => { } }); + describe('active alert payloads replace the stored alert', () => { + // A threshold alert held below its trigger, as the socket sends it: the + // breach stays in value/message and the reading evaluated now rides in + // metricStatus. + const heldTemperatureAlert = (): Record => ({ + id: 'pve2-temperature', + type: 'temperature', + level: 'warning', + resourceId: 'node:pve2', + resourceName: 'pve2', + node: 'pve2', + nodeDisplayName: 'PVE Two', + instance: 'pve2', + message: 'Node temperature at 95°C', + value: 95, + threshold: 80, + startTime: '2026-10-06T10:00:00Z', + acknowledged: false, + metadata: { resourceType: 'Node', unit: '°C', probe: 'cpu.package' }, + metricStatus: { + phase: 'latched', + value: 78, + observedAt: '2026-10-06T10:05:00Z', + trigger: 80, + recovery: 75, + unit: '°C', + }, + }); + + // The same alert as the backend restores it after a restart: no live + // status until the next evaluation. + const restoredTemperatureAlert = (): Record => { + const alert = heldTemperatureAlert(); + delete alert.metricStatus; + return alert; + }; + + const emitActiveAlerts = (lastUpdate: number, activeAlerts: unknown[]) => + emitMessage({ type: 'rawData', data: { lastUpdate, activeAlerts } }); + + const emitInitialAlerts = (activeAlerts: unknown[]) => + emitMessage({ + type: 'initialState', + data: { resources: [], lastUpdate: 100, activeAlerts, recentlyResolved: [] }, + }); + + it('drops a live status, display name and metadata that a snapshot omits', async () => { + const { store, dispose } = await createStoreHarness(); + try { + await waitForOpenTick(); + emitInitialAlerts([heldTemperatureAlert()]); + expect(store.activeAlerts['pve2-temperature']?.metricStatus?.value).toBe(78); + + emitActiveAlerts(200, [restoredTemperatureAlert()]); + const restored = store.activeAlerts['pve2-temperature']; + expect(restored).not.toHaveProperty('metricStatus'); + expect(restored?.value).toBe(95); + expect(store.state.activeAlerts[0]).not.toHaveProperty('metricStatus'); + + const renamed = restoredTemperatureAlert(); + delete renamed.nodeDisplayName; + renamed.metadata = { resourceType: 'Node', unit: '°C' }; + emitActiveAlerts(300, [renamed]); + expect(store.activeAlerts['pve2-temperature']).not.toHaveProperty('nodeDisplayName'); + expect(store.activeAlerts['pve2-temperature']?.metadata).toEqual({ + resourceType: 'Node', + unit: '°C', + }); + + delete renamed.metadata; + emitActiveAlerts(400, [renamed]); + expect(store.activeAlerts['pve2-temperature']).not.toHaveProperty('metadata'); + } finally { + dispose(); + } + }); + + it('drops a live status that a keyed delta deletes', async () => { + const { store, dispose } = await createStoreHarness(); + try { + await waitForOpenTick(); + emitInitialAlerts([heldTemperatureAlert()]); + + emitMessage({ + type: 'rawData', + data: { + lastUpdate: 200, + activeAlertsDelta: { + upserts: [{ id: 'pve2-temperature', metricStatus: null, nodeDisplayName: null }], + }, + }, + }); + const stored = store.activeAlerts['pve2-temperature']; + expect(stored).not.toHaveProperty('metricStatus'); + expect(stored).not.toHaveProperty('nodeDisplayName'); + expect(stored?.message).toBe('Node temperature at 95°C'); + } finally { + dispose(); + } + }); + + it('drops a live status that REST recovery omits', async () => { + const { store, dispose } = await createStoreHarness(); + try { + await waitForOpenTick(); + emitInitialAlerts([heldTemperatureAlert()]); + + apiFetchJSONMock.mockResolvedValueOnce([restoredTemperatureAlert()]); + await expect(store.refreshActiveAlerts()).resolves.toBe(true); + expect(apiFetchJSONMock).toHaveBeenCalledWith('/api/alerts/active'); + expect(store.activeAlerts['pve2-temperature']).not.toHaveProperty('metricStatus'); + } finally { + dispose(); + } + }); + + it('clears acknowledgement fields when the server reports an unacknowledge', async () => { + const { store, dispose } = await createStoreHarness(); + try { + await waitForOpenTick(); + emitInitialAlerts([ + { + ...restoredTemperatureAlert(), + acknowledged: true, + ackTime: '2026-10-06T10:06:00Z', + ackUser: 'admin', + }, + ]); + expect(store.activeAlerts['pve2-temperature']?.ackUser).toBe('admin'); + + // Unacknowledged from another session: the payload omits both fields. + emitActiveAlerts(200, [restoredTemperatureAlert()]); + const stored = store.activeAlerts['pve2-temperature']; + expect(stored?.acknowledged).toBe(false); + expect(stored).not.toHaveProperty('ackTime'); + expect(stored).not.toHaveProperty('ackUser'); + } finally { + dispose(); + } + }); + + it('holds a local acknowledge and unacknowledge until the server confirms each', async () => { + const { store, dispose } = await createStoreHarness(); + try { + await waitForOpenTick(); + const id = 'pve2-temperature'; + const acknowledged = { + ...restoredTemperatureAlert(), + acknowledged: true, + ackTime: '2026-10-06T10:06:01Z', + ackUser: 'admin', + }; + emitInitialAlerts([restoredTemperatureAlert()]); + + store.updateAlert(id, { + acknowledged: true, + ackTime: '2026-10-06T10:06:00Z', + ackUser: undefined, + }); + emitActiveAlerts(200, [restoredTemperatureAlert()]); + expect(store.activeAlerts[id]?.acknowledged).toBe(true); + + emitActiveAlerts(300, [acknowledged]); + expect(store.activeAlerts[id]).toMatchObject({ + acknowledged: true, + ackTime: '2026-10-06T10:06:01Z', + ackUser: 'admin', + }); + + store.updateAlert(id, { acknowledged: false, ackTime: undefined, ackUser: undefined }); + emitActiveAlerts(400, [acknowledged]); + expect(store.activeAlerts[id]?.acknowledged).toBe(false); + expect(store.activeAlerts[id]).not.toHaveProperty('ackUser'); + + emitActiveAlerts(500, [restoredTemperatureAlert()]); + expect(store.activeAlerts[id]?.acknowledged).toBe(false); + expect(store.activeAlerts[id]).not.toHaveProperty('ackTime'); + expect(store.activeAlerts[id]).not.toHaveProperty('ackUser'); + } finally { + dispose(); + } + }); + + it('keeps the record a local update replaced intact for rollback', async () => { + const { store, dispose } = await createStoreHarness(); + try { + await waitForOpenTick(); + const id = 'pve2-temperature'; + const record = { + id: 'alert:pve2-temperature', + canonicalSpecId: 'temperature', + subjectResourceId: 'node:pve2', + state: 'open', + severity: 'warning', + firstObservedAt: '2026-10-06T10:00:00Z', + lastObservedAt: '2026-10-06T10:05:00Z', + stateChangedAt: '2026-10-06T10:00:00Z', + acknowledgement: { at: '2026-10-06T10:01:00Z', by: 'admin' }, + evidenceIds: ['evidence-1'], + causeKey: 'pve2-temperature', + relatedResourceIds: [], + }; + emitInitialAlerts([{ ...restoredTemperatureAlert(), operationalRecord: record }]); + + // As a snooze does: the optimistic record reuses nested values of the + // record it keeps for rollback. + const previous = store.activeAlerts[id]?.operationalRecord; + if (!previous) throw new Error('expected an operational record'); + store.updateAlert(id, { + operationalRecord: { + ...previous, + state: 'suppressed', + suppression: { + at: '2026-10-06T10:06:00Z', + by: 'current-user', + reason: 'user_snooze', + expiresAt: '2026-10-06T11:06:00Z', + }, + }, + }); + + emitActiveAlerts(200, [ + { + ...restoredTemperatureAlert(), + operationalRecord: { + ...record, + acknowledgement: { at: '2026-10-06T10:07:00Z', by: 'operator' }, + evidenceIds: ['evidence-1', 'evidence-2'], + }, + }, + ]); + expect(store.activeAlerts[id]?.operationalRecord?.acknowledgement?.by).toBe('operator'); + expect(previous.acknowledgement).toEqual({ at: '2026-10-06T10:01:00Z', by: 'admin' }); + expect([...previous.evidenceIds]).toEqual(['evidence-1']); + } finally { + dispose(); + } + }); + + it('leaves unchanged alerts and unchanged nested fields quiet', async () => { + const { store, dispose } = await createStoreHarness(); + let disposeTracking = () => {}; + try { + await waitForOpenTick(); + const cpuAlert = (): Record => ({ + ...heldTemperatureAlert(), + id: 'pve1-cpu', + type: 'cpu', + resourceId: 'node:pve1', + metricStatus: { + phase: 'breaching', + value: 93, + observedAt: '2026-10-06T10:05:00Z', + trigger: 90, + recovery: 85, + }, + }); + emitInitialAlerts([heldTemperatureAlert(), cpuAlert()]); + + let temperatureRuns = 0; + let cpuRuns = 0; + createRoot((d) => { + disposeTracking = d; + createComputed(() => { + void store.activeAlerts['pve2-temperature']?.metricStatus?.value; + temperatureRuns += 1; + }); + createComputed(() => { + void store.activeAlerts['pve1-cpu']?.metricStatus?.value; + cpuRuns += 1; + }); + }); + const cpuBefore = store.activeAlerts['pve1-cpu']; + const cpuStatusBefore = cpuBefore?.metricStatus; + + const cooler = heldTemperatureAlert(); + cooler.metricStatus = { ...(cooler.metricStatus as object), value: 77 }; + emitActiveAlerts(200, [cooler, cpuAlert()]); + + expect(store.activeAlerts['pve2-temperature']?.metricStatus?.value).toBe(77); + expect(temperatureRuns).toBe(2); + expect(cpuRuns).toBe(1); + expect(store.activeAlerts['pve1-cpu']).toBe(cpuBefore); + expect(store.activeAlerts['pve1-cpu']?.metricStatus).toBe(cpuStatusBefore); + } finally { + disposeTracking(); + dispose(); + } + }); + + it('drops fields a recently resolved alert payload omits', async () => { + const { store, dispose } = await createStoreHarness(); + try { + await waitForOpenTick(); + const resolved = { + ...restoredTemperatureAlert(), + resolvedTime: '2026-10-06T10:10:00Z', + }; + emitMessage({ + type: 'initialState', + data: { + resources: [], + lastUpdate: 100, + activeAlerts: [], + recentlyResolved: [{ ...resolved, nodeDisplayName: 'PVE Two' }], + }, + }); + expect(store.recentlyResolved['pve2-temperature']?.nodeDisplayName).toBe('PVE Two'); + + const renamed: Record = { ...resolved }; + delete renamed.nodeDisplayName; + emitMessage({ type: 'rawData', data: { lastUpdate: 200, recentlyResolved: [renamed] } }); + expect(store.recentlyResolved['pve2-temperature']).not.toHaveProperty('nodeDisplayName'); + } finally { + dispose(); + } + }); + }); + it('drops scroll-queued deltas when a full snapshot supersedes their baseline', async () => { const { store, dispose } = await createStoreHarness(); try { diff --git a/frontend-modern/src/stores/websocket.ts b/frontend-modern/src/stores/websocket.ts index e3f1f34b11..28023d5c01 100644 --- a/frontend-modern/src/stores/websocket.ts +++ b/frontend-modern/src/stores/websocket.ts @@ -422,7 +422,12 @@ export function createWebSocketStore(url: string) { clearPendingAck(id); } - setActiveAlerts(id, alert); + // Each payload is the server's whole alert. A plain path set would + // shallow-merge it, so a field the server now omits (metricStatus on an + // alert restored after a restart, ackTime/ackUser after an unacknowledge) + // would keep its old value. reconcile removes omitted keys and diffs + // nested values in place, so unchanged fields stay quiet. + setActiveAlerts(id, reconcile(alert)); }); setState('activeAlerts', Object.values(alertsMap)); @@ -1380,9 +1385,10 @@ export function createWebSocketStore(url: string) { } }); - // Add new resolved alerts + // Add new resolved alerts, replacing rather than merging (see + // applyActiveAlerts) Object.entries(newResolvedAlerts).forEach(([id, alert]) => { - setRecentlyResolved(id, alert); + setRecentlyResolved(id, reconcile(alert)); }); setState('recentlyResolved', Object.values(newResolvedAlerts)); @@ -1973,7 +1979,13 @@ export function createWebSocketStore(url: string) { }, 15000); pendingAckTimeouts.set(alertIdentifier, pendingTimeout); } - setActiveAlerts(alertIdentifier, { ...existingAlert, ...updates }); + // Incoming payloads reconcile stored alerts in place, so store a copy + // of the update: an optimistic record can share nested objects with + // the record its caller keeps for rollback. + setActiveAlerts(alertIdentifier, { + ...existingAlert, + ...structuredClone(unwrap(updates)), + }); } }, }; From 8e93360d10228dd3f4dc7ea9fa781d502de7fd10 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:35:50 +0100 Subject: [PATCH 34/53] Poll only the history each metric table renders useWorkloadTableMetricHistory ran two polled estate-wide reads per instance in Trends mode: the workloads history and the infrastructure summary. Since grouped workload rows lost their host metric cells, the Workloads surface reads only guest series, and the Proxmox nodes table never read anything but node series. Each reader passes its own abort signal, which skips the summary caches' in-flight dedupe, so every instance paid for both routes every 30 seconds. A required series option now names the rows a caller draws. The Workloads state passes 'guests' and the Proxmox nodes table passes 'nodes', so each polls one route. On the demo estate in Trends this takes the vSphere overview from one infrastructure and one workloads read per poll to one workloads read, and the Proxmox overview from two of each to one of each. Node table sparklines are unchanged. The performance contract claimed the two readers deduped their reads; it now states that they do not and that each reader polls only what it renders. The adjacent Bars-mode prose in the performance and frontend-primitives contracts described one active-row request that leaving the row cancels; it now describes the bounded warming window the hook has run since the history lens shipped. --- .../subsystems/frontend-primitives.md | 21 ++++-- .../subsystems/performance-and-scalability.md | 36 +++++++--- .../internal/subsystems/unified-resources.md | 5 +- .../4e061179721ef90a.json | 39 +++++++++++ .../HistoryResponse.contract.test.tsx | 1 + ...loadsSurface.performance.contract.test.tsx | 12 ++++ .../useWorkloadTableMetricHistory.test.tsx | 70 ++++++++++++++++++- .../useWorkloadTableMetricHistory.ts | 14 ++-- .../components/Workloads/useWorkloadsState.ts | 3 + .../features/proxmox/ProxmoxNodesTable.tsx | 7 +- .../__tests__/ProxmoxNodesTable.test.tsx | 39 ++++++++++- 11 files changed, 219 insertions(+), 28 deletions(-) create mode 100644 frontend-modern/browser-verification/4e061179721ef90a.json diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 6c07a49441..1f45a407f9 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -688,11 +688,13 @@ same relative point in the selected history range. Leaving the row clears the cursor and restores the bars and I/O readouts together. The lens mounts with a short reduced-motion-safe fade and must not leave both bar and chart semantics in the accessibility tree simultaneously. -Bar mode resolves history only for that active guest through its canonical -metrics target and the selected compact range; it must not start an -estate-wide chart request merely because the range changes. The active request -key is stable across equivalent live guest snapshots, and leaving the row or -selecting another range aborts superseded browser work. Persistent Trends may +Bar mode resolves history per guest through each guest's canonical metrics +target and the selected compact range, warming only a bounded window of +mounted and adjacent rows with the active guest first; it must not start an +estate-wide chart request merely because the range changes. The request key is +stable across equivalent live guest snapshots, and selecting another range +aborts superseded browser work. Leaving a row does not cancel its read, which +settles into the bounded row cache. Persistent Trends may retain the shared estate reader, but range changes must clear prior-range data unless an exact-key cache entry exists. @@ -7983,6 +7985,15 @@ Rendered table proof belongs in drawer grouping and fallback proof belongs in `frontend-modern/src/components/Infrastructure/__tests__/resourceDetailDrawerMetricsHistoryModel.branchcov0712.test.ts`. +### The Proxmox nodes table reads node history only + +`frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx` draws its Trends +sparklines through `useWorkloadTableMetricHistory` with `series: 'nodes'`, so +it polls the infrastructure summary and no guest history. The embedded +workloads table below it owns guest history and passes `series: 'guests'`. +`ProxmoxNodesTable.test.tsx` pins the option; the per-page polling budget +belongs to performance-and-scalability. + ### Proxmox Storage reuses the shared product-family source scope `frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx` mounts the diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 5231c21704..d970748a0f 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -1920,9 +1920,17 @@ internal persistent signal so the toggle and the hosts-table render stay in lockstep. The history range remains inline in both display modes because the default bars mode uses it for the intent-driven guest-row history lens. Embedded sibling tables that opt into sparklines -re-instantiate `useWorkloadTableMetricHistory`; the cache key matches -the workloads-table reader so both readers dedupe their fetches and -the canonical Workloads hot-path budget is preserved. Standalone +re-instantiate `useWorkloadTableMetricHistory` and name the rows they draw +through its required `series` option. The Proxmox nodes table passes +`'nodes'` and polls only the infrastructure summary; the WorkloadsSurface +reader passes `'guests'` and polls only the workloads history. Two readers +on one page do not share reads: each forwards its own abort signal, which +bypasses the summary caches' in-flight dedupe, and retained query values +seed matching mounts and source changes without deduplicating requests. A +reader therefore must not poll a summary its rows do not render. In Trends +the vSphere overview polls only the workloads history, and the Proxmox +overview polls the workloads history and the infrastructure summary once +each. Standalone WorkloadsSurface callers (no override props) keep the original persistent-signal-backed behavior. Every toolbar that controls a `WorkloadsSurface` consumes @@ -1996,14 +2004,20 @@ The Workloads table metric display mode is part of the protected Workloads hot path. Default bar mode keeps compact current-value bars at rest and swaps only the active fine-pointer or keyboard-focused guest row to the original sparklines; persistent Trends mode keeps sparklines visible for every rendered -row. Bar mode does not hydrate estate history at rest: the selected range is -local state until a row is active, then one centralized query reads only that -guest's canonical metrics target from `/api/metrics-store/history`, capped at -36 points. Equivalent live guest snapshots must not restart that request, and -leaving the row disables it; per-row query owners, fan-out, and polling remain -forbidden. Persistent Trends retains the shared, cache-keyed -`fetchWorkloadsSummaryAndCache` / `fetchInfrastructureSummaryAndCache` reader -for all rendered rows. Range-sensitive readers clear prior-range data when no +row. Bar mode does not hydrate estate history at rest. With the history lens +on, one centralized queue warms a bounded window instead: up to the first six +guests of the mounted table window at rest, then the active guest first plus +its next four and previous one, with at most four reads in flight and 18 cached rows. Each read +targets one guest's canonical metrics target on `/api/metrics-store/history`, +is capped at 36 points, and is never polled. Equivalent live guest snapshots +must not restart a read. A range change or leaving the lens mode cancels +pending and in-flight reads, while leaving a row lets its read settle into the +bounded cache. Per-row query owners, estate-wide fan-out, and polling remain +forbidden. Persistent Trends retains one cache-keyed +`fetchWorkloadsSummaryAndCache` reader for all rendered guest rows. Grouped +host rows render no metric cells, so the WorkloadsSurface reader never polls +`fetchInfrastructureSummaryAndCache`; node series belong to the page's hosts +table reader. Range-sensitive readers clear prior-range data when no exact cache entry exists and forward cancellation so superseded range work does not continue occupying browser connections. Sparkline ranges must stay bounded to the governed compact table windows. Expanded history belongs in the existing guest drawer chart diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index d505a37b64..d60d234213 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -1535,7 +1535,10 @@ When every node shares one cluster and none is standalone, the model drops the `cluster` column (it would repeat one value on every row) and `ProxmoxNodesTable` names that cluster in the table header instead, visible during search and independent of the inventory-count preference, so the -cluster identity is never lost. Workload status buckets read the canonical +cluster identity is never lost. Its Trends sparklines read node history +through `useWorkloadTableMetricHistory` with `series: 'nodes'`; guest history +stays with the embedded workloads table below it, so the nodes table adds no +guest-history poll to the page. Workload status buckets read the canonical health too: a running workload whose unified health names an open `warning_alert` or `critical_alert` reason counts under Attention, while `backup_stale` and other non-alert reasons do not. diff --git a/frontend-modern/browser-verification/4e061179721ef90a.json b/frontend-modern/browser-verification/4e061179721ef90a.json new file mode 100644 index 0000000000..2c87631f4c --- /dev/null +++ b/frontend-modern/browser-verification/4e061179721ef90a.json @@ -0,0 +1,39 @@ +{ + "version": 1, + "base_sha": "1fe9f86d55271345492945a25ab9398892ec6bf0", + "verified_at": "2026-10-06T22:35:29Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Workloads/useWorkloadTableMetricHistory.ts", + "frontend-modern/src/components/Workloads/useWorkloadsState.ts", + "frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx" + ], + "content_sha256": { + "frontend-modern/src/components/Workloads/useWorkloadTableMetricHistory.ts": "1879e91c0eb901261d6497cc0245e2a72b8d0acc72414cd69d3369d187e59fc5", + "frontend-modern/src/components/Workloads/useWorkloadsState.ts": "0f710d2544c964ae36e04f5bdb1bb2e3f2db37deb9b1ad804df6909e22256fc9", + "frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx": "32a8c017790a55dc230585e189bbff5cd3e81ee932a80c4379fc1be3ea61172d" + }, + "routes": ["/proxmox/overview", "/vmware"], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Vite dev server serving this tree (based on PR 2622 head 1fe9f86d55) against a private DEMO_MODE mock backend (50 PVE nodes, 929 guests, 7 ESXi hosts, telemetry off), headless Chromium via Playwright; desktop 1440x900 and phone 390x844.", + "Trends mode, 95 s per page counted per 30 s poll tick, before (base files swapped back under the same server) vs after: /vmware went from 1 /api/charts/infrastructure + 1 /api/charts/workloads read per tick to 0 + 1; /proxmox/overview from 2 + 2 to 1 + 1. Same on both viewports (95 s totals: vmware 4+4 to 0+4, proxmox 8+8 to 4+4).", + "Proxmox nodes table in Trends: every rendered node row draws its CPU, memory and disk sparklines (36 of 36 on desktop, 18 of 18 on phone), same as before; guest table sparklines unchanged (Proxmox 420 desktop / 108 phone, vSphere 54).", + "Bars mode on /proxmox/overview: node table shows no sparklines and the page makes no /api/charts reads.", + "Screenshots of both tables in Trends inspected at both viewports. The only failed request is the read-only demo refusing POST /api/usage/workload-history (403), the Trends activity ping, which this change does not touch." + ], + "interactions": [ + "Desktop: opened the View popover and switched Bars to Trends; phone: opened Filters, then View, and switched Bars to Trends. The Trends button reports aria-pressed=true and the switch triggers exactly one workloads read and one infrastructure read.", + "Navigated between /vmware and /proxmox/overview with Trends persisted and watched three poll cycles on each page.", + "Scrolled the nodes table into view before each capture." + ] +} diff --git a/frontend-modern/src/components/Workloads/__tests__/HistoryResponse.contract.test.tsx b/frontend-modern/src/components/Workloads/__tests__/HistoryResponse.contract.test.tsx index f2a43447a8..85cb5ee0f2 100644 --- a/frontend-modern/src/components/Workloads/__tests__/HistoryResponse.contract.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/HistoryResponse.contract.test.tsx @@ -205,6 +205,7 @@ describe('production History consumers reject unbound response evidence', () => onDemand: () => true, range: () => '24h', selectedNode: () => null, + series: 'guests', }); return ( diff --git a/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx b/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx index 7a272f8b5c..70b013dddc 100644 --- a/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx @@ -452,6 +452,9 @@ describe('Workloads platform-page embed contract', () => { expect(nodesTableSource).toContain('useWorkloadTableMetricHistory'); expect(nodesTableSource).toContain('MetricMiniSparkline'); expect(nodesTableSource).toContain('isSparklineMode'); + // The hosts table draws node rows only: it polls the infrastructure + // summary and leaves guest history to the embedded workloads reader. + expect(nodesTableSource).toContain("series: 'nodes',"); }); it('lets platform pages move grouped host drawers to a dedicated host table owner', async () => { @@ -1432,12 +1435,21 @@ describe('Workloads performance contract', () => { expect(workloadsFilterModelSource).toContain('mobileTrailing?: JSX.Element;'); expect(workloadsStateSource).toContain('useWorkloadRouteState'); expect(workloadsStateSource).toContain('range: workloadMetricHistoryRange'); + // Grouped node rows render no metric cells, so Trends on the Workloads + // surface must not poll the infrastructure summary. + expect(workloadsStateSource).toContain("series: 'guests',"); expect(workloadTableMetricHistoryStateSource).toContain( 'fetchWorkloadsSummaryAndCache(parsed.range', ); expect(workloadTableMetricHistoryStateSource).toContain( 'fetchInfrastructureSummaryAndCache(parsed.range', ); + expect(workloadTableMetricHistoryStateSource).toContain( + "if (options.series !== 'guests' || !options.enabled()) return null;", + ); + expect(workloadTableMetricHistoryStateSource).toContain( + "options.series === 'nodes' && options.enabled() ? buildHistoryScope(options.range()) : null", + ); expect( workloadTableMetricHistoryStateSource.match(/retainPreviousValueOnSourceChange: false/g), ).toHaveLength(2); diff --git a/frontend-modern/src/components/Workloads/__tests__/useWorkloadTableMetricHistory.test.tsx b/frontend-modern/src/components/Workloads/__tests__/useWorkloadTableMetricHistory.test.tsx index 9acd3d5d4c..a176ba845b 100644 --- a/frontend-modern/src/components/Workloads/__tests__/useWorkloadTableMetricHistory.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/useWorkloadTableMetricHistory.test.tsx @@ -4,10 +4,15 @@ import { createSignal, type Accessor } from 'solid-js'; import { ChartsAPI, type AllMetricsHistoryResponse } from '@/api/charts'; import { resetCreateNonSuspendingQueryCacheForTest } from '@/hooks/createNonSuspendingQuery'; +import type { Node } from '@/types/api'; import type { WorkloadGuest } from '@/types/workloads'; +import { __resetInfrastructureSummaryFetchesForTests } from '@/utils/infrastructureSummaryCache'; import { useWorkloadTableMetricHistory } from '../useWorkloadTableMetricHistory'; -import type { WorkloadTableMetricHistoryRange } from '../workloadMetricHistoryModel'; +import { + WORKLOAD_TABLE_HISTORY_POLL_MS, + type WorkloadTableMetricHistoryRange, +} from '../workloadMetricHistoryModel'; const guest = { id: 'cluster-a:pve1:101', @@ -47,6 +52,7 @@ function HistoryProbe(props: { prefetchGuests: props.prefetchGuests, range: props.range, selectedNode: () => null, + series: 'guests', }); const activeHistoryValue = () => { @@ -57,13 +63,75 @@ function HistoryProbe(props: { return
{activeHistoryValue()}
; } +const node = { id: 'cluster-a-pve1', name: 'pve1', instance: 'cluster-a' } as Node; +const point = (value: number) => [{ timestamp: 2, value, min: value, max: value }]; + +function SeriesProbe(props: { series: 'guests' | 'nodes' }) { + const reader = useWorkloadTableMetricHistory({ + enabled: () => true, + range: () => '1h', + series: props.series, + }); + const latest = (series: ReturnType) => + series[0]?.points.at(-1)?.value ?? 'none'; + + return ( +
+ {`guest=${latest(reader.getGuestMetricSeries(guest, 'cpu'))} node=${latest( + reader.getNodeMetricSeries(node, 'cpu'), + )}`} +
+ ); +} + afterEach(() => { cleanup(); resetCreateNonSuspendingQueryCacheForTest(); + __resetInfrastructureSummaryFetchesForTests(); vi.restoreAllMocks(); + vi.useRealTimers(); }); describe('useWorkloadTableMetricHistory', () => { + it.each([ + ['guests', 'guest=11 node=none', 'workloads'], + ['nodes', 'guest=none node=22', 'infrastructure'], + ] as const)( + 'polls only the summary its %s rows read', + async (series, expectedValues, polledRoute) => { + vi.useFakeTimers(); + const workloadSpy = vi.spyOn(ChartsAPI, 'getWorkloadCharts').mockResolvedValue({ + data: { [guest.id]: { cpu: point(11) } }, + dockerData: {}, + guestTypes: {}, + timestamp: 2, + stats: { oldestDataTimestamp: 1 }, + }); + const infrastructureSpy = vi + .spyOn(ChartsAPI, 'getInfrastructureSummaryCharts') + .mockResolvedValue({ + nodeData: { [node.id]: { cpu: point(22) } }, + timestamp: 2, + stats: { oldestDataTimestamp: 1 }, + }); + const [polledSpy, idleSpy] = + polledRoute === 'workloads' + ? [workloadSpy, infrastructureSpy] + : [infrastructureSpy, workloadSpy]; + + render(() => ); + await vi.advanceTimersByTimeAsync(0); + + expect(screen.getByTestId('series-values')).toHaveTextContent(expectedValues); + expect(polledSpy).toHaveBeenCalledTimes(1); + expect(idleSpy).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(WORKLOAD_TABLE_HISTORY_POLL_MS * 2); + expect(polledSpy).toHaveBeenCalledTimes(3); + expect(idleSpy).not.toHaveBeenCalled(); + }, + ); + it('bounds slow-estate warming and prioritizes a distant active guest', async () => { const guests = Array.from({ length: 500 }, (_, index) => ({ ...guest, diff --git a/frontend-modern/src/components/Workloads/useWorkloadTableMetricHistory.ts b/frontend-modern/src/components/Workloads/useWorkloadTableMetricHistory.ts index 9a0969b59f..768fdc4b55 100644 --- a/frontend-modern/src/components/Workloads/useWorkloadTableMetricHistory.ts +++ b/frontend-modern/src/components/Workloads/useWorkloadTableMetricHistory.ts @@ -37,7 +37,13 @@ interface WorkloadTableMetricHistoryOptions { onDemand?: Accessor; prefetchGuests?: Accessor; range: Accessor; - selectedNode: Accessor; + selectedNode?: Accessor; + // The rows this caller draws while `enabled`. Each estate-wide history is a + // polled summary read, so a reader polls only the one its rows consume: + // guest rows read the workloads summary, node rows the infrastructure + // summary. Readers do not share in-flight requests, so a second reader on + // the same page pays for its own reads. + series: 'guests' | 'nodes'; } interface ActiveGuestHistoryQueryKey { @@ -111,13 +117,13 @@ const buildActiveGuestHistoryCacheKey = (key: ActiveGuestHistoryQueryKey): strin export function useWorkloadTableMetricHistory( options: WorkloadTableMetricHistoryOptions, ): WorkloadMetricHistoryReader { - const selectedNodeScope = createMemo(() => normalizeNodeScope(options.selectedNode())); + const selectedNodeScope = createMemo(() => normalizeNodeScope(options.selectedNode?.())); const workloadHistoryScope = createMemo(() => { - if (!options.enabled()) return null; + if (options.series !== 'guests' || !options.enabled()) return null; return buildHistoryScope(options.range(), selectedNodeScope()); }); const infrastructureHistoryScope = createMemo(() => - options.enabled() ? buildHistoryScope(options.range()) : null, + options.series === 'nodes' && options.enabled() ? buildHistoryScope(options.range()) : null, ); const workloadHistory = createNonSuspendingQuery({ diff --git a/frontend-modern/src/components/Workloads/useWorkloadsState.ts b/frontend-modern/src/components/Workloads/useWorkloadsState.ts index 195099789b..8b5b954709 100644 --- a/frontend-modern/src/components/Workloads/useWorkloadsState.ts +++ b/frontend-modern/src/components/Workloads/useWorkloadsState.ts @@ -575,6 +575,9 @@ export function useWorkloadsState(props: WorkloadsSurfaceProps) { prefetchGuests: metricHistoryPrefetchGuests, range: workloadMetricHistoryRange, selectedNode, + // Grouped node rows carry no metric cells, so this reader never polls + // the infrastructure summary. + series: 'guests', }); const [workloadHistoryHintSeen, setWorkloadHistoryHintSeen] = usePersistentSignal( STORAGE_KEYS.WORKLOADS_HISTORY_HINT_SEEN, diff --git a/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx b/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx index 191809f431..469b881f1e 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx @@ -258,12 +258,13 @@ export const ProxmoxNodesTable: Component<{ nodePreview.toggle(); }; - // Use the same canonical history reader the workloads table uses; cache - // keys collide so the two readers dedupe their fetches. + // Use the same canonical history reader the workloads table uses. This + // table draws node rows only, so it polls the infrastructure summary and + // leaves the guest history to the workloads table below. const metricHistory = useWorkloadTableMetricHistory({ enabled: isSparklineMode, range: () => props.metricHistoryRange?.() ?? '1h', - selectedNode: () => '', + series: 'nodes', }); return ( diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx b/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx index fc4e8b9ae9..89b48696e1 100644 --- a/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx +++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx @@ -1,4 +1,5 @@ import { cleanup, fireEvent, render, screen } from '@solidjs/testing-library'; +import { createSignal } from 'solid-js'; import { createStore } from 'solid-js/store'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; @@ -9,6 +10,7 @@ const nodeDrawerMock = vi.hoisted(() => vi.fn()); const activeAlertsMock = vi.hoisted(() => ({ value: {} as Record })); const getMetricThresholdsMock = vi.hoisted(() => vi.fn()); const temperatureGaugeMock = vi.hoisted(() => vi.fn()); +const metricHistoryMock = vi.hoisted(() => vi.fn()); vi.mock('@/hooks/useBreakpoint', () => ({ useBreakpoint: () => ({ @@ -63,9 +65,12 @@ vi.mock('@/components/shared/TemperatureGauge', () => ({ })); vi.mock('@/components/Workloads/useWorkloadTableMetricHistory', () => ({ - useWorkloadTableMetricHistory: () => ({ - getNodeMetricSeries: () => [], - }), + useWorkloadTableMetricHistory: (options: unknown) => { + metricHistoryMock(options); + return { + getNodeMetricSeries: () => [], + }; + }, })); vi.mock('@/components/Workloads/NodeDrawer', () => ({ @@ -578,6 +583,34 @@ describe('ProxmoxNodesTable', () => { expect(screen.queryByTestId('temperature-gauge')).not.toBeInTheDocument(); }); + it('reads node history only, and only while Trends is on', () => { + const [mode, setMode] = createSignal<'bars' | 'sparklines'>('bars'); + render(() => ( + } + emptyTitle="No Proxmox VE nodes" + emptyDescription="No nodes" + /> + )); + + expect(metricHistoryMock).toHaveBeenCalledTimes(1); + const options = metricHistoryMock.mock.calls[0][0] as { + enabled: () => boolean; + selectedNode?: unknown; + series: string; + }; + // Guest history belongs to the workloads table below; a second guest + // reader here would poll the workloads route for rows it never draws. + expect(options.series).toBe('nodes'); + expect(options.selectedNode).toBeUndefined(); + expect(options.enabled()).toBe(false); + setMode('sparklines'); + expect(options.enabled()).toBe(true); + }); + it('updates every availability signal when a live row becomes offline', () => { const [state, setState] = createStore({ nodes: [makeNodeResource({ uptime: 187_200, temperature: 76 })], From 344f1f85310199ec7880c722bd2f5a65718e5244 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:38:25 +0100 Subject: [PATCH 35/53] Raise disk health alerts for mock physical disks checkMockAlerts evaluated guests, agents, nodes, storage, PBS, PMG and Docker against the mock estate but never physical disks. The fixture keeps a FAILED cohort and worn SSDs stable on purpose, so mock and demo stacks showed failing and "Replace Now" disks with no disk-health or disk-wearout alert, and the disk alert lifecycle could not be exercised without real hardware. The pass applies the physical disk poller's boundary: only disks on online nodes, matched by instance and node name, with the linked agent's --disk-exclude patterns forcing a matched device healthy. The per-disk evaluation moves into checkPhysicalDiskAlerts so the poller and the mock pass share one rule. The poller's agent-link settle deferral is not applied, because fixture links and exclusions are complete from the first pass. Like the other loops in checkMockAlerts, the disk loop runs after the pass's single mode check, so the existing race with leaving mock mode now covers disk alerts too. That race also involves other fixture evaluation paths and is left for a monitor-wide fix. On the public demo's estate (8 nodes) this adds one warning-level disk-wearout alert and no disk-health alert: that estate has no FAILED Proxmox disk. --- .../v6/internal/subsystems/monitoring.md | 56 ++++++++ .../monitoring/canonical_guardrails_test.go | 3 +- internal/monitoring/monitor_alerts.go | 43 +++++++ .../monitoring/monitor_mock_alerts_test.go | 121 ++++++++++++++++++ internal/monitoring/monitor_pve.go | 25 ++-- 5 files changed, 238 insertions(+), 10 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index a912bfce62..0c751ad9be 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -1679,6 +1679,62 @@ boundaries: `TestMockKubernetesLoneHeavyPodStaysUnderNodeCeiling` in `internal/mock/generator_test.go`. +Mock alert evaluation covers physical disks as well. Live Proxmox disks are +evaluated by the physical disk poller in `maybePollPhysicalDisksAsync`, which +only polls configured Proxmox instances, so fixture disks had no evaluation +path. The fixture deliberately keeps a FAILED health cohort and worn SSDs +stable across restarts, so the estate showed failing and "Replace Now" disks +with no `disk-health` or `disk-wearout` alert. `checkMockAlerts` now evaluates +fixture disks through `checkMockPhysicalDiskAlerts`. Both paths share the +per-disk rule in `checkPhysicalDiskAlerts` (a device matched by the linked +agent's `--disk-exclude` patterns is evaluated as healthy, otherwise +`CheckDiskHealth` judges the disk), and the mock pass applies the poller's +boundary around it: + +- Only disks on online nodes are evaluated, as the poller only evaluates nodes + it reached. A disk on an offline node keeps whatever alert it had. The mock + pass treats every disk on an online fixture node as collected; the poller + also skips a node whose disk query and SMART fallback both failed. Nodes are + matched by instance and name, as the poller only pairs a disk with its own + instance's nodes. +- Excluding a disk resolves the alerts it already raised, because the excluded + device is evaluated as healthy rather than skipped. +- The poller's wait for host-agent links to settle after a restart (#1674) is + not applied: fixture links and exclusions are complete from the first pass. +- A disk whose node name leaves the estate loses its alerts through + `CleanupAlertsForNodes`, which the same pass already runs. That cleanup + matches node names only, so it relies on fixture node names being unique + across instances, which they are. + +The disk loop relies on the pass's single up-front mode check, like every +loop in `checkMockAlerts` except the host-agent step, which re-checks under its +own lock. A pass already past that check when mock mode is switched off can +reopen its fixture alerts after `SetMockMode(false)` clears them, and +other paths that evaluate fixture data (frontend refresh into unified alert +evaluation, backup rollups) have the same window. That mode-exit race predates +the disk loop and is not closed here; a lock around `checkMockAlerts` alone +would leave the other paths open. + +The disk evaluation runs on the mock alert tick, not the poller's disk +interval (five minutes by default, `PhysicalDiskPollingMinutes` per instance). +`CheckDiskHealth` logs at error level whenever it evaluates a FAILED disk +(models with a known firmware bug skip the health check) and at warn level for +a worn SSD, so an evaluated fixture disk in either state logs on every tick. On +2026-10-06, private stacks with the public demo's estate (8 nodes) carried 12 +Proxmox disks, none FAILED. Baseline against patched, the only new disk alert +was one warning-level `disk-wearout` (a 7% SSD on `pve5`), whose resource ID is +one of the disk resource's canonical aliases, and neither stack had a +`disk-health` alert. `CheckDiskHealth` raises only those two alert types, so +the other differences between the two boots (ZFS and Docker service alerts) are +not from the disk pass; those fixtures draw random state each boot. The default +test estate raises five `disk-health` and one `disk-wearout` alert. The tests +pin the boundary: `TestCheckMockAlertsEvaluatesPhysicalDisks` and +`TestCheckMockPhysicalDiskAlertsFollowsPollerBoundary` in +`internal/monitoring/monitor_mock_alerts_test.go`, with +`TestProxmoxDiskAlertsRunOnMergedDiskState` in +`internal/monitoring/canonical_guardrails_test.go` keeping the poller on the +shared helper. + Host and container-runtime disk collection supports an explicit include list for filesystems hidden by Pulse's automatic virtual/container filtering. The include list is bounded to that automatic filter; explicit disk exclusions diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go index 3c398a8cbf..6234aff22a 100644 --- a/internal/monitoring/canonical_guardrails_test.go +++ b/internal/monitoring/canonical_guardrails_test.go @@ -2234,7 +2234,8 @@ func TestProxmoxDiskAlertsRunOnMergedDiskState(t *testing.T) { file: "monitor_pve.go", snippets: []string{ "allDisks = mergeHostAgentSMARTIntoDisks(allDisks, nodesFromState, hosts)", - "m.alertManager.CheckDiskHealth(inst, disk.Node, proxmoxDiskFromPhysicalDisk(disk))", + "m.checkPhysicalDiskAlerts(inst, disk, diskExcludeByNode[disk.Node])", + "m.alertManager.CheckDiskHealth(instance, disk.Node, proxmoxDiskFromPhysicalDisk(disk))", "func proxmoxDiskFromPhysicalDisk(disk models.PhysicalDisk) proxmox.Disk {", }, }, diff --git a/internal/monitoring/monitor_alerts.go b/internal/monitoring/monitor_alerts.go index 89e870836f..a75e7dd557 100644 --- a/internal/monitoring/monitor_alerts.go +++ b/internal/monitoring/monitor_alerts.go @@ -716,6 +716,14 @@ func (m *Monitor) checkMockAlerts() { m.alertManager.CheckStorageWithCapacityTrend(storage, m.storageCapacityTrend(storage, time.Now())) } + // Check alerts for physical disks. Live disks are evaluated by the + // physical disk poller, which only polls configured Proxmox instances, and + // the fixture deliberately keeps a FAILED cohort and worn SSDs, so without + // this pass the estate shows failing disks with no disk-health or + // disk-wearout alert. + log.Debug().Int("diskCount", len(state.PhysicalDisks)).Msg("checking physical disk alerts") + m.checkMockPhysicalDiskAlerts(state.PhysicalDisks, state.Nodes, state.Hosts) + // Check alerts for PBS instances log.Debug().Int("pbsCount", len(state.PBSInstances)).Msg("checking PBS alerts") for _, pbsInst := range state.PBSInstances { @@ -815,3 +823,38 @@ func (m *Monitor) checkMockDockerHostAlerts(host models.DockerHost) { } m.alertManager.CheckDockerHost(host) } + +// checkMockPhysicalDiskAlerts applies the physical disk poller's alert +// boundary to fixture disks. The poller only evaluates disks on nodes it +// reached, so a disk on a node that is not online keeps whatever alert it +// had, and a device matched by the linked agent's --disk-exclude patterns is +// evaluated as healthy. The poller's wait for host-agent links to settle after +// a restart does not apply: fixture links and exclusions are complete from the +// first pass. +func (m *Monitor) checkMockPhysicalDiskAlerts(disks []models.PhysicalDisk, nodes []models.Node, hosts []models.Host) { + type nodeKey struct{ instance, name string } + + excludeByHost := make(map[string][]string, len(hosts)) + for _, host := range hosts { + if len(host.DiskExclude) > 0 { + excludeByHost[host.ID] = host.DiskExclude + } + } + onlineNodes := make(map[nodeKey]bool, len(nodes)) + excludeByNode := make(map[nodeKey][]string) + for _, node := range nodes { + key := nodeKey{instance: node.Instance, name: node.Name} + onlineNodes[key] = node.Status == "online" + if patterns := excludeByHost[node.LinkedAgentID]; node.LinkedAgentID != "" && len(patterns) > 0 { + excludeByNode[key] = patterns + } + } + + for _, disk := range disks { + key := nodeKey{instance: disk.Instance, name: disk.Node} + if !onlineNodes[key] { + continue + } + m.checkPhysicalDiskAlerts(disk.Instance, disk, excludeByNode[key]) + } +} diff --git a/internal/monitoring/monitor_mock_alerts_test.go b/internal/monitoring/monitor_mock_alerts_test.go index 66e69d616a..3c5c933339 100644 --- a/internal/monitoring/monitor_mock_alerts_test.go +++ b/internal/monitoring/monitor_mock_alerts_test.go @@ -239,3 +239,124 @@ func TestMockHostAgentPassAfterLeavingMockModeRegistersNothing(t *testing.T) { } t.Fatal("stale mock pass registered fixture hostname pve1 and suppressed the real node's memory alert") } + +// diskAlertKeys lists the active disk-health and disk-wearout alerts as +// "type instance/node device" so assertions name the disk, not the alert ID. +func diskAlertKeys(manager *alerts.Manager) map[string]bool { + keys := make(map[string]bool) + for _, alert := range manager.GetActiveAlerts() { + if alert.Type != "disk-health" && alert.Type != "disk-wearout" { + continue + } + devPath, _ := alert.Metadata["disk_path"].(string) + keys[alert.Type+" "+alert.Instance+"/"+alert.Node+" "+devPath] = true + } + return keys +} + +func TestCheckMockAlertsEvaluatesPhysicalDisks(t *testing.T) { + // Pin the default estate: a PULSE_MOCK_* environment can select one with + // no FAILED disk (the public demo's eight nodes have none). + previousEnabled, previousConfig := mock.IsMockEnabled(), mock.GetConfig() + t.Cleanup(func() { + mustSetMockEnabled(t, false) + mock.SetMockConfig(previousConfig) + mustSetMockEnabled(t, previousEnabled) + }) + mustSetMockEnabled(t, false) + cfg := mock.DefaultConfig + cfg.UpdateInterval = 5 * time.Minute + mock.SetMockConfig(cfg) + mustSetMockEnabled(t, true) + manager := newMockHostAlertTestManager(t) + + monitor := &Monitor{alertManager: manager} + monitor.checkMockAlerts() + + // The fixture keeps a FAILED cohort and worn SSDs stable across restarts + // so the disk alert lifecycle has evidence to act on. Each must raise the + // alert the physical disk poller raises for a live disk, and a disk on a + // node the poller would not reach raises nothing. + state := mock.CurrentFixtureGraph().State + onlineNodes := make(map[string]bool, len(state.Nodes)) + for _, node := range state.Nodes { + onlineNodes[node.Instance+"/"+node.Name] = node.Status == "online" + } + want := make(map[string]bool) + failed, worn := 0, 0 + for _, disk := range state.PhysicalDisks { + if !onlineNodes[disk.Instance+"/"+disk.Node] { + continue + } + if strings.EqualFold(disk.Health, "FAILED") { + want["disk-health "+disk.Instance+"/"+disk.Node+" "+disk.DevPath] = true + failed++ + } + if disk.Wearout >= 0 && disk.Wearout < 10 { + want["disk-wearout "+disk.Instance+"/"+disk.Node+" "+disk.DevPath] = true + worn++ + } + } + if failed == 0 || worn == 0 { + t.Fatalf("mock fixture has %d FAILED and %d worn disks on online nodes, want at least one of each", failed, worn) + } + + got := diskAlertKeys(manager) + for key := range got { + if !want[key] { + t.Errorf("mock pass raised unexpected disk alert %q", key) + } + } + for key := range want { + if !got[key] { + t.Errorf("mock pass raised no alert %q", key) + } + } +} + +func TestCheckMockPhysicalDiskAlertsFollowsPollerBoundary(t *testing.T) { + manager := newMockHostAlertTestManager(t) + monitor := &Monitor{alertManager: manager} + + nodes := []models.Node{ + {ID: "west-pve1", Name: "pve1", Instance: "west", Status: "online", LinkedAgentID: "host-west-pve1"}, + {ID: "east-pve1", Name: "pve1", Instance: "east", Status: "offline"}, + } + hosts := []models.Host{{ID: "host-west-pve1", Hostname: "pve1", Status: "online"}} + disks := []models.PhysicalDisk{ + {Instance: "west", Node: "pve1", DevPath: "/dev/sda", Model: "Crucial MX500 2TB", Type: "sata", Health: "FAILED", Wearout: -1}, + {Instance: "west", Node: "pve1", DevPath: "/dev/nvme0n1", Model: "Samsung 980 PRO 2TB", Type: "nvme", Health: "PASSED", Wearout: 6}, + // Same node name on another instance: that node is offline, so the + // poller would not reach it and its disk must not alert. + {Instance: "east", Node: "pve1", DevPath: "/dev/sdb", Model: "WD Red Pro 8TB", Type: "sata", Health: "FAILED", Wearout: -1}, + } + + monitor.checkMockPhysicalDiskAlerts(disks, nodes, hosts) + got := diskAlertKeys(manager) + if len(got) != 2 || !got["disk-health west/pve1 /dev/sda"] || !got["disk-wearout west/pve1 /dev/nvme0n1"] { + t.Fatalf("disk alerts = %v, want only FAILED health on west/pve1 /dev/sda and wearout on west/pve1 /dev/nvme0n1", got) + } + + // The linked agent's --disk-exclude forces matched devices healthy, so + // excluding a disk resolves the alerts it already raised. Endurance + // recovery needs three healthy-looking passes. + hosts[0].DiskExclude = []string{"/dev/sda", "nvme0n1"} + for i := 0; i < 3; i++ { + monitor.checkMockPhysicalDiskAlerts(disks, nodes, hosts) + } + if got := diskAlertKeys(manager); len(got) != 0 { + t.Fatalf("excluded devices kept disk alerts %v", got) + } + + // A node that leaves the estate takes its disk alerts with it through the + // node cleanup the mock pass runs first. + hosts[0].DiskExclude = nil + monitor.checkMockPhysicalDiskAlerts(disks, nodes, hosts) + if got := diskAlertKeys(manager); len(got) != 2 { + t.Fatalf("disk alerts after removing the exclusion = %v, want both west/pve1 alerts back", got) + } + manager.CleanupAlertsForNodes(map[string]bool{"pve2": true}) + if got := diskAlertKeys(manager); len(got) != 0 { + t.Fatalf("disk alerts outlived their departed node: %v", got) + } +} diff --git a/internal/monitoring/monitor_pve.go b/internal/monitoring/monitor_pve.go index 8b03fac7e0..581307f263 100644 --- a/internal/monitoring/monitor_pve.go +++ b/internal/monitoring/monitor_pve.go @@ -1175,15 +1175,7 @@ func (m *Monitor) maybePollPhysicalDisksAsync( Int("wearout", disk.Wearout). Msg("Checking disk health") - if excludePatterns, ok := diskExcludeByNode[disk.Node]; ok && fsfilters.MatchesDeviceExclude(disk.DevPath, excludePatterns) { - healthyDisk := proxmoxDiskFromPhysicalDisk(disk) - healthyDisk.Health = "PASSED" - healthyDisk.Wearout = 100 - m.alertManager.CheckDiskHealth(inst, disk.Node, healthyDisk) - continue - } - - m.alertManager.CheckDiskHealth(inst, disk.Node, proxmoxDiskFromPhysicalDisk(disk)) + m.checkPhysicalDiskAlerts(inst, disk, diskExcludeByNode[disk.Node]) } // Write SMART metrics to persistent store @@ -1380,6 +1372,21 @@ func preserveUnavailablePhysicalDiskEvidence(current, previous models.PhysicalDi return current } +// checkPhysicalDiskAlerts evaluates one Proxmox physical disk's health and +// wearout alerts. A device matched by the linked host agent's --disk-exclude +// patterns is evaluated as healthy, so excluding a disk also resolves the +// alerts it already raised. +func (m *Monitor) checkPhysicalDiskAlerts(instance string, disk models.PhysicalDisk, excludePatterns []string) { + if len(excludePatterns) > 0 && fsfilters.MatchesDeviceExclude(disk.DevPath, excludePatterns) { + healthyDisk := proxmoxDiskFromPhysicalDisk(disk) + healthyDisk.Health = "PASSED" + healthyDisk.Wearout = 100 + m.alertManager.CheckDiskHealth(instance, disk.Node, healthyDisk) + return + } + m.alertManager.CheckDiskHealth(instance, disk.Node, proxmoxDiskFromPhysicalDisk(disk)) +} + func proxmoxDiskFromPhysicalDisk(disk models.PhysicalDisk) proxmox.Disk { return proxmox.Disk{ DevPath: disk.DevPath, From c06af69fcb225f64c7823bcb9536fd2f810714e8 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 00:14:49 +0100 Subject: [PATCH 36/53] Show a host whose agent stopped reporting as offline A host agent past its reporting lease is offline: the monitor marks it so and raises "Host is offline". The registry's stale pass ranked the agent's stale sighting above offline, though, so the machine reached every consumer as a warning. The Machines table showed it amber and rendered its last CPU, memory, disk and temperature as current readings, while its own drawer led with the critical offline alert. A Docker host was offline only until its sighting crossed the 120-second stale threshold, then flipped to warning, and Kubernetes clusters likewise. Each sighting now records when a push reporter (host agent, Docker, Kubernetes) delivered its machine, Docker host or cluster as offline, and status aggregation counts that as offline whatever the sighting's age. The sighting itself still reads stale, and a live source still carries a merged row, so a Proxmox node with a dead agent stays online. A copy that went through JSON re-derives the marker from its stored status, and a manual link joining two resources from one source keeps the fresher sighting, so a retired agent cannot pull a live one down. Offline rows then take the existing danger gate. The Machines Uptime and Temp cells and the Docker hosts Temp cell had no gate and now follow it. The drawer drops a silent agent's uptime and marks its non-disk Thermals rows "(last known)", as retained disk rows already are. --- .../subsystems/frontend-primitives.md | 8 + .../subsystems/performance-and-scalability.md | 5 + .../internal/subsystems/storage-recovery.md | 10 + .../internal/subsystems/unified-resources.md | 38 +- .../3b34c585696e9d41.json | 49 +++ .../Infrastructure/ResourceDetailSummary.tsx | 6 +- .../ResourceDetailSummary.offline.test.tsx | 64 ++++ .../__tests__/resourceDetailMappers.test.ts | 41 +++ .../Infrastructure/resourceDetailMappers.ts | 33 +- .../useResourceDetailDrawerDerivedState.ts | 9 +- .../src/features/docker/DockerHostsTable.tsx | 6 +- .../__tests__/DockerHostsTable.test.tsx | 52 +++ .../standalone/AgentsMachinesTable.tsx | 27 +- .../__tests__/AgentsMachinesTable.test.tsx | 97 ++++- internal/unifiedresources/clone_test.go | 6 + internal/unifiedresources/registry.go | 55 ++- .../registry_merge_policy_test.go | 348 ++++++++++++++++++ internal/unifiedresources/types.go | 6 + 18 files changed, 831 insertions(+), 29 deletions(-) create mode 100644 frontend-modern/browser-verification/3b34c585696e9d41.json create mode 100644 frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.offline.test.tsx diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index f585ee45a4..378d00185c 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -5591,6 +5591,14 @@ Standalone, TrueNAS, and vSphere platform tables and their table-model helpers must compose those helpers instead of declaring local `metricFallback` / `finiteMetric` helpers or inlining centered muted dash fallback markup in metric cells. +A row whose status indicator is `danger` (offline) blanks every reading from +its last report, not only the metric bars. Proxmox node rows already gate +uptime and temperature on online. The Machines table gates its Uptime and +Temperature cells, and the Docker hosts table its temperature cell, on the same +check. A machine another source keeps up stays rendered, since its cell may +carry that source's current reading. Numeric cells keep their own +right-aligned empty dash; the centred `PlatformTableMetricFallback` marker +stays specific to metric-bar cells. Platform table metric severity coloring is alert-backed, not hardcoded. The Docker host and container, Proxmox node, Kubernetes cluster and node, TrueNAS system and app, and vSphere host tables must resolve display thresholds diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index a7f0088b89..5087716ff4 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -2051,6 +2051,11 @@ in `frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts`. That module stays separate from `diskPresentation.ts` so the drawer and Machines chunks take the decision without the Storage presenter, and the rows add no reads, polling or aggregation work. +A silent host agent's other Thermals rows (pressure, limits, GPU, temperatures, +additional sensors, fans, power) take the same "(last known)" suffix and reason +title. `buildTemperatureRows` gets them through its `lastKnownReason` option, +which the drawer sets from the resource's own `agent.stale`, so the drawer +still reads nothing extra. The Proxmox node drawer overview should follow the existing guest drawer compact detail-section pattern and expose node-specific context such as platform, kernel, hardware, raw capacity, telemetry, and thermal facts rather than diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 2397cd4bb5..e96a2af8cf 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -3169,6 +3169,16 @@ pointer isolation are pinned by `TestResourceFromHostPreservesCustomSensorMeta` and `TestCloneResourceIsolatesCustomSensorValues`. +### Expired reporting leases change status only + +`SourceStatus` now carries an unexported marker that a push reporter (host +agent, Docker, Kubernetes) delivered its machine, Docker host or cluster as +offline at lease expiry. Status aggregation reads it, so such a row is +`offline` instead of `warning`. The marker is not serialized, persisted or +exposed on the wire, and the public `status`, `lastSeen` and cadence fields +are unchanged. No storage path, recovery read or protection judgement depends +on it. `TestCloneResource_MutateSourceStatusMap` pins that clones keep it. + ### Shared system-settings boundary dropped dead auto-update schedule fields The shared `internal/api` system-settings surface this subsystem consumes diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index d14cbc296c..f5e20f3eca 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -2270,10 +2270,12 @@ and temperature facts must also compose the frontend-primitives desktop rows use the shared fixed label track so labels and values remain visually adjacent instead of spanning the full drawer width. A standalone host agent past its reporting lease reaches the Machines table as -`warning` with `agent.stale`, not `offline`: by the time the lease expires the -registry has marked its only source stale, which outranks offline. The offline -metric fallback therefore does not blank its cells, and the temperature cell -has no such gate. For SMART disk temperatures the provenance travels on +`offline`, as described in "A push reporter past its lease is offline". The +offline metric fallback blanks its CPU, memory, disk, network, disk I/O, uptime +and temperature cells. The machine drawer drops a silent agent's uptime and +marks its non-disk Thermals rows "(last known)" with the reason as title, +keyed on `agent.stale` because those rows are that agent's own sensors. For SMART disk +temperatures on rows that still render, the provenance travels on `agent.sensors.smart[].collection`: the Machines temperature cell (`AgentsMachinesTable.tsx`, `agentMachineTableModel.ts`), its tooltip and the drawer's Thermals rows (`resourceDetailMappers.ts`) read it through @@ -6030,6 +6032,34 @@ to that boolean. Identity succession continues to rekey the same state row. round-trip persistence, attention suppression, and the retired remediation lock. +### A push reporter past its lease is offline + +The host agent, Docker and Kubernetes collectors push their own reports, and +the monitor marks the machine, Docker host or cluster offline only once its +reporting lease runs out. Ingest records that verdict on the source's +sighting, and `aggregateStatus` counts such a sighting as `offline` whatever +its age. The stale pass used to rank the stale sighting above `offline`, so a +silent standalone agent reached every consumer as `warning`. The Machines +table then showed it amber with its last report rendered as current readings, +while its own drawer led with a critical "Host is offline" alert. A Docker host +was offline only until its sighting crossed the 120-second stale threshold, +then flipped to `warning`. With this rule, such rows take the existing +offline treatment: frontend danger gates, the health verdict's `offline` +reason, offline filters and counts. + +`SourceStatus.Status` keeps describing delivery freshness, so the sighting +still reads `stale`. Health keeps its `telemetry_stale` reason, and +monitored-system reasons are unchanged. A live source still carries a merged +row: a Proxmox node whose linked agent stopped reporting stays `online` through +the PVE poll. Pull sources (Proxmox, PBS, PMG, TrueNAS, vSphere) and the +resources push collectors report about (guests, containers, pods, disks) keep +the stale-to-warning rule. A JSON copy re-derives the unexported marker from +its stored status in `IngestResources`, and a manual link that joins two +resources reported by one source keeps the fresher sighting. The tests in +`registry_merge_policy_test.go` pin the agent, Docker, Kubernetes, +mixed-source, round-trip and manual-link cases, plus the boundaries that stay +unchanged. + ### Canonical object drawer hierarchy The unified resource drawer is the shared operator-facing detail contract for diff --git a/frontend-modern/browser-verification/3b34c585696e9d41.json b/frontend-modern/browser-verification/3b34c585696e9d41.json new file mode 100644 index 0000000000..f9cb8ca29a --- /dev/null +++ b/frontend-modern/browser-verification/3b34c585696e9d41.json @@ -0,0 +1,49 @@ +{ + "version": 1, + "base_sha": "723f3165a1292268634b6054ab93d3015805d990", + "verified_at": "2026-10-06T23:14:41Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx", + "frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts", + "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts", + "frontend-modern/src/features/docker/DockerHostsTable.tsx", + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx" + ], + "content_sha256": { + "frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx": "f141220216a968f5953b7646035108f015a9f7de3c5ae5a9e3251ec6e7046fa5", + "frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts": "07b26eafecbd1ec93fb1dd90d4af6629778cef59a2d9e7450426cb88c90bdfdb", + "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts": "70955e370c5b2611557a5de8ba4df44b8f6dce47c1b08c611a154d232c786501", + "frontend-modern/src/features/docker/DockerHostsTable.tsx": "450cce7a4723ce7fe7b01ffe9d10c7d6439c34a1a6dda4fc0af6689ceb4741b5", + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx": "8d63a04f6a475f39de90cb9813b8bd4315525bf63318cfaf958ac195ae07950f" + }, + "routes": [ + "/standalone/machines", + "/docker" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Two private mock stacks (8-node estate) built from this tree: one with this commit's registry.go/types.go, one with origin/main's, both with a local-only go build -overlay of internal/mock/generator.go that holds standalone Linux agent host-linux-1 (Apollo-114) and Docker host edge-proxy-01 past their reporting leases the way evaluateHostAgents/evaluateDockerAgents leave them: status offline, LastSeen frozen 14m/9m before the first tick, last readings kept. Mock mode skips the host health sweep, hence the overlay.", + "Before (origin/main backend and frontend): /api/resources served Apollo-114 as status warning with agent.stale true and edge-proxy-01 as warning with health stale. The Machines row showed an amber dot, CPU 32%, Memory 74%, Disk 80%, Net and Disk I/O rates, Up 29d and Temp 45\u00b0C in threshold green beside 'Seen 14m ago', while the drawer led with the critical 'Host Apollo-114 is offline' alert and read 'Observed state Degraded, Uptime 29d 12h'.", + "After (this commit): both resources serve status offline, the agent/docker sightings still read stale, and health reads critical with reasons offline then telemetry_stale. The Apollo-114 row shows a red dot and '\u2014' in CPU, Memory, Disk, Net I/O, Disk I/O, Up and Temp at 1440 and 390; the status filter counts it under Offline. Spectrum-709 (reporting) keeps every value, Temp 47\u00b0C included.", + "Machine drawer at 1440 and 390: Runtime context reads 'Observed state Offline' and 'Last seen N mins ago' with no Uptime row; Platform details > Machine 'Show details' drops the System card's Uptime and the Thermals card reads 'Cpu.package 45\u00b0C (last known)' with title 'Last known reading, not current: host agent stopped reporting'.", + "Docker page at 1440 and 390: edge-proxy-01 shows a red dot, '\u2014' for CPU, Memory, Disk and Temp (the hold seeds a retained 51\u00b0C), and the existing 'No report for 16m' marker in Uptime; Edge Apps 01 keeps its values, Temp 38.5\u00b0C included.", + "Bundle check: WorkloadsSurface unchanged (70.20 kB), Standalone +0.04 kB, Docker +0.01 kB, InfrastructureWorkspace flat versus this tree without the frontend diff; all chunks within budget.", + "Not covered: a real agent going silent on hardware, light/dark differences (no new colour classes), and the drawer Disks card, which still threshold-colours a silent agent's retained filesystem usage (left for a follow-up)." + ], + "interactions": [ + "Enabled the Network, Disk I/O, Uptime and Temp columns through the Machines column preferences (localStorage init) and read every cell of the offline and reporting rows plus the status dot class at 1440 and 390.", + "Opened the Apollo-114 drawer by row click at both widths, read the Runtime context rows, expanded Platform details and the Machine 'Show details', and read the System and Thermals cards with their value titles.", + "Opened /docker at both widths and read every cell of edge-proxy-01 and Edge Apps 01.", + "Queried /api/resources on both stacks for status, per-source sighting status and health reasons of the held agent and Docker host." + ] +} diff --git a/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx b/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx index adb1a78581..43975c192f 100644 --- a/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx +++ b/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx @@ -217,9 +217,13 @@ export const InlineResourceSummaryTables: Component { const docker = dockerContainerMeta(props.resource); + // An offline resource has no current uptime: the retained value is from + // its last report, and the Last seen row already says when that was. const uptimeSeconds = docker ? getDockerContainerUptimeSeconds(props.resource) - : props.resource.uptime; + : props.resource.status === 'offline' + ? undefined + : props.resource.uptime; const identityRows = compactDetailRows([ ...props.drawer.primaryIdentityRows().map((row) => makeDetailRow(row.label, row.value)), props.showPlatformId ? makeDetailRow('Platform ID', props.resource.platformId) : null, diff --git a/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.offline.test.tsx b/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.offline.test.tsx new file mode 100644 index 0000000000..530048ca63 --- /dev/null +++ b/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.offline.test.tsx @@ -0,0 +1,64 @@ +import { cleanup, render, screen, within } from '@solidjs/testing-library'; +import { afterEach, describe, expect, it } from 'vitest'; + +import type { Resource } from '@/types/resource'; +import { InlineResourceSummaryTables } from '../ResourceDetailSummary'; +import type { UseResourceDetailDrawerStateResult } from '../useResourceDetailDrawerState'; + +const drawerState = { + primaryIdentityRows: () => [], + identityIpValues: () => [], + identityAliasValues: () => [], + aliasPreviewValues: () => [], + hasAliasOverflow: () => false, + identityCardHasRichData: () => true, + sourceSummary: () => null, + lastSeen: () => '14 mins ago', + lastSeenAbsolute: () => '', +} as unknown as UseResourceDetailDrawerStateResult; + +const machine = (status: Resource['status']): Resource => ({ + id: 'agent-apollo', + name: 'Apollo-114', + displayName: 'Apollo-114', + platformId: 'host-linux-1', + platformType: 'agent', + sourceType: 'agent', + type: 'agent', + status, + uptime: 2_500_000, + lastSeen: Date.parse('2026-10-06T22:00:00Z'), +}); + +afterEach(cleanup); + +describe('resource drawer runtime context for an offline resource', () => { + it('drops the uptime of an offline machine and keeps when it was last seen', () => { + render(() => ( + + )); + + const section = screen.getByTestId('resource-runtime-context-section'); + expect(within(section).getByText('offline')).toBeInTheDocument(); + expect(within(section).queryByText('Uptime')).toBeNull(); + expect(within(section).getByText('Last seen')).toBeInTheDocument(); + expect(within(section).getByText('14 mins ago')).toBeInTheDocument(); + }); + + it('keeps the uptime of a machine that is up', () => { + render(() => ( + + )); + + const section = screen.getByTestId('resource-runtime-context-section'); + expect(within(section).getByText('Uptime')).toBeInTheDocument(); + }); +}); diff --git a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts index 5596569845..9a3f4849f0 100644 --- a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts +++ b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { buildCustomSensorRows, buildTemperatureRows, + HOST_AGENT_STOPPED_REPORTING_REASON, formatInteger, formatSensorName, formatSourceType, @@ -248,6 +249,37 @@ describe('resourceDetailMappers', () => { ]); }); + it("marks a silent agent's sensor readings last known and leaves disks to their own state", () => { + const rows = buildTemperatureRows( + { + thermalState: { pressure: 'serious' }, + temperatureCelsius: { 'cpu.package': 45 }, + fanRpm: { chassis_fan: 1200 }, + smart: [ + { + device: 'sda', + temperature: 38, + collection: { temperature: { state: 'available', source: 'smartctl' } }, + }, + ], + }, + { lastKnownReason: HOST_AGENT_STOPPED_REPORTING_REASON }, + ); + + const lastKnown = 'Last known reading, not current: host agent stopped reporting'; + expect(rows).toEqual([ + { label: 'Thermal pressure', value: 'Serious (last known)', valueTitle: lastKnown }, + { label: 'Cpu.package', value: '45°C (last known)', valueTitle: lastKnown }, + { label: 'Chassis Fan', value: '1,200 RPM (last known)', valueTitle: lastKnown }, + // A disk row follows its own collection state, which the backend + // withdraws when the lease expires. + { label: 'Disk sda', value: '38°C', valueTitle: '38.0°C' }, + ]); + expect(buildTemperatureRows({ temperatureCelsius: { 'cpu.package': 45 } })).toEqual([ + { label: 'Cpu.package', value: '45°C', valueTitle: '45.0°C' }, + ]); + }); + it('lists a SMART row only for a positive reading in any state, never a standby disk', () => { const states: (PhysicalDiskCollectionStatus | undefined)[] = [ undefined, @@ -442,6 +474,15 @@ describe('resourceDetailMappers', () => { expect(agent?.id).not.toBe('resource:host:hash-1'); }); + it('reports no uptime for a silent agent, whose last report is not how long it has been up', () => { + const base = createHybridHostResource(); + const reporting = { ...base, uptime: 2_500_000 } as Resource; + expect(toAgentFromResource(reporting)?.uptimeSeconds).toBe(2_500_000); + + const silent = { ...reporting, agent: { stale: true } } as Resource; + expect(toAgentFromResource(silent)?.uptimeSeconds).toBe(0); + }); + it('preserves the local infrastructure display name for governed resources', () => { const agent = toAgentFromResource({ ...createHybridHostResource(), diff --git a/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts b/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts index b6b94a7ea1..07e3b11301 100644 --- a/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts +++ b/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts @@ -317,7 +317,9 @@ export const toAgentFromResource = ( sensors: agent.sensors, raid: agent.raid, status: resource.status, - uptimeSeconds: agent.uptimeSeconds ?? resource.uptime ?? 0, + // A silent agent's uptime is from its last report, not how long the + // machine has been up. + uptimeSeconds: resource.agent?.stale ? 0 : (agent.uptimeSeconds ?? resource.uptime ?? 0), lastSeen: resource.lastSeen, agentVersion: agent.agentVersion, commandsEnabled: agent.commandsEnabled, @@ -400,7 +402,23 @@ const buildTypedGPUTemperatureKeys = (gpus?: HostSensorSummary['gpu']) => { return keys; }; -export const buildTemperatureRows = (sensors?: HostSensorSummary) => { +// Matches the backend's reason for readings a host agent past its reporting +// lease no longer collects (models.HostAgentStoppedReportingReason). +export const HOST_AGENT_STOPPED_REPORTING_REASON = 'host agent stopped reporting'; + +export type TemperatureRowsOptions = { + /** + * Why the machine's own sensor readings are retained rather than current, + * such as a host agent past its reporting lease. Disk rows keep their own + * collection state instead. + */ + lastKnownReason?: string; +}; + +export const buildTemperatureRows = ( + sensors?: HostSensorSummary, + options: TemperatureRowsOptions = {}, +) => { const rows: { label: string; value: string; valueTitle?: string }[] = []; const thermalState = sensors?.thermalState; const typedGPUTemperatureKeys = buildTypedGPUTemperatureKeys(sensors?.gpu); @@ -500,6 +518,17 @@ export const buildTemperatureRows = (sensors?: HostSensorSummary) => { }); } + // This card has no other cue that a value is not current, so a retained + // sensor reading says so in its value, as a retained disk reading does. + const lastKnownReason = options.lastKnownReason?.trim(); + if (lastKnownReason) { + const title = `Last known reading, not current: ${lastKnownReason}`; + for (const row of rows) { + row.value = `${row.value} (last known)`; + row.valueTitle = title; + } + } + const smart = sensors?.smart; if (smart) { smart diff --git a/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts b/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts index 1b33f7d387..a71b909b29 100644 --- a/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts +++ b/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts @@ -35,6 +35,7 @@ import type { ResourceIntelligence } from '@/types/aiIntelligence'; import { buildCustomSensorRows, buildTemperatureRows, + HOST_AGENT_STOPPED_REPORTING_REASON, toAgentFromResource, toNodeFromProxmox, type AgentPlatformData, @@ -201,7 +202,13 @@ export const useResourceDetailDrawerDerivedState = ( [], ); const agentInfo = createMemo(() => toAgentFromResource(resource(), agentMeta())); - const temperatureRows = createMemo(() => buildTemperatureRows(agentInfo()?.sensors)); + // A silent agent's sensors are its last report, even on a row another + // source keeps online. + const temperatureRows = createMemo(() => + buildTemperatureRows(agentInfo()?.sensors, { + lastKnownReason: resource().agent?.stale ? HOST_AGENT_STOPPED_REPORTING_REASON : undefined, + }), + ); const customSensorRows = createMemo(() => buildCustomSensorRows(agentInfo()?.sensors)); const dockerHostData = createMemo(() => platformData()?.docker as DockerPlatformData | undefined); diff --git a/frontend-modern/src/features/docker/DockerHostsTable.tsx b/frontend-modern/src/features/docker/DockerHostsTable.tsx index fbaa7b0858..0c798eb6f6 100644 --- a/frontend-modern/src/features/docker/DockerHostsTable.tsx +++ b/frontend-modern/src/features/docker/DockerHostsTable.tsx @@ -517,7 +517,11 @@ export const DockerHostsTable: Component<{ class={`${getPlatformTableCellClassForKind('numeric-value')} platform-table-narrow-hidden hidden text-base-content min-[360px]:table-cell md:table-cell`} > diff --git a/frontend-modern/src/features/docker/__tests__/DockerHostsTable.test.tsx b/frontend-modern/src/features/docker/__tests__/DockerHostsTable.test.tsx index f5a4d1b7d3..7e24587e0b 100644 --- a/frontend-modern/src/features/docker/__tests__/DockerHostsTable.test.tsx +++ b/frontend-modern/src/features/docker/__tests__/DockerHostsTable.test.tsx @@ -217,6 +217,58 @@ describe('DockerHostsTable', () => { ); }); + it("drops an offline host's retained temperature with its other readings", () => { + // A Docker host past its reporting lease is offline, and its last + // temperature is no more current than its last CPU, memory and disk. + const { container } = render(() => ( + } + emptyTitle="No Docker hosts" + emptyDescription="No hosts" + showToolbar={false} + /> + )); + + const headers = Array.from(container.querySelectorAll('thead th')).map((th) => + (th.textContent ?? '').trim(), + ); + const tempIndex = headers.findIndex((header) => header.startsWith('Temp')); + expect(tempIndex).toBeGreaterThan(-1); + const temp = (id: string) => + ( + container.querySelectorAll(`[data-docker-host-row="${id}"] td`)[tempIndex]?.textContent ?? + '' + ).trim(); + expect(temp('agent:docker-01')).toBe('—'); + expect(temp('agent:docker-02')).toBe('51.0°C'); + const offlineRow = container.querySelector('[data-docker-host-row="agent:docker-01"]')!; + expect(within(offlineRow as HTMLElement).queryAllByTestId('stacked-memory-bar')).toHaveLength( + 0, + ); + expect(offlineRow.querySelector('[data-docker-host-stale] .sr-only')).toHaveTextContent( + 'No report for 9m', + ); + }); + it('tints a host row for the Docker alerts its drawer lists, not by display name', () => { // Docker alerts are keyed on the Docker host id and carry the hostname // as their node; neither is the row's id or its display name. diff --git a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx index 2609a62fa7..fb4ee7772f 100644 --- a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx +++ b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx @@ -1806,7 +1806,9 @@ export const AgentsMachinesTable: Component<{ class={`${getPlatformTableCellClassForKind('numeric-value')} ${machineColumnWidthClass('uptime')} text-base-content`} > {formatPlatformTableUptimeValue( - machine.uptime ?? machine.agent?.uptimeSeconds, + canRenderMetrics() + ? (machine.uptime ?? machine.agent?.uptimeSeconds) + : undefined, )} @@ -1814,15 +1816,20 @@ export const AgentsMachinesTable: Component<{ - + —} + > + + diff --git a/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx b/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx index 39b476ff18..3c7f0a3506 100644 --- a/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx +++ b/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx @@ -1160,14 +1160,15 @@ describe('AgentsMachinesTable', () => { expect(screen.getByText('1400 RPM')).toBeInTheDocument(); }); - it('shows a silent agent disk temperature as last known, never as a current reading', async () => { - // A host agent past its reporting lease is stale and keeps its last SMART - // temperatures with a non-available collection state. + it('shows a retained disk temperature as last known, never as a current reading', async () => { + // A reporting agent whose SMART probe returned no temperature keeps the + // last reading with a non-available collection state. (A host agent past + // its lease is offline, and its row shows no temperature at all.) const stoppedReporting = { temperature: { state: 'unavailable' as const, source: 'smartctl', - reason: 'host agent stopped reporting', + reason: 'SMART probe returned no usable temperature data', }, }; const { container } = render(() => ( @@ -1176,9 +1177,7 @@ describe('AgentsMachinesTable', () => { resource({ id: 'silent-nas', name: 'Silent NAS', - status: 'warning', agent: { - stale: true, sensors: { smart: [ { device: '/dev/sda', model: 'Archive HDD', temperature: 71 }, @@ -1249,6 +1248,92 @@ describe('AgentsMachinesTable', () => { expect(within(mixed).queryByText('69°C')).toBeNull(); }); + it("blanks an offline machine's retained readings, temperature and uptime included", async () => { + // A host agent past its reporting lease is offline: CPU, memory, disk and + // the sensors, uptime and disks it last reported are not current. + const stoppedReporting = { + temperature: { + state: 'unavailable' as const, + source: 'smartctl', + reason: 'host agent stopped reporting', + }, + }; + const lastReport: Partial = { + cpu: { current: 32 }, + memory: { total: 100, used: 73, free: 27, current: 73 }, + disk: { total: 100, used: 80, free: 20, current: 80 }, + }; + const sensors = { + temperatureCelsius: { 'cpu.package': 45 }, + thermalState: { pressure: 'serious' }, + smart: [ + { + device: '/dev/sda', + model: 'Archive HDD', + temperature: 41, + collection: stoppedReporting, + }, + ], + }; + const { container } = render(() => ( + + )); + + await openMachineColumnPicker(); + await fireEvent.click(screen.getByLabelText('Temp')); + await fireEvent.click(screen.getByLabelText('Uptime')); + + const cells = (id: string) => { + const row = container.querySelector(`[data-agents-machine-row="${id}"]`) as HTMLElement; + const headers = Array.from(container.querySelectorAll('thead th')).map((th) => + (th.textContent ?? '').replace(/[▲▼]/g, '').trim(), + ); + const values = Array.from(row.querySelectorAll('td')).map((td) => + (td.textContent ?? '').trim(), + ); + return (label: string) => values[headers.indexOf(label)]; + }; + + const silent = cells('silent-host'); + expect(silent('CPU')).toBe('—'); + expect(silent('Up')).toBe('—'); + expect(silent('Temp')).toBe('—'); + expect( + container.querySelector( + '[data-agents-machine-row="silent-host"] [data-agent-machine-temperature-trigger="true"]', + ), + ).toBeNull(); + + const live = cells('live-host'); + expect(live('CPU')).not.toBe('—'); + expect(live('Up')).toBe('28d'); + expect(live('Temp')).toBe('45°C'); + }); + it('moves a machine temperature between current and last known as its disk reports change', async () => { // Fresh objects per update, like parsed API payloads: the store reconciles // into the objects it was given, so a reused fixture would be rewritten. diff --git a/internal/unifiedresources/clone_test.go b/internal/unifiedresources/clone_test.go index 1ad90ad974..151f411769 100644 --- a/internal/unifiedresources/clone_test.go +++ b/internal/unifiedresources/clone_test.go @@ -221,12 +221,18 @@ func TestCloneResource_MutateSourceStatusMap(t *testing.T) { ID: "r-1", SourceStatus: map[DataSource]SourceStatus{ SourceProxmox: {Status: "online", ExpectedUpdateIntervalSeconds: 300}, + SourceAgent: {Status: "stale", leaseExpired: true}, }, } cloned := cloneResource(original) if cloned.SourceStatus[SourceProxmox].ExpectedUpdateIntervalSeconds != 300 { t.Fatal("clone lost collector cadence") } + // A clone that dropped the lease verdict would aggregate a silent agent + // back to warning. + if !cloned.SourceStatus[SourceAgent].leaseExpired { + t.Fatal("clone lost the agent's expired reporting lease") + } cloned.SourceStatus[SourceDocker] = SourceStatus{Status: "online"} if _, exists := original.SourceStatus[SourceDocker]; exists { diff --git a/internal/unifiedresources/registry.go b/internal/unifiedresources/registry.go index cb2f10a48a..c4a31be483 100644 --- a/internal/unifiedresources/registry.go +++ b/internal/unifiedresources/registry.go @@ -809,6 +809,14 @@ func (rr *ResourceRegistry) ingestResources(resources []Resource, thresholds map } } } + // The lease marker is unexported, so a persisted or serialized copy + // arrives without it. Its stored status still carries the verdict. + for source, status := range resource.SourceStatus { + if !status.leaseExpired && reportingLeaseExpired(source, *resource) { + status.leaseExpired = true + resource.SourceStatus[source] = status + } + } rr.mu.Lock() rr.canonicalIdentityIndex = nil @@ -1756,6 +1764,29 @@ func sourceSightingStatus(lastSeen time.Time) string { return "online" } +// reportingLeaseExpired reports whether a source delivered the machine, +// Docker host or Kubernetes cluster it reports for as offline. Those +// collectors push their own reports, and Pulse marks the reporter offline +// only once its reporting lease runs out (evaluateHostAgents, +// evaluateDockerAgents, evaluateKubernetesAgents). The registry's stale +// threshold is a different clock: it can mark the sighting stale before +// the lease ends, or leave it fresh after, so it must not decide the status. +// Resources those sources describe (guests, containers, pods, disks) are +// not lease holders and keep the stale-to-warning rule. +func reportingLeaseExpired(source DataSource, resource Resource) bool { + if resource.Status != StatusOffline { + return false + } + switch source { + case SourceAgent, SourceDocker: + return resource.Type == ResourceTypeAgent + case SourceK8s: + return resource.Type == ResourceTypeK8sCluster + default: + return false + } +} + func (rr *ResourceRegistry) markStaleLocked(now time.Time, thresholds map[DataSource]time.Duration) { thresholds = effectiveStaleThresholds(thresholds) @@ -2848,6 +2879,7 @@ func (rr *ResourceRegistry) ingestRecord(source DataSource, sourceID string, res sighting := resource.SourceStatus[source] sighting.Status = sourceSightingStatus(resource.LastSeen) sighting.LastSeen = resource.LastSeen + sighting.leaseExpired = reportingLeaseExpired(source, resource) resource.SourceStatus = map[DataSource]SourceStatus{source: sighting} resource.parentBySource = make(map[DataSource]string) rr.setSourceParent(&resource, source, resource.ParentID) @@ -3787,6 +3819,7 @@ func (rr *ResourceRegistry) mergeInto(existing *Resource, incoming Resource, sou sighting := incoming.SourceStatus[source] sighting.Status = sourceSightingStatus(incoming.LastSeen) sighting.LastSeen = incoming.LastSeen + sighting.leaseExpired = reportingLeaseExpired(source, incoming) existing.SourceStatus[source] = sighting if incoming.LastSeen.After(existing.LastSeen) { @@ -4425,6 +4458,12 @@ func (rr *ResourceRegistry) mergeResourceData(primary *Resource, other *Resource primary.SourceStatus = make(map[DataSource]SourceStatus) } for source, status := range other.SourceStatus { + // Linked resources can share a source, such as an agent reinstalled + // under a new id. Keep the fresher sighting so a stale or expired one + // cannot override a reporter that is still live. + if current, ok := primary.SourceStatus[source]; ok && current.LastSeen.After(status.LastSeen) { + continue + } primary.SourceStatus[source] = status } if other.LastSeen.After(primary.LastSeen) { @@ -5793,14 +5832,22 @@ func aggregateStatus(resource *Resource) ResourceStatus { best := StatusUnknown bestScore := 0 for _, status := range resource.SourceStatus { - score := statusPriority[strings.ToLower(status.Status)] + state := strings.ToLower(strings.TrimSpace(status.Status)) + // A reporter past its lease is offline, not merely late. Ranking its + // stale sighting above offline would show a silent machine as a + // warning while its retained readings render as current. + if status.leaseExpired { + state = "offline" + } + score := statusPriority[state] if score > bestScore { bestScore = score - if status.Status == "online" { + switch state { + case "online": best = StatusOnline - } else if status.Status == "stale" { + case "stale": best = StatusWarning - } else if status.Status == "offline" { + case "offline": best = StatusOffline } } diff --git a/internal/unifiedresources/registry_merge_policy_test.go b/internal/unifiedresources/registry_merge_policy_test.go index 83c8fda61a..cc78d4b7a1 100644 --- a/internal/unifiedresources/registry_merge_policy_test.go +++ b/internal/unifiedresources/registry_merge_policy_test.go @@ -1,6 +1,7 @@ package unifiedresources import ( + "encoding/json" "strings" "testing" "time" @@ -131,6 +132,353 @@ func TestHostnameIPDoesNotAutoMerge(t *testing.T) { } } +func onlyResourceOfType(t *testing.T, rr *ResourceRegistry, resourceType ResourceType) Resource { + t.Helper() + resources := rr.ListByType(resourceType) + if len(resources) != 1 { + t.Fatalf("expected one %s resource, got %d: %+v", resourceType, len(resources), resources) + } + return resources[0] +} + +// A host agent past its reporting lease is offline. Its source sighting is +// stale too, and the stale pass used to rank that above offline, so the +// machine reached the frontend as a warning with its last report rendered +// as current readings. +func TestLeaseExpiredHostAgentStaysOffline(t *testing.T) { + lastReport := time.Now().UTC().Add(-14 * time.Minute) + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-silent", + MachineID: "machine-silent", + Hostname: "silent", + Status: "offline", + LastSeen: lastReport, + IntervalSeconds: 30, + CPUUsage: 32, + }}, + }) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if resource.Status != StatusOffline { + t.Fatalf("status = %q, want offline for an agent past its lease", resource.Status) + } + if resource.Agent == nil || !resource.Agent.Stale { + t.Fatalf("expected the agent facet to stay flagged stale, got %+v", resource.Agent) + } + // Status describes the resource; the sighting keeps describing delivery + // freshness, which health and monitored-system reasons read. + if got := resource.SourceStatus[SourceAgent].Status; got != "stale" { + t.Fatalf("agent sighting = %q, want stale", got) + } + + // The monitor adapter runs the stale pass again after record sources. + rr.MarkStale(time.Now().UTC(), nil) + if got, _ := rr.Get(resource.ID); got.Status != StatusOffline { + t.Fatalf("second stale pass changed status to %q, want offline", got.Status) + } + + health := EvaluateResourceHealth(resource, nil, time.Now().UTC()) + if health.Verdict != HealthCritical || len(health.Reasons) < 2 || + health.Reasons[0].Code != "offline" || health.Reasons[1].Code != "telemetry_stale" { + t.Fatalf("health = %+v, want critical offline with the stale reason after it", health) + } +} + +// An agent that is late but still inside its lease keeps the warning: the +// monitor has not decided it is gone, so its last readings are only stale. +func TestLateHostAgentInsideLeaseStaysWarning(t *testing.T) { + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-late", + MachineID: "machine-late", + Hostname: "late", + Status: "online", + LastSeen: time.Now().UTC().Add(-90 * time.Second), + IntervalSeconds: 30, + }}, + }) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if resource.Status != StatusWarning { + t.Fatalf("status = %q, want warning for a late agent inside its lease", resource.Status) + } +} + +// A Proxmox node whose linked agent stopped reporting stays online through +// the PVE poll (issue #1515). Only the agent sighting carries the lease. +func TestLeaseExpiredAgentOnLiveProxmoxNodeStaysOnline(t *testing.T) { + now := time.Now().UTC() + const machineID = "machine-pve1" + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Nodes: []models.Node{{ + ID: "homelab-pve1", + Name: "pve1", + Instance: "homelab", + ClusterName: "homelab", + Status: "online", + LastSeen: now, + }}, + Hosts: []models.Host{{ + ID: machineID, + MachineID: machineID, + Hostname: "pve1", + LinkedNodeID: "homelab-pve1", + Status: "offline", + LastSeen: now.Add(-14 * time.Minute), + IntervalSeconds: 30, + }}, + }) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if _, ok := resource.SourceStatus[SourceProxmox]; !ok { + t.Fatalf("expected the node and its agent to share one resource, got sources %+v", resource.SourceStatus) + } + if resource.Status != StatusOnline { + t.Fatalf("status = %q, want online through the live PVE poll", resource.Status) + } + if resource.Agent == nil || !resource.Agent.Stale { + t.Fatalf("expected the dead agent to stay flagged stale, got %+v", resource.Agent) + } +} + +// Docker hosts hold a shorter lease than the registry's Docker stale +// threshold, so a silent Docker host was offline for a while and then +// flipped to warning once its sighting went stale. +func TestLeaseExpiredDockerHostStaysOfflineAfterSightingGoesStale(t *testing.T) { + lastReport := time.Now().UTC().Add(-5 * time.Minute) + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + DockerHosts: []models.DockerHost{{ + ID: "docker-silent", + AgentID: "docker-agent-silent", + Hostname: "docker-silent", + MachineID: "machine-docker-silent", + Status: "offline", + LastSeen: lastReport, + IntervalSeconds: 30, + Containers: []models.DockerContainer{{ + ID: "container-web", + Name: "web", + State: "running", + }}, + }}, + }) + + host := onlyResourceOfType(t, rr, ResourceTypeAgent) + if host.Status != StatusOffline { + t.Fatalf("docker host status = %q, want offline after its sighting went stale", host.Status) + } + if got := host.SourceStatus[SourceDocker].Status; got != "stale" { + t.Fatalf("docker sighting = %q, want stale", got) + } + + // Containers are what the host reports about, not lease holders: a + // running container on a silent host is stale, not known to be down. + container := onlyResourceOfType(t, rr, ResourceTypeAppContainer) + if container.Status != StatusWarning { + t.Fatalf("container status = %q, want warning on a silent host", container.Status) + } +} + +// A machine reporting through both the host agent and Docker is offline once +// both leases expire. The Docker lease ends first, so its sighting can still +// be fresh when the agent's goes stale. +func TestMachineWithExpiredAgentAndDockerLeasesIsOffline(t *testing.T) { + now := time.Now().UTC() + const machineID = "machine-both" + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-both", + MachineID: machineID, + Hostname: "both", + Status: "offline", + LastSeen: now.Add(-3 * time.Minute), + IntervalSeconds: 30, + }}, + DockerHosts: []models.DockerHost{{ + ID: "docker-both", + AgentID: "host-both", + Hostname: "both", + MachineID: machineID, + Status: "offline", + LastSeen: now.Add(-90 * time.Second), + IntervalSeconds: 10, + }}, + }) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if _, ok := resource.SourceStatus[SourceDocker]; !ok { + t.Fatalf("expected agent and Docker to share one resource, got sources %+v", resource.SourceStatus) + } + if got := resource.SourceStatus[SourceDocker].Status; got != "online" { + t.Fatalf("docker sighting = %q, want it still inside the stale threshold", got) + } + if resource.Status != StatusOffline { + t.Fatalf("status = %q, want offline once both leases expired", resource.Status) + } +} + +// A live reporter still carries the machine when the other lease expired. +func TestMachineWithLiveDockerReporterStaysOnline(t *testing.T) { + now := time.Now().UTC() + const machineID = "machine-split" + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-split", + MachineID: machineID, + Hostname: "split", + Status: "offline", + LastSeen: now.Add(-14 * time.Minute), + IntervalSeconds: 30, + }}, + DockerHosts: []models.DockerHost{{ + ID: "docker-split", + AgentID: "docker-agent-split", + Hostname: "split", + MachineID: machineID, + Status: "online", + LastSeen: now, + IntervalSeconds: 10, + }}, + }) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if resource.Status != StatusOnline { + t.Fatalf("status = %q, want online through the live Docker reporter", resource.Status) + } +} + +// The lease marker is unexported, so a resource that went through JSON (a +// persisted or remote copy) must recover it from its stored status. +func TestLeaseExpiredHostAgentStaysOfflineAfterJSONRoundTrip(t *testing.T) { + source := NewRegistry(nil) + source.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-silent", + MachineID: "machine-silent", + Hostname: "silent", + Status: "offline", + LastSeen: time.Now().UTC().Add(-14 * time.Minute), + IntervalSeconds: 30, + }}, + }) + payload, err := json.Marshal(source.List()) + if err != nil { + t.Fatalf("marshal resources: %v", err) + } + var copied []Resource + if err := json.Unmarshal(payload, &copied); err != nil { + t.Fatalf("unmarshal resources: %v", err) + } + + rr := NewRegistry(nil) + rr.IngestResources(copied) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if resource.Status != StatusOffline { + t.Fatalf("status = %q, want offline after a JSON round trip", resource.Status) + } +} + +// A manual link can join two resources reported by the same source, such as +// an agent reinstalled under a new id. The expired sighting of the old one +// must not replace the live sighting of the new one. +func TestManualLinkKeepsTheLiveSightingOfASharedSource(t *testing.T) { + now := time.Now().UTC() + store := NewMemoryStore() + if err := store.AddLink(ResourceLink{ + ResourceA: "agent-reinstalled", + ResourceB: "agent-retired", + PrimaryID: "agent-reinstalled", + }); err != nil { + t.Fatalf("add link: %v", err) + } + rr := NewRegistry(store) + rr.IngestResources([]Resource{ + { + ID: "agent-reinstalled", + Type: ResourceTypeAgent, + Name: "tower", + Status: StatusOnline, + LastSeen: now, + Sources: []DataSource{SourceAgent}, + SourceStatus: map[DataSource]SourceStatus{ + SourceAgent: {Status: "online", LastSeen: now}, + }, + Agent: &AgentData{AgentID: "host-new", Hostname: "tower"}, + }, + { + ID: "agent-retired", + Type: ResourceTypeAgent, + Name: "tower", + Status: StatusOffline, + LastSeen: now.Add(-14 * time.Minute), + Sources: []DataSource{SourceAgent}, + SourceStatus: map[DataSource]SourceStatus{ + SourceAgent: {Status: "stale", LastSeen: now.Add(-14 * time.Minute)}, + }, + Agent: &AgentData{AgentID: "host-old", Hostname: "tower", Stale: true}, + }, + }) + + resource, ok := rr.Get("agent-reinstalled") + if !ok { + t.Fatal("linked resource missing") + } + if resource.Status != StatusOnline { + t.Fatalf("status = %q, want online through the live reinstalled agent", resource.Status) + } + if got := resource.SourceStatus[SourceAgent]; !got.LastSeen.Equal(now) { + t.Fatalf("agent sighting = %+v, want the live sighting", got) + } +} + +func TestLeaseExpiredKubernetesClusterStaysOffline(t *testing.T) { + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + KubernetesClusters: []models.KubernetesCluster{{ + ID: "cluster-silent", + AgentID: "k8s-agent-silent", + Name: "silent", + Status: "offline", + LastSeen: time.Now().UTC().Add(-5 * time.Minute), + IntervalSeconds: 30, + }}, + }) + + cluster := onlyResourceOfType(t, rr, ResourceTypeK8sCluster) + if cluster.Status != StatusOffline { + t.Fatalf("cluster status = %q, want offline after its sighting went stale", cluster.Status) + } +} + +// Pull sources keep the stale-to-warning rule: a stale PVE sighting means +// Pulse lost its poller, not that the node reported itself gone. +func TestStaleProxmoxNodeStaysWarning(t *testing.T) { + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Nodes: []models.Node{{ + ID: "homelab-pve9", + Name: "pve9", + Instance: "homelab", + ClusterName: "homelab", + Status: "online", + LastSeen: time.Now().UTC().Add(-5 * time.Minute), + }}, + }) + + node := onlyResourceOfType(t, rr, ResourceTypeAgent) + if node.Status != StatusWarning { + t.Fatalf("node status = %q, want warning for a stale PVE sighting", node.Status) + } +} + func TestCloneDockerDataPreservesContainerRuntimeMetadata(t *testing.T) { startedAt := time.Date(2026, 6, 11, 13, 15, 30, 0, time.UTC) finishedAt := startedAt.Add(45 * time.Minute) diff --git a/internal/unifiedresources/types.go b/internal/unifiedresources/types.go index 31b01a0e85..d64e288bef 100644 --- a/internal/unifiedresources/types.go +++ b/internal/unifiedresources/types.go @@ -295,6 +295,12 @@ type SourceStatus struct { LastSeen time.Time `json:"lastSeen"` Error string `json:"error,omitempty"` ExpectedUpdateIntervalSeconds int64 `json:"expectedUpdateIntervalSeconds,omitempty"` // Collector-authored cadence, zero uses source default + + // leaseExpired records that this source delivered the machine, Docker + // host or cluster it reports for as offline because its reporting lease + // ran out. Status aggregation counts that as offline whatever the + // sighting's age, while Status keeps describing delivery freshness. + leaseExpired bool } // ResourceIdentity holds identifiers used for matching. From 3fd5411ecc324a02d444c40ea349437c69d08dee Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:50:39 +0100 Subject: [PATCH 37/53] Judge disk heat by the alert disk temperature thresholds Pulse decided whether a physical disk was too hot in three places with three thresholds. Disk risk added temperature_high at a flat 60/70C for every disk type and drove the disk status and the Physical Disks Health verdict. Patrol flagged anything over 55C and the AI chat context listed disks from 50C. Alerts and the Temp column used the per-type thresholds users tune in Alerts (NVMe 70, SAS 65, SATA 55 by default). An NVMe at 63C read Running Hot beside a green Temp cell. A SATA disk at 56C read Healthy beside a red one. A user who raised the NVMe trigger to 75 still got a hot verdict at 70. The alert disk temperature policy now decides disk heat. alerts.Manager.DiskTemperatureThreshold exposes the per-type resolution CheckHost already applies. - Disk risk no longer judges temperature. Every registry that rebuilds risk lacks the alert configuration, so no rule there could honour it. A hot disk keeps an online status. - The Health verdict reads Running Hot when a current reading reaches the disk type's trigger. It uses the thresholds object that colours the Temp cell, so the two cannot disagree. The amber band below the trigger stays a cell colour, not a verdict. - The TrueNAS storage table lost the heat it used to get from risk. It now judges heat the same way: an Attention row with the reason, and a warning dot for phones. - Patrol triage, its disk issue filter, finding recovery and the chat context's attention list judge the collected reading against the same trigger, including scoped runs and a provider set before Patrol starts. - A switched-off agent Disk Temp default resolves to no thresholds, which the displays that read them now treat as off. Per-host Disk Temp overrides do not reach these surfaces yet. TrueNAS disk alerts, the TrueNAS disk drawer and PDF reports keep their own fixed temperatures for now. --- .../v6/internal/subsystems/ai-runtime.md | 16 + .../v6/internal/subsystems/alerts.md | 29 ++ .../subsystems/frontend-primitives.md | 5 +- .../v6/internal/subsystems/monitoring.md | 23 +- .../internal/subsystems/storage-recovery.md | 44 +- .../internal/subsystems/unified-resources.md | 27 +- .../b24ffeda6578c122.json | 60 +++ .../src/components/Storage/DiskList.tsx | 5 +- .../Storage/__tests__/DiskList.test.tsx | 97 ++-- .../components/Storage/useDiskDetailModel.ts | 5 +- .../components/Storage/useDiskListModel.ts | 11 +- .../Workloads/GuestPhysicalDisks.tsx | 4 +- .../__tests__/diskDetailPresentation.test.ts | 2 + .../__tests__/diskPresentation.test.ts | 413 +++++++++--------- .../storageBackups/diskDetailPresentation.ts | 7 +- .../storageBackups/diskPresentation.ts | 101 +++-- .../diskTemperaturePresentation.ts | 27 ++ .../truenas/TrueNASStorageTopologyTable.tsx | 36 +- .../TrueNASStorageTopologyTable.test.tsx | 107 ++++- .../__tests__/truenasPageModel.test.ts | 67 +++ .../src/features/truenas/truenasPageModel.ts | 86 +++- .../utils/__tests__/metricThresholds.test.ts | 23 + .../src/utils/__tests__/temperature.test.ts | 6 + frontend-modern/src/utils/metricThresholds.ts | 10 +- frontend-modern/src/utils/temperature.ts | 5 + internal/ai/alert_threshold_adapter.go | 16 + internal/ai/alert_threshold_adapter_test.go | 25 ++ internal/ai/mock_test.go | 13 + internal/ai/patrol.go | 4 + internal/ai/patrol_ai.go | 19 +- internal/ai/patrol_disk_temperature.go | 50 +++ internal/ai/patrol_disk_temperature_test.go | 228 ++++++++++ internal/ai/patrol_findings.go | 18 +- internal/ai/patrol_run.go | 1 + internal/ai/patrol_state.go | 23 +- internal/ai/patrol_thresholds_test.go | 3 + internal/ai/patrol_triage.go | 13 +- internal/ai/resource_context.go | 8 +- internal/ai/service.go | 9 +- internal/alerts/host.go | 39 +- .../threshold_resolution_shared_test.go | 58 +++ internal/api/ai_handlers_concurrency_test.go | 3 + internal/api/ai_handlers_more_test.go | 3 + internal/storagehealth/risk.go | 40 +- .../storagehealth/risk_branchcov0718_test.go | 48 +- internal/storagehealth/risk_test.go | 110 +---- internal/truenas/provider.go | 7 +- internal/truenas/provider_test.go | 21 +- .../unifiedresources/adapter_coverage_test.go | 12 +- internal/unifiedresources/adapters.go | 18 +- .../unifiedresources/physical_disk_risk.go | 1 - .../physical_disk_risk_test.go | 11 +- internal/unifiedresources/registry_test.go | 156 +++---- .../unifiedresources/storage_risk_test.go | 4 +- 54 files changed, 1514 insertions(+), 663 deletions(-) create mode 100644 frontend-modern/browser-verification/b24ffeda6578c122.json create mode 100644 internal/ai/patrol_disk_temperature.go create mode 100644 internal/ai/patrol_disk_temperature_test.go diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index da8d664e67..216666a720 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -306,6 +306,22 @@ retained value as "last known 41C (disk is in standby)". Proofs: `TestPatrolPhysicalDiskRowsJudgeHeatOnCollectedTemperature` in `internal/ai/patrol_ai_more_test.go`. +They judge that collected reading against the alert disk temperature policy, +not a fixed line. `ThresholdProvider.GetDiskTemperatureThreshold(diskType)` +(the alert manager's per-type trigger and clear) resolves each disk's limits +once in `patrolPhysicalDiskRows`, and the run state carries the provider +(`patrolRuntimeState.thresholdProvider`). With no provider the factory alert +configuration applies. A disk is hot from its trigger: a triage warning, a disk +issue and an AI chat "needing attention" entry, as its temperature alert and +Physical Disks Running Hot verdict are. A disk-high finding recovers at or +below the clear value (under the trigger when there is no band below it), as +the alert does. Scoped runs keep the provider, and a +provider set before Patrol starts is handed to it. Below the trigger Patrol flags nothing, so an NVMe at 63C stays +quiet and a SATA disk at 56C is flagged. Proofs: +`internal/ai/patrol_disk_temperature_test.go` and +`TestAlertThresholdAdapter_DiskTemperatureFollowsAlertPolicy` in +`internal/ai/alert_threshold_adapter_test.go`. + Retained summaries disclose that point and bucket timestamps describe returned history inside the requested window. Their spacing does not measure collection uptime or explain missing history. Collector lifecycle and retention settings diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index c80daddd90..692424c56d 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -1682,6 +1682,35 @@ an alert the next evaluation raises again. `TestConfigSaveKeepsAlertTheEvaluatorKeepsWithoutRecoveryBand` in `internal/alerts/host_unraid_lifecycle_test.go` pin these paths. +### The disk temperature policy decides disk heat + +The policy `CheckHost` applies to a SMART disk (the agent Disk Temp default, +refined by the disk type's `DiskTempByType` entry, and switched off for every +type when that default is off) is the canonical answer to "is this disk too +hot". `diskTemperatureThresholdForType` in `internal/alerts/host.go` holds it, +and `Manager.DiskTemperatureThreshold(diskType)` exposes it, without host +overrides, to Patrol through `AlertThresholdAdapter.GetDiskTemperatureThreshold`. +`DefaultDiskTemperatureThreshold` serves callers with no manager. Disk risk +(`internal/storagehealth`) judges no temperature. The Physical Disks Temp cell +and Health verdict mirror the per-type triggers and a switched-off default in +`resolveDiskTemperatureDisplayThresholds`; host overrides do not reach that +page yet. That resolver returns null when the agent Disk Temp default is off. +The displays that read it treat null as off: no threshold colour +(`getTemperatureTextClass` with `diskTemperature`) and never hot. A +`DiskTempByType` entry cannot be switched off on its own, because +normalization restores a non-positive entry to its default. +A disk is hot from its trigger. Alerts and Patrol findings stay open until the +reading falls to the clear value, or under the trigger when there is no band +below it. The Physical Disks verdict and the TrueNAS Health cell judge only the +current reading against the trigger. Judges that still differ: TrueNAS disk +temperature alerts use `TrueNASDiskDefaults.Temperature` (a flat 55/50), the +TrueNAS disk drawer tones the reading from a fixed 55C, and PDF reports colour +disk temperatures at a fixed 50/60C. +`TestDiskTemperatureThresholdMatchesCheckHostPolicy` in +`internal/alerts/threshold_resolution_shared_test.go` pins per-type resolution, a raised +NVMe trigger that `CheckHost` also honours, the copy, the disabled default and +the nil manager. + ### Agent disk temperature alerts clear when their disk leaves the report `CheckHost` also clears a disk temperature alert, and drops its pending diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 5464d8b393..d58b392ed1 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -414,8 +414,9 @@ available. Stable disk History catalog/organisation/access ownership is unchange The physical-disk verdict words and their phone forms are `Needs Attention` (`Attention`), `Running Hot` (`Hot`) and `Replace Now` (`Replace`), all from -`getPhysicalDiskHealthCompactLabel` in `diskPresentation.ts`. A hot disk keeps -the same label at either tier; its tone carries the severity. +`getPhysicalDiskHealthCompactLabel` in `diskPresentation.ts`. `Running Hot` is +red and starts at the disk's alert trigger, from the same thresholds object +that colours its Temp cell (`getPhysicalDiskTemperatureThresholds`). Controller expansions carry the supplied kind-specific targets, absolute times, duration and cluster/namespace when the phone row omits those columns. Proxmox diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index f5ed572cc3..c233ab1ee2 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -4501,17 +4501,18 @@ emit structured storage topology such as Unraid per-disk state, the shared assessment layer must derive canonical risk and alert severity from that richer disk topology instead of letting coarser aggregate counters override it and flap the operator-facing storage alert surface. -That shared assessment judges only a disk temperature that was collected now. -`storagehealth.CollectedTemperature` passes a reading through when its -`collection.temperature` state is `available`, or when the source predates -collection state, and otherwise drops it before `AssessPhysicalDisk` and -`AssessHostSMARTDisk` build their sample. A last known value retained for a -disk in standby or a host agent past its reporting lease therefore raises no -`temperature_high` reason and cannot turn the disk's verdict or status into -a warning, while the value itself stays on the resource as last-known -history. Proof: `TestDiskAssessmentsIgnoreRetainedTemperature` and -`TestCollectedTemperatureKeepsOnlyCurrentReadings` in -`internal/storagehealth/risk_test.go`. +That shared assessment never judges disk temperature. Heat is a metric owned +by the alert disk temperature policy (`alerts.Manager.DiskTemperatureThreshold`: +the per-type `diskTempByType` trigger, else the agent Disk Temp default), which +users tune per disk type. Disk risk is rebuilt by registries that cannot see +that configuration, so a heat rule here would contradict it: a flat 60/70C rule +once called a healthy 63C NVMe hot and a 56C SATA disk fine. No disk type, +temperature or collection state reaches `storagehealth.Sample`, the TrueNAS +provider and the Unraid inventory assessment included, and a hot disk keeps an +`online` status. Proof: `TestDiskAssessmentsNeverJudgeTemperature` in +`internal/storagehealth/risk_test.go` and +`TestRecordsLeaveDiskHeatToTheAlertPolicy` in +`internal/truenas/provider_test.go`. That same monitoring-owned storage polling boundary also owns cluster-shared Proxmox storage status coherence. `internal/monitoring/monitor_polling_storage.go` must merge shared storage observations across nodes into one cluster-scoped diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 06020140f3..079fabf486 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -6759,23 +6759,33 @@ not live appliance collection or recovery success. ### Physical-disk heat is not a replacement verdict -The Physical Disks Health verdict names the action it asks for. Risk whose only -reason is `temperature_high` reads `Running Hot` (compact `Hot`), red at the -critical tier and amber at warning, with the temperature reason as its summary. -`Replace Now` stays reserved for FAILED health, any other critical reason, and -a critical level that no listed reason explains. `Needs Attention` keeps the -remaining warning evidence. When heat and other evidence both apply, the more -severe class names the verdict and failure evidence wins a tie, whatever order -merged reasons arrive in. Sorting follows the verdict (replace, critical heat, -attention, warning heat), and the health filter places a hot disk at its -temperature tier, so `Needs attention` still lists it. This matches alerting, -which raises disk heat as a Temperature metric and never as disk health. -Thresholds are unchanged: the tier still comes from the server's disk risk -(flat 60/70C), not the per-type alert thresholds that colour the Temp column. -Reconciling those is separate work. `frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts` -and `frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx` cover -the split, merged reason order, the unexplained critical level, sorting and -filtering. +The Physical Disks Health verdict names the action it asks for. A disk whose +current temperature has reached its alert trigger reads `Running Hot` (compact +`Hot`), red, with the reading and the trigger as its summary. `Replace Now` +stays reserved for FAILED health, any critical risk reason, and a critical +level that no listed reason explains. `Needs Attention` keeps the remaining +warning evidence. Heat outranks warning evidence and failure evidence outranks +heat. Sorting follows the verdict (replace, heat, attention), and the health +filter places a hot disk with the critical disks, as its Temp cell reads red. + +Heat has one owner: the alert disk temperature policy (the per-type triggers in +Alerts > Thresholds, `diskTempByType`, else the agent Disk Temp default). +Disk risk never carries heat, because every registry that rebuilds it lacks the +alert configuration. `extractPhysicalDiskPresentationData` takes the alerts +store's `getDiskTemperatureThresholds` resolver, so the Temp cell colour and the +verdict read one thresholds object. The band below the trigger is a Temp cell +colour only, and a retained (not current) reading is never heat. The verdict +judges the current reading, with no memory of an alert. A disk temperature +alert held between its clear value and trigger shows an amber Temp cell, not +`Running Hot`. A switched-off agent Disk Temp default resolves to null, which +leaves the reading uncoloured and never hot, as alerting is off for it. +Per-host Disk Temp overrides do not yet reach Physical Disks. +`frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts` +pins cell-and-verdict agreement for every type from 40C to 80C under factory and +raised triggers, plus retained readings, failure evidence, sorting and +filtering. `frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx` +covers NVMe 63C, SATA 56C and a raised NVMe trigger loaded through the alerts +store. ### Pool-to-physical-disk ownership in Storage details diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 0424cd7e79..6a00c0a3c0 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -1111,6 +1111,15 @@ The first reason renders in the desktop Health column on the row's single line, coloured by bucket, with a count for the rest and the full list in the title and screen-reader text. Phones keep the status dot and reach the reason through the row drawer. Health sorts by severity rank, not label text. +Disk risk carries no heat, so the table judges it itself. A disk whose current +reading has reached its type's alert disk temperature trigger is an Attention +row with the reason "Disk temperature is 57°C, at or above its 55°C alert +threshold.". The trigger comes from the alerts store's +`getDiskTemperatureThresholds`, which the table passes to the row build, +filter, counts, sort and issue. `isPhysicalDiskRunningHot` in +`features/storageBackups/diskTemperaturePresentation.ts` makes the same call +for the Physical Disks verdict. The status dot keeps the source's own state. +`truenasPageModel.test.ts` and `TrueNASStorageTopologyTable.test.tsx` pin it. The vSphere Datastores and Networks tables follow the same exception-first Health column. `getVmwareDatastoreIssue` and `getVmwareNetworkIssue` return nothing for a green row; otherwise the reasons are, in order, an impaired @@ -3564,8 +3573,8 @@ and otherwise `available`. The value is kept as last-known context; it is collected again only when a reporting host sends a positive reading for a disk that is no longer spun down. The Unraid-native disk row carries that state, and so does a SMART row that falls back to the inventory reading because it has -none of its own. The adapter's risk assessment of an Unraid row no longer counts -a withdrawn inventory temperature as heat. +none of its own. The adapter's risk assessment of an Unraid row judges no +temperature at all; heat belongs to the alert disk temperature policy. Cross-source correlation compares normalized serial and WWN values across fields without truncation, allowing a PVE bare-hex array-volume serial to join @@ -4111,13 +4120,13 @@ incidents during cross-source merges. A provider alert such as TrueNAS `truenas_smart` is not presentation-only context; it must become a canonical `physicalDisk.risk.reasons` entry so hybrid agent/API disk resources keep one shared disk-health truth after deduplication. -That shared risk contract judges only a collected temperature. Both the -adapter assessment and the registry's recompute from merged metadata -(`physicalDiskAssessmentFromMeta`) pass the temperature through -`storagehealth.CollectedTemperature`, so a retained `unavailable`, -`unsupported` or `missing` temperature stays visible as last-known evidence -but adds no `temperature_high` reason and leaves the disk `online` rather -than `warning`. Proof: `TestPhysicalDiskRiskIgnoresRetainedTemperature` in +That shared risk contract never judges temperature, collected or retained. +Heat belongs to the alert disk temperature policy, which no registry can see, +so neither the adapter assessment nor the registry's recompute from merged +metadata (`physicalDiskAssessmentFromMeta`) passes a temperature into +`storagehealth`. A 72C disk keeps its reading on the resource, adds no +`temperature_high` reason and stays `online`. Proof: +`TestPhysicalDiskRiskNeverJudgesTemperature` in `internal/unifiedresources/registry_test.go`. That same canonical disk contract now also owns recent aggregate temperature history. When a provider such as TrueNAS can supply `disk.temperature_agg` diff --git a/frontend-modern/browser-verification/b24ffeda6578c122.json b/frontend-modern/browser-verification/b24ffeda6578c122.json new file mode 100644 index 0000000000..8b1f018cc9 --- /dev/null +++ b/frontend-modern/browser-verification/b24ffeda6578c122.json @@ -0,0 +1,60 @@ +{ + "version": 1, + "base_sha": "1b6a9b45d7a85ae2add1ff4213ef80f6b35f7309", + "verified_at": "2026-10-06T22:49:34Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Storage/DiskList.tsx", + "frontend-modern/src/components/Storage/useDiskDetailModel.ts", + "frontend-modern/src/components/Storage/useDiskListModel.ts", + "frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx", + "frontend-modern/src/features/storageBackups/diskDetailPresentation.ts", + "frontend-modern/src/features/storageBackups/diskPresentation.ts", + "frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts", + "frontend-modern/src/features/truenas/TrueNASStorageTopologyTable.tsx", + "frontend-modern/src/features/truenas/truenasPageModel.ts", + "frontend-modern/src/utils/metricThresholds.ts", + "frontend-modern/src/utils/temperature.ts" + ], + "content_sha256": { + "frontend-modern/src/components/Storage/DiskList.tsx": "9fbcdb4659cfdf443f521c9b15969f485876cbdde8d7503110e12325b1899538", + "frontend-modern/src/components/Storage/useDiskDetailModel.ts": "51dea57998a0c0d88f4a8d6410ff16dc0559674cfecf456b61706bac145321af", + "frontend-modern/src/components/Storage/useDiskListModel.ts": "778db34c9def3a921e4bb6f9a36a79dcaa804962e9fb5bc3c03d908c585d8503", + "frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx": "ef649564bd14366aa71121e9c09ac4309bcd299fd1d388ceff5d5969671ae471", + "frontend-modern/src/features/storageBackups/diskDetailPresentation.ts": "dac5b69aa5c603b719946fc3262d810a5a9bdc86469f33c3097542516f73d5cd", + "frontend-modern/src/features/storageBackups/diskPresentation.ts": "4210a0e7d61113f774462571700582d2f32cfe4474f04048e03d728272bd2247", + "frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts": "f983ea771d8f21ab018ca38a802aa7fa9795860c0ff010adcb7507cf9c47678c", + "frontend-modern/src/features/truenas/TrueNASStorageTopologyTable.tsx": "168af9011efa41fc545dfb198f6eff5e9d5dbb003a0cd330c1238a820979c6b9", + "frontend-modern/src/features/truenas/truenasPageModel.ts": "36497cd8a519e3de9564eefb4b1a245fac6b0478c9f9a44a262c24228fc33dc9", + "frontend-modern/src/utils/metricThresholds.ts": "227994b089456c54489d64c90d3a3111421ec0c1b13adddc2744ab2d3749cfc2", + "frontend-modern/src/utils/temperature.ts": "a67d4ec80af39b6057ed155dc8ce5ce4c7e697345f7b9fadb798dcbd31605f9f" + }, + "routes": ["/proxmox/storage?tab=disks&q=West%20Production", "/truenas/storage"], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack built from this exact staged content (embedded UI and backend from one tree). A local-only generator overlay, never committed, pinned Proxmox disks from a control file: Samsung 990 PRO NVMe 63C, Crucial MX500 SATA 56C, WD Black SN850X NVMe 72C, Seagate IronWolf SATA 52C, plus TrueNAS mock disk sda (SATA) at 57C. The API returned no physicalDisk.risk and status online for every pinned disk", + "Same-session control on a second private stack built from the base without this change (main plus #2623): NVMe 63C read 'Running Hot' (amber) beside a green 63C cell with risk warning [temperature_high] and status warning; SATA 56C read 'Healthy' beside a red 56C cell; with the NVMe trigger raised to 75 the NVMe 72C still read 'Running Hot' from the flat 70C rule beside an amber cell", + "After, factory thresholds, 1440: Crucial MX500 56C 'Running Hot' red, title 'Disk temperature is 56\u00b0C, at or above its 55\u00b0C alert threshold.', red 56C cell, sorted first; WD Black 72C 'Running Hot' red with its 70C threshold; Samsung 990 PRO 63C 'Healthy' with a green cell; Crucial MX500 51C 'Healthy' with an amber cell (the band below the trigger stays colour only)", + "After, factory thresholds, 390: same rows with compact 'Hot' in red beside red cells, 'Healthy' beside the green 63C NVMe; no horizontal overflow (scrollWidth 390)", + "After, NVMe trigger raised to 75/70 through PUT /api/alerts/config: WD Black NVMe 72C is no longer Running Hot (its mock SMART health is UNKNOWN so it reads 'Unknown') and its cell is amber; SATA 56C still Running Hot", + "After, agent Disk Temp default switched off (trigger 0) through PUT /api/alerts/config: no disk reads Running Hot and every Temp cell is uncoloured (text-base-content); TrueNAS sda reads healthy with an Online dot", + "After, disk drawer for the 56C SATA disk: header 'Running Hot Disk temperature is 56\u00b0C, at or above its 55\u00b0C alert threshold.' and the Temperature card reads 56\u00b0C", + "After, TrueNAS Storage at 1440: sda (57C) is an Attention row with Health 'Disk temperature is 57\u00b0C, at or above its 55\u00b0C alert threshold.'; the Attention status filter lists it; other disks healthy and unchanged", + "After, TrueNAS Storage at 390: the Health column is phone-hidden, and sda's status dot reads Warning (amber) while cool disks stay Online (green)" + ], + "interactions": [ + "Loaded Physical Disks filtered to West Production at 1440x900 and 390x844 and read each row's health word, title, colour and Temp cell class on both the before and after stacks", + "Changed diskTempByType.nvme to 75/70, then agentDefaults.diskTemperature to 0, through GET/PUT /api/alerts/config and reloaded Physical Disks and TrueNAS Storage after each, then restored the factory config", + "Opened the inline drawer of the 56C SATA disk by clicking its row", + "Loaded TrueNAS Storage at 1440 and 390, clicked the Attention status filter, and read sda's Health cell, title and status dot" + ] +} diff --git a/frontend-modern/src/components/Storage/DiskList.tsx b/frontend-modern/src/components/Storage/DiskList.tsx index 4d463bb5c2..db5c65b4b0 100644 --- a/frontend-modern/src/components/Storage/DiskList.tsx +++ b/frontend-modern/src/components/Storage/DiskList.tsx @@ -13,7 +13,6 @@ import { TableHeader, TableRow, } from '@/components/shared/Table'; -import { useAlertsActivation } from '@/stores/alertsActivation'; import { formatBytes } from '@/utils/format'; import { getTemperatureTextClass } from '@/utils/temperature'; import { @@ -77,6 +76,7 @@ import { getPhysicalDiskEmptyStatePresentation, getPhysicalDiskHealthCompactLabel, getPhysicalDiskHealthStatus, + getPhysicalDiskTemperatureThresholds, getPhysicalDiskHealthSummary, getPhysicalDiskHostLabel, getPhysicalDiskLifeLabel, @@ -132,7 +132,6 @@ export const DiskList: Component = (props) => { }; const columnStyle = (columnId: PhysicalDiskTableColumnId) => getPhysicalDiskColumnWidthStyle(layoutMode(), columnId); - const { getDiskTemperatureThresholds } = useAlertsActivation(); const model = useDiskListModel({ disks: () => props.disks, nodes: () => props.nodes, @@ -523,7 +522,7 @@ export const DiskList: Component = (props) => { reading().current ? getTemperatureTextClass( data().temperature, - getDiskTemperatureThresholds(data().type), + getPhysicalDiskTemperatureThresholds(data()), 'diskTemperature', ) : PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS diff --git a/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx b/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx index 887fe416aa..e91f30ba64 100644 --- a/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx +++ b/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx @@ -2,8 +2,12 @@ import { cleanup, fireEvent, render, screen, waitFor, within } from '@solidjs/te import { createSignal } from 'solid-js'; import { createStore, reconcile } from 'solid-js/store'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import { AlertsAPI } from '@/api/alerts'; +import type { AlertConfig } from '@/types/alerts'; import type { Resource } from '@/types/resource'; import { DiskList } from '@/components/Storage/DiskList'; +import { useAlertsActivation } from '@/stores/alertsActivation'; +import { eventBus } from '@/stores/events'; vi.mock('@/components/Storage/DiskDetail', () => ({ DiskDetail: (props: { disk: Resource }) =>
{props.disk.id}
, @@ -348,19 +352,9 @@ describe('DiskList', () => { disks: [ buildDisk('sda', 'pve3', { model: 'Crucial MX500 2TB', - diskType: 'ssd', + diskType: 'sata', wearout: 95, - temperature: 72, - risk: { - level: 'critical', - reasons: [ - { - code: 'temperature_high', - severity: 'critical', - summary: 'Disk temperature is 72C', - }, - ], - }, + temperature: 56, }), buildDisk('sdb', 'pve3', { temperature: 72, @@ -368,11 +362,6 @@ describe('DiskList', () => { risk: { level: 'critical', reasons: [ - { - code: 'temperature_high', - severity: 'critical', - summary: 'Disk temperature is 72C', - }, { code: 'pending_sectors', severity: 'critical', @@ -381,6 +370,11 @@ describe('DiskList', () => { ], }, }), + buildDisk('nvme0n1', 'pve3', { + model: 'Samsung 990 PRO', + diskType: 'nvme', + temperature: 63, + }), ], nodes: [buildNode('node-pve3', 'pve3')], selectedNode: null, @@ -389,29 +383,34 @@ describe('DiskList', () => { const rows = Array.from(view.container.querySelectorAll('[data-row-id]')); // The disk that needs replacing sorts above the one that needs cooling. - expect(rows.map((row) => row.getAttribute('data-row-id'))).toEqual(['sdb', 'sda']); + expect(rows.map((row) => row.getAttribute('data-row-id'))).toEqual(['sdb', 'sda', 'nvme0n1']); const hotRow = within(rows[1] as HTMLElement); expect(hotRow.queryByText('Replace Now')).not.toBeInTheDocument(); expect(hotRow.getByText('Running Hot')).toHaveClass( 'platform-table-label-full', 'text-red-700', ); - expect(hotRow.getByText('Running Hot')).toHaveAttribute('title', 'Disk temperature is 72C'); + expect(hotRow.getByText('Running Hot')).toHaveAttribute( + 'title', + 'Disk temperature is 56°C, at or above its 55°C alert threshold.', + ); expect(hotRow.getByText('Hot')).toHaveClass('platform-table-label-compact', 'text-red-700'); + // The Temp cell beside it reads red too: the same SATA alert trigger. + expect(hotRow.getByText('56°C')).toHaveClass('text-red-600'); const failingRow = within(rows[0] as HTMLElement); expect(failingRow.getByText('Replace Now')).toHaveAttribute( 'title', 'Pending sectors detected (2)', ); + // An NVMe at 63C is under its 70C trigger and its 65C warning colour. + const nvmeRow = within(rows[2] as HTMLElement); + expect(nvmeRow.getByText('Healthy')).toBeInTheDocument(); + expect(nvmeRow.getByText('63°C')).toHaveClass('text-green-600'); }); - it('follows a disk from warm to critically hot and back to healthy', async () => { - const heat = (severity: 'warning' | 'critical', celsius: number) => ({ - level: severity, - reasons: [{ code: 'temperature_high', severity, summary: `Disk temperature is ${celsius}C` }], - }); + it('follows a disk from warm to hot and back to healthy', async () => { const [disks, setDisks] = createStore({ - items: [buildDisk('sda', 'pve3', { temperature: 63, risk: heat('warning', 63) })], + items: [buildDisk('sda', 'pve3', { temperature: 52 })], }); const [healthFilter, setHealthFilter] = createSignal<'all' | 'critical'>('all'); const view = render(() => ( @@ -426,21 +425,15 @@ describe('DiskList', () => { /> )); const row = () => view.container.querySelector('[data-row-id="sda"]') as HTMLElement | null; - expect(within(row()!).getByText('Running Hot')).toHaveClass('text-amber-700'); - expect(within(row()!).getByText('Hot')).toHaveClass('text-amber-700'); + // Warm: the Temp cell warns in amber, the verdict stays Healthy. + expect(within(row()!).getByText('52°C')).toHaveClass('text-amber-600'); + expect(within(row()!).getByText('Healthy')).toBeInTheDocument(); setHealthFilter('critical'); await waitFor(() => expect(row()).toBeNull()); - setDisks( - 'items', - reconcile([buildDisk('sda', 'pve3', { temperature: 72, risk: heat('critical', 72) })]), - ); + setDisks('items', reconcile([buildDisk('sda', 'pve3', { temperature: 56 })])); await waitFor(() => expect(row()).not.toBeNull()); expect(within(row()!).getByText('Running Hot')).toHaveClass('text-red-700'); - expect(within(row()!).getByText('Running Hot')).toHaveAttribute( - 'title', - 'Disk temperature is 72C', - ); setHealthFilter('all'); setDisks('items', reconcile([buildDisk('sda', 'pve3', { temperature: 41 })])); @@ -448,6 +441,40 @@ describe('DiskList', () => { expect(within(row()!).queryByText('Running Hot')).not.toBeInTheDocument(); }); + it('judges heat by the NVMe trigger the user raised in Alerts', async () => { + const getConfig = vi.spyOn(AlertsAPI, 'getConfig').mockResolvedValue({ + enabled: true, + activationState: 'active', + agentDefaults: { diskTemperature: { trigger: 55, clear: 50 } }, + diskTempByType: { + nvme: { trigger: 75, clear: 70 }, + sas: { trigger: 65, clear: 60 }, + sata: { trigger: 55, clear: 50 }, + }, + } as unknown as AlertConfig); + try { + const view = renderDiskList({ + disks: [buildDisk('nvme0n1', 'pve3', { diskType: 'nvme', temperature: 72 })], + nodes: [], + selectedNode: null, + searchTerm: '', + }); + const row = () => + within(view.container.querySelector('[data-row-id="nvme0n1"]') as HTMLElement); + // Factory thresholds until the user's configuration loads: 72C passes + // the 70C NVMe trigger. + expect(row().getByText('Running Hot')).toBeInTheDocument(); + expect(row().getByText('72°C')).toHaveClass('text-red-600'); + + await useAlertsActivation().refreshConfig(); + await waitFor(() => expect(row().getByText('Healthy')).toBeInTheDocument()); + expect(row().getByText('72°C')).toHaveClass('text-amber-600'); + } finally { + getConfig.mockRestore(); + eventBus.emit('org_switched', 'default'); + } + }); + it('renders SSD life and falls back to the Proxmox usage string for Belongs', () => { renderDiskList({ disks: [ diff --git a/frontend-modern/src/components/Storage/useDiskDetailModel.ts b/frontend-modern/src/components/Storage/useDiskDetailModel.ts index 2038b38433..e8c31c049b 100644 --- a/frontend-modern/src/components/Storage/useDiskDetailModel.ts +++ b/frontend-modern/src/components/Storage/useDiskDetailModel.ts @@ -3,6 +3,7 @@ import type { HistoryTimeRange } from '@/api/charts'; import { getPhysicalDiskCollectionMessages, extractPhysicalDiskPresentationData, + getPhysicalDiskTemperatureThresholds, type PhysicalDiskPresentationData, } from '@/features/storageBackups/diskPresentation'; import { @@ -23,11 +24,11 @@ export const useDiskDetailModel = (options: UseDiskDetailModelOptions) => { const { getDiskTemperatureThresholds } = useAlertsActivation(); const diskData = createMemo(() => - extractPhysicalDiskPresentationData(options.disk()), + extractPhysicalDiskPresentationData(options.disk(), getDiskTemperatureThresholds), ); const historyResourceId = createMemo(() => resolvePhysicalDiskHistoryResourceId(options.disk())); const attributeCards = createMemo(() => - getDiskDetailAttributeCards(diskData(), getDiskTemperatureThresholds(diskData().type)), + getDiskDetailAttributeCards(diskData(), getPhysicalDiskTemperatureThresholds(diskData())), ); const historyCharts = createMemo(() => getDiskDetailHistoryCharts(diskData())); const healthPresentation = createMemo(() => getDiskDetailHealthPresentation(diskData())); diff --git a/frontend-modern/src/components/Storage/useDiskListModel.ts b/frontend-modern/src/components/Storage/useDiskListModel.ts index a4aec530bf..93428b1a15 100644 --- a/frontend-modern/src/components/Storage/useDiskListModel.ts +++ b/frontend-modern/src/components/Storage/useDiskListModel.ts @@ -9,6 +9,7 @@ import { filterAndSortPhysicalDisks, type PhysicalDiskPresentationData, } from '@/features/storageBackups/diskPresentation'; +import { useAlertsActivation } from '@/stores/alertsActivation'; import { matchesPhysicalDiskNode } from './diskResourceUtils'; type UseDiskListModelOptions = { @@ -26,13 +27,19 @@ type UseDiskListModelOptions = { export const useDiskListModel = (options: UseDiskListModelOptions) => { const hasPVENodes = createMemo(() => options.nodes().length > 0); + const { getDiskTemperatureThresholds } = useAlertsActivation(); - const diskDataById = createMemo(() => buildPhysicalDiskPresentationDataMap(options.disks())); + // Each disk's heat is judged by the user's alert disk temperature thresholds, + // so the presentation data recomputes when the alert configuration loads. + const diskDataById = createMemo(() => + buildPhysicalDiskPresentationDataMap(options.disks(), getDiskTemperatureThresholds), + ); const roleFilterOptions = createMemo(() => buildPhysicalDiskRoleFilterOptions(options.disks())); const groupFilterOptions = createMemo(() => buildPhysicalDiskGroupFilterOptions(options.disks())); const getDiskData = (disk: Resource): PhysicalDiskPresentationData => - diskDataById().get(disk.id) ?? extractPhysicalDiskPresentationData(disk); + diskDataById().get(disk.id) ?? + extractPhysicalDiskPresentationData(disk, getDiskTemperatureThresholds); const selectedNodeResource = createMemo( () => options.nodes().find((node) => node.id === options.selectedNode()) ?? null, diff --git a/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx b/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx index ceeaa26a50..2b057ff711 100644 --- a/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx +++ b/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx @@ -9,6 +9,7 @@ import { PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS, getPhysicalDiskTemperaturePresentation, } from '@/features/storageBackups/diskTemperaturePresentation'; +import { useAlertsActivation } from '@/stores/alertsActivation'; import type { Resource } from '@/types/resource'; import { formatBytes } from '@/utils/format'; @@ -20,7 +21,8 @@ const LazyDiskDetail = lazy(() => const GuestPhysicalDiskRow: Component<{ disk: Resource }> = (props) => { const [expanded, setExpanded] = createSignal(false); - const data = () => extractPhysicalDiskPresentationData(props.disk); + const { getDiskTemperatureThresholds } = useAlertsActivation(); + const data = () => extractPhysicalDiskPresentationData(props.disk, getDiskTemperatureThresholds); const health = () => getPhysicalDiskHealthStatus(data()); const temperature = () => getPhysicalDiskTemperaturePresentation(data()); const label = () => data().model || data().devPath || props.disk.name; diff --git a/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.test.ts b/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.test.ts index 392e467d21..a5f615a718 100644 --- a/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.test.ts +++ b/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.test.ts @@ -69,6 +69,8 @@ describe('diskDetailPresentation', () => { expect(getLinkedDiskTemperatureTextClass(65)).toBe('text-red-500'); expect(getLinkedDiskTemperatureTextClass(52)).toBe('text-yellow-500'); expect(getLinkedDiskTemperatureTextClass(45)).toBe('text-muted'); + // Disk temperature alerting off for the disk: no reading is warm or hot. + expect(getLinkedDiskTemperatureTextClass(80, null)).toBe('text-muted'); // Explicit thresholds (e.g. resolved for an NVMe disk) shift the colors. expect(getLinkedDiskTemperatureTextClass(55, { warning: 65, critical: 70 })).toBe('text-muted'); expect(getLinkedDiskTemperatureTextClass(67, { warning: 65, critical: 70 })).toBe( diff --git a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts index e3ebb4c861..cb94ecd9f7 100644 --- a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts +++ b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts @@ -40,6 +40,7 @@ import { getPhysicalDiskSourceBadgePresentation, getPhysicalDiskTableLayoutModeForContainer, getPhysicalDiskTemperaturePresentation, + getPhysicalDiskTemperatureThresholds, hasUnraidPhysicalDiskFaultSignal, hasPhysicalDiskSmartWarning, isPhysicalDiskWearoutReported, @@ -51,6 +52,11 @@ import { type PhysicalDiskPresentationData, } from '@/features/storageBackups/diskPresentation'; import * as diskTemperaturePresentation from '@/features/storageBackups/diskTemperaturePresentation'; +import { + resolveDiskTemperatureDisplayThresholds, + type MetricDisplayThresholds, +} from '@/utils/metricThresholds'; +import { getTemperatureTextClass } from '@/utils/temperature'; function makeDiskData( overrides: Partial = {}, @@ -293,131 +299,188 @@ describe('diskPresentation', () => { expect(getPhysicalDiskHealthCompactLabel('Unknown')).toBe('Unknown'); }); - describe('heat versus replacement evidence', () => { - const buildRiskDisk = ( - id: string, - risk: { level: string; reasons: { code: string; severity: string; summary: string }[] }, - overrides: Record = {}, - ): Resource => + describe('heat judged by the alert disk temperature policy', () => { + // Disk risk never carries heat. The verdict judges a current reading + // against the same per-type alert thresholds the Temp cell colours by. + const buildDisk = (id: string, overrides: Record = {}): Resource => ({ id, name: id, type: 'physical_disk', - status: 'warning', + status: 'online', physicalDisk: { devPath: `/dev/${id}`, model: 'Crucial MX500 2TB', diskType: 'sata', health: 'PASSED', wearout: 95, - temperature: 72, - risk, + temperature: 40, + collection: { temperature: { state: 'available', source: 'smartctl' } }, ...overrides, }, identity: { hostname: 'pve3' }, canonicalIdentity: { hostname: 'pve3' }, platformType: 'proxmox-pve', }) as unknown as Resource; - const hot = (severity: string, celsius = 72) => ({ - code: 'temperature_high', - severity, - summary: `Disk temperature is ${celsius}C`, - }); + const verdictOf = ( + disk: Resource, + resolve?: (diskType: string) => MetricDisplayThresholds | null, + ) => getPhysicalDiskHealthStatus(extractPhysicalDiskPresentationData(disk, resolve)); + // A user who raised the NVMe trigger to 75C in Alerts; warning colours + // start at its 70C clear value. + const raisedNVMe = (diskType: string) => + diskType === 'nvme' + ? { warning: 70, critical: 75 } + : resolveDiskTemperatureDisplayThresholds(null, diskType); - it('calls a disk that is only hot Running Hot at the tier of its reading', () => { - const criticalDisk = buildRiskDisk('sda', { level: 'critical', reasons: [hot('critical')] }); - const criticalData = extractPhysicalDiskPresentationData(criticalDisk); - expect(criticalData.riskReasonDetails).toEqual([hot('critical')]); - expect(getPhysicalDiskHealthStatus(criticalData)).toEqual({ + it('calls a disk Running Hot from its type alert trigger, not a flat line', () => { + expect(verdictOf(buildDisk('nvme0n1', { diskType: 'nvme', temperature: 63 })).label).toBe( + 'Healthy', + ); + expect(verdictOf(buildDisk('nvme0n1', { diskType: 'nvme', temperature: 72 }))).toEqual({ label: 'Running Hot', - summary: 'Disk temperature is 72C', + summary: 'Disk temperature is 72°C, at or above its 70°C alert threshold.', tone: 'text-red-700 dark:text-red-300', }); - expect(getPhysicalDiskNormalizedHealth(criticalDisk, criticalData)).toBe('critical'); - - const warmDisk = buildRiskDisk( - 'sdb', - { level: 'warning', reasons: [hot('warning', 63)] }, - { temperature: 63 }, - ); - const warmData = extractPhysicalDiskPresentationData(warmDisk); - expect(getPhysicalDiskHealthStatus(warmData)).toEqual({ + expect(verdictOf(buildDisk('sda', { temperature: 56 }))).toEqual({ label: 'Running Hot', - summary: 'Disk temperature is 63C', - tone: 'text-amber-700 dark:text-amber-300', + summary: 'Disk temperature is 56°C, at or above its 55°C alert threshold.', + tone: 'text-red-700 dark:text-red-300', }); - expect(getPhysicalDiskNormalizedHealth(warmDisk, warmData)).toBe('warning'); + // The amber band below the trigger is a Temp cell colour, not a verdict. + expect(verdictOf(buildDisk('sdb', { temperature: 52 })).label).toBe('Healthy'); + + const hotDisk = buildDisk('sdc', { temperature: 56 }); + expect( + getPhysicalDiskNormalizedHealth(hotDisk, extractPhysicalDiskPresentationData(hotDisk)), + ).toBe('critical'); }); - it('keeps Replace Now for failure evidence even when heat is listed first', () => { - // Merged risks keep insertion order, so heat can precede the reason - // that actually calls for a new disk. - const disk = buildRiskDisk('sda', { - level: 'critical', - reasons: [ - hot('critical'), - { - code: 'pending_sectors', - severity: 'critical', - summary: 'Pending sectors detected (2)', - }, - ], + it('follows a raised trigger from the alert configuration', () => { + const nvme72 = buildDisk('nvme0n1', { diskType: 'nvme', temperature: 72 }); + expect(verdictOf(nvme72, raisedNVMe).label).toBe('Healthy'); + expect( + verdictOf(buildDisk('nvme1n1', { diskType: 'nvme', temperature: 76 }), raisedNVMe), + ).toEqual({ + label: 'Running Hot', + summary: 'Disk temperature is 76°C, at or above its 75°C alert threshold.', + tone: 'text-red-700 dark:text-red-300', }); - const data = extractPhysicalDiskPresentationData(disk); - expect(getPhysicalDiskHealthStatus(data)).toEqual({ + expect(verdictOf(buildDisk('sda', { temperature: 56 }), raisedNVMe).label).toBe( + 'Running Hot', + ); + expect(extractPhysicalDiskPresentationData(nvme72, raisedNVMe).temperatureThresholds).toEqual( + { warning: 70, critical: 75 }, + ); + }); + + it('reads red in the Temp cell exactly when the verdict says Running Hot', () => { + const red = getTemperatureTextClass(99, { warning: 1, critical: 2 }, 'diskTemperature'); + // A user who switched disk temperature alerting off resolves to null. + const alertingOff = () => null; + for (const resolve of [undefined, raisedNVMe, alertingOff]) { + for (const diskType of ['nvme', 'sata', 'sas', 'hdd', '']) { + for (let celsius = 40; celsius <= 80; celsius += 1) { + const data = extractPhysicalDiskPresentationData( + buildDisk('sda', { diskType, temperature: celsius }), + resolve, + ); + const cellIsRed = + getTemperatureTextClass( + celsius, + getPhysicalDiskTemperatureThresholds(data), + 'diskTemperature', + ) === red; + expect([ + diskType, + celsius, + getPhysicalDiskHealthStatus(data).label === 'Running Hot', + ]).toEqual([diskType, celsius, cellIsRed]); + } + } + } + }); + + it('calls no disk hot when its temperature alerting is off', () => { + const off = () => null; + const data = extractPhysicalDiskPresentationData(buildDisk('sda', { temperature: 80 }), off); + expect(data.temperatureThresholds).toBeNull(); + expect(getPhysicalDiskHealthStatus(data).label).toBe('Healthy'); + expect( + getTemperatureTextClass(80, getPhysicalDiskTemperatureThresholds(data), 'diskTemperature'), + ).toBe('text-base-content'); + }); + + it('never calls a retained reading hot', () => { + const silent = buildDisk('sda', { + temperature: 80, + collection: { + temperature: { + state: 'unavailable', + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }, + }); + expect(verdictOf(silent).label).toBe('Healthy'); + // A source that predates collection state is current. + expect(verdictOf(buildDisk('sdb', { temperature: 80, collection: undefined })).label).toBe( + 'Running Hot', + ); + }); + + it('keeps Replace Now for failure evidence on a hot disk', () => { + const pending = verdictOf( + buildDisk('sda', { + temperature: 72, + risk: { + level: 'critical', + reasons: [ + { + code: 'pending_sectors', + severity: 'critical', + summary: 'Pending sectors detected (2)', + }, + ], + }, + }), + ); + expect(pending).toEqual({ label: 'Replace Now', summary: 'Pending sectors detected (2)', tone: 'text-red-700 dark:text-red-300', }); - - const failed = extractPhysicalDiskPresentationData( - buildRiskDisk( - 'sdb', - { - level: 'critical', - reasons: [ - hot('critical'), - { - code: 'health_status', - severity: 'critical', - summary: 'Disk reports health status FAILED', - }, - ], - }, - { health: 'FAILED' }, - ), + expect(verdictOf(buildDisk('sdb', { temperature: 72, health: 'FAILED' })).label).toBe( + 'Replace Now', ); - expect(getPhysicalDiskHealthStatus(failed).label).toBe('Replace Now'); - expect(getPhysicalDiskHealthStatus(failed).summary).toBe('Disk reports health status FAILED'); }); - it('lets the more severe of heat and wear evidence name the verdict', () => { - const hotAndWorn = extractPhysicalDiskPresentationData( - buildRiskDisk('sda', { - level: 'critical', - reasons: [ - hot('critical'), - { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, - ], - }), - ); - expect(getPhysicalDiskHealthStatus(hotAndWorn).label).toBe('Running Hot'); - expect(getPhysicalDiskHealthStatus(hotAndWorn).summary).toBe('Disk temperature is 72C'); - - const warmAndWorn = extractPhysicalDiskPresentationData( - buildRiskDisk( - 'sdb', - { + it('lets a hot disk outrank warning-level wear and SMART evidence', () => { + const wornAndHot = verdictOf( + buildDisk('sda', { + temperature: 72, + risk: { level: 'warning', reasons: [ - hot('warning', 63), { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, ], }, - { temperature: 63 }, - ), + }), ); - expect(getPhysicalDiskHealthStatus(warmAndWorn)).toEqual({ + expect(wornAndHot.label).toBe('Running Hot'); + + const wornAndWarm = verdictOf( + buildDisk('sdb', { + temperature: 52, + risk: { + level: 'warning', + reasons: [ + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + ], + }, + }), + ); + expect(wornAndWarm).toEqual({ label: 'Needs Attention', summary: 'SSD life remaining is 8%', tone: 'text-amber-700 dark:text-amber-300', @@ -426,7 +489,15 @@ describe('diskPresentation', () => { it('keeps a critical level that no listed reason explains as Replace Now', () => { const data = extractPhysicalDiskPresentationData( - buildRiskDisk('sda', { level: 'critical', reasons: [hot('warning', 63)] }), + buildDisk('sda', { + risk: { + level: 'critical', + reasons: [ + { code: 'crc_errors', severity: 'monitor', summary: 'UDMA CRC errors detected (2)' }, + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + ], + }, + }), ); expect(getPhysicalDiskHealthStatus(data)).toEqual({ label: 'Replace Now', @@ -435,42 +506,19 @@ describe('diskPresentation', () => { }); }); - it('decides the verdict from codes and severities even without display text', () => { - const pendingWithoutText = extractPhysicalDiskPresentationData( - buildRiskDisk('sda', { - level: 'critical', - reasons: [ - hot('critical'), - { code: 'pending_sectors', severity: 'critical', summary: '' }, - ], - }), - ); - expect(pendingWithoutText.riskReasons).toEqual(['Disk temperature is 72C']); - expect(getPhysicalDiskHealthStatus(pendingWithoutText)).toEqual({ - label: 'Replace Now', - summary: 'Disk health has degraded to a critical state.', - tone: 'text-red-700 dark:text-red-300', - }); - - const heatWithoutText = extractPhysicalDiskPresentationData( - buildRiskDisk('sdb', { - level: 'critical', - reasons: [{ code: 'temperature_high', severity: 'critical', summary: '' }], - }), - ); - expect(getPhysicalDiskHealthStatus(heatWithoutText).label).toBe('Running Hot'); - }); - - it('does not let a weaker reason stand in for an unexplained critical level', () => { + it('decides the verdict from severities even without display text', () => { const data = extractPhysicalDiskPresentationData( - buildRiskDisk('sda', { - level: 'critical', - reasons: [ - { code: 'crc_errors', severity: 'monitor', summary: 'UDMA CRC errors detected (2)' }, - hot('warning', 63), - ], + buildDisk('sda', { + risk: { + level: 'critical', + reasons: [ + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + { code: 'pending_sectors', severity: 'critical', summary: '' }, + ], + }, }), ); + expect(data.riskReasons).toEqual(['SSD life remaining is 8%']); expect(getPhysicalDiskHealthStatus(data)).toEqual({ label: 'Replace Now', summary: 'Disk health has degraded to a critical state.', @@ -479,90 +527,61 @@ describe('diskPresentation', () => { }); it('reads severities case-insensitively and in any order within a class', () => { - const shouting = extractPhysicalDiskPresentationData( - buildRiskDisk('sda', { - level: 'CRITICAL', - reasons: [{ ...hot('critical'), severity: ' Critical ' }], + const shouting = verdictOf( + buildDisk('sda', { + risk: { + level: 'CRITICAL', + reasons: [ + { + code: 'media_errors', + severity: ' Critical ', + summary: 'Media errors detected (4)', + }, + ], + }, }), ); - expect(getPhysicalDiskHealthStatus(shouting).label).toBe('Running Hot'); - expect(getPhysicalDiskHealthStatus(shouting).tone).toBe('text-red-700 dark:text-red-300'); + expect(shouting.label).toBe('Replace Now'); + expect(shouting.summary).toBe('Media errors detected (4)'); - const unsorted = extractPhysicalDiskPresentationData( - buildRiskDisk('sdb', { - level: 'critical', - reasons: [ - { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, - { - code: 'pending_sectors', - severity: 'critical', - summary: 'Pending sectors detected (2)', - }, - ], + const unsorted = verdictOf( + buildDisk('sdb', { + risk: { + level: 'critical', + reasons: [ + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + { + code: 'pending_sectors', + severity: 'critical', + summary: 'Pending sectors detected (2)', + }, + ], + }, }), ); - expect(getPhysicalDiskHealthStatus(unsorted).summary).toBe('Pending sectors detected (2)'); - }); - - it('lets SMART counters and low life outrank warning heat', () => { - const warmAndLow = extractPhysicalDiskPresentationData( - buildRiskDisk( - 'sda', - { level: 'warning', reasons: [hot('warning', 63)] }, - { diskType: 'ssd', wearout: 8, temperature: 63 }, - ), - ); - expect(getPhysicalDiskHealthStatus(warmAndLow)).toEqual({ - label: 'Needs Attention', - summary: 'SSD life is running low.', - tone: 'text-amber-700 dark:text-amber-300', - }); - - const warmAndReallocating = extractPhysicalDiskPresentationData( - buildRiskDisk( - 'sdb', - { level: 'warning', reasons: [hot('warning', 63)] }, - { temperature: 63, smart: { reallocatedSectors: 3 } }, - ), - ); - expect(getPhysicalDiskHealthStatus(warmAndReallocating).label).toBe('Needs Attention'); - }); - - it('sorts a FAILED disk with no risk payload ahead of a critically hot one', () => { - const failedDisk = buildRiskDisk( - 'sdb', - { level: 'healthy', reasons: [] }, - { risk: undefined, health: 'FAILED', temperature: 40 }, - ); - const hotDisk = buildRiskDisk('sda', { level: 'critical', reasons: [hot('critical')] }); - const failedData = extractPhysicalDiskPresentationData(failedDisk); - const hotData = extractPhysicalDiskPresentationData(hotDisk); - expect(getPhysicalDiskHealthStatus(failedData).label).toBe('Replace Now'); - expect( - comparePhysicalDiskPresentation(failedDisk, failedData, hotDisk, hotData), - ).toBeLessThan(0); + expect(unsorted.summary).toBe('Pending sectors detected (2)'); }); it('sorts and filters hot disks by the action their verdict asks for', () => { - const warm = buildRiskDisk('sdc', { level: 'warning', reasons: [hot('warning', 63)] }); - const worn = buildRiskDisk('sdd', { - level: 'warning', - reasons: [ - { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, - ], + const warm = buildDisk('sdc', { temperature: 52 }); + const worn = buildDisk('sdd', { + risk: { + level: 'warning', + reasons: [ + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + ], + }, }); - const hotDisk = buildRiskDisk('sda', { level: 'critical', reasons: [hot('critical')] }); - const failing = buildRiskDisk('sdb', { - level: 'critical', - reasons: [ - { code: 'media_errors', severity: 'critical', summary: 'Media errors detected (4)' }, - ], + const hotDisk = buildDisk('sda', { temperature: 72 }); + const failing = buildDisk('sdb', { + risk: { + level: 'critical', + reasons: [ + { code: 'media_errors', severity: 'critical', summary: 'Media errors detected (4)' }, + ], + }, }); - const healthy = buildRiskDisk( - 'sde', - { level: 'healthy', reasons: [] }, - { risk: undefined, temperature: 38 }, - ); + const healthy = buildDisk('sde', { temperature: 38 }); const disks = [healthy, warm, worn, hotDisk, failing]; const dataMap = buildPhysicalDiskPresentationDataMap(disks); const sorted = (healthFilter: 'all' | 'attention' | 'critical' | 'warning') => @@ -576,9 +595,9 @@ describe('diskPresentation', () => { // Device order alone would put each hot disk first. expect(sorted('all')).toEqual(['sdb', 'sda', 'sdd', 'sdc', 'sde']); - expect(sorted('attention')).toEqual(['sdb', 'sda', 'sdd', 'sdc']); + expect(sorted('attention')).toEqual(['sdb', 'sda', 'sdd']); expect(sorted('critical')).toEqual(['sdb', 'sda']); - expect(sorted('warning')).toEqual(['sdd', 'sdc']); + expect(sorted('warning')).toEqual(['sdd']); }); }); diff --git a/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts b/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts index 75f66c2cce..4e58c0f28f 100644 --- a/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts +++ b/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts @@ -25,7 +25,8 @@ export function getLinkedDiskTemperatureTextClass( tempCelsius: number, thresholds?: MetricDisplayThresholds | null, ): string { - if (!Number.isFinite(tempCelsius) || tempCelsius <= 0) { + // Null thresholds mean disk temperature alerting is off for this disk. + if (!Number.isFinite(tempCelsius) || tempCelsius <= 0 || thresholds === null) { return 'text-muted'; } const severity = getMetricSeverity(tempCelsius, 'diskTemperature', thresholds); @@ -137,7 +138,9 @@ export function getDiskDetailAttributeCards( cards.push({ label: 'Temperature', value: temperature.label, - ok: getMetricSeverity(disk.temperature, 'diskTemperature', diskTempThresholds) !== 'critical', + ok: + diskTempThresholds === null || + getMetricSeverity(disk.temperature, 'diskTemperature', diskTempThresholds) !== 'critical', }); } else if (temperature) { // A retained reading is neither healthy nor hot now. The collection diff --git a/frontend-modern/src/features/storageBackups/diskPresentation.ts b/frontend-modern/src/features/storageBackups/diskPresentation.ts index 1f25850810..4d8c996f08 100644 --- a/frontend-modern/src/features/storageBackups/diskPresentation.ts +++ b/frontend-modern/src/features/storageBackups/diskPresentation.ts @@ -14,7 +14,15 @@ import { import { getAllFilterOptionLabel } from '@/components/shared/filterOptionPresentation'; import { getPhysicalDiskNodeIdentity } from '@/components/Storage/diskResourceUtils'; import { getInfrastructureSettingsLocationLabel } from '@/utils/infrastructureSettingsPresentation'; +import { + resolveDiskTemperatureDisplayThresholds, + type MetricDisplayThresholds, +} from '@/utils/metricThresholds'; import { normalizeStorageSourceKey, storageSourceMatchesFilter } from '@/utils/storageSources'; +import { + getPhysicalDiskHeatSummary, + isPhysicalDiskRunningHot, +} from './diskTemperaturePresentation'; import type { NormalizedHealth, StorageHealthFilter } from './models'; import { matchesStorageNodeTerms, parseStorageSearchQuery } from './storageSearchQuery'; @@ -67,6 +75,12 @@ export interface PhysicalDiskPresentationData { controller?: string; target?: string; temperature: number; + /** + * The alert disk temperature thresholds for this disk's type. The Temp cell + * colours the reading by them and the Health verdict judges heat by them. + * Absent means the factory thresholds for the type. + */ + temperatureThresholds?: MetricDisplayThresholds | null; rpm: number; used: string; collection?: PhysicalDiskCollectionStatus; @@ -395,13 +409,35 @@ export function getPhysicalDiskHostLabel( return (disk.node || resource.parentName || '').trim(); } +/** + * Resolves the alert disk temperature thresholds for a disk type. Callers with + * the user's alert configuration pass `getDiskTemperatureThresholds` from the + * alerts activation store; the default is the factory configuration. + */ +export type PhysicalDiskTemperatureThresholdResolver = ( + diskType: string, +) => MetricDisplayThresholds | null; + +const resolveFactoryDiskTemperatureThresholds: PhysicalDiskTemperatureThresholdResolver = ( + diskType, +) => resolveDiskTemperatureDisplayThresholds(null, diskType); + +/** The thresholds the Temp cell colours this disk's reading by. */ +export const getPhysicalDiskTemperatureThresholds = ( + disk: Pick, +): MetricDisplayThresholds | null => + disk.temperatureThresholds !== undefined + ? disk.temperatureThresholds + : resolveFactoryDiskTemperatureThresholds(disk.type); + export function extractPhysicalDiskPresentationData( resource: Resource, + resolveTemperatureThresholds: PhysicalDiskTemperatureThresholdResolver = resolveFactoryDiskTemperatureThresholds, ): PhysicalDiskPresentationData { const pd = resource.physicalDisk || ((resource.platformData as any)?.physicalDisk ?? {}); const diskNode = getPhysicalDiskNodeIdentity(resource); - // A reason's code and severity decide the verdict even when it carries no - // display text, so only the summary list drops empty summaries. + // A reason's severity decides the verdict even when it carries no display + // text, so only the summary list drops empty summaries. const riskReasonDetails: ResourceStorageRiskReason[] = Array.isArray(pd.risk?.reasons) ? pd.risk.reasons .filter((reason: unknown) => reason !== null && typeof reason === 'object') @@ -431,6 +467,7 @@ export function extractPhysicalDiskPresentationData( health: pd.health || 'UNKNOWN', wearout: pd.wearout ?? -1, temperature: pd.temperature ?? 0, + temperatureThresholds: resolveTemperatureThresholds(pd.diskType || ''), rpm: pd.rpm ?? 0, used: pd.used || '', storageRole: pd.storageRole, @@ -503,23 +540,24 @@ export function getPhysicalDiskCollectionMessages(disk: PhysicalDiskPresentation export function buildPhysicalDiskPresentationDataMap( disks: Resource[], + resolveTemperatureThresholds?: PhysicalDiskTemperatureThresholdResolver, ): Map { const map = new Map(); for (const disk of disks || []) { - map.set(disk.id, extractPhysicalDiskPresentationData(disk)); + map.set(disk.id, extractPhysicalDiskPresentationData(disk, resolveTemperatureThresholds)); } return map; } // Disks sort in the order their verdicts ask for action: replacement first, -// then a disk running critically hot, then other warnings, then a warm disk. +// then a disk running hot, then other warnings. const getPhysicalDiskVerdictPriority = (disk: PhysicalDiskPresentationData): number => { const verdict = getPhysicalDiskHealthVerdict(disk); switch (verdict.kind) { case 'replace': return 400; case 'hot': - return verdict.critical ? 300 : 100; + return 300; case 'attention': return 200; default: @@ -677,11 +715,6 @@ export function hasPhysicalDiskSmartWarning(disk: PhysicalDiskPresentationData): ); } -// Heat is the one disk risk whose fix is cooling, not a replacement disk. -// Alerts raise it as a temperature metric rather than as disk health, so the -// verdict names it apart from the evidence that does mean replacing the disk. -const PHYSICAL_DISK_TEMPERATURE_RISK_CODE = 'temperature_high'; - const PHYSICAL_DISK_RISK_SEVERITY_RANK: Record = { monitor: 1, warning: 2, @@ -696,20 +729,14 @@ interface PhysicalDiskRiskEvidence { summary?: string; } -// Splits the disk's risk into heat and everything else, keeping the most -// severe reason of each. A risk level that none of its reasons explains, or -// one sent without reason codes, stays with the disk-health evidence. -function splitPhysicalDiskRisk(disk: PhysicalDiskPresentationData): { - health: PhysicalDiskRiskEvidence; - heat: PhysicalDiskRiskEvidence; -} { - const health: PhysicalDiskRiskEvidence = { rank: 0 }; - const heat: PhysicalDiskRiskEvidence = { rank: 0 }; +// Keeps the disk's most severe risk reason. A risk level that none of its +// reasons explains, or one sent without reasons, still sets the rank. +function getPhysicalDiskRiskEvidence(disk: PhysicalDiskPresentationData): PhysicalDiskRiskEvidence { + const evidence: PhysicalDiskRiskEvidence = { rank: 0 }; const details = disk.riskReasonDetails ?? disk.riskReasons.map((summary) => ({ code: '', severity: disk.riskLevel || '', summary })); for (const reason of details) { - const evidence = reason.code === PHYSICAL_DISK_TEMPERATURE_RISK_CODE ? heat : health; const rank = getPhysicalDiskRiskSeverityRank(reason.severity); if (rank > evidence.rank) { evidence.rank = rank; @@ -719,25 +746,25 @@ function splitPhysicalDiskRisk(disk: PhysicalDiskPresentationData): { } } const levelRank = getPhysicalDiskRiskSeverityRank(disk.riskLevel); - if (levelRank > Math.max(health.rank, heat.rank)) { + if (levelRank > evidence.rank) { // A reason with a known, lower severity does not explain this level, so // its text must not stand in as the verdict's reason. - if (health.rank > 0) health.summary = undefined; - health.rank = levelRank; + if (evidence.rank > 0) evidence.summary = undefined; + evidence.rank = levelRank; } - return { health, heat }; + return evidence; } type PhysicalDiskHealthVerdict = | { kind: 'replace'; summary: string } - | { kind: 'hot'; critical: boolean; summary: string } + | { kind: 'hot'; summary: string } | { kind: 'attention'; summary: string } | { kind: 'clear' }; function getPhysicalDiskHealthVerdict( disk: PhysicalDiskPresentationData, ): PhysicalDiskHealthVerdict { - const { health, heat } = splitPhysicalDiskRisk(disk); + const health = getPhysicalDiskRiskEvidence(disk); const critical = PHYSICAL_DISK_RISK_SEVERITY_RANK.critical; const warning = PHYSICAL_DISK_RISK_SEVERITY_RANK.warning; const lowLife = isPhysicalDiskWearoutReported(disk) && disk.wearout < 10; @@ -748,8 +775,12 @@ function getPhysicalDiskHealthVerdict( summary: health.summary || 'Disk health has degraded to a critical state.', }; } - if (heat.rank >= critical) { - return { kind: 'hot', critical: true, summary: heat.summary || 'Disk temperature is high.' }; + const temperatureThresholds = getPhysicalDiskTemperatureThresholds(disk); + if (isPhysicalDiskRunningHot(disk, temperatureThresholds)) { + return { + kind: 'hot', + summary: getPhysicalDiskHeatSummary(disk.temperature, temperatureThresholds), + }; } if (health.rank >= warning || hasPhysicalDiskSmartWarning(disk) || lowLife) { return { @@ -759,9 +790,6 @@ function getPhysicalDiskHealthVerdict( (lowLife ? 'SSD life is running low.' : 'SMART counters indicate elevated risk.'), }; } - if (heat.rank >= warning) { - return { kind: 'hot', critical: false, summary: heat.summary || 'Disk temperature is high.' }; - } return { kind: 'clear' }; } @@ -778,11 +806,7 @@ export function getPhysicalDiskHealthStatus( } if (verdict.kind === 'hot') { - return { - label: 'Running Hot', - summary: verdict.summary, - tone: verdict.critical ? PHYSICAL_DISK_CRITICAL_TONE : PHYSICAL_DISK_WARNING_TONE, - }; + return { label: 'Running Hot', summary: verdict.summary, tone: PHYSICAL_DISK_CRITICAL_TONE }; } if (verdict.kind === 'attention') { @@ -814,8 +838,9 @@ export function getPhysicalDiskNormalizedHealth( if (clearLabel === 'Online') return 'healthy'; if (resource.status === 'offline') return 'offline'; if (verdict.kind === 'replace') return 'critical'; - // A disk running hot sits in the health filter at its temperature tier. - if (verdict.kind === 'hot') return verdict.critical ? 'critical' : 'warning'; + // A disk running hot sits in the health filter with the critical disks, as + // its Temp cell reads red. + if (verdict.kind === 'hot') return 'critical'; if (verdict.kind === 'attention') return 'warning'; return clearLabel === 'Healthy' ? 'healthy' : 'unknown'; } diff --git a/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts b/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts index acb63c612a..b4571083ea 100644 --- a/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts +++ b/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts @@ -1,4 +1,5 @@ import type { PhysicalDiskCollectionStatus, PhysicalDiskFieldStatus } from '@/types/resource'; +import type { MetricDisplayThresholds } from '@/utils/metricThresholds'; import { formatTemperature } from '@/utils/temperature'; // The canonical decision on whether a disk temperature is a current reading. @@ -51,3 +52,29 @@ export function getPhysicalDiskTemperaturePresentation(disk: { : getPhysicalDiskLastKnownTemperatureTitle(disk.collection?.temperature), }; } + +// Disk heat is judged by the alert disk temperature policy, through the +// thresholds the Temp column colours a reading by: a disk runs hot from its +// type's alert trigger (`critical`), the reading its disk temperature alert +// fires at. The band just below the trigger is a colour, not heat. A retained +// reading is not current, and null thresholds mean disk temperature alerting +// is off for the disk, so neither is ever hot. Disk risk never carries heat. +export function isPhysicalDiskRunningHot( + disk: { temperature: number; collection?: PhysicalDiskCollectionStatus | null }, + thresholds: MetricDisplayThresholds | null, +): boolean { + if (!thresholds || !(thresholds.critical > 0)) return false; + if (!Number.isFinite(disk.temperature) || disk.temperature <= 0) return false; + if (!isPhysicalDiskTemperatureCurrent(disk.collection)) return false; + return disk.temperature >= thresholds.critical; +} + +export function getPhysicalDiskHeatSummary( + temperature: number, + thresholds: MetricDisplayThresholds | null, +): string { + const reading = formatTemperature(temperature); + return thresholds && thresholds.critical > 0 + ? `Disk temperature is ${reading}, at or above its ${formatTemperature(thresholds.critical)} alert threshold.` + : `Disk temperature is ${reading}.`; +} diff --git a/frontend-modern/src/features/truenas/TrueNASStorageTopologyTable.tsx b/frontend-modern/src/features/truenas/TrueNASStorageTopologyTable.tsx index 8f3e627337..27c9caf59a 100644 --- a/frontend-modern/src/features/truenas/TrueNASStorageTopologyTable.tsx +++ b/frontend-modern/src/features/truenas/TrueNASStorageTopologyTable.tsx @@ -37,13 +37,15 @@ import { createPlatformResourceLabelResolver, getPlatformResourceDetailRowClass, } from '@/features/platformPage/PlatformResourceDetailTableRow'; +import { useAlertsActivation } from '@/stores/alertsActivation'; import type { Resource } from '@/types/resource'; import { buildTrueNASStorageTopologyRows, filterTrueNASStorageTopologyRows, filterTrueNASStorageTopologyRowsByKind, - getTrueNASResourceDisplayStatus, + getTrueNASStorageDotStatus, getTrueNASStorageIssue, + type TrueNASDiskTemperatureThresholdResolver, type TrueNASStorageIssue, type TrueNASStorageStatusFilter, type TrueNASStorageKindFilter, @@ -232,6 +234,7 @@ type TrueNASStorageSortKey = (typeof TRUENAS_STORAGE_SORT_KEYS)[number]; const getTrueNASStorageSortValue = ( row: TrueNASStorageTopologyRow, key: TrueNASStorageSortKey, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, ): PlatformTableSortValue => { switch (key) { case 'resource': @@ -255,7 +258,9 @@ const getTrueNASStorageSortValue = ( return typeof temperature === 'number' && Number.isFinite(temperature) ? temperature : null; } case 'health': - return HEALTH_SORT_RANK[getTrueNASStorageIssue(row.resource)?.status ?? 'healthy']; + return HEALTH_SORT_RANK[ + getTrueNASStorageIssue(row.resource, resolveDiskTemperatureThresholds)?.status ?? 'healthy' + ]; default: key satisfies never; return null; @@ -271,6 +276,7 @@ const getTrueNASStorageSortValue = ( const sortTrueNASStorageTopologyRows = ( rows: readonly TrueNASStorageTopologyRow[], sort: PlatformTableSortState, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, ): readonly TrueNASStorageTopologyRow[] => { if (!sort.sortKey()) return rows; const present = new Set(rows.map((row) => row.id)); @@ -288,7 +294,9 @@ const sortTrueNASStorageTopologyRows = ( } const ordered: TrueNASStorageTopologyRow[] = []; const emit = (siblings: TrueNASStorageTopologyRow[]) => { - for (const row of sort.sortRows(siblings, getTrueNASStorageSortValue)) { + for (const row of sort.sortRows(siblings, (sibling, key) => + getTrueNASStorageSortValue(sibling, key, resolveDiskTemperatureThresholds), + )) { ordered.push(row); const children = childrenByParent.get(row.id); if (children) emit(children); @@ -308,7 +316,9 @@ export const getTrueNASStorageTopologyIndentClass = (depth: number): string => { const ResourceCell: Component<{ row: TrueNASStorageTopologyRow; detailToggle?: JSX.Element }> = ( props, ) => { - const displayStatus = () => getTrueNASResourceDisplayStatus(props.row.resource); + const { getDiskTemperatureThresholds } = useAlertsActivation(); + const displayStatus = () => + getTrueNASStorageDotStatus(props.row.resource, getDiskTemperatureThresholds); const indicator = () => getSimpleStatusIndicator(displayStatus()); const name = () => resourceName(props.row.resource); return ( @@ -342,7 +352,11 @@ export const TrueNASStorageTopologyTable: Component<{ kindFilter?: TrueNASStorageKindFilter; onKindFilterChange?: (value: TrueNASStorageKindFilter) => void; }> = (props) => { - const rows = createMemo(() => buildTrueNASStorageTopologyRows(props.resources)); + // Disk heat follows the user's alert disk temperature thresholds. + const { getDiskTemperatureThresholds } = useAlertsActivation(); + const rows = createMemo(() => + buildTrueNASStorageTopologyRows(props.resources, getDiskTemperatureThresholds), + ); const [internalKindFilter, setInternalKindFilter] = createSignal('all'); const kindFilter = () => props.kindFilter ?? internalKindFilter(); const setKindFilter = (value: TrueNASStorageKindFilter) => { @@ -356,7 +370,8 @@ export const TrueNASStorageTopologyTable: Component<{ const tableState = createPlatformTableFilterState({ resources: scopedRows, initialStatus: 'all' as TrueNASStorageStatusFilter, - filter: filterTrueNASStorageTopologyRows, + filter: (rows, search, status) => + filterTrueNASStorageTopologyRows(rows, search, status, getDiskTemperatureThresholds), }); const kindFilters = createMemo(() => [ { @@ -373,6 +388,7 @@ export const TrueNASStorageTopologyTable: Component<{ filterTrueNASStorageTopologyRowsByKind(rows(), option.value), tableState.search(), tableState.status(), + getDiskTemperatureThresholds, ).length, })), value: kindFilter, @@ -392,7 +408,9 @@ export const TrueNASStorageTopologyTable: Component<{ sortKeys: TRUENAS_STORAGE_SORT_KEYS, descendingFirst: ['usage', 'disks', 'temp'], }); - const sortedRows = createMemo(() => sortTrueNASStorageTopologyRows(tableState.filtered(), sort)); + const sortedRows = createMemo(() => + sortTrueNASStorageTopologyRows(tableState.filtered(), sort, getDiskTemperatureThresholds), + ); return ( {(row) => { const resource = () => row.resource; - const issue = createMemo(() => getTrueNASStorageIssue(resource())); + const issue = createMemo(() => + getTrueNASStorageIssue(resource(), getDiskTemperatureThresholds), + ); const detailRowId = () => drawer.detailRowId(resource()); const isExpanded = () => drawer.isExpanded(resource()); return ( diff --git a/frontend-modern/src/features/truenas/__tests__/TrueNASStorageTopologyTable.test.tsx b/frontend-modern/src/features/truenas/__tests__/TrueNASStorageTopologyTable.test.tsx index 57982bab5f..07cdc03e00 100644 --- a/frontend-modern/src/features/truenas/__tests__/TrueNASStorageTopologyTable.test.tsx +++ b/frontend-modern/src/features/truenas/__tests__/TrueNASStorageTopologyTable.test.tsx @@ -1,5 +1,10 @@ -import { cleanup, fireEvent, render, screen } from '@solidjs/testing-library'; -import { afterEach, describe, expect, it } from 'vitest'; +import { cleanup, fireEvent, render, screen, waitFor } from '@solidjs/testing-library'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { AlertsAPI } from '@/api/alerts'; +import { useAlertsActivation } from '@/stores/alertsActivation'; +import { eventBus } from '@/stores/events'; +import type { AlertConfig } from '@/types/alerts'; import { TrueNASStorageTopologyTable, @@ -215,4 +220,102 @@ describe('TrueNASStorageTopologyTable', () => { expect(healthyRow?.querySelector('[data-truenas-storage-health]')).toBeNull(); expect(healthyRow).not.toHaveTextContent('Healthy'); }); + + it('flags a disk running hot by its type alert trigger', () => { + // Disk risk carries no heat, so the table judges it with the alerts + // store's per-type thresholds (factory here: SATA 55C, NVMe 70C). + const disk = (id: string, diskType: string, temperature: number) => + makeStorageResource({ + id, + type: 'physical_disk', + name: id, + storage: undefined, + physicalDisk: { devPath: `/dev/${id}`, serial: `serial-${id}`, diskType, temperature }, + }); + const resources = [disk('sda', 'sata', 56), disk('nvme0n1', 'nvme', 63)]; + const { container } = render(() => ( + } + emptyTitle="No storage" + emptyDescription="No storage" + showToolbar={false} + /> + )); + + const hotRow = container.querySelector('[data-truenas-storage-resource="sda"]'); + expect(hotRow?.querySelector('[data-truenas-storage-health="attention"]')).toHaveAttribute( + 'title', + 'Disk temperature is 56°C, at or above its 55°C alert threshold.', + ); + const warmRow = container.querySelector('[data-truenas-storage-resource="nvme0n1"]'); + expect(warmRow?.querySelector('[data-truenas-storage-health]')).toBeNull(); + // Phones hide the Health cell, so the status dot carries the heat too. + expect(hotRow?.querySelector('[title="Warning"]')).not.toBeNull(); + expect(warmRow?.querySelector('[title="Online"]')).not.toBeNull(); + // The hot disk sorts ahead of its cooler sibling. + expect( + [...container.querySelectorAll('[data-truenas-storage-resource]')].map((row) => + row.getAttribute('data-truenas-storage-resource'), + ), + ).toEqual(['sda', 'nvme0n1']); + }); + + it('follows a disk temperature trigger the user raised in Alerts', async () => { + const getConfig = vi.spyOn(AlertsAPI, 'getConfig').mockResolvedValue({ + enabled: true, + activationState: 'active', + agentDefaults: { diskTemperature: { trigger: 55, clear: 50 } }, + diskTempByType: { + nvme: { trigger: 70, clear: 65 }, + sas: { trigger: 65, clear: 60 }, + sata: { trigger: 60, clear: 55 }, + }, + } as unknown as AlertConfig); + try { + const resources = [ + makeStorageResource({ + id: 'sda', + type: 'physical_disk', + name: 'sda', + storage: undefined, + physicalDisk: { + devPath: '/dev/sda', + serial: 'serial-sda', + diskType: 'sata', + temperature: 56, + }, + }), + ]; + const { container } = render(() => ( + } + emptyTitle="No storage" + emptyDescription="No storage" + /> + )); + const health = () => + container.querySelector( + '[data-truenas-storage-resource="sda"] [data-truenas-storage-health]', + ); + expect(health()).toHaveAttribute('data-truenas-storage-health', 'attention'); + expect(screen.getByRole('button', { name: /Attention/ })).toHaveTextContent('1'); + // With the Attention filter on, the storage-type counts judge heat too. + await fireEvent.click(screen.getByRole('button', { name: /Attention/ })); + expect(screen.getByRole('button', { name: 'Physical disks, 1' })).toBeInTheDocument(); + + await useAlertsActivation().refreshConfig(); + await waitFor(() => + expect(screen.getByRole('button', { name: 'Physical disks, 0' })).toBeInTheDocument(), + ); + // The status filter counts follow the same thresholds. + expect(screen.getByRole('button', { name: /Attention/ })).toHaveTextContent('0'); + } finally { + getConfig.mockRestore(); + eventBus.emit('org_switched', 'default'); + } + }); }); diff --git a/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts b/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts index 8f62772d95..1d29985b7c 100644 --- a/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts +++ b/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import type { RecoveryPoint } from '@/types/recovery'; import type { Resource } from '@/types/resource'; +import { resolveDiskTemperatureDisplayThresholds } from '@/utils/metricThresholds'; import { TRUENAS_TAB_SPECS, buildTrueNASPageModel, @@ -630,6 +631,72 @@ describe('truenasPageModel', () => { expect(mapTrueNASStorageStatus(healthyByState)).toBe('healthy'); }); + it('flags a disk whose current reading reaches its alert trigger', () => { + // Disk risk carries no heat, so a hot disk stays `online`; the page judges + // it by the alert disk temperature thresholds, as Physical Disks does. + const resolveFactory = (diskType: string) => + resolveDiskTemperatureDisplayThresholds(null, diskType); + const hotDisk = makeResource({ + id: 'disk-hot', + type: 'physical_disk', + name: 'sda', + status: 'online', + physicalDisk: { + devPath: '/dev/sda', + serial: 'serial-hot', + diskType: 'sata', + health: 'UNKNOWN', + temperature: 56, + }, + }); + expect(mapTrueNASStorageStatus(hotDisk, resolveFactory)).toBe('attention'); + expect(getTrueNASStorageIssue(hotDisk, resolveFactory)).toEqual({ + status: 'attention', + label: 'Attention', + reasons: ['Disk temperature is 56°C, at or above its 55°C alert threshold.'], + }); + // A cool sibling that sorts first by name: heat moves the hot disk ahead + // of it, and the status filter splits them. + const coolDisk = makeResource({ + ...hotDisk, + id: 'disk-cool', + name: 'ada0', + physicalDisk: { ...hotDisk.physicalDisk!, serial: 'serial-cool', temperature: 40 }, + }); + const rows = buildTrueNASStorageTopologyRows([coolDisk, hotDisk], resolveFactory); + expect(rows.map((row) => row.id)).toEqual(['disk:disk-hot', 'disk:disk-cool']); + expect(buildTrueNASStorageTopologyRows([coolDisk, hotDisk]).map((row) => row.id)).toEqual([ + 'disk:disk-cool', + 'disk:disk-hot', + ]); + expect( + filterTrueNASStorageTopologyRows(rows, '', 'attention', resolveFactory).map((row) => row.id), + ).toEqual(['disk:disk-hot']); + expect( + filterTrueNASStorageTopologyRows(rows, '', 'healthy', resolveFactory).map((row) => row.id), + ).toEqual(['disk:disk-cool']); + + // An NVMe at 63C is under its 70C trigger. + const warmNVMe = makeResource({ + ...hotDisk, + id: 'disk-nvme', + physicalDisk: { ...hotDisk.physicalDisk!, diskType: 'nvme', temperature: 63 }, + }); + expect(mapTrueNASStorageStatus(warmNVMe, resolveFactory)).toBe('healthy'); + // A trigger the user raised, alerting switched off, and a retained reading + // are never heat. + expect(mapTrueNASStorageStatus(hotDisk, () => ({ warning: 55, critical: 60 }))).toBe('healthy'); + expect(mapTrueNASStorageStatus(hotDisk, () => null)).toBe('healthy'); + const retained = makeResource({ + ...hotDisk, + physicalDisk: { + ...hotDisk.physicalDisk!, + collection: { temperature: { state: 'unavailable', source: 'truenas', reason: 'stale' } }, + }, + }); + expect(mapTrueNASStorageStatus(retained, resolveFactory)).toBe('healthy'); + }); + it('renders expected receive-side replication readonly posture as healthy', () => { const replicatedDataset = makeResource({ id: 'dataset-replica', diff --git a/frontend-modern/src/features/truenas/truenasPageModel.ts b/frontend-modern/src/features/truenas/truenasPageModel.ts index 8279c78afe..ceca216eae 100644 --- a/frontend-modern/src/features/truenas/truenasPageModel.ts +++ b/frontend-modern/src/features/truenas/truenasPageModel.ts @@ -2,6 +2,11 @@ import { resolveResourcePlatformType } from '@/utils/sourcePlatforms'; import { asTrimmedString } from '@/utils/stringUtils'; import { hasImpairedResourceSource } from '@/utils/resourceSourceHealth'; import { getTrueNASDatasetStateSummary } from '@/utils/truenasDatasetState'; +import type { MetricDisplayThresholds } from '@/utils/metricThresholds'; +import { + getPhysicalDiskHeatSummary, + isPhysicalDiskRunningHot, +} from '@/features/storageBackups/diskTemperaturePresentation'; import type { Resource, ResourceIncident, @@ -402,7 +407,10 @@ export function buildTrueNASStorageChildCounts( export function buildTrueNASStorageTopologyRows( resources: Resource[], + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, ): TrueNASStorageTopologyRow[] { + const compareStorageResources = (left: Resource, right: Resource): number => + compareStorageResourcesByStatus(left, right, resolveDiskTemperatureThresholds); const storageResources = resources.filter( (resource) => isTrueNASPlatform(resource) && @@ -556,8 +564,14 @@ function filterTrueNASStorageTopologyRow( row: TrueNASStorageTopologyRow, search: string, status: TrueNASStorageStatusFilter, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, ): boolean { - if (status !== 'all' && mapTrueNASStorageStatus(row.resource) !== status) return false; + if ( + status !== 'all' && + mapTrueNASStorageStatus(row.resource, resolveDiskTemperatureThresholds) !== status + ) { + return false; + } if (!search.trim()) return true; if (row.kind.includes(search.trim().toLowerCase())) return true; return matchesTrueNASStorageSearch(row.resource, search); @@ -567,9 +581,14 @@ export function filterTrueNASStorageTopologyRows( rows: TrueNASStorageTopologyRow[], search: string, status: TrueNASStorageStatusFilter, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, ): TrueNASStorageTopologyRow[] { const directMatches = new Set( - rows.filter((row) => filterTrueNASStorageTopologyRow(row, search, status)).map((row) => row.id), + rows + .filter((row) => + filterTrueNASStorageTopologyRow(row, search, status, resolveDiskTemperatureThresholds), + ) + .map((row) => row.id), ); if (directMatches.size === rows.length) return rows; @@ -628,8 +647,11 @@ const titleize = (value: string): string => .map((part) => part.charAt(0).toUpperCase() + part.slice(1).toLowerCase()) .join(' '); -const storageStatusRank = (resource: Resource): number => { - switch (mapTrueNASStorageStatus(resource)) { +const storageStatusRank = ( + resource: Resource, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, +): number => { + switch (mapTrueNASStorageStatus(resource, resolveDiskTemperatureThresholds)) { case 'attention': return 0; case 'offline': @@ -641,8 +663,14 @@ const storageStatusRank = (resource: Resource): number => { } }; -const compareStorageResources = (left: Resource, right: Resource): number => { - const rankDelta = storageStatusRank(left) - storageStatusRank(right); +const compareStorageResourcesByStatus = ( + left: Resource, + right: Resource, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, +): number => { + const rankDelta = + storageStatusRank(left, resolveDiskTemperatureThresholds) - + storageStatusRank(right, resolveDiskTemperatureThresholds); if (rankDelta !== 0) return rankDelta; const nameDelta = resourceDisplayName(left).localeCompare(resourceDisplayName(right)); if (nameDelta !== 0) return nameDelta; @@ -699,8 +727,45 @@ export function mapTrueNASIncidentSeverity( return 'info'; } +/** + * Resolves a disk type's alert disk temperature thresholds: the alerts store's + * `getDiskTemperatureThresholds`. Without one, disk heat is not judged. + */ +export type TrueNASDiskTemperatureThresholdResolver = ( + diskType: string, +) => MetricDisplayThresholds | null; + +// Disk risk carries no heat, so a disk whose current reading has reached its +// type's alert trigger is judged here, by the same policy as the Physical +// Disks Running Hot verdict. Returns the heat reason, or null. +function getTrueNASDiskHeatSummary( + resource: Resource, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, +): string | null { + const disk = resource.physicalDisk; + if (!resolveDiskTemperatureThresholds || resource.type !== 'physical_disk' || !disk) return null; + const temperature = disk.temperature ?? 0; + const thresholds = resolveDiskTemperatureThresholds(disk.diskType || ''); + return isPhysicalDiskRunningHot({ temperature, collection: disk.collection }, thresholds) + ? getPhysicalDiskHeatSummary(temperature, thresholds) + : null; +} + +// A hot disk reports `online` (disk risk carries no heat), so its status dot +// takes the warning tone its Health cell gives it. On phones the dot is the +// row's only health signal. +export function getTrueNASStorageDotStatus( + resource: Resource, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, +): string { + const status = getTrueNASResourceDisplayStatus(resource); + if (normalize(status) !== 'online') return status; + return getTrueNASDiskHeatSummary(resource, resolveDiskTemperatureThresholds) ? 'warning' : status; +} + export function mapTrueNASStorageStatus( resource: Resource, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, ): Exclude | 'unknown' { const status = normalize(getTrueNASResourceDisplayStatus(resource)); const zfsState = normalize(resource.storage?.zfsPoolState); @@ -719,6 +784,7 @@ export function mapTrueNASStorageStatus( if (diskHealth && !['passed', 'healthy', 'ok', 'unknown', 'unavailable'].includes(diskHealth)) { return 'attention'; } + if (getTrueNASDiskHeatSummary(resource, resolveDiskTemperatureThresholds)) return 'attention'; if (['online', 'running', 'healthy'].includes(status)) return 'healthy'; return 'unknown'; } @@ -753,8 +819,11 @@ const reasonKey = (text: string): string => // the failing member or test, so it leads. Pulse's derived risk summaries // restate the same pool and disk state, so they only stand in when TrueNAS // raised no alert; the row drawer still lists every one of them. -export function getTrueNASStorageIssue(resource: Resource): TrueNASStorageIssue | null { - const status = mapTrueNASStorageStatus(resource); +export function getTrueNASStorageIssue( + resource: Resource, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, +): TrueNASStorageIssue | null { + const status = mapTrueNASStorageStatus(resource, resolveDiskTemperatureThresholds); if (status === 'healthy') return null; const reasons: string[] = []; @@ -775,6 +844,7 @@ export function getTrueNASStorageIssue(resource: Resource): TrueNASStorageIssue if (reasons.length === leading) { for (const reason of resource.storage?.risk?.reasons ?? []) push(reason.summary); for (const reason of resource.physicalDisk?.risk?.reasons ?? []) push(reason.summary); + push(getTrueNASDiskHeatSummary(resource, resolveDiskTemperatureThresholds)); } // A read-only dataset raises no incident, so its state tag is the only // reason the provider gives. The drawer reads the same shared mapping. diff --git a/frontend-modern/src/utils/__tests__/metricThresholds.test.ts b/frontend-modern/src/utils/__tests__/metricThresholds.test.ts index 76b8e749d5..59cd271bc2 100644 --- a/frontend-modern/src/utils/__tests__/metricThresholds.test.ts +++ b/frontend-modern/src/utils/__tests__/metricThresholds.test.ts @@ -374,6 +374,29 @@ describe('metricThresholds', () => { }); }); + it('switches every disk type off with the agent disk temperature default', () => { + const config = { + enabled: true, + guestDefaults: {}, + nodeDefaults: {}, + agentDefaults: { diskTemperature: { trigger: 0, clear: 0 } }, + diskTempByType: { + nvme: { trigger: 70, clear: 65 }, + sata: { trigger: 55, clear: 50 }, + }, + storageDefault: { trigger: 85, clear: 80 }, + overrides: { 'host-1': { diskTemperature: { trigger: 80, clear: 75 } } }, + } as AlertConfig; + expect(resolveDiskTemperatureDisplayThresholds(config, 'nvme')).toBeNull(); + expect(resolveDiskTemperatureDisplayThresholds(config, 'sata')).toBeNull(); + expect(resolveDiskTemperatureDisplayThresholds(config, '')).toBeNull(); + // An explicit host override still applies, as CheckHost applies it. + expect(resolveDiskTemperatureDisplayThresholds(config, 'nvme', 'host-1')).toEqual({ + warning: 75, + critical: 80, + }); + }); + it('falls back to seeded per-type disk temperature defaults without config', () => { expect(resolveDiskTemperatureDisplayThresholds(null, 'nvme')).toEqual({ warning: 65, diff --git a/frontend-modern/src/utils/__tests__/temperature.test.ts b/frontend-modern/src/utils/__tests__/temperature.test.ts index 5224735935..6f601bbf0a 100644 --- a/frontend-modern/src/utils/__tests__/temperature.test.ts +++ b/frontend-modern/src/utils/__tests__/temperature.test.ts @@ -130,6 +130,12 @@ describe('temperature', () => { expect(getTemperatureTextClass(80)).toContain('text-red-600'); }); + it('colours no disk reading whose disk temperature alerting is off', () => { + expect(getTemperatureTextClass(80, null, 'diskTemperature')).toBe('text-base-content'); + // Node temperatures keep the factory colours without configured thresholds. + expect(getTemperatureTextClass(80, null, 'temperature')).toContain('text-red-600'); + }); + it('honors configured warning and critical thresholds', () => { const thresholds = { warning: 80, critical: 85 }; diff --git a/frontend-modern/src/utils/metricThresholds.ts b/frontend-modern/src/utils/metricThresholds.ts index 3cc016b000..ee533c19ae 100644 --- a/frontend-modern/src/utils/metricThresholds.ts +++ b/frontend-modern/src/utils/metricThresholds.ts @@ -269,7 +269,9 @@ export const resolveMetricDisplayThresholds = ( * Resolve display thresholds for a physical disk's SMART temperature. * Mirrors the backend precedence: an explicit diskTemperature override on the * host (or inherited linked resource) wins, then the per-type map - * (diskTempByType: nvme/sas/sata), then the global agent default. + * (diskTempByType: nvme/sas/sata), then the global agent default. A + * switched-off agent default switches every type off, as disk temperature + * alerts do. Null means disk temperature alerting is off for this disk. */ export const resolveDiskTemperatureDisplayThresholds = ( config: AlertConfig | null, @@ -285,6 +287,11 @@ export const resolveDiskTemperatureDisplayThresholds = ( return resolveThreshold(overrideValue, FACTORY_AGENT_DEFAULTS.diskTemperature, margin); } + const baseValue = getBaseThresholdValue(config?.agentDefaults, 'diskTemperature'); + if (baseValue !== undefined && resolveThreshold(baseValue, undefined, margin) === null) { + return null; + } + const byTypeFallback = normalizedType ? FACTORY_DISK_TEMP_BY_TYPE[normalizedType] : undefined; if (normalizedType) { const byType = config?.diskTempByType?.[normalizedType]; @@ -293,7 +300,6 @@ export const resolveDiskTemperatureDisplayThresholds = ( } } - const baseValue = getBaseThresholdValue(config?.agentDefaults, 'diskTemperature'); return resolveThreshold( baseValue, byTypeFallback ?? FACTORY_AGENT_DEFAULTS.diskTemperature, diff --git a/frontend-modern/src/utils/temperature.ts b/frontend-modern/src/utils/temperature.ts index 15f608fbac..6a033e47bd 100644 --- a/frontend-modern/src/utils/temperature.ts +++ b/frontend-modern/src/utils/temperature.ts @@ -123,6 +123,11 @@ export const getTemperatureTextClass = ( if (celsius === null || celsius === undefined || !Number.isFinite(celsius)) { return 'text-muted'; } + // Null disk thresholds mean disk temperature alerting is off for that disk, + // so no reading of it is warm or hot. + if (metric === 'diskTemperature' && thresholds === null) { + return 'text-base-content'; + } const readingSeverity = getMetricSeverity( celsius, metric, diff --git a/internal/ai/alert_threshold_adapter.go b/internal/ai/alert_threshold_adapter.go index c10e5e3960..8509f06899 100644 --- a/internal/ai/alert_threshold_adapter.go +++ b/internal/ai/alert_threshold_adapter.go @@ -68,6 +68,22 @@ func (a *AlertThresholdAdapter) GetGuestDiskThreshold() float64 { }, 90) } +// GetDiskTemperatureThreshold returns the disk temperature alert trigger and +// clear values for a disk type, resolved exactly as disk temperature alerts +// resolve them (alerts.Manager.DiskTemperatureThreshold). +func (a *AlertThresholdAdapter) GetDiskTemperatureThreshold(diskType string) (float64, float64) { + var threshold *alerts.HysteresisThreshold + if a.manager != nil { + threshold = a.manager.DiskTemperatureThreshold(diskType) + } else { + threshold = alerts.DefaultDiskTemperatureThreshold(diskType) + } + if threshold == nil || threshold.Trigger <= 0 { + return 0, 0 + } + return threshold.Trigger, threshold.Clear +} + // GetStorageThreshold returns the usage alert trigger threshold for storage (0-100%) func (a *AlertThresholdAdapter) GetStorageThreshold() float64 { return a.getThreshold(func(cfg alerts.AlertConfig) (float64, bool) { diff --git a/internal/ai/alert_threshold_adapter_test.go b/internal/ai/alert_threshold_adapter_test.go index 99a40bd800..f65840ecbd 100644 --- a/internal/ai/alert_threshold_adapter_test.go +++ b/internal/ai/alert_threshold_adapter_test.go @@ -82,3 +82,28 @@ func TestAlertThresholdAdapter_Fallbacks(t *testing.T) { t.Fatalf("GetStorageThreshold default = %v", a.GetStorageThreshold()) } } + +func TestAlertThresholdAdapter_DiskTemperatureFollowsAlertPolicy(t *testing.T) { + if trigger, clear := NewAlertThresholdAdapter(nil).GetDiskTemperatureThreshold("nvme"); trigger != 70 || clear != 65 { + t.Fatalf("no manager: nvme = %v/%v, want the factory 70/65", trigger, clear) + } + + mgr := alerts.NewManager() + cfg := mgr.GetConfig() + cfg.DiskTempByType["nvme"] = alerts.HysteresisThreshold{Trigger: 75, Clear: 70} + mgr.UpdateConfig(cfg) + a := NewAlertThresholdAdapter(mgr) + if trigger, clear := a.GetDiskTemperatureThreshold("nvme"); trigger != 75 || clear != 70 { + t.Fatalf("raised nvme = %v/%v, want 75/70", trigger, clear) + } + if trigger, clear := a.GetDiskTemperatureThreshold("sata"); trigger != 55 || clear != 50 { + t.Fatalf("sata = %v/%v, want 55/50", trigger, clear) + } + + cfg = mgr.GetConfig() + cfg.AgentDefaults.DiskTemperature = &alerts.HysteresisThreshold{Trigger: 0, Clear: 0} + mgr.UpdateConfig(cfg) + if trigger, clear := a.GetDiskTemperatureThreshold("nvme"); trigger != 0 || clear != 0 { + t.Fatalf("disk temperature alerting off: nvme = %v/%v, want 0/0", trigger, clear) + } +} diff --git a/internal/ai/mock_test.go b/internal/ai/mock_test.go index ad2c2836bd..91524141ab 100644 --- a/internal/ai/mock_test.go +++ b/internal/ai/mock_test.go @@ -5,6 +5,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/agentexec" "github.com/rcourtman/pulse-go-rewrite/internal/ai/providers" + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" ) @@ -51,6 +52,9 @@ type mockThresholdProvider struct { guestMem float64 guestDisk float64 storage float64 + // diskTemperature holds trigger/clear by disk type; a type it does not + // list resolves to the factory disk temperature policy. + diskTemperature map[string][2]float64 } func (m *mockThresholdProvider) GetNodeCPUThreshold() float64 { return m.nodeCPU } @@ -59,6 +63,15 @@ func (m *mockThresholdProvider) GetGuestCPUThreshold() float64 { return 0 } func (m *mockThresholdProvider) GetGuestMemoryThreshold() float64 { return m.guestMem } func (m *mockThresholdProvider) GetGuestDiskThreshold() float64 { return m.guestDisk } func (m *mockThresholdProvider) GetStorageThreshold() float64 { return m.storage } +func (m *mockThresholdProvider) GetDiskTemperatureThreshold(diskType string) (float64, float64) { + if limits, ok := m.diskTemperature[diskType]; ok { + return limits[0], limits[1] + } + if threshold := alerts.DefaultDiskTemperatureThreshold(diskType); threshold != nil { + return threshold.Trigger, threshold.Clear + } + return 0, 0 +} type mockUnifiedResourceProvider struct { UnifiedResourceProvider diff --git a/internal/ai/patrol.go b/internal/ai/patrol.go index eb93e20b73..a630554703 100644 --- a/internal/ai/patrol.go +++ b/internal/ai/patrol.go @@ -66,6 +66,10 @@ type ThresholdProvider interface { GetGuestDiskThreshold() float64 // GetStorageThreshold returns the usage alert trigger threshold for storage (0-100%) GetStorageThreshold() float64 + // GetDiskTemperatureThreshold returns the disk temperature alert trigger + // and clear values (°C) for a disk type. A zero trigger means disk + // temperature alerting is off. + GetDiskTemperatureThreshold(diskType string) (trigger, clear float64) } // AlertResolver provides the ability to review and resolve alerts diff --git a/internal/ai/patrol_ai.go b/internal/ai/patrol_ai.go index 8757ac5d0f..a14000ab2e 100644 --- a/internal/ai/patrol_ai.go +++ b/internal/ai/patrol_ai.go @@ -1930,8 +1930,9 @@ type patrolPhysicalDiskRow struct { wearout int // temperature.Collected is the only temperature Patrol may judge as // heat; a retained last-known value is shown as context. - temperature tools.DiskTemperature - smartEvidence []string + temperature tools.DiskTemperature + temperatureLimits diskTemperatureLimits + smartEvidence []string } type patrolPrecomputeNodeSource struct { @@ -2464,7 +2465,7 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool) health = "UNKNOWN" } - rows = append(rows, patrolPhysicalDiskRow{ + row := patrolPhysicalDiskRow{ id: r.ID, name: name, node: strings.TrimSpace(r.ParentName), @@ -2477,7 +2478,9 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool) wearout: r.PhysicalDisk.Wearout, temperature: tools.SplitDiskTemperature(r.PhysicalDisk.Temperature, r.PhysicalDisk.Collection), smartEvidence: unifiedPhysicalDiskSMARTIssueParts(r.PhysicalDisk.SMART), - }) + } + row.temperatureLimits = snap.diskTemperatureLimits(row.diskType) + rows = append(rows, row) } return rows } @@ -2505,7 +2508,7 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool) health = "UNKNOWN" } - rows = append(rows, patrolPhysicalDiskRow{ + row := patrolPhysicalDiskRow{ id: d.ID, name: name, node: strings.TrimSpace(d.Node), @@ -2518,7 +2521,9 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool) wearout: d.Wearout, temperature: tools.SplitDiskTemperature(d.Temperature, d.Collection), smartEvidence: modelPhysicalDiskSMARTIssueParts(d.SmartAttributes), - }) + } + row.temperatureLimits = snap.diskTemperatureLimits(row.diskType) + rows = append(rows, row) } return rows } @@ -2534,7 +2539,7 @@ func patrolPhysicalDiskHealthIssue(row patrolPhysicalDiskRow) bool { (row.wearout == 0 && (strings.EqualFold(row.diskType, "ssd") || strings.EqualFold(row.diskType, "nvme")))) && row.wearout < 20) || - row.temperature.Collected > 55 || + row.temperatureLimits.hot(row.temperature.Collected) || len(row.smartEvidence) > 0 } diff --git a/internal/ai/patrol_disk_temperature.go b/internal/ai/patrol_disk_temperature.go new file mode 100644 index 0000000000..53e58fabe3 --- /dev/null +++ b/internal/ai/patrol_disk_temperature.go @@ -0,0 +1,50 @@ +package ai + +import "github.com/rcourtman/pulse-go-rewrite/internal/alerts" + +// diskTemperatureLimits is the alert disk temperature policy for one disk +// type, the same policy disk temperature alerts and the Physical Disks Health +// verdict judge heat by. A disk is hot at the trigger and stays hot until it +// cools to the clear value. A zero trigger means disk temperature alerting is +// off, so no reading counts as heat. +type diskTemperatureLimits struct { + trigger float64 + clear float64 +} + +// diskTemperatureLimitsFor resolves the policy from the user's alert +// configuration, or from the factory alert configuration when Patrol has no +// threshold provider. +func diskTemperatureLimitsFor(provider ThresholdProvider, diskType string) diskTemperatureLimits { + var trigger, clear float64 + if provider != nil { + trigger, clear = provider.GetDiskTemperatureThreshold(diskType) + } else if threshold := alerts.DefaultDiskTemperatureThreshold(diskType); threshold != nil { + trigger, clear = threshold.Trigger, threshold.Clear + } + if trigger <= 0 { + return diskTemperatureLimits{} + } + if clear <= 0 || clear > trigger { + clear = trigger + } + return diskTemperatureLimits{trigger: trigger, clear: clear} +} + +// hot reports a reading at or above the alert trigger. +func (l diskTemperatureLimits) hot(temperature int) bool { + return l.trigger > 0 && temperature > 0 && float64(temperature) >= l.trigger +} + +// cooled reports a reading at or below the clear value, where a disk +// temperature alert recovers. With no band below the trigger, a reading at +// the trigger still fires, so only a reading under it has cooled. +func (l diskTemperatureLimits) cooled(temperature int) bool { + if l.trigger <= 0 || temperature <= 0 { + return true + } + if l.clear >= l.trigger { + return float64(temperature) < l.trigger + } + return float64(temperature) <= l.clear +} diff --git a/internal/ai/patrol_disk_temperature_test.go b/internal/ai/patrol_disk_temperature_test.go new file mode 100644 index 0000000000..e38358380e --- /dev/null +++ b/internal/ai/patrol_disk_temperature_test.go @@ -0,0 +1,228 @@ +package ai + +import ( + "strings" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +// raisedNVMeThresholds is a user who raised the NVMe disk temperature trigger +// to 75C (clear 70) in Alerts and left every other type at its default. +func raisedNVMeThresholds() *mockThresholdProvider { + return &mockThresholdProvider{diskTemperature: map[string][2]float64{"nvme": {75, 70}}} +} + +func TestDiskTemperatureLimitsFollowTheAlertPolicy(t *testing.T) { + for _, tc := range []struct { + name string + provider ThresholdProvider + diskType string + want diskTemperatureLimits + }{ + {name: "factory nvme", diskType: "nvme", want: diskTemperatureLimits{trigger: 70, clear: 65}}, + {name: "factory sata", diskType: "sata", want: diskTemperatureLimits{trigger: 55, clear: 50}}, + {name: "factory untyped", diskType: "", want: diskTemperatureLimits{trigger: 55, clear: 50}}, + {name: "raised nvme", provider: raisedNVMeThresholds(), diskType: "nvme", want: diskTemperatureLimits{trigger: 75, clear: 70}}, + {name: "raised nvme leaves sata", provider: raisedNVMeThresholds(), diskType: "sata", want: diskTemperatureLimits{trigger: 55, clear: 50}}, + {name: "alerting off", provider: &mockThresholdProvider{diskTemperature: map[string][2]float64{"sata": {0, 0}}}, diskType: "sata", want: diskTemperatureLimits{}}, + {name: "clear above trigger", provider: &mockThresholdProvider{diskTemperature: map[string][2]float64{"sas": {60, 64}}}, diskType: "sas", want: diskTemperatureLimits{trigger: 60, clear: 60}}, + } { + if got := diskTemperatureLimitsFor(tc.provider, tc.diskType); got != tc.want { + t.Errorf("%s: limits = %+v, want %+v", tc.name, got, tc.want) + } + } + + off := diskTemperatureLimits{} + if off.hot(95) || !off.cooled(95) { + t.Fatal("with disk temperature alerting off no reading is heat") + } +} + +func diskTemperatureTriageState(provider ThresholdProvider) patrolRuntimeState { + state := newPatrolRuntimeState(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{ + {ID: "nvme-63", DevPath: "/dev/nvme0n1", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 63}, + {ID: "nvme-66", DevPath: "/dev/nvme1n1", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 66}, + {ID: "nvme-72", DevPath: "/dev/nvme2n1", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 72}, + {ID: "sata-52", DevPath: "/dev/sda", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 52}, + {ID: "sata-56", DevPath: "/dev/sdb", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 56}, + }}) + state.thresholdProvider = provider + return state +} + +// Patrol judges disk heat by the alert disk temperature policy: a warning from +// the per-type trigger and nothing below it, the band under the trigger being +// a Temp cell colour rather than a finding. An NVMe at 63C is not hot; a SATA +// disk at 56C is. +func TestTriageDiskTemperatureFollowsTheAlertPolicy(t *testing.T) { + temperatureFlag := func(flags []TriageFlag, id string) *TriageFlag { + return triageFindFlag(flags, func(f TriageFlag) bool { + return f.ResourceID == id && strings.Contains(f.Reason, "Disk temperature") + }) + } + expect := func(flags []TriageFlag, id, severity string, threshold float64) { + t.Helper() + flag := temperatureFlag(flags, id) + if severity == "" { + if flag != nil { + t.Errorf("%s: unexpected temperature flag %+v", id, *flag) + } + return + } + if flag == nil || flag.Severity != severity || flag.Threshold != threshold { + t.Errorf("%s: temperature flag = %+v, want %s at threshold %.0f", id, flag, severity, threshold) + } + } + + flags := triageDiskHealthChecksState(diskTemperatureTriageState(nil), nil) + expect(flags, "nvme-63", "", 0) + expect(flags, "nvme-66", "", 0) + expect(flags, "nvme-72", "warning", 70) + expect(flags, "sata-52", "", 0) + expect(flags, "sata-56", "warning", 55) + if flag := temperatureFlag(flags, "sata-56"); flag == nil || flag.Reason != "Disk temperature 56°C (alert threshold: 55°C)" { + t.Errorf("sata-56 reason = %+v, want the reading and the alert trigger", flag) + } + + flags = triageDiskHealthChecksState(diskTemperatureTriageState(raisedNVMeThresholds()), nil) + expect(flags, "nvme-72", "", 0) + expect(flags, "sata-56", "warning", 55) + + off := &mockThresholdProvider{diskTemperature: map[string][2]float64{"nvme": {0, 0}, "sata": {0, 0}}} + if flags := triageDiskHealthChecksState(diskTemperatureTriageState(off), nil); len(flags) != 0 { + t.Fatalf("disk temperature alerting off should raise no heat flags, got %+v", flags) + } +} + +// The seed's disk issue list and finding verification share the policy: a +// disk is an issue from the trigger and recovers below the clear value. +func TestPatrolDiskHeatIssueAndRecoveryFollowTheAlertPolicy(t *testing.T) { + issues := map[string]bool{} + for _, row := range patrolPhysicalDiskRows(diskTemperatureTriageState(nil), nil) { + issues[row.id] = patrolPhysicalDiskHealthIssue(row) + } + for id, want := range map[string]bool{"nvme-63": false, "nvme-66": false, "nvme-72": true, "sata-52": false, "sata-56": true} { + if issues[id] != want { + t.Errorf("%s: health issue = %v, want %v", id, issues[id], want) + } + } + for _, row := range patrolPhysicalDiskRows(diskTemperatureTriageState(raisedNVMeThresholds()), nil) { + if row.id == "nvme-72" && patrolPhysicalDiskHealthIssue(row) { + t.Error("nvme-72 under a raised 75C trigger should not be an issue") + } + } + + state := diskTemperatureTriageState(nil) + for id, want := range map[string]bool{"nvme-63": true, "nvme-66": false, "nvme-72": false, "sata-52": false} { + recovered, err := verifyMetricRecoveredState(state, PatrolThresholds{}, "disk-high", id, "physical_disk") + if err != nil { + t.Fatalf("%s: verification error %v", id, err) + } + if recovered != want { + t.Errorf("%s: recovered = %v, want %v", id, recovered, want) + } + } +} + +func TestResourceContextListsDisksHotByTheAlertPolicy(t *testing.T) { + registry := unifiedresources.NewRegistry(nil) + now := time.Now() + registry.IngestSnapshot(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{ + {ID: "pve-nvme", Node: "pve", Instance: "lab", DevPath: "/dev/nvme0n1", Model: "Samsung 980", Serial: "N63", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 63, LastChecked: now}, + {ID: "pve-nvme-72", Node: "pve", Instance: "lab", DevPath: "/dev/nvme1n1", Model: "Samsung 990", Serial: "N72", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 72, LastChecked: now}, + {ID: "pve-sata", Node: "pve", Instance: "lab", DevPath: "/dev/sda", Model: "WDC WD80EFAX", Serial: "S56", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 56, LastChecked: now}, + }}) + + s := &Service{} + s.SetUnifiedResourceProvider(unifiedresources.NewUnifiedAIAdapter(registry)) + attention := func() string { + got := s.buildUnifiedResourceContext() + _, section, found := strings.Cut(got, "Physical Disks Needing Attention") + if !found { + return "" + } + section, _, _ = strings.Cut(section, "\n\n") + return section + } + + // Disk names are redacted in this context, so the readings identify them. + section := attention() + if !strings.Contains(section, "Temp: 56C") || !strings.Contains(section, "Temp: 72C") || strings.Contains(section, "Temp: 63C") { + t.Fatalf("factory policy: attention section = %q, want the SATA 56C and NVMe 72C disks only", section) + } + + s.SetPatrolThresholdProvider(raisedNVMeThresholds()) + section = attention() + if !strings.Contains(section, "Temp: 56C") || strings.Contains(section, "Temp: 72C") || strings.Contains(section, "Temp: 63C") { + t.Fatalf("raised NVMe trigger: attention section = %q, want only the SATA 56C disk", section) + } +} + +// A Patrol run judges disk heat by the thresholds the user configured, through +// both ways a run builds its state. +func TestPatrolRunStateCarriesTheConfiguredDiskTemperaturePolicy(t *testing.T) { + snapshot := models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{ + {ID: "nvme-72", DevPath: "/dev/nvme0n1", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 72}, + }} + ps := NewPatrolService(nil, &mockStateProvider{state: snapshot}) + if got := ps.currentPatrolRuntimeState().diskTemperatureLimits("nvme"); got.trigger != 70 { + t.Fatalf("no provider: nvme trigger = %v, want the factory 70", got.trigger) + } + + ps.SetThresholdProvider(raisedNVMeThresholds()) + for name, state := range map[string]patrolRuntimeState{ + "current": ps.currentPatrolRuntimeState(), + "snapshot": ps.patrolRuntimeStateForSnapshot(snapshot), + } { + if got := state.diskTemperatureLimits("nvme"); got.trigger != 75 || got.clear != 70 { + t.Errorf("%s state: nvme limits = %+v, want the configured 75/70", name, got) + } + if flags := triageDiskHealthChecksState(state, nil); len(flags) != 0 { + t.Errorf("%s state: nvme at 72C under a 75C trigger flagged %+v", name, flags) + } + } +} + +// A scoped Patrol run filters its state; the filter must keep the configured +// thresholds, or a scoped run would judge heat by the factory triggers. +func TestScopedPatrolStateKeepsTheConfiguredDiskTemperaturePolicy(t *testing.T) { + state := diskTemperatureTriageState(raisedNVMeThresholds()) + scoped := filterPatrolStateByScopeState(state, PatrolScope{ResourceIDs: []string{"nvme-72"}}) + if got := scoped.diskTemperatureLimits("nvme"); got.trigger != 75 || got.clear != 70 { + t.Fatalf("scoped state nvme limits = %+v, want the configured 75/70", got) + } + if flags := triageDiskHealthChecksState(scoped, nil); len(flags) != 0 { + t.Fatalf("scoped run flagged nvme at 72C under a 75C trigger: %+v", flags) + } +} + +// The alert threshold provider can arrive before Patrol exists; Patrol must +// still start with it. +func TestPatrolStartsWithAThresholdProviderSetBeforeIt(t *testing.T) { + s := NewService(nil, nil) + s.SetPatrolThresholdProvider(raisedNVMeThresholds()) + s.SetStateProvider(&mockStateProvider{}) + patrol := s.GetPatrolService() + if patrol == nil { + t.Fatal("expected Patrol to start once a state provider exists") + } + if got := patrol.currentPatrolRuntimeState().diskTemperatureLimits("nvme"); got.trigger != 75 { + t.Fatalf("Patrol nvme trigger = %v, want the provider's 75", got.trigger) + } +} + +// A heat finding recovers at the clear value, where its alert recovers. +func TestDiskHeatRecoversAtTheClearValue(t *testing.T) { + limits := diskTemperatureLimitsFor(nil, "nvme") + if !limits.cooled(65) || limits.cooled(66) { + t.Fatalf("nvme limits %+v: want 65C cooled and 66C still warm", limits) + } + // With no band below the trigger, a reading at the trigger is still hot. + noBand := diskTemperatureLimitsFor(&mockThresholdProvider{diskTemperature: map[string][2]float64{"sas": {70, 70}}}, "sas") + if !noBand.hot(70) || noBand.cooled(70) || !noBand.cooled(69) { + t.Fatalf("no-band limits %+v: want 70C hot and not cooled, 69C cooled", noBand) + } +} diff --git a/internal/ai/patrol_findings.go b/internal/ai/patrol_findings.go index d72a73f0e8..1bb5e87af0 100644 --- a/internal/ai/patrol_findings.go +++ b/internal/ai/patrol_findings.go @@ -2291,7 +2291,9 @@ func verifyMetricRecoveredState(snap patrolRuntimeState, thresholds PatrolThresh if disk.wearout >= 0 && disk.wearout < 20 { return false, nil } - if disk.temperature > 55 { + // A hot disk recovers once it cools to the alert clear + // value, as its disk temperature alert does. + if !disk.temperatureLimits.cooled(disk.temperature) { return false, nil } return true, nil @@ -2315,9 +2317,10 @@ func verifyMetricRecoveredState(snap patrolRuntimeState, thresholds PatrolThresh } type patrolPhysicalDiskVerification struct { - health string - wearout int - temperature int + health string + wearout int + temperature int + temperatureLimits diskTemperatureLimits } type patrolPhysicalDiskVisitor func(identifiers []string, verification patrolPhysicalDiskVerification) bool @@ -2461,9 +2464,10 @@ func patrolVisitPhysicalDiskVerification(snap patrolRuntimeState, visit patrolPh rows := patrolPhysicalDiskRows(snap, nil) for _, disk := range rows { if !visit([]string{disk.id, disk.name, disk.devPath, disk.model}, patrolPhysicalDiskVerification{ - health: strings.TrimSpace(disk.health), - wearout: disk.wearout, - temperature: disk.temperature.Collected, + health: strings.TrimSpace(disk.health), + wearout: disk.wearout, + temperature: disk.temperature.Collected, + temperatureLimits: disk.temperatureLimits, }) { return true } diff --git a/internal/ai/patrol_run.go b/internal/ai/patrol_run.go index 67c08b5e60..db7c3decc2 100644 --- a/internal/ai/patrol_run.go +++ b/internal/ai/patrol_run.go @@ -1464,6 +1464,7 @@ func newPatrolScopedFilterState(snap patrolRuntimeState) patrolScopedFilterState filtered: patrolRuntimeState{ readState: snap.readState, unifiedResourceProvider: snap.unifiedResourceProvider, + thresholdProvider: snap.thresholdProvider, }, includedResourceIDs: make(map[string]bool), includedGuestVMIDs: make(map[int]bool), diff --git a/internal/ai/patrol_state.go b/internal/ai/patrol_state.go index 822728ba0a..1d0d7e8493 100644 --- a/internal/ai/patrol_state.go +++ b/internal/ai/patrol_state.go @@ -89,6 +89,15 @@ type patrolRuntimeState struct { ConnectionHealth map[string]bool ActiveAlerts []models.Alert RecentlyResolved []models.ResolvedAlert + // thresholdProvider carries the user's alert thresholds into the run. Nil + // means the factory alert configuration. + thresholdProvider ThresholdProvider +} + +// diskTemperatureLimits resolves the alert disk temperature policy for one +// disk type. +func (s patrolRuntimeState) diskTemperatureLimits(diskType string) diskTemperatureLimits { + return diskTemperatureLimitsFor(s.thresholdProvider, diskType) } func newPatrolRuntimeState(snapshot models.StateSnapshot) patrolRuntimeState { @@ -171,11 +180,16 @@ func (p *PatrolService) currentPatrolRuntimeState() patrolRuntimeState { stateProvider := p.stateProvider readState := p.readState unifiedResourceProvider := p.unifiedResourceProvider + thresholdProvider := p.thresholdProvider p.mu.RUnlock() + var state patrolRuntimeState if stateProvider == nil { - return emptyPatrolRuntimeState(readState, unifiedResourceProvider) + state = emptyPatrolRuntimeState(readState, unifiedResourceProvider) + } else { + state = newPatrolRuntimeStateWithProviders(stateProvider.ReadSnapshot(), readState, unifiedResourceProvider) } - return newPatrolRuntimeStateWithProviders(stateProvider.ReadSnapshot(), readState, unifiedResourceProvider) + state.thresholdProvider = thresholdProvider + return state } func (p *PatrolService) hasPatrolRuntimeInputs() bool { @@ -194,8 +208,11 @@ func (p *PatrolService) patrolRuntimeStateForSnapshot(snapshot models.StateSnaps p.mu.RLock() readState := p.readState unifiedResourceProvider := p.unifiedResourceProvider + thresholdProvider := p.thresholdProvider p.mu.RUnlock() - return newPatrolRuntimeStateWithProviders(snapshot, readState, unifiedResourceProvider) + state := newPatrolRuntimeStateWithProviders(snapshot, readState, unifiedResourceProvider) + state.thresholdProvider = thresholdProvider + return state } func patrolVisitRuntimeResources(s patrolRuntimeState, visit func(patrolRuntimeResourceRecord) bool) { diff --git a/internal/ai/patrol_thresholds_test.go b/internal/ai/patrol_thresholds_test.go index fc92810a6f..53ad2439db 100644 --- a/internal/ai/patrol_thresholds_test.go +++ b/internal/ai/patrol_thresholds_test.go @@ -20,6 +20,9 @@ func (m MockThresholdProvider) GetNodeMemoryThreshold() float64 { return m.Node func (m MockThresholdProvider) GetGuestMemoryThreshold() float64 { return m.GuestMem } func (m MockThresholdProvider) GetGuestDiskThreshold() float64 { return m.GuestDisk } func (m MockThresholdProvider) GetStorageThreshold() float64 { return m.Storage } +func (m MockThresholdProvider) GetDiskTemperatureThreshold(string) (float64, float64) { + return 0, 0 +} func TestCalculatePatrolThresholds_Default(t *testing.T) { // Test default behavior (exact mode) diff --git a/internal/ai/patrol_triage.go b/internal/ai/patrol_triage.go index f3d3d635e2..04822aa4f6 100644 --- a/internal/ai/patrol_triage.go +++ b/internal/ai/patrol_triage.go @@ -345,13 +345,16 @@ func triageDiskHealthChecksState(snap patrolRuntimeState, scopedSet map[string]b flags := make([]TriageFlag, 0) for _, disk := range patrolPhysicalDiskRows(snap, scopedSet) { - flags = append(flags, triagePhysicalDiskFlags(disk.id, disk.name, disk.health, disk.wearout, disk.temperature.Collected)...) + flags = append(flags, triagePhysicalDiskFlags(disk.id, disk.name, disk.health, disk.wearout, disk.temperature.Collected, disk.temperatureLimits)...) } return flags } -func triagePhysicalDiskFlags(resourceID, resourceName, health string, wearout, temperature int) []TriageFlag { +// triagePhysicalDiskFlags judges heat by the alert disk temperature policy: a +// warning from the alert trigger, the reading the disk's temperature alert and +// its Running Hot verdict start at. +func triagePhysicalDiskFlags(resourceID, resourceName, health string, wearout, temperature int, temperatureLimits diskTemperatureLimits) []TriageFlag { flags := make([]TriageFlag, 0, 3) if health != "" && !strings.EqualFold(health, "PASSED") && !strings.EqualFold(health, "UNKNOWN") && !strings.EqualFold(health, "OK") { flags = append(flags, TriageFlag{ @@ -377,17 +380,17 @@ func triagePhysicalDiskFlags(resourceID, resourceName, health string, wearout, t Threshold: 20, }) } - if temperature > 55 { + if temperatureLimits.hot(temperature) { flags = append(flags, TriageFlag{ ResourceID: resourceID, ResourceName: resourceName, ResourceType: "physical_disk", Category: "health", Severity: "warning", - Reason: fmt.Sprintf("Disk temperature %d°C", temperature), + Reason: fmt.Sprintf("Disk temperature %d°C (alert threshold: %.0f°C)", temperature, temperatureLimits.trigger), Metric: "disk", Value: float64(temperature), - Threshold: 55, + Threshold: temperatureLimits.trigger, }) } return flags diff --git a/internal/ai/resource_context.go b/internal/ai/resource_context.go index 1702ac0a05..31218ace81 100644 --- a/internal/ai/resource_context.go +++ b/internal/ai/resource_context.go @@ -52,6 +52,7 @@ func (s *Service) buildUnifiedResourceContextForModel(destinationModel string) s urp := s.unifiedResourceProvider ap := s.alertProvider agentServer := s.agentServer + thresholdProvider := s.thresholdProvider s.mu.RUnlock() if urp != nil { @@ -410,13 +411,14 @@ func (s *Service) buildUnifiedResourceContextForModel(destinationModel string) s attention := make([]unifiedresources.Resource, 0) for _, disk := range physicalDisks { health := "" - temperature := 0 + hot := false if disk.PhysicalDisk != nil { health = strings.ToUpper(strings.TrimSpace(disk.PhysicalDisk.Health)) // A retained last-known reading is not current heat. - temperature = tools.SplitDiskTemperature(disk.PhysicalDisk.Temperature, disk.PhysicalDisk.Collection).Collected + collected := tools.SplitDiskTemperature(disk.PhysicalDisk.Temperature, disk.PhysicalDisk.Collection).Collected + hot = diskTemperatureLimitsFor(thresholdProvider, disk.PhysicalDisk.DiskType).hot(collected) } - if disk.Status != unifiedresources.StatusOnline || (health != "" && health != "PASSED" && health != "UNKNOWN") || temperature >= 50 { + if disk.Status != unifiedresources.StatusOnline || (health != "" && health != "PASSED" && health != "UNKNOWN") || hot { attention = append(attention, disk) } } diff --git a/internal/ai/service.go b/internal/ai/service.go index ddd974b733..9b19fd6cb4 100644 --- a/internal/ai/service.go +++ b/internal/ai/service.go @@ -261,6 +261,7 @@ type Service struct { patrolAutopilotPolicy func() unifiedresources.PatrolAutopilotServerPolicy patrolObjectiveStore *PatrolObjectiveStore patrolService *PatrolService // Background AI monitoring service + thresholdProvider ThresholdProvider // User-configured alert thresholds; nil means factory defaults metadataProvider MetadataProvider // Enables AI to update resource URLs incidentStore *memory.IncidentStore // Alert-scoped investigation memory; not the canonical durable resource history chatService ChatServiceProvider // Chat service for investigation orchestrator @@ -584,6 +585,9 @@ func (s *Service) initPatrolServiceLocked() { s.patrolService = NewPatrolService(s, s.stateProvider) s.patrolService.SetObjectiveStore(s.patrolObjectiveStore) + if s.thresholdProvider != nil { + s.patrolService.SetThresholdProvider(s.thresholdProvider) + } if s.knowledgeStore != nil { s.patrolService.SetKnowledgeStore(s.knowledgeStore) } @@ -831,9 +835,10 @@ func (s *Service) ClearCostHistory() error { // SetPatrolThresholdProvider sets the threshold provider for patrol // This should be called with an AlertThresholdAdapter to connect patrol to user-configured thresholds func (s *Service) SetPatrolThresholdProvider(provider ThresholdProvider) { - s.mu.RLock() + s.mu.Lock() + s.thresholdProvider = provider patrol := s.patrolService - s.mu.RUnlock() + s.mu.Unlock() if patrol != nil { patrol.SetThresholdProvider(provider) diff --git a/internal/alerts/host.go b/internal/alerts/host.go index 157cc4c5bb..9347925507 100644 --- a/internal/alerts/host.go +++ b/internal/alerts/host.go @@ -990,21 +990,50 @@ func hostDiskTemperatureResourceID(hostID, device string) string { } // hostDiskTemperatureThresholdNoLock returns the threshold for one SMART disk -// given the host's resolved disk temperature threshold. An enabled threshold -// gives way to the disk type's DiskTempByType entry unless an explicit host or -// linked-resource override set it. Callers must hold m.mu. +// given the host's resolved disk temperature threshold. Callers must hold m.mu. func (m *Manager) hostDiskTemperatureThresholdNoLock(hostThreshold *HysteresisThreshold, overridden bool, diskType string) *HysteresisThreshold { + return diskTemperatureThresholdForType(m.config.DiskTempByType, hostThreshold, overridden, diskType) +} + +// diskTemperatureThresholdForType is the disk temperature policy. An enabled +// host threshold gives way to the disk type's DiskTempByType entry unless an +// explicit host or linked-resource override set it; a disabled one switches +// disk temperature alerting off for every type. +func diskTemperatureThresholdForType(byType map[string]HysteresisThreshold, hostThreshold *HysteresisThreshold, overridden bool, diskType string) *HysteresisThreshold { if hostThreshold == nil || hostThreshold.Trigger <= 0 || overridden { return hostThreshold } if diskType = strings.ToLower(strings.TrimSpace(diskType)); diskType != "" { - if th, ok := m.config.DiskTempByType[diskType]; ok { + if th, ok := byType[diskType]; ok { return &th } } return hostThreshold } +// DiskTemperatureThreshold returns the threshold that judges a physical disk's +// temperature when no host override applies: the disk type's DiskTempByType +// entry, else the agent default. This is the canonical disk heat policy. Disk +// temperature alerts, Patrol and the Physical Disks Health verdict all judge +// heat against it; disk risk does not. The disk is hot at the trigger, and the +// clear value is where it stops being hot. A nil threshold or a non-positive +// trigger means disk temperature alerting is off. +func (m *Manager) DiskTemperatureThreshold(diskType string) *HysteresisThreshold { + if m == nil { + return DefaultDiskTemperatureThreshold(diskType) + } + m.mu.RLock() + defer m.mu.RUnlock() + return cloneThreshold(m.hostDiskTemperatureThresholdNoLock(m.config.AgentDefaults.DiskTemperature, false, diskType)) +} + +// DefaultDiskTemperatureThreshold is DiskTemperatureThreshold under the +// factory alert configuration, for callers with no alert manager. +func DefaultDiskTemperatureThreshold(diskType string) *HysteresisThreshold { + config := defaultAlertConfig() + return cloneThreshold(diskTemperatureThresholdForType(config.DiskTempByType, config.AgentDefaults.DiskTemperature, false, diskType)) +} + // lowestHostDiskTemperatureThresholdNoLock returns the lowest enabled // threshold any SMART disk of the host can be evaluated against. An alert // whose disk type is unknown is judged against it, so a config save never @@ -1547,8 +1576,6 @@ func splitSMARTAlertReasons(reasons []storagehealth.Reason) ([]storagehealth.Rea switch reason.Code { case "wearout_low", "nvme_available_spare_low", "nvme_percentage_used_high": wearReasons = append(wearReasons, reason) - case "temperature_high": - continue default: healthReasons = append(healthReasons, reason) } diff --git a/internal/alerts/threshold_resolution_shared_test.go b/internal/alerts/threshold_resolution_shared_test.go index bdffd75d22..673a1bab50 100644 --- a/internal/alerts/threshold_resolution_shared_test.go +++ b/internal/alerts/threshold_resolution_shared_test.go @@ -1387,3 +1387,61 @@ func TestCheckMetricResolveRemovesCanonicallyKeyedAlert(t *testing.T) { t.Fatal("expected a recently-resolved entry for the cleared alert") } } + +// DiskTemperatureThreshold is the policy Patrol and the Physical Disks Health +// verdict judge disk heat by, so it must resolve every disk type exactly as +// CheckHost does: the per-type trigger the user set, else the agent default, +// and nothing at all once the agent default is switched off. +func TestDiskTemperatureThresholdMatchesCheckHostPolicy(t *testing.T) { + m := configureDiskTempTypeHostManager(t) + + for diskType, want := range map[string]HysteresisThreshold{ + "nvme": {Trigger: 70, Clear: 65}, + " NVMe": {Trigger: 70, Clear: 65}, + "sas": {Trigger: 65, Clear: 60}, + "sata": {Trigger: 55, Clear: 50}, + "hdd": {Trigger: 55, Clear: 50}, + "": {Trigger: 55, Clear: 50}, + } { + if got := m.DiskTemperatureThreshold(diskType); got == nil || *got != want { + t.Errorf("DiskTemperatureThreshold(%q) = %+v, want %+v", diskType, got, want) + } + if got := DefaultDiskTemperatureThreshold(diskType); got == nil || *got != want { + t.Errorf("DefaultDiskTemperatureThreshold(%q) = %+v, want the factory %+v", diskType, got, want) + } + } + + // A user who raised the NVMe trigger to 75 gets no alert at 72C, and the + // policy says the same disk is not hot. + m.mu.Lock() + m.config.DiskTempByType["nvme"] = HysteresisThreshold{Trigger: 75, Clear: 70} + m.mu.Unlock() + if got := m.DiskTemperatureThreshold("nvme"); got == nil || got.Trigger != 75 || got.Clear != 70 { + t.Fatalf("raised nvme threshold = %+v, want 75/70", got) + } + host := hostWithSMARTDiskTemp("host-temp-raised-nvme", "nvme", 72) + m.CheckHost(host) + if _, exists := testLookupActiveAlert(t, m, hostDiskTempAlertID(host)); exists { + t.Fatalf("expected no alert for nvme at 72C under a raised 75C trigger, active: %v", alertKeys(m)) + } + + // The returned threshold is a copy. + m.DiskTemperatureThreshold("nvme").Trigger = 1 + if got := m.DiskTemperatureThreshold("nvme"); got.Trigger != 75 { + t.Fatalf("caller mutation reached the config: %+v", got) + } + + m.mu.Lock() + m.config.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0, Clear: 0} + m.mu.Unlock() + for _, diskType := range []string{"nvme", "sata", "hdd"} { + if got := m.DiskTemperatureThreshold(diskType); got == nil || got.Trigger > 0 { + t.Errorf("DiskTemperatureThreshold(%q) with disk temperature alerting off = %+v, want a disabled threshold", diskType, got) + } + } + + var nilManager *Manager + if got := nilManager.DiskTemperatureThreshold("nvme"); got == nil || got.Trigger != 70 { + t.Fatalf("nil manager threshold = %+v, want the factory nvme 70", got) + } +} diff --git a/internal/api/ai_handlers_concurrency_test.go b/internal/api/ai_handlers_concurrency_test.go index 7659b34fa0..ca04c61635 100644 --- a/internal/api/ai_handlers_concurrency_test.go +++ b/internal/api/ai_handlers_concurrency_test.go @@ -67,6 +67,9 @@ func (concurrencyThresholdProvider) GetGuestMemoryThreshold() float64 { } func (concurrencyThresholdProvider) GetGuestDiskThreshold() float64 { return 95 } func (concurrencyThresholdProvider) GetStorageThreshold() float64 { return 92 } +func (concurrencyThresholdProvider) GetDiskTemperatureThreshold(string) (float64, float64) { + return 70, 65 +} type concurrencyMetricsHistoryProvider struct{} diff --git a/internal/api/ai_handlers_more_test.go b/internal/api/ai_handlers_more_test.go index f1d8973d81..128483e0c1 100644 --- a/internal/api/ai_handlers_more_test.go +++ b/internal/api/ai_handlers_more_test.go @@ -154,6 +154,9 @@ func (stubThresholdProvider) GetGuestMemoryThreshold() float64 { } func (stubThresholdProvider) GetGuestDiskThreshold() float64 { return 95 } func (stubThresholdProvider) GetStorageThreshold() float64 { return 92 } +func (stubThresholdProvider) GetDiskTemperatureThreshold(string) (float64, float64) { + return 70, 65 +} type stubMetricsHistoryProvider struct{} diff --git a/internal/storagehealth/risk.go b/internal/storagehealth/risk.go index de165bbb5b..990bb6143e 100644 --- a/internal/storagehealth/risk.go +++ b/internal/storagehealth/risk.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/rcourtman/pulse-go-rewrite/internal/models" - "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) type RiskLevel string @@ -31,7 +30,12 @@ type Assessment struct { // SMARTThresholds controls the discrete health evidence that becomes a disk // risk. Counter values at zero disable that rule; the endurance percentages -// use the same convention. Temperature remains a separate metric threshold. +// use the same convention. +// +// Disk risk never judges temperature. Heat is a metric judged by the alert +// disk temperature policy (alerts.Manager.DiskTemperatureThreshold), which +// users tune per disk type and per host. Risk is rebuilt by registries that +// cannot see that configuration, so a heat rule here would disagree with it. type SMARTThresholds struct { HealthFailure bool ReallocatedSectors int64 @@ -61,7 +65,6 @@ func DefaultSMARTThresholds() SMARTThresholds { type Sample struct { Model string Health string - Temperature int Wearout int WearoutKnown bool PowerOnHours int64 @@ -77,29 +80,10 @@ type Sample struct { UnsafeShutdowns int64 } -// CollectedTemperature returns a disk temperature only when its collection -// state marks it as observed now. Normalization may keep the last known value -// when the current observation is not available, such as a disk in standby or -// a host agent past its reporting lease. That retained value is history, not -// evidence that the disk is hot now, so risk must not judge it. A source that -// predates collection state counts as collected. -func CollectedTemperature(temperature int, collection *diskinventory.CollectionStatus) int { - if collection == nil { - return temperature - } - switch collection.Temperature.State { - case "", diskinventory.FieldAvailable: - return temperature - default: - return 0 - } -} - func AssessPhysicalDisk(disk models.PhysicalDisk) Assessment { sample := Sample{ Model: disk.Model, Health: disk.Health, - Temperature: CollectedTemperature(disk.Temperature, disk.Collection), Wearout: disk.Wearout, WearoutKnown: WearoutReported(disk.Wearout, disk.Type), } @@ -113,10 +97,9 @@ func AssessHostSMARTDisk(disk models.HostDiskSMART) Assessment { func AssessHostSMARTDiskWithThresholds(disk models.HostDiskSMART, thresholds SMARTThresholds) Assessment { sample := Sample{ - Model: disk.Model, - Health: disk.Health, - Temperature: CollectedTemperature(disk.Temperature, disk.Collection), - Wearout: -1, + Model: disk.Model, + Health: disk.Health, + Wearout: -1, } applySMARTAttributes(&sample, disk.Attributes) return AssessSampleWithThresholds(sample, thresholds) @@ -216,11 +199,6 @@ func AssessSampleWithThresholds(sample Sample, thresholds SMARTThresholds) Asses } else if percentageWarning { addReason("nvme_percentage_used_high", RiskWarning, fmt.Sprintf("NVMe endurance used is %d%%", sample.PercentageUsed)) } - if sample.Temperature >= 70 { - addReason("temperature_high", RiskCritical, fmt.Sprintf("Disk temperature is %dC", sample.Temperature)) - } else if sample.Temperature >= 60 { - addReason("temperature_high", RiskWarning, fmt.Sprintf("Disk temperature is %dC", sample.Temperature)) - } if thresholds.ReallocatedSectors > 0 && sample.ReallocatedSectors >= thresholds.ReallocatedSectors { addReason("reallocated_sectors", RiskWarning, fmt.Sprintf("Reallocated sectors detected (%d)", sample.ReallocatedSectors)) } diff --git a/internal/storagehealth/risk_branchcov0718_test.go b/internal/storagehealth/risk_branchcov0718_test.go index 52b5eb4165..65200993ae 100644 --- a/internal/storagehealth/risk_branchcov0718_test.go +++ b/internal/storagehealth/risk_branchcov0718_test.go @@ -33,7 +33,7 @@ func TestAssessPhysicalDisk_Branches(t *testing.T) { }{ { // nil SmartAttributes arm: every *int64 pointer is skipped; only - // Model/Health/Temperature/Wearout are sourced from the disk. + // Model/Health/Wearout are sourced from the disk. name: "healthy disk with nil smart attributes stays healthy", disk: models.PhysicalDisk{Model: "Crucial MX500", Health: "PASSED", Temperature: 35, Wearout: 80}, wantLevel: RiskHealthy, @@ -165,18 +165,10 @@ func TestAssessPhysicalDisk_Branches(t *testing.T) { wantReasons: []string{"nvme_percentage_used_high"}, }, { - // Temperature >= 70 -> critical. - name: "temperature critical", - disk: models.PhysicalDisk{Health: "PASSED", Temperature: 72}, - wantLevel: RiskCritical, - wantReasons: []string{"temperature_high"}, - }, - { - // Temperature 60..69 -> warning. - name: "temperature warning", - disk: models.PhysicalDisk{Health: "PASSED", Temperature: 63}, - wantLevel: RiskWarning, - wantReasons: []string{"temperature_high"}, + // Heat is the alert disk temperature policy's to judge, never risk's. + name: "hot disk stays healthy", + disk: models.PhysicalDisk{Health: "PASSED", Temperature: 72}, + wantLevel: RiskHealthy, }, { // Reallocated sectors > 0 via SMART -> warning. @@ -247,20 +239,20 @@ func TestAssessPhysicalDisk_NilSmartAttributesDoesNotPanic(t *testing.T) { // is preserved through the SMART-attribute mapping. func TestAssessPhysicalDisk_MultipleReasonsTakeHighest(t *testing.T) { got := AssessPhysicalDisk(models.PhysicalDisk{ - Health: "PASSED", - Temperature: 63, // warning + Health: "PASSED", SmartAttributes: &models.SMARTAttributes{ - PendingSectors: int64Ptr(1), // critical - UDMACRCErrors: int64Ptr(2), // monitor + ReallocatedSectors: int64Ptr(3), // warning + PendingSectors: int64Ptr(1), // critical + UDMACRCErrors: int64Ptr(2), // monitor }, }) if got.Level != RiskCritical { t.Fatalf("expected critical (highest), got %q; reasons=%+v", got.Level, got.Reasons) } if len(got.Reasons) != 3 { - t.Fatalf("expected 3 reasons (temperature_high, pending_sectors, crc_errors), got %d: %+v", len(got.Reasons), got.Reasons) + t.Fatalf("expected 3 reasons (reallocated_sectors, pending_sectors, crc_errors), got %d: %+v", len(got.Reasons), got.Reasons) } - for _, code := range []string{"pending_sectors", "temperature_high", "crc_errors"} { + for _, code := range []string{"pending_sectors", "reallocated_sectors", "crc_errors"} { if _, ok := reasonSeverity(got, code); !ok { t.Errorf("expected reason %q present, got %+v", code, got.Reasons) } @@ -282,7 +274,7 @@ func TestAssessHostSMARTDisk_Branches(t *testing.T) { wantReasons []string }{ { - // nil Attributes arm: only Model/Health/Temperature are sourced; + // nil Attributes arm: only Model/Health are sourced; // Wearout stays -1 so the wearout branch is skipped. name: "nil attributes stays healthy", disk: models.HostDiskSMART{Model: "WD Blue", Health: "PASSED", Temperature: 35}, @@ -402,18 +394,10 @@ func TestAssessHostSMARTDisk_Branches(t *testing.T) { wantReasons: []string{"nvme_available_spare_low"}, }, { - // Temperature >= 70 -> critical. - name: "temperature critical", - disk: models.HostDiskSMART{Health: "PASSED", Temperature: 75}, - wantLevel: RiskCritical, - wantReasons: []string{"temperature_high"}, - }, - { - // Temperature 60..69 -> warning. - name: "temperature warning", - disk: models.HostDiskSMART{Health: "PASSED", Temperature: 60}, - wantLevel: RiskWarning, - wantReasons: []string{"temperature_high"}, + // Heat is the alert disk temperature policy's to judge, never risk's. + name: "hot disk stays healthy", + disk: models.HostDiskSMART{Health: "PASSED", Temperature: 75}, + wantLevel: RiskHealthy, }, { // Reallocated sectors > 0 via host SMART -> warning. diff --git a/internal/storagehealth/risk_test.go b/internal/storagehealth/risk_test.go index e044189efd..508bfab988 100644 --- a/internal/storagehealth/risk_test.go +++ b/internal/storagehealth/risk_test.go @@ -11,9 +11,8 @@ import ( func TestAssessSample_HealthyDisk(t *testing.T) { assessment := AssessSample(Sample{ - Health: "PASSED", - Temperature: 35, - Wearout: 80, + Health: "PASSED", + Wearout: 80, }) if assessment.Level != RiskHealthy { t.Errorf("expected healthy, got %s", assessment.Level) @@ -308,38 +307,6 @@ func TestAssessSample_NVMePercentageUsedWarning(t *testing.T) { } } -func TestAssessSample_TemperatureCritical(t *testing.T) { - assessment := AssessSample(Sample{ - Health: "PASSED", - Temperature: 72, // >=70 critical - }) - if assessment.Level != RiskCritical { - t.Errorf("expected critical for temp=72, got %s", assessment.Level) - } -} - -func TestAssessSample_TemperatureWarning(t *testing.T) { - assessment := AssessSample(Sample{ - Health: "PASSED", - Temperature: 63, // >=60, <70 warning - }) - if assessment.Level != RiskWarning { - t.Errorf("expected warning for temp=63, got %s", assessment.Level) - } -} - -func TestAssessSample_TemperatureHealthy(t *testing.T) { - assessment := AssessSample(Sample{ - Health: "PASSED", - Temperature: 45, - }) - for _, r := range assessment.Reasons { - if r.Code == "temperature_high" { - t.Error("temp=45 should not trigger temperature_high") - } - } -} - func TestAssessSample_ReallocatedSectors(t *testing.T) { assessment := AssessSample(Sample{ Health: "PASSED", @@ -371,10 +338,10 @@ func TestAssessSample_CRCErrors(t *testing.T) { func TestAssessSample_MultipleIssuesTakesHighest(t *testing.T) { assessment := AssessSample(Sample{ - Health: "PASSED", - Temperature: 63, // warning - PendingSectors: 1, // critical - UDMACRCErrors: 10, // monitor + Health: "PASSED", + ReallocatedSectors: 1, // warning + PendingSectors: 1, // critical + UDMACRCErrors: 10, // monitor }) if assessment.Level != RiskCritical { t.Errorf("expected critical (highest), got %s", assessment.Level) @@ -387,10 +354,11 @@ func TestAssessSample_MultipleIssuesTakesHighest(t *testing.T) { func TestAssessSample_ReasonsSortedBySeverityDescending(t *testing.T) { assessment := AssessSample(Sample{ Health: "PASSED", - Temperature: 63, // warning PendingSectors: 1, // critical UDMACRCErrors: 10, // monitor ReallocatedSectors: 1, // warning + WearoutKnown: true, + Wearout: 8, // warning }) for i := 1; i < len(assessment.Reasons); i++ { prevRank := severityRank(assessment.Reasons[i-1].Severity) @@ -540,58 +508,20 @@ func TestAssessPhysicalDisk_RotationalZeroIsNotRisk(t *testing.T) { } } -func TestCollectedTemperatureKeepsOnlyCurrentReadings(t *testing.T) { - cases := []struct { - name string - collection *diskinventory.CollectionStatus - want int - }{ - {name: "no collection state", collection: nil, want: 72}, - {name: "empty state", collection: &diskinventory.CollectionStatus{}, want: 72}, - {name: "available", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}, want: 72}, - {name: "unavailable", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}, want: 0}, - {name: "unsupported", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "")}, want: 0}, - {name: "missing", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Missing("smartctl", "")}, want: 0}, - } - for _, tc := range cases { - if got := CollectedTemperature(72, tc.collection); got != tc.want { - t.Errorf("%s: CollectedTemperature = %d, want %d", tc.name, got, tc.want) - } - } -} - -func TestDiskAssessmentsIgnoreRetainedTemperature(t *testing.T) { - retained := &diskinventory.CollectionStatus{ - Temperature: diskinventory.Unavailable("host_agent", "host agent stopped reporting"), - } +// Heat belongs to the alert disk temperature policy, which users tune per disk +// type and per host. Disk risk cannot see that configuration, so it must not +// judge temperature at all, however hot and however current the reading. +func TestDiskAssessmentsNeverJudgeTemperature(t *testing.T) { collected := &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")} - - hasTemperatureReason := func(assessment Assessment) bool { - for _, reason := range assessment.Reasons { - if reason.Code == "temperature_high" { - return true - } + for _, temperature := range []int{56, 63, 72, 95} { + disk := models.PhysicalDisk{Model: "WDC WD80EFAX", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: temperature, Collection: collected} + if got := AssessPhysicalDisk(disk); got.Level != RiskHealthy || len(got.Reasons) != 0 { + t.Errorf("%dC physical disk = %+v, want healthy with no reasons", temperature, got) } - return false - } - disk := models.PhysicalDisk{Model: "WDC WD80EFAX", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 72} - disk.Collection = collected - if got := AssessPhysicalDisk(disk); got.Level != RiskCritical || !hasTemperatureReason(got) { - t.Fatalf("collected 72C physical disk = %+v, want critical temperature_high", got) - } - disk.Collection = retained - if got := AssessPhysicalDisk(disk); got.Level != RiskHealthy || hasTemperatureReason(got) { - t.Fatalf("retained 72C physical disk = %+v, want healthy", got) - } - - smart := models.HostDiskSMART{Device: "/dev/sda", Model: "WDC WD80EFAX", Type: "sata", Health: "PASSED", Temperature: 72} - smart.Collection = collected - if got := AssessHostSMARTDisk(smart); got.Level != RiskCritical || !hasTemperatureReason(got) { - t.Fatalf("collected 72C host SMART disk = %+v, want critical temperature_high", got) - } - smart.Collection = retained - if got := AssessHostSMARTDisk(smart); got.Level != RiskHealthy || hasTemperatureReason(got) { - t.Fatalf("retained 72C host SMART disk = %+v, want healthy", got) + smart := models.HostDiskSMART{Device: "/dev/nvme0n1", Model: "Samsung 990 PRO", Type: "nvme", Health: "PASSED", Temperature: temperature, Collection: collected} + if got := AssessHostSMARTDisk(smart); got.Level != RiskHealthy || len(got.Reasons) != 0 { + t.Errorf("%dC host SMART disk = %+v, want healthy with no reasons", temperature, got) + } } } diff --git a/internal/truenas/provider.go b/internal/truenas/provider.go index f24f0efe4b..c20af6a3be 100644 --- a/internal/truenas/provider.go +++ b/internal/truenas/provider.go @@ -1953,10 +1953,9 @@ func zfsPoolRecommendation(assessment storagehealth.Assessment) string { func assessDisk(disk Disk) storagehealth.Assessment { sampleAssessment := storagehealth.AssessSample(storagehealth.Sample{ - Model: strings.TrimSpace(disk.Model), - Health: healthForAssessment(disk), - Temperature: disk.Temperature, - Wearout: -1, + Model: strings.TrimSpace(disk.Model), + Health: healthForAssessment(disk), + Wearout: -1, }) stateUpper := normalizedDiskStatus(disk) diff --git a/internal/truenas/provider_test.go b/internal/truenas/provider_test.go index cd268a2660..34dc1b6451 100644 --- a/internal/truenas/provider_test.go +++ b/internal/truenas/provider_test.go @@ -745,8 +745,9 @@ func TestRecordsIncludeDiskResourcesWithCorrectParentChain(t *testing.T) { foundSmartReason = true } } - if !foundTemperatureReason { - t.Fatalf("expected sdc physical-disk risk to include temperature_high, got %+v", sdc.Resource.PhysicalDisk.Risk.Reasons) + // Heat is the alert disk temperature policy's to judge, not disk risk's. + if foundTemperatureReason { + t.Fatalf("expected sdc physical-disk risk to leave its 63C temperature to the alert policy, got %+v", sdc.Resource.PhysicalDisk.Risk.Reasons) } if !foundSmartReason { t.Fatalf("expected sdc physical-disk risk to include truenas_smart, got %+v", sdc.Resource.PhysicalDisk.Risk.Reasons) @@ -931,7 +932,9 @@ func TestRecordsIncludeTrueNASVMsAsCanonicalWorkloads(t *testing.T) { } } -func TestRecordsElevateOnlineDiskWhenTemperatureCritical(t *testing.T) { +// A hot disk stays online with no disk risk. Its temperature is judged by the +// alert disk temperature policy, which users tune per disk type. +func TestRecordsLeaveDiskHeatToTheAlertPolicy(t *testing.T) { previous := IsFeatureEnabled() SetFeatureEnabled(true) t.Cleanup(func() { @@ -967,14 +970,14 @@ func TestRecordsElevateOnlineDiskWhenTemperatureCritical(t *testing.T) { if diskRecord == nil { t.Fatal("expected physical disk record") } - if diskRecord.Resource.Status != unifiedresources.StatusWarning { - t.Fatalf("expected hot disk status warning, got %s", diskRecord.Resource.Status) + if diskRecord.Resource.Status != unifiedresources.StatusOnline { + t.Fatalf("expected hot disk status online, got %s", diskRecord.Resource.Status) } - if diskRecord.Resource.PhysicalDisk == nil || diskRecord.Resource.PhysicalDisk.Risk == nil { - t.Fatalf("expected hot disk physical risk, got %+v", diskRecord.Resource.PhysicalDisk) + if diskRecord.Resource.PhysicalDisk == nil || diskRecord.Resource.PhysicalDisk.Temperature != 72 { + t.Fatalf("expected the hot disk to keep its 72C reading, got %+v", diskRecord.Resource.PhysicalDisk) } - if diskRecord.Resource.PhysicalDisk.Risk.Level != storagehealth.RiskCritical { - t.Fatalf("expected hot disk critical risk, got %+v", diskRecord.Resource.PhysicalDisk.Risk) + if diskRecord.Resource.PhysicalDisk.Risk != nil { + t.Fatalf("expected no disk risk from temperature, got %+v", diskRecord.Resource.PhysicalDisk.Risk) } } diff --git a/internal/unifiedresources/adapter_coverage_test.go b/internal/unifiedresources/adapter_coverage_test.go index 8953fb4a49..439ae536cf 100644 --- a/internal/unifiedresources/adapter_coverage_test.go +++ b/internal/unifiedresources/adapter_coverage_test.go @@ -1051,7 +1051,8 @@ func TestMetricPercent(t *testing.T) { // derives their state from the host's reporting lease and the disk's spin // state. The native row and a SMART row that borrows the inventory reading // both carry it, and a SMART row with its own reading keeps the agent's state. -// A withdrawn reading never counts as heat in the disk's risk. +// Disk risk never judges heat in any state: the alert disk temperature policy +// owns that. func TestUnraidDiskTemperatureFollowsTheHostLease(t *testing.T) { smartWithout := models.HostDiskSMART{Device: "/dev/sdc", Serial: "UNRAID-1", Collection: &diskinventory.CollectionStatus{ Serial: diskinventory.Available("smartctl"), @@ -1088,7 +1089,6 @@ func TestUnraidDiskTemperatureFollowsTheHostLease(t *testing.T) { Temperature: 72, SpunDown: tc.spunDown} host := models.Host{ID: "agent-tower", Hostname: "tower", Status: tc.status, Unraid: &models.HostUnraidStorage{Disks: []models.HostUnraidDisk{inventory}}} - collected := tc.borrowed.State == diskinventory.FieldAvailable native, _ := resourceFromHostUnraidPhysicalDisk(host, inventory) got := native.PhysicalDisk @@ -1099,8 +1099,8 @@ func TestUnraidDiskTemperatureFollowsTheHostLease(t *testing.T) { if got.Temperature != 72 || nativeStatus != tc.native { t.Fatalf("%s: Unraid row temperature=%d state=%+v, want 72 with %+v", tc.name, got.Temperature, nativeStatus, tc.native) } - if hotRisk(got.Risk) != collected { - t.Fatalf("%s: Unraid row temperature risk = %v, want %v: %+v", tc.name, hotRisk(got.Risk), collected, got.Risk) + if hotRisk(got.Risk) { + t.Fatalf("%s: Unraid row risk judged its 72C temperature: %+v", tc.name, got.Risk) } borrowed, _ := resourceFromHostSMARTDisk(host, smartWithout) @@ -1109,8 +1109,8 @@ func TestUnraidDiskTemperatureFollowsTheHostLease(t *testing.T) { got.Collection.Serial != diskinventory.Available("smartctl") { t.Fatalf("%s: SMART row borrowing the inventory reading: temperature=%d collection=%+v", tc.name, got.Temperature, got.Collection) } - if hotRisk(got.Risk) != collected { - t.Fatalf("%s: borrowed temperature risk = %v, want %v: %+v", tc.name, hotRisk(got.Risk), collected, got.Risk) + if hotRisk(got.Risk) { + t.Fatalf("%s: SMART row risk judged the borrowed 72C temperature: %+v", tc.name, got.Risk) } if smartWithout.Collection.Temperature.State != diskinventory.FieldUnsupported { t.Fatal("the adapter mutated the SMART row's own collection state") diff --git a/internal/unifiedresources/adapters.go b/internal/unifiedresources/adapters.go index da65047d35..e9386c81f0 100644 --- a/internal/unifiedresources/adapters.go +++ b/internal/unifiedresources/adapters.go @@ -804,7 +804,7 @@ func resourceFromHostUnraidPhysicalDisk(host models.Host, disk models.HostUnraid model := strings.TrimSpace(disk.Model) name := firstNonEmpty(model, disk.Name, disk.Device, host.Hostname) health := unraidPhysicalDiskHealth(disk) - assessment := assessUnraidPhysicalDisk(host, disk) + assessment := assessUnraidPhysicalDisk(disk) sizeBytes := unraidDiskSizeBytes(host, disk) var collection *diskinventory.CollectionStatus if status := unraidDiskTemperatureStatus(host, disk); status.State != "" { @@ -939,7 +939,7 @@ func resourceFromHostSMARTDisk(host models.Host, disk models.HostDiskSMART) (Res } assessment := storagehealth.AssessHostSMARTDisk(disk) if unraidDisk != nil { - assessment = storagehealth.SummarizeAssessments(assessment, assessUnraidPhysicalDisk(host, *unraidDisk)) + assessment = storagehealth.SummarizeAssessments(assessment, assessUnraidPhysicalDisk(*unraidDisk)) } wearout := physicalDiskWearoutFromSMARTAttributes(disk.Attributes) @@ -1324,17 +1324,11 @@ func unraidPhysicalDiskHealth(disk models.HostUnraidDisk) string { } } -func assessUnraidPhysicalDisk(host models.Host, disk models.HostUnraidDisk) storagehealth.Assessment { - // A retained inventory temperature is not current evidence of heat. - temperature := 0 - if status := unraidDiskTemperatureStatus(host, disk); status.State == diskinventory.FieldAvailable { - temperature = disk.Temperature - } +func assessUnraidPhysicalDisk(disk models.HostUnraidDisk) storagehealth.Assessment { assessment := storagehealth.AssessSample(storagehealth.Sample{ - Model: disk.Model, - Health: unraidPhysicalDiskHealth(disk), - Temperature: temperature, - Wearout: -1, + Model: disk.Model, + Health: unraidPhysicalDiskHealth(disk), + Wearout: -1, }) addReason := func(code string, severity storagehealth.RiskLevel, summary string) { if strings.TrimSpace(summary) == "" { diff --git a/internal/unifiedresources/physical_disk_risk.go b/internal/unifiedresources/physical_disk_risk.go index 444a62af1d..20ef7fd878 100644 --- a/internal/unifiedresources/physical_disk_risk.go +++ b/internal/unifiedresources/physical_disk_risk.go @@ -82,7 +82,6 @@ func physicalDiskAssessmentFromMeta(meta *PhysicalDiskMeta) storagehealth.Assess sample := storagehealth.Sample{ Model: meta.Model, Health: meta.Health, - Temperature: storagehealth.CollectedTemperature(meta.Temperature, meta.Collection), Wearout: meta.Wearout, WearoutKnown: storagehealth.WearoutReported(meta.Wearout, meta.DiskType), } diff --git a/internal/unifiedresources/physical_disk_risk_test.go b/internal/unifiedresources/physical_disk_risk_test.go index 28087ea2f2..17dd1f2918 100644 --- a/internal/unifiedresources/physical_disk_risk_test.go +++ b/internal/unifiedresources/physical_disk_risk_test.go @@ -41,7 +41,7 @@ func TestPhysicalDiskRiskFromAssessment_ConvertsReasons(t *testing.T) { Level: storagehealth.RiskCritical, Reasons: []storagehealth.Reason{ {Code: "pending_sectors", Severity: storagehealth.RiskCritical, Summary: "Pending sectors detected (5)"}, - {Code: "temperature_high", Severity: storagehealth.RiskWarning, Summary: "Disk temperature is 65C"}, + {Code: "reallocated_sectors", Severity: storagehealth.RiskWarning, Summary: "Reallocated sectors detected (4)"}, }, } result := PhysicalDiskRiskFromAssessment(assessment) @@ -66,7 +66,7 @@ func TestPhysicalDiskRiskFromAssessmentAndIncidents_IncludesDiskHealthIncidents( assessment := storagehealth.Assessment{ Level: storagehealth.RiskWarning, Reasons: []storagehealth.Reason{ - {Code: "temperature_high", Severity: storagehealth.RiskWarning, Summary: "Disk temperature is 63C"}, + {Code: "reallocated_sectors", Severity: storagehealth.RiskWarning, Summary: "Reallocated sectors detected (2)"}, }, } incidents := []ResourceIncident{ @@ -137,14 +137,15 @@ func TestPhysicalDiskAssessmentFromMeta_PendingSectors(t *testing.T) { } } -func TestPhysicalDiskAssessmentFromMeta_HighTemperature(t *testing.T) { +// Heat is the alert disk temperature policy's to judge, never disk risk's. +func TestPhysicalDiskAssessmentFromMeta_HighTemperatureIsNotRisk(t *testing.T) { meta := &PhysicalDiskMeta{ Health: "PASSED", Temperature: 72, } assessment := physicalDiskAssessmentFromMeta(meta) - if assessment.Level != storagehealth.RiskCritical { - t.Errorf("72°C should produce critical assessment, got %s", assessment.Level) + if assessment.Level != storagehealth.RiskHealthy || len(assessment.Reasons) != 0 { + t.Errorf("72°C should leave the assessment healthy, got %+v", assessment) } } diff --git a/internal/unifiedresources/registry_test.go b/internal/unifiedresources/registry_test.go index 0256304351..7c98c19bca 100644 --- a/internal/unifiedresources/registry_test.go +++ b/internal/unifiedresources/registry_test.go @@ -7035,10 +7035,11 @@ func TestRegistryProjectionSnapshotDetachesAndTracksMappings(t *testing.T) { } } -// A retained temperature is last-known history, not evidence that the disk is -// hot now. The risk verdict (and the warning status it drives) must judge only -// a temperature whose collection state says it was observed now. -func TestPhysicalDiskRiskIgnoresRetainedTemperature(t *testing.T) { +// Disk risk (and the warning status it drives) never judges temperature, +// collected or retained. Heat belongs to the alert disk temperature policy, +// which users tune per disk type and per host and which no registry can see. +// The reading itself stays on the resource for that policy to judge. +func TestPhysicalDiskRiskNeverJudgesTemperature(t *testing.T) { // A fresh check time keeps the registry's source-staleness sweep out of // the status this test reads. now := time.Now() @@ -7047,15 +7048,12 @@ func TestPhysicalDiskRiskIgnoresRetainedTemperature(t *testing.T) { cases := []struct { name string collection *diskinventory.CollectionStatus - wantHot bool }{ - {name: "collected now", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("proxmox_node_smart")}, wantHot: true}, - {name: "source predates collection state", collection: nil, wantHot: true}, - {name: "state left empty", collection: &diskinventory.CollectionStatus{}, wantHot: true}, - {name: "retained after the agent went silent", collection: &diskinventory.CollectionStatus{Temperature: retained}, wantHot: false}, - {name: "retained while in standby", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}, wantHot: false}, - {name: "retained after the source stopped exposing it", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature")}, wantHot: false}, - {name: "retained while unexpectedly missing", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Missing("smartctl", "")}, wantHot: false}, + {name: "collected now", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("proxmox_node_smart")}}, + {name: "source predates collection state", collection: nil}, + {name: "state left empty", collection: &diskinventory.CollectionStatus{}}, + {name: "retained after the agent went silent", collection: &diskinventory.CollectionStatus{Temperature: retained}}, + {name: "retained while in standby", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -7089,100 +7087,60 @@ func TestPhysicalDiskRiskIgnoresRetainedTemperature(t *testing.T) { if disk.PhysicalDisk.Temperature != 72 { t.Fatalf("temperature = %d, want the reading kept at 72", disk.PhysicalDisk.Temperature) } - hot := false - if disk.PhysicalDisk.Risk != nil { - for _, reason := range disk.PhysicalDisk.Risk.Reasons { - hot = hot || reason.Code == "temperature_high" - } - } - if hot != tc.wantHot { - t.Fatalf("temperature_high = %v, want %v (risk %+v)", hot, tc.wantHot, disk.PhysicalDisk.Risk) - } - if tc.wantHot && disk.Status != StatusWarning { - t.Fatalf("status = %q, want warning for a collected 72C", disk.Status) - } - if !tc.wantHot && (disk.PhysicalDisk.Risk != nil || disk.Status != StatusOnline) { - t.Fatalf("retained 72C: status = %q risk = %+v, want online with no risk", disk.Status, disk.PhysicalDisk.Risk) + if disk.PhysicalDisk.Risk != nil || disk.Status != StatusOnline { + t.Fatalf("72C disk: status = %q risk = %+v, want online with no risk", disk.Status, disk.PhysicalDisk.Risk) } }) } // A host agent's SMART row merged with the Proxmox inventory row for the - // same disk: the agent's retained reading keeps its unavailable state - // through the registry merge and recompute, so it stays out of risk. - for _, tc := range []struct { - name string - state diskinventory.FieldStatus - wantHot bool - }{ - {name: "agent reading collected", state: diskinventory.Available("host_agent"), wantHot: true}, - {name: "agent reading retained", state: retained, wantHot: false}, - } { - t.Run("merged "+tc.name, func(t *testing.T) { - adapter := NewMonitorAdapter(NewRegistry(nil)) - adapter.PopulateFromSnapshot(models.StateSnapshot{ - Hosts: []models.Host{{ - ID: "host-pve", - Hostname: "pve", - Status: "online", - LastSeen: now, - Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{ - Device: "/dev/sda", - Model: "WDC WD80EFAX", - Serial: "SERIAL-SDA", - Type: "sata", - Temperature: 72, - Health: "PASSED", - Collection: &diskinventory.CollectionStatus{Temperature: tc.state}, - }}}, - }}, - PhysicalDisks: []models.PhysicalDisk{{ - ID: "homelab-pve--dev-sda", - Node: "pve", - Instance: "homelab", - DevPath: "/dev/sda", - Model: "WDC WD80EFAX", - Serial: "SERIAL-SDA", - Type: "sata", - Health: "PASSED", - Wearout: -1, - Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature")}, - LastChecked: now, - }}, - }) - var disks []Resource - for _, resource := range adapter.GetAll() { - if resource.Type == ResourceTypePhysicalDisk { - disks = append(disks, resource) - } - } - if len(disks) != 1 || disks[0].PhysicalDisk == nil { - t.Fatalf("physical disks = %d, want the agent and Proxmox rows merged into one", len(disks)) - } - disk := disks[0] - if disk.PhysicalDisk.Temperature != 72 { - t.Fatalf("merged temperature = %d, want 72", disk.PhysicalDisk.Temperature) - } - hot := false - if disk.PhysicalDisk.Risk != nil { - for _, reason := range disk.PhysicalDisk.Risk.Reasons { - hot = hot || reason.Code == "temperature_high" - } - } - if hot != tc.wantHot { - t.Fatalf("merged temperature_high = %v, want %v (collection %+v, risk %+v)", hot, tc.wantHot, disk.PhysicalDisk.Collection, disk.PhysicalDisk.Risk) - } - }) + // same disk goes through the registry merge and its risk recompute. + adapter := NewMonitorAdapter(NewRegistry(nil)) + adapter.PopulateFromSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-pve", + Hostname: "pve", + Status: "online", + LastSeen: now, + Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{ + Device: "/dev/sda", + Model: "WDC WD80EFAX", + Serial: "SERIAL-SDA", + Type: "sata", + Temperature: 72, + Health: "PASSED", + Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")}, + }}}, + }}, + PhysicalDisks: []models.PhysicalDisk{{ + ID: "homelab-pve--dev-sda", + Node: "pve", + Instance: "homelab", + DevPath: "/dev/sda", + Model: "WDC WD80EFAX", + Serial: "SERIAL-SDA", + Type: "sata", + Health: "PASSED", + Wearout: -1, + Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature")}, + LastChecked: now, + }}, + }) + var merged []Resource + for _, resource := range adapter.GetAll() { + if resource.Type == ResourceTypePhysicalDisk { + merged = append(merged, resource) + } + } + if len(merged) != 1 || merged[0].PhysicalDisk == nil { + t.Fatalf("physical disks = %d, want the agent and Proxmox rows merged into one", len(merged)) + } + if disk := merged[0].PhysicalDisk; disk.Temperature != 72 || disk.Risk != nil { + t.Fatalf("merged disk temperature = %d risk = %+v, want 72 with no risk", disk.Temperature, disk.Risk) } - // The registry recomputes risk from merged metadata after a source merge; - // that path follows the same rule. - meta := &PhysicalDiskMeta{Health: "PASSED", Temperature: 72, Collection: &diskinventory.CollectionStatus{Temperature: retained}} + meta := &PhysicalDiskMeta{Health: "PASSED", Temperature: 72, Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")}} if risk := physicalDiskRiskFromMeta(meta, nil); risk != nil { - t.Fatalf("meta recompute risk = %+v, want none for a retained temperature", risk) - } - meta.Collection = &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")} - if risk := physicalDiskRiskFromMeta(meta, nil); risk == nil || risk.Level != storagehealth.RiskCritical { - t.Fatalf("meta recompute risk = %+v, want critical for a collected 72C", risk) + t.Fatalf("meta recompute risk = %+v, want none for a collected 72C", risk) } } diff --git a/internal/unifiedresources/storage_risk_test.go b/internal/unifiedresources/storage_risk_test.go index d88b2d5b15..06eb5bc6d8 100644 --- a/internal/unifiedresources/storage_risk_test.go +++ b/internal/unifiedresources/storage_risk_test.go @@ -35,7 +35,7 @@ func TestStorageRiskFromAssessment_CriticalConvertsReasons(t *testing.T) { Level: storagehealth.RiskCritical, Reasons: []storagehealth.Reason{ {Code: "pending_sectors", Severity: storagehealth.RiskCritical, Summary: "Bad sectors"}, - {Code: "temperature_high", Severity: storagehealth.RiskWarning, Summary: "Hot disk"}, + {Code: "reallocated_sectors", Severity: storagehealth.RiskWarning, Summary: "Reallocated sectors"}, }, }) if got == nil { @@ -47,7 +47,7 @@ func TestStorageRiskFromAssessment_CriticalConvertsReasons(t *testing.T) { if len(got.Reasons) != 2 { t.Fatalf("expected 2 reasons, got %d", len(got.Reasons)) } - if got.Reasons[0].Code != "pending_sectors" || got.Reasons[1].Code != "temperature_high" { + if got.Reasons[0].Code != "pending_sectors" || got.Reasons[1].Code != "reallocated_sectors" { t.Error("reasons should preserve order and codes") } } From 35bf55af23860aa1cbec348cc40ac07c4e9dc4a1 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 00:09:01 +0100 Subject: [PATCH 38/53] Colour report disk temperatures by the alert thresholds The PDF performance report's disk table coloured a temperature amber from 50C and red from 60C for every disk type, the last fixed disk heat line after disk heat moved to the alert disk temperature policy. An NVMe at 63C, well under its 70C alert trigger, printed red. A user's raised trigger changed nothing. Each reported disk now carries the tenant's alert thresholds for its type (the clear value and the trigger, from alerts.Manager.DiskTemperatureThreshold). The PDF colours the reading amber from the clear value and red from the trigger, as the Physical Disks Temp column does, and leaves it plain when disk temperature alerting is off. --- .../v6/internal/subsystems/agent-lifecycle.md | 3 +- .../v6/internal/subsystems/alerts.md | 8 +-- .../v6/internal/subsystems/api-contracts.md | 9 +++ .../internal/subsystems/storage-recovery.md | 4 +- internal/api/contract_test.go | 64 +++++++++++++++++++ internal/api/metrics_reporting_handlers.go | 40 +++++++++--- pkg/reporting/pdf.go | 22 ++++++- pkg/reporting/pdf_additional_test.go | 23 +++++++ pkg/reporting/reporting.go | 6 ++ 9 files changed, 163 insertions(+), 16 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 419c34448e..a16a714d8b 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -764,7 +764,8 @@ Downstream readers keep that distinction too. The disk temperature charts do not pad a series to now with the retained value. The AI chat context, Patrol and the AI disk tools present it only as a last-known value with its reason. The performance report and the reporting runtime snapshot leave it out of their -disk tables. +disk tables. The performance report colours a collected reading by the +tenant's alert disk temperature thresholds for the disk type, not a fixed line. An enabled availability target assigned to a host agent creates an agent-lifecycle lease for that exact target/agent pairing. First assignment diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 692424c56d..24d5382258 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -1702,10 +1702,10 @@ normalization restores a non-positive entry to its default. A disk is hot from its trigger. Alerts and Patrol findings stay open until the reading falls to the clear value, or under the trigger when there is no band below it. The Physical Disks verdict and the TrueNAS Health cell judge only the -current reading against the trigger. Judges that still differ: TrueNAS disk -temperature alerts use `TrueNASDiskDefaults.Temperature` (a flat 55/50), the -TrueNAS disk drawer tones the reading from a fixed 55C, and PDF reports colour -disk temperatures at a fixed 50/60C. +current reading against the trigger. PDF reports colour disk temperatures by +the same thresholds. Judges that still differ: TrueNAS disk temperature alerts +use `TrueNASDiskDefaults.Temperature` (a flat 55/50), and the TrueNAS disk +drawer tones the reading from a fixed 55C. `TestDiskTemperatureThresholdMatchesCheckHostPolicy` in `internal/alerts/threshold_resolution_shared_test.go` pins per-type resolution, a raised NVMe trigger that `CheckHost` also honours, the copy, the disabled default and diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index e04dff160b..21b9ae0ad6 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -6349,6 +6349,15 @@ when `diskinventory.TemperatureCollected` holds, so a retained value is reported as 0 (no reading) rather than tabulated as measured. Proof: `TestContract_ReportsOmitRetainedDiskTemperatures` in `internal/api/contract_test.go`. +Each reported disk also carries its alert disk temperature thresholds +(`reporting.DiskInfo.TemperatureWarning` / `TemperatureCritical`, the clear +value and the trigger). `reportDiskTemperatureThresholds` resolves them from +the tenant's alert manager (`alerts.Manager.DiskTemperatureThreshold`, the +factory policy when there is none). The PDF disk table colours a reading amber +from the clear value and red from the trigger, as the Physical Disks Temp +column does, and leaves it plain when disk temperature alerting is off. It no +longer uses a fixed 50/60C. Proof: +`TestContract_ReportsCarryDiskTemperatureAlertThresholds`. That same metrics-history contract also owns Kubernetes pod identity normalization. `/api/metrics-store/history` must accept legacy bare pod IDs such as `cluster-1:pod:pod-1`, canonicalize them onto the unified pod metrics diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 079fabf486..6607d9eed8 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -6779,7 +6779,9 @@ judges the current reading, with no memory of an alert. A disk temperature alert held between its clear value and trigger shows an amber Temp cell, not `Running Hot`. A switched-off agent Disk Temp default resolves to null, which leaves the reading uncoloured and never hot, as alerting is off for it. -Per-host Disk Temp overrides do not yet reach Physical Disks. +Per-host Disk Temp overrides do not yet reach Physical Disks. PDF performance +reports colour a disk reading by the same per-type thresholds, which +`reporting.DiskInfo` carries. `frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts` pins cell-and-verdict agreement for every type from 40C to 80C under factory and raised triggers, plus retained readings, failure evidence, sorting and diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go index 17faf9067b..1e0a0292d1 100644 --- a/internal/api/contract_test.go +++ b/internal/api/contract_test.go @@ -5286,6 +5286,70 @@ func TestContract_ReportsOmitRetainedDiskTemperatures(t *testing.T) { } } +// Report disk tables colour a reading by the disk's alert disk temperature +// thresholds (the tenant's per-type policy), not a fixed 50/60C: amber from the +// clear value, red from the trigger, as the Physical Disks Temp column does. +func TestContract_ReportsCarryDiskTemperatureAlertThresholds(t *testing.T) { + engine := &stubReportingEngine{data: []byte("report"), contentType: "application/pdf"} + original := reporting.GetEngine() + reporting.SetEngine(engine) + t.Cleanup(func() { reporting.SetEngine(original) }) + + state := models.NewState() + state.Nodes = []models.Node{{ID: "node-1", Name: "node-a", Status: "online"}} + disk := func(id, devPath, diskType string, temperature int) unifiedresources.Resource { + return unifiedresources.Resource{ + ID: id, Type: unifiedresources.ResourceTypePhysicalDisk, Name: id, ParentName: "node-a", + Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"node-a"}}, + PhysicalDisk: &unifiedresources.PhysicalDiskMeta{ + DevPath: devPath, DiskType: diskType, Health: "PASSED", Wearout: -1, Temperature: temperature, + Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}, + }, + } + } + monitor := newReportingMonitorForTest(t, state, []unifiedresources.Resource{ + disk("disk-nvme", "/dev/nvme0n1", "nvme", 63), + disk("disk-sata", "/dev/sda", "sata", 56), + }) + handler := NewReportingHandlers(newReportingMTMForTest(t, monitor), nil) + thresholdsByDevice := func() map[string][2]float64 { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/api/reporting?format=pdf&resourceType=node&resourceId=node-1", nil) + req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "default")) + rec := httptest.NewRecorder() + handler.HandleGenerateReport(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rec.Code, rec.Body.String()) + } + got := map[string][2]float64{} + for _, d := range engine.lastReq.Disks { + got[d.Device] = [2]float64{d.TemperatureWarning, d.TemperatureCritical} + } + return got + } + + // No alert manager: the factory per-type policy. + if got, want := thresholdsByDevice(), map[string][2]float64{"/dev/nvme0n1": {65, 70}, "/dev/sda": {50, 55}}; !reflect.DeepEqual(got, want) { + t.Fatalf("factory report disk thresholds = %v, want %v", got, want) + } + + manager := alerts.NewManager() + cfg := manager.GetConfig() + cfg.DiskTempByType["nvme"] = alerts.HysteresisThreshold{Trigger: 75, Clear: 70} + manager.UpdateConfig(cfg) + setUnexportedField(t, monitor, "alertManager", manager) + if got, want := thresholdsByDevice(), map[string][2]float64{"/dev/nvme0n1": {70, 75}, "/dev/sda": {50, 55}}; !reflect.DeepEqual(got, want) { + t.Fatalf("configured report disk thresholds = %v, want %v", got, want) + } + + cfg = manager.GetConfig() + cfg.AgentDefaults.DiskTemperature = &alerts.HysteresisThreshold{Trigger: 0, Clear: 0} + manager.UpdateConfig(cfg) + if got, want := thresholdsByDevice(), map[string][2]float64{"/dev/nvme0n1": {0, 0}, "/dev/sda": {0, 0}}; !reflect.DeepEqual(got, want) { + t.Fatalf("report disk thresholds with disk temperature alerting off = %v, want %v", got, want) + } +} + func TestContract_ReportingRequestCarriesEntitledReportBranding(t *testing.T) { engine := &stubReportingEngine{data: []byte("report"), contentType: "application/pdf"} original := reporting.GetEngine() diff --git a/internal/api/metrics_reporting_handlers.go b/internal/api/metrics_reporting_handlers.go index f363c6c0b2..3cf94d3f68 100644 --- a/internal/api/metrics_reporting_handlers.go +++ b/internal/api/metrics_reporting_handlers.go @@ -14,6 +14,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/ai" + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/monitoring" @@ -334,6 +335,9 @@ type reportingEnrichmentSnapshot struct { RecentlyResolved []models.ResolvedAlert LegacyBackups models.PVEBackups Resources []unifiedresources.Resource + // alertManager supplies the tenant's disk temperature thresholds. Nil + // means the factory alert configuration. + alertManager *alerts.Manager } func emptyReportingEnrichmentSnapshot() reportingEnrichmentSnapshot { @@ -405,6 +409,7 @@ func (h *ReportingHandlers) getReportingEnrichmentSnapshot(ctx context.Context, RecentlyResolved: monitor.RecentlyResolvedSnapshot(), LegacyBackups: monitor.PVEBackupsSnapshot(), Resources: unifiedResources, + alertManager: monitor.GetAlertManager(), } snapshot.normalizeCollections() return snapshot, true @@ -876,20 +881,39 @@ func (h *ReportingHandlers) enrichNodeReport(req *reporting.MetricReportRequest, continue } pd := r.PhysicalDisk + temperatureWarning, temperatureCritical := reportDiskTemperatureThresholds(snapshot.alertManager, pd.DiskType) req.Disks = append(req.Disks, reporting.DiskInfo{ - Device: pd.DevPath, - Model: pd.Model, - Serial: pd.Serial, - Type: pd.DiskType, - Size: pd.SizeBytes, - Health: pd.Health, - Temperature: reportDiskTemperature(*pd), - WearLevel: pd.Wearout, + Device: pd.DevPath, + Model: pd.Model, + Serial: pd.Serial, + Type: pd.DiskType, + Size: pd.SizeBytes, + Health: pd.Health, + Temperature: reportDiskTemperature(*pd), + WearLevel: pd.Wearout, + TemperatureWarning: temperatureWarning, + TemperatureCritical: temperatureCritical, }) } } } +// reportDiskTemperatureThresholds returns the alert disk temperature thresholds +// a report colours a disk reading by: the clear value and the trigger, from +// the same policy as disk temperature alerts and the Physical Disks Temp +// column. Zero means disk temperature alerting is off for the disk. +func reportDiskTemperatureThresholds(manager *alerts.Manager, diskType string) (float64, float64) { + threshold := manager.DiskTemperatureThreshold(diskType) + if threshold == nil || threshold.Trigger <= 0 { + return 0, 0 + } + warning := threshold.Clear + if warning <= 0 || warning > threshold.Trigger { + warning = threshold.Trigger + } + return warning, threshold.Trigger +} + // reportDiskTemperature returns the disk temperature a report may tabulate: // one the current observation collected. Normalization may keep a last-known // temperature it did not collect (a disk in standby, a host agent past its diff --git a/pkg/reporting/pdf.go b/pkg/reporting/pdf.go index b1db00fdc1..1cf9f3d575 100644 --- a/pkg/reporting/pdf.go +++ b/pkg/reporting/pdf.go @@ -1670,9 +1670,10 @@ func (g *PDFGenerator) writeDisksSection(pdf *fpdf.Fpdf, data *ReportData) { tempStr := "-" if disk.Temperature > 0 { tempStr = fmt.Sprintf("%dC", disk.Temperature) - if disk.Temperature >= 60 { + switch diskTemperatureSeverity(disk) { + case "critical": pdf.SetTextColor(colorDanger[0], colorDanger[1], colorDanger[2]) - } else if disk.Temperature >= 50 { + case "warning": pdf.SetTextColor(colorWarning[0], colorWarning[1], colorWarning[2]) } } @@ -2808,3 +2809,20 @@ func formatDuration(d time.Duration) string { } return fmt.Sprintf("%d %s", minutes, minWord) } + +// diskTemperatureSeverity colours a disk reading by its alert disk temperature +// thresholds: "critical" from the trigger, "warning" from the clear value, and +// "" below it or when the disk carries no thresholds. +func diskTemperatureSeverity(disk DiskInfo) string { + if disk.Temperature <= 0 || disk.TemperatureCritical <= 0 { + return "" + } + reading := float64(disk.Temperature) + if reading >= disk.TemperatureCritical { + return "critical" + } + if disk.TemperatureWarning > 0 && reading >= disk.TemperatureWarning { + return "warning" + } + return "" +} diff --git a/pkg/reporting/pdf_additional_test.go b/pkg/reporting/pdf_additional_test.go index 49ca2c3fc9..d190c3a4b2 100644 --- a/pkg/reporting/pdf_additional_test.go +++ b/pkg/reporting/pdf_additional_test.go @@ -57,3 +57,26 @@ func TestFormatDuration(t *testing.T) { t.Fatalf("expected 2 days, 1 hour, got %q", got) } } + +// Disk temperatures in a report take the disk's alert thresholds, not a fixed +// 50/60C: an NVMe at 63C under a 70C trigger stays uncoloured, a SATA disk at +// 56C past its 55C trigger is red, and a disk without thresholds is plain. +func TestDiskTemperatureSeverityFollowsAlertThresholds(t *testing.T) { + cases := []struct { + disk DiskInfo + want string + }{ + {DiskInfo{Temperature: 63, TemperatureWarning: 65, TemperatureCritical: 70}, ""}, + {DiskInfo{Temperature: 66, TemperatureWarning: 65, TemperatureCritical: 70}, "warning"}, + {DiskInfo{Temperature: 70, TemperatureWarning: 65, TemperatureCritical: 70}, "critical"}, + {DiskInfo{Temperature: 56, TemperatureWarning: 50, TemperatureCritical: 55}, "critical"}, + {DiskInfo{Temperature: 72, TemperatureWarning: 70, TemperatureCritical: 75}, "warning"}, + {DiskInfo{Temperature: 80}, ""}, + {DiskInfo{Temperature: 0, TemperatureWarning: 50, TemperatureCritical: 55}, ""}, + } + for _, tc := range cases { + if got := diskTemperatureSeverity(tc.disk); got != tc.want { + t.Errorf("diskTemperatureSeverity(%+v) = %q, want %q", tc.disk, got, tc.want) + } + } +} diff --git a/pkg/reporting/reporting.go b/pkg/reporting/reporting.go index cc424e6bc8..66d8e6babf 100644 --- a/pkg/reporting/reporting.go +++ b/pkg/reporting/reporting.go @@ -153,6 +153,12 @@ type DiskInfo struct { Health string // PASSED, FAILED, UNKNOWN Temperature int // Celsius WearLevel int // 0-100, percentage of life REMAINING (100 = healthy, 0 = end of life, -1 = unknown) + // TemperatureWarning and TemperatureCritical are the disk's alert disk + // temperature thresholds in Celsius (the clear value and the trigger). The + // reading is amber from Warning and red from Critical, as in the Physical + // Disks Temp column. A zero Critical leaves the reading uncoloured. + TemperatureWarning float64 + TemperatureCritical float64 } // MultiReportRequest defines the parameters for generating a multi-resource report. From 396bb5d517f14466795a2ccd2ff438fdba0ab077 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 00:17:32 +0100 Subject: [PATCH 39/53] Keep a disk heat finding open without a current reading Patrol verifies a disk-high finding as recovered once the disk's temperature cools to its alert clear value. A disk in standby, or one whose host agent stopped reporting, has no current reading, and a missing reading counted as cooled. A finding raised for a SATA disk at 56C would close as soon as the disk spun down, while its temperature alert stayed open. When a disk has no current reading and its last known reading is still at or above the clear value, verification now reports the recovery as unknown instead of proven. A disk last seen cool still recovers. --- .../v6/internal/subsystems/ai-runtime.md | 3 ++- internal/ai/patrol_disk_temperature_test.go | 21 ++++++++++++++++ internal/ai/patrol_findings.go | 25 +++++++++++++------ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 216666a720..c6402b3694 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -315,7 +315,8 @@ configuration applies. A disk is hot from its trigger: a triage warning, a disk issue and an AI chat "needing attention" entry, as its temperature alert and Physical Disks Running Hot verdict are. A disk-high finding recovers at or below the clear value (under the trigger when there is no band below it), as -the alert does. Scoped runs keep the provider, and a +the alert does. With no current reading, a disk last seen hot leaves that +verification unknown rather than recovered. Scoped runs keep the provider, and a provider set before Patrol starts is handed to it. Below the trigger Patrol flags nothing, so an NVMe at 63C stays quiet and a SATA disk at 56C is flagged. Proofs: `internal/ai/patrol_disk_temperature_test.go` and diff --git a/internal/ai/patrol_disk_temperature_test.go b/internal/ai/patrol_disk_temperature_test.go index e38358380e..d8c433eae4 100644 --- a/internal/ai/patrol_disk_temperature_test.go +++ b/internal/ai/patrol_disk_temperature_test.go @@ -1,12 +1,15 @@ package ai import ( + "errors" "strings" "testing" "time" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) // raisedNVMeThresholds is a user who raised the NVMe disk temperature trigger @@ -226,3 +229,21 @@ func TestDiskHeatRecoversAtTheClearValue(t *testing.T) { t.Fatalf("no-band limits %+v: want 70C hot and not cooled, 69C cooled", noBand) } } + +// With no current reading (standby, a silent agent) a disk last seen hot has +// not shown that it cooled, so heat recovery is unknown rather than proven. +func TestDiskHeatRecoveryNeedsACurrentReading(t *testing.T) { + retained := &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")} + state := newPatrolRuntimeState(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{ + {ID: "sata-retained-hot", DevPath: "/dev/sda", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 56, Collection: retained}, + {ID: "sata-retained-cool", DevPath: "/dev/sdb", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 40, Collection: retained}, + }}) + + if _, err := verifyMetricRecoveredState(state, PatrolThresholds{}, "disk-high", "sata-retained-hot", "physical_disk"); !errors.Is(err, aicontracts.ErrVerificationUnknown) { + t.Fatalf("retained 56C: err = %v, want ErrVerificationUnknown", err) + } + recovered, err := verifyMetricRecoveredState(state, PatrolThresholds{}, "disk-high", "sata-retained-cool", "physical_disk") + if err != nil || !recovered { + t.Fatalf("retained 40C: recovered = %v, err = %v, want recovered", recovered, err) + } +} diff --git a/internal/ai/patrol_findings.go b/internal/ai/patrol_findings.go index 1bb5e87af0..d4f5fbc2b7 100644 --- a/internal/ai/patrol_findings.go +++ b/internal/ai/patrol_findings.go @@ -2296,6 +2296,11 @@ func verifyMetricRecoveredState(snap patrolRuntimeState, thresholds PatrolThresh if !disk.temperatureLimits.cooled(disk.temperature) { return false, nil } + // A disk last seen hot with no current reading has not + // shown that it cooled, so its recovery is unknown. + if disk.temperature <= 0 && !disk.temperatureLimits.cooled(disk.lastKnownTemperature) { + return false, fmt.Errorf("%w: no current temperature reading for %s", aicontracts.ErrVerificationUnknown, rid) + } return true, nil } break @@ -2317,10 +2322,13 @@ func verifyMetricRecoveredState(snap patrolRuntimeState, thresholds PatrolThresh } type patrolPhysicalDiskVerification struct { - health string - wearout int - temperature int - temperatureLimits diskTemperatureLimits + health string + wearout int + temperature int + // lastKnownTemperature is a retained reading the current observation + // did not collect (a disk in standby, a silent agent). + lastKnownTemperature int + temperatureLimits diskTemperatureLimits } type patrolPhysicalDiskVisitor func(identifiers []string, verification patrolPhysicalDiskVerification) bool @@ -2464,10 +2472,11 @@ func patrolVisitPhysicalDiskVerification(snap patrolRuntimeState, visit patrolPh rows := patrolPhysicalDiskRows(snap, nil) for _, disk := range rows { if !visit([]string{disk.id, disk.name, disk.devPath, disk.model}, patrolPhysicalDiskVerification{ - health: strings.TrimSpace(disk.health), - wearout: disk.wearout, - temperature: disk.temperature.Collected, - temperatureLimits: disk.temperatureLimits, + health: strings.TrimSpace(disk.health), + wearout: disk.wearout, + temperature: disk.temperature.Collected, + lastKnownTemperature: disk.temperature.LastKnown, + temperatureLimits: disk.temperatureLimits, }) { return true } From e3e743b076db00da139c04a6daba20fb8e594d82 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 00:21:46 +0100 Subject: [PATCH 40/53] List disk heat beside TrueNAS alert text The TrueNAS storage table's Health reasons put native TrueNAS alert text first and used Pulse's derived summaries only when TrueNAS raised no alert. Disk heat landed in that fallback, so a hot disk with any incident, even an informational one, showed only the incident's text under Attention, and nothing said the disk was hot. Heat is Pulse's own judgement against the alert disk temperature thresholds, not a restatement of TrueNAS state, so it is now always listed after any native alert text. --- .../internal/subsystems/unified-resources.md | 4 ++- .../e3956fd0c9818451.json | 30 +++++++++++++++++++ .../__tests__/truenasPageModel.test.ts | 10 +++++++ .../src/features/truenas/truenasPageModel.ts | 4 ++- 4 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 frontend-modern/browser-verification/e3956fd0c9818451.json diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 6a00c0a3c0..c9fb0f7d27 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -1118,7 +1118,9 @@ threshold.". The trigger comes from the alerts store's `getDiskTemperatureThresholds`, which the table passes to the row build, filter, counts, sort and issue. `isPhysicalDiskRunningHot` in `features/storageBackups/diskTemperaturePresentation.ts` makes the same call -for the Physical Disks verdict. The status dot keeps the source's own state. +for the Physical Disks verdict. The heat reason follows any native TrueNAS +alert text, and a hot disk's status dot turns warning though its source state +stays `online`, because phones show only the dot. `truenasPageModel.test.ts` and `TrueNASStorageTopologyTable.test.tsx` pin it. The vSphere Datastores and Networks tables follow the same exception-first Health column. `getVmwareDatastoreIssue` and `getVmwareNetworkIssue` return diff --git a/frontend-modern/browser-verification/e3956fd0c9818451.json b/frontend-modern/browser-verification/e3956fd0c9818451.json new file mode 100644 index 0000000000..4e012c9ea7 --- /dev/null +++ b/frontend-modern/browser-verification/e3956fd0c9818451.json @@ -0,0 +1,30 @@ +{ + "version": 1, + "base_sha": "396bb5d517f14466795a2ccd2ff438fdba0ab077", + "verified_at": "2026-10-06T23:21:38Z", + "result": "passed", + "changed_paths": ["frontend-modern/src/features/truenas/truenasPageModel.ts"], + "content_sha256": { + "frontend-modern/src/features/truenas/truenasPageModel.ts": "91d7afa5bfb5b47692def18ae0a6e773ca47194d41faf0b57fa5c5c82d57d278" + }, + "routes": ["/truenas/storage"], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack built from this exact content, with a local-only overlay (never committed) pinning TrueNAS mock disks: sdc (SAS, carries the native incident 'Device /dev/sdc has SMART test failures.') at 67C and sda (SATA, no incident) at 57C, factory disk temperature thresholds (SAS 65C, SATA 55C)", + "1440: sdc's Health cell reads 'Device /dev/sdc has SMART test failures.' with '+1', and its title lists that reason then 'Disk temperature is 67\u00b0C, at or above its 65\u00b0C alert threshold.'; before this change the heat reason was dropped whenever an incident had text. sda still reads 'Disk temperature is 57\u00b0C, at or above its 55\u00b0C alert threshold.'", + "390: the Health column stays phone-hidden; sda's status dot reads Warning and the cool sdd reads Online; no horizontal overflow (scrollWidth 390)" + ], + "interactions": [ + "Loaded TrueNAS Storage at 1440x900 and 390x844 and read the sdc and sda Health cells, their titles and visible reason text, and the status dot titles", + "Moved sdc's pinned reading from 57C (under its SAS trigger: incident only, no heat) to 67C and reloaded to confirm the heat reason joins the incident text" + ] +} diff --git a/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts b/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts index 1d29985b7c..3a88ea9308 100644 --- a/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts +++ b/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts @@ -676,6 +676,16 @@ describe('truenasPageModel', () => { filterTrueNASStorageTopologyRows(rows, '', 'healthy', resolveFactory).map((row) => row.id), ).toEqual(['disk:disk-cool']); + // Native TrueNAS alert text leads, and the heat reason stays listed. + const hotWithIncident = makeResource({ + ...hotDisk, + incidents: [{ code: 'truenas_smart', severity: 'info', summary: 'SMART test scheduled' }], + }); + expect(getTrueNASStorageIssue(hotWithIncident, resolveFactory)?.reasons).toEqual([ + 'SMART test scheduled', + 'Disk temperature is 56°C, at or above its 55°C alert threshold.', + ]); + // An NVMe at 63C is under its 70C trigger. const warmNVMe = makeResource({ ...hotDisk, diff --git a/frontend-modern/src/features/truenas/truenasPageModel.ts b/frontend-modern/src/features/truenas/truenasPageModel.ts index ceca216eae..75c13bcf0b 100644 --- a/frontend-modern/src/features/truenas/truenasPageModel.ts +++ b/frontend-modern/src/features/truenas/truenasPageModel.ts @@ -844,8 +844,10 @@ export function getTrueNASStorageIssue( if (reasons.length === leading) { for (const reason of resource.storage?.risk?.reasons ?? []) push(reason.summary); for (const reason of resource.physicalDisk?.risk?.reasons ?? []) push(reason.summary); - push(getTrueNASDiskHeatSummary(resource, resolveDiskTemperatureThresholds)); } + // Heat is Pulse's own judgement, not a restatement of TrueNAS state, so it + // stays listed beside any native alert text. + push(getTrueNASDiskHeatSummary(resource, resolveDiskTemperatureThresholds)); // A read-only dataset raises no incident, so its state tag is the only // reason the provider gives. The drawer reads the same shared mapping. if (reasons.length === leading && resource.storage?.topology === 'dataset') { From 4fd6b483cbb905c7caf132c23396ae83936015d8 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 01:42:54 +0100 Subject: [PATCH 41/53] Use a SMART warning, not heat, in the cluster member disk test TestContract_ConnectionSystemMembersIgnoreOwnedPhysicalDisks proves a disk's health never folds into its cluster node's liveness. Its warning disk came from heat: an NVMe at 63C, which the flat 60C disk risk rule flagged. Disk risk no longer judges temperature, and 63C is under the NVMe alert trigger, so that disk is now online and the precondition failed. Reallocated sectors give the disk the same warning status through real SMART evidence, which keeps the test on its subject. --- internal/api/contract_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go index 1e0a0292d1..fa09df5ea3 100644 --- a/internal/api/contract_test.go +++ b/internal/api/contract_test.go @@ -16889,9 +16889,9 @@ func TestContract_PBSHostAgentComposesIntoOwningSystem(t *testing.T) { // An agent on a Proxmox cluster node reports SMART data for the node's disks, // and the registry stamps each disk with the owning node's identity so it // stays discoverable in the Proxmox workspace. A disk is not a cluster member: -// projecting it as one folded its health into the node row, so a warm NVMe -// drive rendered an actively reporting node as Stale whenever the agent's -// report was newer than the last Proxmox poll. +// projecting it as one folded its health into the node row, so a disk with a +// SMART warning rendered an actively reporting node as Stale whenever the +// agent's report was newer than the last Proxmox poll. func TestContract_ConnectionSystemMembersIgnoreOwnedPhysicalDisks(t *testing.T) { cfg := &config.Config{DataPath: t.TempDir()} monitor, err := monitoring.New(cfg) @@ -16902,6 +16902,7 @@ func TestContract_ConnectionSystemMembersIgnoreOwnedPhysicalDisks(t *testing.T) polledAt := time.Now().UTC().Add(-10 * time.Second) reportedAt := polledAt.Add(6 * time.Second) + reallocatedSectors := int64(3) adapter := unifiedresources.NewMonitorAdapter(nil) adapter.PopulateFromSnapshot(models.StateSnapshot{ Nodes: []models.Node{{ @@ -16930,8 +16931,9 @@ func TestContract_ConnectionSystemMembersIgnoreOwnedPhysicalDisks(t *testing.T) Model: "WD_BLACK SN7100 4TB", Serial: "SN7100-AURORA", Type: "nvme", - Temperature: 63, + Temperature: 41, Health: "PASSED", + Attributes: &models.SMARTAttributes{ReallocatedSectors: &reallocatedSectors}, }}, }, }}, From a570d9f335cf8cd13a9df00a123a52234ec04a23 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:48:47 +0100 Subject: [PATCH 42/53] Say the alert card's 16rem floor covers its icon column The frontend-primitives note said the card's text column keeps a 16rem floor, but the floor sits on the left column that holds the status icon and the text, so the text itself gets about 230px before wrapping. --- .../v6/internal/subsystems/frontend-primitives.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 93e7cbcca1..53b15f10dd 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -39,8 +39,9 @@ guesses or the breach message. `useAlertHistoryState` exposes its active-alert accessor so the incident panel matches its open occurrence to the live alert without a second store read, and the panel reads `useRelativeTimeNow` so a reading that stops updating turns stale on screen. From the `sm` breakpoint the -card's text column keeps a 16rem floor and breaks long words, so the action -buttons wrap instead of squeezing the reading to its longest word beside them. +card's left column (status icon and text) keeps a 16rem floor and the text +breaks long words, so the action buttons wrap instead of squeezing the reading +to its longest word beside them. ### Canonical drawer History preserves guest read provenance From 04e4f64a6e1b0fa7f02c093624e2bfdbe659831b Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:49:00 +0100 Subject: [PATCH 43/53] Pin the per-change presentation wrapper in the resource type guard The moved-alert change routes the resource change summary and the drawer timeline through getResourceChangePresentation, which wraps getResourceChangeKindPresentation so an alert close that was not a recovery reads as a move. The raw-source boundary guard still required the summary component to name the kind helper, so the required Frontend check failed on that one assertion. The guard now pins the wrapper call in both readers, keeps the summary off the bare kind call, and requires the shared module to export the wrapper. --- .../__tests__/frontendResourceTypeBoundaries.test.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/frontend-modern/src/utils/__tests__/frontendResourceTypeBoundaries.test.ts b/frontend-modern/src/utils/__tests__/frontendResourceTypeBoundaries.test.ts index dd056e05e3..95d18b5412 100644 --- a/frontend-modern/src/utils/__tests__/frontendResourceTypeBoundaries.test.ts +++ b/frontend-modern/src/utils/__tests__/frontendResourceTypeBoundaries.test.ts @@ -2143,12 +2143,22 @@ describe('frontend resource type boundaries', () => { expect(resourceDetailDrawerSource).toContain('getResourceChangeKindPresentation'); expect(resourceDetailDrawerSource).toContain('getResourceChangeSourceTypePresentation'); expect(resourceDetailDrawerSource).toContain('getResourceChangeSourceAdapterPresentation'); - expect(resourceChangeSummarySource).toContain('getResourceChangeKindPresentation'); + // Per-change rows read getResourceChangePresentation, which wraps the + // kind presentation so an alert close that was not a recovery reads as + // a move. + expect(resourceDetailDrawerSource).toContain('getResourceChangePresentation(change)'); + expect(resourceChangeSummarySource).toContain('getResourceChangePresentation(change)'); + expect(resourceChangeSummarySource).not.toContain( + 'getResourceChangeKindPresentation(change.kind)', + ); expect(resourceChangeSummarySource).toContain('getResourceChangeSourceTypePresentation'); expect(resourceChangeSummarySource).toContain('getResourceChangeSourceAdapterPresentation'); expect(resourceDetailDrawerSource).not.toContain('healthToneClass('); expect(resourceDetailDrawerSource).not.toContain('normalizeHealthLabel('); expect(resourceChangePresentationSource).toContain('getResourceChangeKindPresentation'); + expect(resourceChangePresentationSource).toContain( + 'export function getResourceChangePresentation', + ); expect(resourceChangePresentationSource).toContain('getResourceChangeSourceTypePresentation'); expect(resourceChangePresentationSource).toContain( 'getResourceChangeSourceAdapterPresentation', From 79b6a0974e7ba963747ad88e703f3d8e8a72c2b2 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:57:20 +0100 Subject: [PATCH 44/53] Name disk wearout as life remaining and log disk alerts once The Proxmox disk wearout alert read "SSD has less than 10% life remaining (7% wearout)". The number is percent life remaining, so "7% wearout" reads as 7% worn and contradicts the first clause, and the Alerts overview shows that text verbatim. It now reads "SSD life remaining is 7%", the wording host-agent SMART wearout alerts already use. The threshold stays in the alert's Threshold field, which the card and notifications show beside the message; spelling it into the sentence as "below 10%" would be false at exactly 10%, which alerts. CheckDiskHealth also logged a failed disk at error and a worn disk at warn on every evaluation, not when the alert opened. The physical disk poll runs every 5 minutes by default, so one failed disk logged an error 288 times a day for as long as it stayed failed, and mock mode repeated it on every alert tick. Both now log at error or warn only on the evaluation that opens the alert and at debug afterwards. The new log test captures through a hook installed in TestMain: assigning log.Logger inside a test would race with managers that earlier tests leave running. The alerts contract also said wearout recovery needs readings of at least 10%; a 10% reading still matches, so recovery needs readings above it. --- .../v6/internal/subsystems/alerts.md | 24 +++- internal/alerts/alerts_test.go | 111 ++++++++++++++++++ internal/alerts/disk_health.go | 58 ++++++--- 3 files changed, 176 insertions(+), 17 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index e00fd188b7..1e96e73aae 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -407,9 +407,10 @@ own inline boundary. A wearout arm keyed on `> 0` silently exempts the single worst reading a disk can publish, which let a spent SSD read critical on the Physical Disks surface while raising no alert at all. -Proxmox wearout recovery requires three consecutive reported readings of at -least 10% remaining life. An absent reading does not prove recovery, and a -new low reading resets the recovery run; until recovery is confirmed the +Proxmox wearout recovery requires three consecutive reported readings above +10% remaining life; a reading of exactly 10% still matches the alert. An +absent reading does not prove recovery, and a new low reading resets the +recovery run; until recovery is confirmed the existing alert retains its low-life value and occurrence. This bounds the resolved-then-refired notification loop when endurance data briefly looks healthy (#2112), while a sustained replacement/corrected reading can still @@ -418,6 +419,23 @@ resolve. `TestCheckDiskHealthWearoutFlappingDoesNotRepeatNotifications` and deduplication and confirmed recovery. This is synthetic source proof, not a claim that the reporter's installed image has been repaired. +The Proxmox wearout value is percent life remaining, so its alert message names +it that way (`SSD life remaining is 7%`), matching the host-agent SMART +`wearout_low` summary; the threshold travels in the alert's `Threshold` field, +which the Alerts card and notifications render beside the message. Calling the +number "wearout" read as 7% worn and contradicted the low-life warning. +`CheckDiskHealth` logs a failed disk at error and a worn disk at warn only on +the evaluation that opens the alert (the canonical `activated` transition); +later polls that still find the disk failed or at or below the wearout +threshold, including mock mode's per-tick evaluation, log at debug. Logging on +every evaluation repeated one failed disk's error on every poll for as long as +it stayed failed. `TestCheckDiskHealthLowWearoutCreatesAlert` pins the message +and `TestCheckDiskHealthLogsAlertOpeningOnce` pins the log-once boundary at the +threshold, including a fresh error when a recovered disk fails again. That test +captures logs through a hook installed in the package `TestMain`, because +assigning `log.Logger` inside a test races with managers earlier tests left +running. + Host SMART counter growth is an event boundary rather than a warning on every historical non-zero value. For an agent-only disk, the first reported UDMA CRC error count establishes an in-memory baseline and raises no alert. A later diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go index 677d046d37..f67cbee911 100644 --- a/internal/alerts/alerts_test.go +++ b/internal/alerts/alerts_test.go @@ -25,6 +25,8 @@ import ( unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" "github.com/rcourtman/pulse-go-rewrite/internal/utils" "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" ) // testEnvMu protects concurrent access to PULSE_DATA_DIR during parallel tests. @@ -176,6 +178,61 @@ func newTestManager(t *testing.T) *Manager { return m } +// testLogCapture collects log events for captureTestLogs. TestMain installs +// its hook before any test runs: writing log.Logger inside a test would race +// with goroutines of managers that earlier tests left running. +var testLogCapture struct { + mu sync.Mutex + active bool + lines []capturedLogLine +} + +type capturedLogLine struct { + level zerolog.Level + message string +} + +func TestMain(m *testing.M) { + log.Logger = log.Logger.Hook(zerolog.HookFunc(func(_ *zerolog.Event, level zerolog.Level, message string) { + testLogCapture.mu.Lock() + defer testLogCapture.mu.Unlock() + if testLogCapture.active { + testLogCapture.lines = append(testLogCapture.lines, capturedLogLine{level: level, message: message}) + } + })) + os.Exit(m.Run()) +} + +// captureTestLogs records log events, debug included, until the test ends. +// The returned func counts the events seen so far with a level and message. +func captureTestLogs(t *testing.T) func(zerolog.Level, string) int { + t.Helper() + originalLevel := zerolog.GlobalLevel() + zerolog.SetGlobalLevel(zerolog.DebugLevel) + testLogCapture.mu.Lock() + testLogCapture.active = true + testLogCapture.lines = nil + testLogCapture.mu.Unlock() + t.Cleanup(func() { + testLogCapture.mu.Lock() + testLogCapture.active = false + testLogCapture.lines = nil + testLogCapture.mu.Unlock() + zerolog.SetGlobalLevel(originalLevel) + }) + return func(level zerolog.Level, message string) int { + testLogCapture.mu.Lock() + defer testLogCapture.mu.Unlock() + n := 0 + for _, line := range testLogCapture.lines { + if line.level == level && line.message == message { + n++ + } + } + return n + } +} + func TestAcknowledgePersistsThroughCheckMetric(t *testing.T) { m := newTestManager(t) m.ClearActiveAlerts() @@ -4498,6 +4555,10 @@ func TestCheckDiskHealthLowWearoutCreatesAlert(t *testing.T) { if alert.Threshold != 10.0 { t.Errorf("expected threshold 10.0, got %f", alert.Threshold) } + // Wearout is life remaining: the message must not read as 5% worn. + if want := "SSD life remaining is 5%"; alert.Message != want { + t.Errorf("message = %q, want %q", alert.Message, want) + } if got := alert.Metadata["canonicalAlertKind"]; got != "severity-threshold" { t.Fatalf("canonicalAlertKind = %v, want severity-threshold", got) } @@ -4554,6 +4615,56 @@ func TestCheckDiskHealthWearoutAlertUpdatesOnSubsequentChecks(t *testing.T) { if alert.Value != 6 { t.Errorf("expected value to be updated to 6, got %f", alert.Value) } + if want := "SSD life remaining is 6%"; alert.Message != want { + t.Errorf("message = %q, want %q", alert.Message, want) + } +} + +// The physical-disk poller re-evaluates every disk each cycle (and mock mode +// on every alert tick), so a failed or worn disk must log at error/warn once, +// when its alert opens, and at debug on every later poll. +func TestCheckDiskHealthLogsAlertOpeningOnce(t *testing.T) { + count := captureTestLogs(t) + m := newTestManager(t) + m.ClearActiveAlerts() + + // Exactly at the wearout threshold, which still alerts. + disk := proxmox.Disk{ + DevPath: "/dev/sdf", + Model: "Crucial CT500MX500SSD1", + Serial: "2034E4A1B2C3", + Type: "ssd", + Health: "FAILED", + Wearout: 10, + Size: 500107862016, + } + const polls = 3 + for i := 0; i < polls; i++ { + m.CheckDiskHealth("test-instance", "pve-node1", disk) + } + + if got := count(zerolog.ErrorLevel, "Disk health alert created"); got != 1 { + t.Errorf("error-level health log count after %d polls = %d, want 1", polls, got) + } + if got := count(zerolog.DebugLevel, "Disk health check still failing"); got != polls-1 { + t.Errorf("debug-level health log count after %d polls = %d, want %d", polls, got, polls-1) + } + if got := count(zerolog.WarnLevel, "Disk wearout alert created"); got != 1 { + t.Errorf("warn-level wearout log count after %d polls = %d, want 1", polls, got) + } + if got := count(zerolog.DebugLevel, "Disk life remaining still at or below wearout threshold"); got != polls-1 { + t.Errorf("debug-level wearout log count after %d polls = %d, want %d", polls, got, polls-1) + } + + // A disk that passes and then fails again opens a new alert, which logs + // at error again. + disk.Health = "PASSED" + m.CheckDiskHealth("test-instance", "pve-node1", disk) + disk.Health = "FAILED" + m.CheckDiskHealth("test-instance", "pve-node1", disk) + if got := count(zerolog.ErrorLevel, "Disk health alert created"); got != 2 { + t.Errorf("error-level health log count after the disk failed again = %d, want 2", got) + } } func TestCheckDiskHealthWearoutRecoveryAlertCleared(t *testing.T) { diff --git a/internal/alerts/disk_health.go b/internal/alerts/disk_health.go index 4afe7de5c4..5864403747 100644 --- a/internal/alerts/disk_health.go +++ b/internal/alerts/disk_health.go @@ -26,6 +26,10 @@ func proxmoxDiskAlertMetadata(disk proxmox.Disk) map[string]interface{} { } } +// proxmoxDiskWearoutThreshold is the percent life remaining at or below +// which a Proxmox SSD raises a wearout alert. +const proxmoxDiskWearoutThreshold = 10 + // CheckDiskHealth checks disk health and creates alerts if needed func (m *Manager) CheckDiskHealth(instance, node string, disk proxmox.Disk) { // Create unique alert ID for this disk @@ -84,7 +88,7 @@ func (m *Manager) CheckDiskHealth(instance, node string, disk proxmox.Disk) { metadata := proxmoxDiskAlertMetadata(disk) metadata["disk_health"] = disk.Health - _, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{ + result, _ := m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{ Spec: spec, Evidence: alertspecs.AlertEvidence{ ObservedAt: time.Now(), @@ -106,12 +110,22 @@ func (m *Manager) CheckDiskHealth(instance, node string, disk proxmox.Disk) { AddToHistory: true, }) - log.Error(). - Str("node", node). - Str("disk", disk.DevPath). - Str("model", disk.Model). - Str("health", disk.Health). - Msg("Disk health alert created") + // Every disk poll re-evaluates a failed disk, so log at error + // only when the alert opens, not on each poll after it. + if diskAlertOpened(result) { + log.Error(). + Str("node", node). + Str("disk", disk.DevPath). + Str("model", disk.Model). + Str("health", disk.Health). + Msg("Disk health alert created") + } else { + log.Debug(). + Str("node", node). + Str("disk", disk.DevPath). + Str("health", disk.Health). + Msg("Disk health check still failing") + } } } else { // Disk is healthy, clear alert if it exists @@ -122,8 +136,11 @@ func (m *Manager) CheckDiskHealth(instance, node string, disk proxmox.Disk) { // cannot prove recovery; 0 from an SSD is a real, fully spent reading, // whereas -1 and rotational 0 are absent evidence. if storagehealth.WearoutReported(disk.Wearout, disk.Type) { - message := fmt.Sprintf("SSD has less than 10%% life remaining (%d%% wearout)", disk.Wearout) - spec, err := buildCanonicalSeverityThresholdSpecWithDirection(canonicalResourceID+"-wearout", canonicalResourceID, resourceName, resourceType, "wearout-remaining", alertspecs.ThresholdDirectionBelow, 10, 0, false) + // Wearout is percent life remaining, not percent worn, so the + // message names it as life remaining, in the host-agent SMART + // wording. The threshold travels in the alert's Threshold field. + message := fmt.Sprintf("SSD life remaining is %d%%", disk.Wearout) + spec, err := buildCanonicalSeverityThresholdSpecWithDirection(canonicalResourceID+"-wearout", canonicalResourceID, resourceName, resourceType, "wearout-remaining", alertspecs.ThresholdDirectionBelow, proxmoxDiskWearoutThreshold, 0, false) if err != nil { log.Warn(). Err(err). @@ -134,7 +151,7 @@ func (m *Manager) CheckDiskHealth(instance, node string, disk proxmox.Disk) { metadata := proxmoxDiskAlertMetadata(disk) metadata["disk_wearout"] = disk.Wearout - _, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{ + result, _ := m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{ Spec: spec, // Endurance cannot normally improve on the same disk. Require // sustained healthy-looking polls before resolving a worn-disk @@ -156,20 +173,33 @@ func (m *Manager) CheckDiskHealth(instance, node string, disk proxmox.Disk) { Instance: instance, Message: message, Value: float64(disk.Wearout), - Threshold: 10.0, + Threshold: proxmoxDiskWearoutThreshold, Metadata: metadata, AddToRecent: true, AddToHistory: true, }) - if disk.Wearout < 10 { + if diskAlertOpened(result) { log.Warn(). Str("node", node). Str("disk", disk.DevPath). Str("model", disk.Model). - Int("wearout", disk.Wearout). - Msg("Disk wearout alert observed") + Int("lifeRemaining", disk.Wearout). + Msg("Disk wearout alert created") + } else if disk.Wearout <= proxmoxDiskWearoutThreshold { + log.Debug(). + Str("node", node). + Str("disk", disk.DevPath). + Int("lifeRemaining", disk.Wearout). + Msg("Disk life remaining still at or below wearout threshold") } } } } + +// diskAlertOpened reports whether this evaluation opened the alert, as +// opposed to re-evaluating one that is already active, pending, or +// suppressed. +func diskAlertOpened(result alertspecs.EvaluationResult) bool { + return result.Transition != nil && result.Transition.Kind == alertspecs.EvaluationTransitionActivated +} From 579ff5ca0f1a424b560f293462adfba0722043e7 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:10:52 +0100 Subject: [PATCH 45/53] Move static inline styles to classes the production CSP allows The production server sends style-src 'self' 'nonce-...' with no 'unsafe-inline'. Solid compiles static style values (style="..." or a literal entry of a style={{ ... }} object) into template HTML, so every such style is a style attribute the browser reports as a style-src-attr violation. Opening a row's inline detail on Machines or Docker Images logged eight of these, and the HelpIcon popover, the Assistant typing dots and the threshold slider logged their own. Chromium still applied the copied styles, so the visible effect there was console noise; whether a stricter browser applies them is not guaranteed. The Vite dev server allows inline styles, so none of this showed in development. The 24 drawer tab-panel overflow-anchor styles become the [overflow-anchor:none] class, the typing-dot delays become arbitrary animation-delay classes (animate-bounce already sets the 1s duration), the slider uses touch-none, the HelpIcon popover keeps its dynamic position in style and takes min-w-[200px], and the boot-failure fallback is built with DOM nodes, which also stops the error text being parsed as markup. SharedPrimitives.guardrails.test.ts now compiles every runtime .tsx that mentions style with the client-build Solid preset and fails on any style attribute or + + ); + `; + + expect(findTemplateStyleAttributes(fixture, 'Fixture.tsx')).toEqual([ + '
', + '', + '

', + '', + '', + '