From 5becf5953676d14ccdd99d3a289255ae1281bf7e Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:46:05 +0100 Subject: [PATCH 01/13] Remove the unreachable host metric cells from grouped workload rows Both production mounts of the Workloads surface, the Proxmox and vSphere overviews, set compactGroupHeaders unconditionally. WorkloadPanel's per-column node cells (CPU, memory, disk and I/O bars or sparklines, uptime, version, temperature) and NodeGroupHeader's column mode and inline host facts therefore never rendered. They were left over from the retired standalone Workloads page, whose group rows had no hosts table beside them. Drop the option, the dead renderers and the NodeGroupHeader props that only served them. Grouped host rows keep exactly the shape both pages already render: one spanning cell with status, linked name, cluster and agent badges, while host stats stay in each page's own hosts table. Tests that only pinned the option are removed. The NodeGroupHeader test now covers the spanning-cell shape and fails against the old default facts, and the PVE version formatter guardrail moves to proxmoxPageModel, the remaining version renderer. --- .../subsystems/frontend-primitives.md | 20 +- .../subsystems/performance-and-scalability.md | 30 +- .../internal/subsystems/storage-recovery.md | 5 + .../internal/subsystems/unified-resources.md | 5 +- .../8ada2c9fe2438ea7.json | 49 +++ .../components/Workloads/WorkloadPanel.tsx | 304 ------------------ .../components/Workloads/WorkloadsSurface.tsx | 1 - .../components/Workloads/WorkloadsTable.tsx | 4 - ...loadsSurface.performance.contract.test.tsx | 26 -- .../components/Workloads/useWorkloadsState.ts | 7 - .../src/components/shared/NodeGroupHeader.tsx | 71 +--- .../SharedPrimitives.guardrails.test.ts | 7 +- .../shared/__tests__/NodeGroupHeader.test.tsx | 39 +-- .../features/proxmox/ProxmoxPageSurface.tsx | 2 - .../ProxmoxPageSurface.contract.test.tsx | 10 + .../src/features/vmware/VmwarePageSurface.tsx | 2 - .../temperatureDisplayGuardrails.test.ts | 7 +- 17 files changed, 120 insertions(+), 469 deletions(-) create mode 100644 frontend-modern/browser-verification/8ada2c9fe2438ea7.json diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 85c281c011..6c07a49441 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -2911,9 +2911,9 @@ Agent`), with the plain-language source phrase available through accessible `pveVersion` or a Pulse Agent report whose OS identity resolves to Unraid or Proxmox VE. They must omit the version rather than showing unrelated collector OS versions, such as Debian 12, beside an API-backed PVE badge. - Shared row primitives that render Proxmox node identity, including - `frontend-modern/src/components/shared/NodeGroupHeader.tsx`, must route raw - PVE manager payloads through + Surfaces that render Proxmox node versions, including the Proxmox page + model at `frontend-modern/src/features/proxmox/proxmoxPageModel.ts`, must + route raw PVE manager payloads through `frontend-modern/src/utils/proxmoxVersion.ts` rather than inlining page-local parsing or falling back to unrelated agent OS versions. System title metadata must apply the same identity rule: once the primary @@ -4395,12 +4395,14 @@ The shared table chrome now allows `TableCardHeader` to expose a right-aligned action slot, currently used by the Workloads/Proxmox metric display control. That slot belongs to the table header band and must not reintroduce nested cards or page-local toolbar wrappers inside `TableCard`. Proxmox host grouping -also extends the shared `NodeGroupHeader` row pattern: host metrics may align -with workload table columns, but the shared primitive owns the header/table -shell boundary rather than platform pages copying their own card headers. -Compact PVE version text in that header must come from the shared Proxmox -version formatter so raw `pve-manager/...` payloads and platform-page host -version cells stay consistent. +also extends the shared `NodeGroupHeader` row pattern, which owns the +header/table shell boundary rather than platform pages copying their own card +headers. `frontend-modern/src/components/shared/NodeGroupHeader.tsx` renders +node identity only (status dot, linked name, cluster and agent badges); in +table-row mode that identity sits in one cell spanning the table. It carries no +per-column cell renderer and no inline +version, temperature, or uptime facts: host metrics and versions belong to the +platform page's host table, so a group row must not duplicate them. Mobile navigation now recognizes `proxmox` as a first-class platform tab in the shared priority model so app-shell ordering remains centralized. diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 87517e7399..5231c21704 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -1948,22 +1948,20 @@ query persistence once per resource, or run on the steady-state alert hot path; ancestor lookup remains bounded by canonical hierarchy depth for each active alert actually evaluated. -The embedded WorkloadsSurface exposes a `compactGroupHeaders` prop on -`frontend-modern/src/components/Workloads/useWorkloadsState.ts` that -platform pages owning their own hosts table (Proxmox overview today) set -to strip per-host metric cells out of the `NodeGroupHeader` rows in -grouped mode. The flag is threaded through -`frontend-modern/src/components/Workloads/WorkloadsSurface.tsx`, -`frontend-modern/src/components/Workloads/WorkloadsTable.tsx`, and -`frontend-modern/src/components/Workloads/WorkloadPanel.tsx`; when set, -`WorkloadPanel` calls `NodeGroupHeader` without `columns` / -`renderColumnCell` so the existing colspan layout renders just the -status dot, linked node name, cluster badge, and agent badge — the -duplicate CPU / Memory / Disk / uptime / temperature / version stats -the top-of-page hosts table already owns are dropped. New embedded -platform-page consumers that render their own hosts summary must set -this flag; standalone Workloads surfaces (no top hosts table) keep the -original verbose group rows by default. +Grouped host rows in the embedded WorkloadsSurface are identity-only +dividers. `frontend-modern/src/components/Workloads/WorkloadPanel.tsx` +renders each grouped node through `NodeGroupHeader` as one cell spanning +the table: status dot, linked node name, cluster badge, and agent badge. +Per-host CPU, memory, disk, network and disk I/O, uptime, temperature, +and version stats belong to the platform page's own hosts table (the +Proxmox nodes table today), so the workloads table must not render them +per column or inline in the group row. The former `compactGroupHeaders` +option and the per-column node metric cells it switched off were removed +because both remaining production mounts (Proxmox and vSphere overview) +set it unconditionally, so the non-compact path no longer rendered +anywhere. A new embedded consumer that needs host +stats beside its workloads must render its own hosts table rather than +re-growing metric cells in the shared group row. WorkloadsSurface stays monitoring-first. It must not render a persistent aggregate banner just because running VMs or system containers lack an diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index cd2ec5bf9e..fb2158b9ac 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -970,6 +970,11 @@ the page supplies it through the Workloads `tableTitle` slot and the shared states. This heading alignment changes neither workload backup scope nor the storage/recovery evidence carried by the adjacent Backup column and Backups tab. +The guest table's grouped node rows are identity-only dividers. Node stats and +the node drawer stay with the Proxmox nodes table above it, so `ProxmoxPageSurface` +passes `groupNodeDrawerMode: 'disabled'` and no host-metric option to the +embedded Workloads state. Those dividers carry no backup or recovery evidence; +the per-guest Backup column remains the overview's protection signal. The overview's node-row action selects the shared workload node scope and reveals the guest section. The title names the selected node and reports the filtered guest count, including zero. Activating the selected node again or diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index f3c5938c4c..d505a37b64 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -2973,7 +2973,10 @@ canonical WebSocket snapshot, including the public `vmware-vsphere` alias for the raw `vmware` source, rather than degrading a source-scoped page to periodic REST refreshes. VMware Overview passes that same source-scoped snapshot into the embedded Workloads state, so hosts and VMs share one inventory generation -and one explicit refresh path. +and one explicit refresh path. Beyond disabling the grouped host drawer, it +passes no option that adds host metrics to grouped rows: the shared Workloads +group row carries host identity only, and per-host stats stay in the page's +own hosts table. Its page-owned workload toolbar consumes the complete shared `getWorkloadsMetricFilterProps` binding, so vSphere VMs expose the same Bars, Trends, Details, History, range, and first-use discovery contract as every diff --git a/frontend-modern/browser-verification/8ada2c9fe2438ea7.json b/frontend-modern/browser-verification/8ada2c9fe2438ea7.json new file mode 100644 index 0000000000..4f8e022c8f --- /dev/null +++ b/frontend-modern/browser-verification/8ada2c9fe2438ea7.json @@ -0,0 +1,49 @@ +{ + "version": 1, + "base_sha": "c3adf0c7b9fa7f4283f7fd832a806659b009b51b", + "verified_at": "2026-10-06T21:45:33Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Workloads/WorkloadPanel.tsx", + "frontend-modern/src/components/Workloads/WorkloadsSurface.tsx", + "frontend-modern/src/components/Workloads/WorkloadsTable.tsx", + "frontend-modern/src/components/Workloads/useWorkloadsState.ts", + "frontend-modern/src/components/shared/NodeGroupHeader.tsx", + "frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx", + "frontend-modern/src/features/vmware/VmwarePageSurface.tsx" + ], + "content_sha256": { + "frontend-modern/src/components/Workloads/WorkloadPanel.tsx": "67c2c51ffceb96893f968dc141dbce54f0affe66fb389ffeeba8c8c94988968c", + "frontend-modern/src/components/Workloads/WorkloadsSurface.tsx": "e49032445f566b4e6a4329314f4ada41ed6f7652ba52e9e3fc413047a49f7878", + "frontend-modern/src/components/Workloads/WorkloadsTable.tsx": "a340532f9e3e0e28d83e9a0fb369c8be973acb0dea7b321fd646345ff924db38", + "frontend-modern/src/components/Workloads/useWorkloadsState.ts": "a29034c5149b7323e86bc2fe795d9b18db805b1ad32e32013fb71547256d61f5", + "frontend-modern/src/components/shared/NodeGroupHeader.tsx": "e435c1e6ab04b85a022459cf19a67645bc8339f15ef986ffcb5b42ba27ce667f", + "frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx": "80c860be3bee37251c16d21044074267e48991b9cb39b87d335cb2155066ec96", + "frontend-modern/src/features/vmware/VmwarePageSurface.tsx": "6c74063a298a259ce5f60485174529e9bffbe7293ae0710f53dc95d6f3c11265" + }, + "routes": ["/proxmox/overview", "/vmware"], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Production vite build of this tree (rebased on c3adf0c7b9) embedded in a private DEMO_MODE mock backend (50 PVE nodes, 929 guests, 7 ESXi hosts), headless Chromium via Playwright; desktop 1440x900 and phone 390x844.", + "Proxmox Overview guest table, Grouped layout: every grouped node row (8 desktop, 2 phone in the rendered window) is one td whose colspan equals the guest table's visible header count (8 desktop, 5 phone), with status dot, linked node name, cluster badge and Agent badge only; no data-workload-col cells, no PVE version, temperature or uptime text.", + "vSphere Overview VM table: the 7 host group rows render as one spanning cell (colspan 6 desktop, 5 phone) with host name and cluster badge, unchanged shape.", + "Bars and Trends metric modes both inspected: Trends adds four sparklines per guest row (Proxmox 140 to 560 images, vSphere 18 to 72) while each grouped host row keeps exactly its one status-dot image.", + "Screenshots of grouped rows in both modes and of open guest drawers inspected at both viewports; no page errors.", + "This is the shape both pages already rendered on main, where compactGroupHeaders was always set; it is not a check of host drawers, which both pages disable." + ], + "interactions": [ + "Opened the View popover (desktop) and the Filters popover (phone) and toggled Bars to Trends and back.", + "Clicked a grouped host row on each page: no inline node drawer row appears and nothing else changes.", + "Clicked the first guest row under a group on each page: its inline guest drawer opens.", + "Scrolled the first grouped host row to the viewport centre before each capture." + ] +} diff --git a/frontend-modern/src/components/Workloads/WorkloadPanel.tsx b/frontend-modern/src/components/Workloads/WorkloadPanel.tsx index 9caf56f762..de8a3cab1d 100644 --- a/frontend-modern/src/components/Workloads/WorkloadPanel.tsx +++ b/frontend-modern/src/components/Workloads/WorkloadPanel.tsx @@ -1,10 +1,6 @@ import { createMemo, For, Show } from 'solid-js'; import { ComponentErrorBoundary } from '@/components/ErrorBoundary'; -import { EnhancedCPUBar } from '@/components/Workloads/EnhancedCPUBar'; -import { MetricMiniSparkline } from '@/components/Workloads/MetricMiniSparkline'; -import { StackedDiskBar } from '@/components/Workloads/StackedDiskBar'; -import { StackedMemoryBar } from '@/components/Workloads/StackedMemoryBar'; import { GROUPED_TABLE_ROW_BADGE_CLASS, getGroupedTableRowCellClass, @@ -16,12 +12,9 @@ import { NodeGroupHeader } from '@/components/shared/NodeGroupHeader'; import { createSummaryInteractiveRowPreviewHandlers } from '@/components/shared/summaryInteractionA11y'; import { buildSummaryDisclosureControlsId } from '@/components/shared/summaryInteractionA11y'; import { TableBody, TableCell, TableRow } from '@/components/shared/Table'; -import type { Node } from '@/types/api'; import { getAlertsForResource, getAlertStyles } from '@/utils/alerts'; import { guestOverrideIdCandidates } from '@/features/alerts/guestOverrideIdentity'; -import { formatSpeed, formatUptime } from '@/utils/format'; import { isNodeOnline } from '@/utils/status'; -import { formatTemperature, getCpuTemperature, getTemperatureTextClass } from '@/utils/temperature'; import { getCanonicalWorkloadId, getWorkloadMetadataIdCandidates } from '@/utils/workloads'; import { resolveSummaryGroupMemberInteractionState, @@ -34,14 +27,12 @@ import { NodeDrawer } from './NodeDrawer'; import { buildWorkloadSummaryGroupScope } from './workloadSelectors'; import { getWorkloadGuestMetadataRecord } from './workloadGuestMetadataRecord'; import type { WorkloadsState } from './useWorkloadsState'; -import type { WorkloadTableMetric } from './workloadMetricHistoryModel'; type WorkloadPanelProps = Pick< WorkloadsState, | 'activeAlerts' | 'alertsEnabled' | 'bottomSpacerHeight' - | 'compactGroupHeaders' | 'getGroupLabel' | 'getNodeTemperatureThresholds' | 'groupedGuests' @@ -58,7 +49,6 @@ type WorkloadPanelProps = Pick< | 'focusedSummaryWorkloadGroupScope' | 'focusedSummaryWorkloadGroupId' | 'hoveredSummaryWorkloadGroupScope' - | 'isMobile' | 'nestedWorkloadContextByGuestId' | 'nodeByInstance' | 'search' @@ -80,289 +70,9 @@ type WorkloadPanelProps = Pick< | 'workloadTableLayoutMode' | 'workloadTableVisibleColumnIds' | 'workloadColumnWidths' - | 'workloadTableVisibleColumns' >; -const GROUP_NODE_METRIC_CELL_CLASS = 'px-1.5 sm:px-2 py-0.5 align-middle'; -const GROUP_NODE_NAME_CELL_CLASS = getGroupedTableRowCellClass( - 'py-1! pl-2! sm:pl-3! pr-1.5! sm:pr-2!', -); - -const getGroupNodeColumnCellClass = (_columnId: string, isNameColumn: boolean): string => { - if (isNameColumn) return GROUP_NODE_NAME_CELL_CLASS; - return GROUP_NODE_METRIC_CELL_CLASS; -}; - -const normalizeMetricPercent = (value: number | null | undefined): number => { - if (typeof value !== 'number' || !Number.isFinite(value)) return 0; - return value <= 1 ? Math.max(0, value * 100) : Math.max(0, value); -}; - -const formatMetricPercent = (value: number | null | undefined): string => - `${Math.round(normalizeMetricPercent(value))}%`; - -const getUsedPercent = (used?: number | null, total?: number | null): number => { - if (typeof used !== 'number' || typeof total !== 'number' || total <= 0) return 0; - return normalizeMetricPercent((used / total) * 100); -}; - -const getGroupNodeMetricKey = (node: Node): string => node.linkedAgentId || node.id || node.name; - -const getGroupNodePveVersion = (node: Node): string => { - const version = (node.pveVersion || '').trim(); - if (!version || version.toLowerCase() === 'unknown') return ''; - return ( - version.match(/pve-manager\/([^/\s]+)/i)?.[1] || version.match(/\d+(?:\.\d+)+/)?.[0] || version - ); -}; - -const renderGroupNodeEmptyCell = () => ( -
- -
-); - export function WorkloadPanel(props: WorkloadPanelProps) { - const renderGroupNodeSparkline = ( - node: Node, - metric: WorkloadTableMetric, - valueLabel: string, - title: string, - unit = '%', - valueLabelMode: 'inline' | 'tooltip' | 'hidden' = 'inline', - formatValue?: (value: number) => string, - ) => ( - - ); - - const renderGroupNodeRatePair = ( - leftLabel: string, - leftClass: string, - leftValue: number | null | undefined, - rightLabel: string, - rightClass: string, - rightValue: number | null | undefined, - ) => { - const left = Math.max(0, leftValue ?? 0); - const right = Math.max(0, rightValue ?? 0); - if (left <= 0 && right <= 0) return renderGroupNodeEmptyCell(); - - return ( -
- {leftLabel} - - {formatSpeed(left)} - - {rightLabel} - - {formatSpeed(right)} - -
- ); - }; - - const renderGroupNodeInfoCell = (node: Node) => { - const version = getGroupNodePveVersion(node); - const temperature = getCpuTemperature(node.temperature); - if (!version && temperature === null) return renderGroupNodeEmptyCell(); - - return ( -
- - PVE {version} - - - - {formatTemperature(temperature)} - - -
- ); - }; - - const renderGroupNodeColumnCell = (columnId: string, node: Node) => { - switch (columnId) { - case 'type': - return ( -
- - PVE - -
- ); - case 'info': - case 'vmid': - return renderGroupNodeInfoCell(node); - case 'cpu': - if (props.workloadMetricDisplayMode() === 'sparklines') { - return renderGroupNodeSparkline( - node, - 'cpu', - formatMetricPercent(node.cpu), - `${node.name} CPU history`, - '%', - 'inline', - formatMetricPercent, - ); - } - return ( -
- -
- ); - case 'memory': - if (props.workloadMetricDisplayMode() === 'sparklines') { - return renderGroupNodeSparkline( - node, - 'memory', - node.memory?.usageUnavailable - ? 'N/A' - : formatMetricPercent(getUsedPercent(node.memory?.used, node.memory?.total)), - `${node.name} memory history`, - '%', - 'inline', - formatMetricPercent, - ); - } - return ( -
- -
- ); - case 'disk': - if (props.workloadMetricDisplayMode() === 'sparklines') { - return renderGroupNodeSparkline( - node, - 'disk', - formatMetricPercent(node.disk?.usage), - `${node.name} disk usage history`, - '%', - 'inline', - formatMetricPercent, - ); - } - return ( -
- -
- ); - case 'uptime': - return ( -
- 0} fallback={renderGroupNodeEmptyCell()}> - 0 && node.uptime < 3600 ? 'text-orange-500' : 'text-muted' - }`} - > - {formatUptime(node.uptime, props.isMobile())} - - -
- ); - case 'node': - case 'context': - return ( -
- - {node.name} - -
- ); - case 'os': - return ( -
- PVE -
- ); - case 'netIo': - if (props.workloadMetricDisplayMode() === 'sparklines') { - return renderGroupNodeSparkline( - node, - 'netIo', - `${formatSpeed(node.networkIn ?? 0)} / ${formatSpeed(node.networkOut ?? 0)}`, - `${node.name} network I/O history`, - 'B/s', - 'tooltip', - formatSpeed, - ); - } - return renderGroupNodeRatePair( - '↓', - 'text-emerald-500', - node.networkIn, - '↑', - 'text-orange-400', - node.networkOut, - ); - case 'diskIo': - if (props.workloadMetricDisplayMode() === 'sparklines') { - return renderGroupNodeSparkline( - node, - 'diskIo', - `${formatSpeed(node.diskRead ?? 0)} / ${formatSpeed(node.diskWrite ?? 0)}`, - `${node.name} disk I/O history`, - 'B/s', - 'tooltip', - formatSpeed, - ); - } - return renderGroupNodeRatePair( - 'R', - 'font-mono text-blue-500', - node.diskRead, - 'W', - 'font-mono text-amber-500', - node.diskWrite, - ); - default: - return renderGroupNodeEmptyCell(); - } - }; - return ( 0}> @@ -485,22 +195,8 @@ export function WorkloadPanel(props: WorkloadPanelProps) { > td>div]:flex-nowrap' diff --git a/frontend-modern/src/components/Workloads/WorkloadsSurface.tsx b/frontend-modern/src/components/Workloads/WorkloadsSurface.tsx index 0f413620b6..fe11021d39 100644 --- a/frontend-modern/src/components/Workloads/WorkloadsSurface.tsx +++ b/frontend-modern/src/components/Workloads/WorkloadsSurface.tsx @@ -192,7 +192,6 @@ export function WorkloadsSurface(props: WorkloadsSurfaceComponentProps) { activeAlerts={state.activeAlerts} alertsEnabled={state.alertsEnabled} bottomSpacerHeight={state.bottomSpacerHeight} - compactGroupHeaders={state.compactGroupHeaders} getGroupLabel={state.getGroupLabel} getNodeTemperatureThresholds={state.getNodeTemperatureThresholds} groupedGuests={state.groupedGuests} diff --git a/frontend-modern/src/components/Workloads/WorkloadsTable.tsx b/frontend-modern/src/components/Workloads/WorkloadsTable.tsx index 2f25d5598b..5193c87d1c 100644 --- a/frontend-modern/src/components/Workloads/WorkloadsTable.tsx +++ b/frontend-modern/src/components/Workloads/WorkloadsTable.tsx @@ -17,7 +17,6 @@ type WorkloadsTableProps = Pick< | 'activeAlerts' | 'alertsEnabled' | 'bottomSpacerHeight' - | 'compactGroupHeaders' | 'getGroupLabel' | 'getNodeTemperatureThresholds' | 'groupedGuests' @@ -133,7 +132,6 @@ export function WorkloadsTable(props: WorkloadsTableProps) { activeAlerts={props.activeAlerts} alertsEnabled={props.alertsEnabled} bottomSpacerHeight={props.bottomSpacerHeight} - compactGroupHeaders={props.compactGroupHeaders} getGroupLabel={props.getGroupLabel} getNodeTemperatureThresholds={props.getNodeTemperatureThresholds} groupedGuests={props.groupedGuests} @@ -150,7 +148,6 @@ export function WorkloadsTable(props: WorkloadsTableProps) { focusedSummaryWorkloadGroupScope={props.focusedSummaryWorkloadGroupScope} focusedSummaryWorkloadGroupId={props.focusedSummaryWorkloadGroupId} hoveredSummaryWorkloadGroupScope={props.hoveredSummaryWorkloadGroupScope} - isMobile={props.isMobile} nestedWorkloadContextByGuestId={props.nestedWorkloadContextByGuestId} nodeByInstance={props.nodeByInstance} search={props.search} @@ -171,7 +168,6 @@ export function WorkloadsTable(props: WorkloadsTableProps) { workloadMetricHistory={props.workloadMetricHistory} workloadTableLayoutMode={props.workloadTableLayoutMode} workloadTableVisibleColumnIds={props.workloadTableVisibleColumnIds} - workloadTableVisibleColumns={props.workloadTableVisibleColumns} workloadColumnWidths={props.workloadColumnWidths} /> diff --git a/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx b/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx index d2aa75fff9..7a272f8b5c 100644 --- a/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx @@ -454,32 +454,6 @@ describe('Workloads platform-page embed contract', () => { expect(nodesTableSource).toContain('isSparklineMode'); }); - it('exposes compactGroupHeaders so platform pages can strip duplicate host stats from group rows', async () => { - const stateSource = (await import('../useWorkloadsState.ts?raw')).default; - expect(stateSource).toContain('compactGroupHeaders?: boolean;'); - expect(stateSource).toContain('compactGroupHeaders: () => props.compactGroupHeaders === true,'); - - const tableSource = (await import('../WorkloadsTable.tsx?raw')).default; - expect(tableSource).toContain(`| 'compactGroupHeaders'`); - expect(tableSource).toContain('compactGroupHeaders={props.compactGroupHeaders}'); - - const panelSource = (await import('../WorkloadPanel.tsx?raw')).default; - expect(panelSource).toContain(`| 'compactGroupHeaders'`); - // When the flag is set the panel falls back to NodeGroupHeader's - // colspan layout: no per-column metric cells, no inline node facts. - expect(panelSource).toContain( - 'props.compactGroupHeaders() ? undefined : props.workloadTableVisibleColumns()', - ); - expect(panelSource).toContain( - 'props.compactGroupHeaders() ? undefined : renderGroupNodeColumnCell', - ); - expect(panelSource).toContain('!props.compactGroupHeaders() &&'); - - const proxmoxSource = (await import('../../../features/proxmox/ProxmoxPageSurface.tsx?raw')) - .default; - expect(proxmoxSource).toContain('compactGroupHeaders'); - }); - it('lets platform pages move grouped host drawers to a dedicated host table owner', async () => { const stateSource = (await import('../useWorkloadsState.ts?raw')).default; expect(stateSource).toContain("groupNodeDrawerMode?: 'inline' | 'disabled';"); diff --git a/frontend-modern/src/components/Workloads/useWorkloadsState.ts b/frontend-modern/src/components/Workloads/useWorkloadsState.ts index e041d9d20c..195099789b 100644 --- a/frontend-modern/src/components/Workloads/useWorkloadsState.ts +++ b/frontend-modern/src/components/Workloads/useWorkloadsState.ts @@ -105,12 +105,6 @@ export interface WorkloadsSurfaceProps { }>; resourceSnapshotRefetch?: () => Promise; statusModeStorageScope?: string; - // Platform pages that render their own hosts table above the embedded - // workloads surface (e.g. Proxmox overview) own the per-host CPU / Memory - // / Disk / Temperature / uptime / version stats. Setting - // `compactGroupHeaders` strips those stats from the NodeGroupHeader rows - // in grouped mode so the section dividers don't duplicate the info. - compactGroupHeaders?: boolean; // Default Workloads behavior owns grouped host row drawers inline. Platform // pages with a dedicated host table can disable that drawer so host details // open from the host-owned table instead of the embedded guest table. @@ -728,7 +722,6 @@ export function useWorkloadsState(props: WorkloadsSurfaceProps) { workloadsNoInventoryState, ws, groupingMode, - compactGroupHeaders: () => props.compactGroupHeaders === true, groupNodeDrawerMode: () => props.groupNodeDrawerMode ?? 'inline', groupLabelBadges, } as const; diff --git a/frontend-modern/src/components/shared/NodeGroupHeader.tsx b/frontend-modern/src/components/shared/NodeGroupHeader.tsx index 0c3b9feb69..2dfa8db0fc 100644 --- a/frontend-modern/src/components/shared/NodeGroupHeader.tsx +++ b/frontend-modern/src/components/shared/NodeGroupHeader.tsx @@ -1,12 +1,8 @@ -import { Component, For, Show, type JSX } from 'solid-js'; +import { Component, Show, type JSX } from 'solid-js'; import type { Node } from '@/types/api'; -import type { MetricDisplayThresholds } from '@/utils/metricThresholds'; import { getNodeDisplayName, getNodeExternalUrl, hasAlternateDisplayName } from '@/utils/nodes'; import { StatusDot } from '@/components/shared/StatusDot'; import { getNodeStatusIndicator } from '@/utils/status'; -import { formatUptime } from '@/utils/format'; -import { formatTemperature, getCpuTemperature, getTemperatureTextClass } from '@/utils/temperature'; -import { formatProxmoxVersion } from '@/utils/proxmoxVersion'; import { GROUPED_TABLE_ROW_BADGE_CLASS, getGroupedTableRowCellClass, @@ -17,13 +13,8 @@ import { ResourceNameWithWebInterfaceLink } from './WebInterfaceLink'; interface NodeGroupHeaderProps { node: Node; colspan?: number; - columns?: Array<{ id: string }>; - columnCellClass?: (columnId: string, isNameColumn: boolean) => string; leadingAction?: JSX.Element; - renderColumnCell?: (columnId: string, node: Node) => JSX.Element; renderAs?: 'tr' | 'div'; - showFactsInName?: boolean; - temperatureThresholds?: MetricDisplayThresholds | null; trClass?: string; trProps?: JSX.HTMLAttributes & Partial>; @@ -35,12 +26,6 @@ export const NodeGroupHeader: Component = (props) => { const nodeUrl = () => getNodeExternalUrl(props.node); const displayName = () => getNodeDisplayName(props.node); const showActualName = () => hasAlternateDisplayName(props.node); - const pveVersion = () => formatProxmoxVersion(props.node.pveVersion); - const cpuTemperature = () => getCpuTemperature(props.node.temperature); - const hasNodeFacts = () => - Boolean(pveVersion()) || - cpuTemperature() !== null || - (typeof props.node.uptime === 'number' && props.node.uptime > 0); const InnerContent = () => (
= (props) => { Agent - - - -
); @@ -118,38 +84,9 @@ export const NodeGroupHeader: Component = (props) => { } > - 0 && Boolean(props.renderColumnCell)} - fallback={ - - - - } - > - - {(column) => { - const isNameColumn = () => column.id === 'name'; - return ( - - - - - - ); - }} - - + + +
); diff --git a/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts b/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts index 679ab96c40..7697db0332 100644 --- a/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts +++ b/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts @@ -216,6 +216,7 @@ import proxmoxCoverageTableSource from '@/features/proxmox/ProxmoxCoverageTable. import proxmoxMailGatewayTableSource from '@/features/proxmox/ProxmoxMailGatewayTable.tsx?raw'; import proxmoxNodesTableSource from '@/features/proxmox/ProxmoxNodesTable.tsx?raw'; import proxmoxHostTableModelSource from '@/features/proxmox/proxmoxHostTableModel.ts?raw'; +import proxmoxPageModelSource from '@/features/proxmox/proxmoxPageModel.ts?raw'; import proxmoxRecoverableTableSource from '@/features/proxmox/ProxmoxRecoverableTable.tsx?raw'; import proxmoxReplicationTableSource from '@/features/proxmox/ProxmoxReplicationTable.tsx?raw'; import vsphereHostsTableSource from '@/features/vmware/VsphereHostsTable.tsx?raw'; @@ -1406,9 +1407,9 @@ describe('shared primitive guardrails', () => { }); it('routes Proxmox node version presentation through the shared formatter', () => { - expect(nodeGroupHeaderSource).toContain("from '@/utils/proxmoxVersion'"); - expect(nodeGroupHeaderSource).toContain('formatProxmoxVersion(props.node.pveVersion)'); - expect(nodeGroupHeaderSource).not.toContain('pve-manager\\/'); + expect(proxmoxPageModelSource).toContain("from '@/utils/proxmoxVersion'"); + expect(proxmoxPageModelSource).toContain('formatProxmoxVersion(resource.proxmox?.pveVersion)'); + expect(proxmoxPageModelSource).not.toContain('pve-manager\\/'); expect(proxmoxVersionSource).toContain('formatProxmoxVersion'); expect(proxmoxVersionSource).toContain('pve-manager\\/'); diff --git a/frontend-modern/src/components/shared/__tests__/NodeGroupHeader.test.tsx b/frontend-modern/src/components/shared/__tests__/NodeGroupHeader.test.tsx index 88f23a0606..f31f9a4ea3 100644 --- a/frontend-modern/src/components/shared/__tests__/NodeGroupHeader.test.tsx +++ b/frontend-modern/src/components/shared/__tests__/NodeGroupHeader.test.tsx @@ -1,9 +1,8 @@ import { render, screen } from '@solidjs/testing-library'; -import { beforeEach, describe, expect, it } from 'vitest'; +import { describe, expect, it } from 'vitest'; import { NodeGroupHeader } from '@/components/shared/NodeGroupHeader'; import type { Node } from '@/types/api'; -import { temperatureStore } from '@/utils/temperature'; const makeNode = (overrides: Partial = {}): Node => ({ id: 'cluster-a-pve1', @@ -34,18 +33,11 @@ const makeNode = (overrides: Partial = {}): Node => ({ }); describe('NodeGroupHeader', () => { - beforeEach(() => { - temperatureStore.setUnit('celsius'); - }); - - it('renders compact infrastructure facts in the grouped workload header', () => { + it('renders node identity in the grouped workload header', () => { render(() => ); expect(screen.getByText('pve1')).toBeInTheDocument(); expect(screen.getByText('homelab')).toBeInTheDocument(); - expect(screen.getByText('PVE 9.1.9')).toBeInTheDocument(); - expect(screen.getByText('62°C')).toBeInTheDocument(); - expect(screen.getByText('2h')).toBeInTheDocument(); const link = screen.getByRole('link', { name: 'Open web interface for pve1' }); expect(link).toHaveAttribute('href', 'https://pve1:8006'); @@ -53,20 +45,21 @@ describe('NodeGroupHeader', () => { expect(link).toHaveAttribute('rel', 'noopener noreferrer'); }); - it('colors grouped header temperatures from configured thresholds', () => { - render(() => ( - + it('spans the table in one cell and leaves host stats to the platform host table', () => { + const { container } = render(() => ( + + + + +
)); - expect(screen.getByText('76°C')).toHaveClass('text-green-600'); + const cells = container.querySelectorAll('tr > td'); + expect(cells).toHaveLength(1); + expect(cells[0]).toHaveAttribute('colspan', '9'); + expect(cells[0]).toHaveTextContent('pve1'); + expect(screen.queryByText('PVE 9.1.9')).not.toBeInTheDocument(); + expect(screen.queryByText(/°[CF]/)).not.toBeInTheDocument(); + expect(screen.queryByText('2h')).not.toBeInTheDocument(); }); }); diff --git a/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx b/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx index 0cf4d4c4ba..e34418e333 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx @@ -495,7 +495,6 @@ function ProxmoxOverview(props: ProxmoxOverviewProps) { // surface, so label it with the platform vocabulary instead of 'Info'. columnLabelOverrides: { info: 'ID' }, statusModeStorageScope: PROXMOX_WORKLOAD_STATUS_STORAGE_SCOPE, - compactGroupHeaders: true, groupNodeDrawerMode: 'disabled', metricDisplayMode: props.metricDisplayMode, onMetricDisplayModeChange: props.setMetricDisplayMode, @@ -638,7 +637,6 @@ function ProxmoxOverview(props: ProxmoxOverviewProps) { forcedPlatform={PROXMOX_PLATFORM_FILTER} excludedWorkloadTypes={PROXMOX_WORKLOAD_EXCLUDED_TYPES} showNestedExcludedWorkloads - compactGroupHeaders groupNodeDrawerMode="disabled" suppressFilterToolbar emptyStateTitle="No Proxmox workloads" diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx b/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx index 0979b3a487..198c6b273c 100644 --- a/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx +++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx @@ -214,6 +214,16 @@ describe('ProxmoxPageSurface contract', () => { ); }); + it('leaves host details to the nodes table instead of the embedded guest table', () => { + setResources([makeResource({ id: 'vm-1', type: 'vm' })]); + + renderSurface(); + + expect(mockWorkloadsOptions).toHaveBeenCalledWith( + expect.objectContaining({ groupNodeDrawerMode: 'disabled' }), + ); + }); + afterEach(() => { cleanup(); vi.clearAllMocks(); diff --git a/frontend-modern/src/features/vmware/VmwarePageSurface.tsx b/frontend-modern/src/features/vmware/VmwarePageSurface.tsx index 0e8d015897..1a34ea8be5 100644 --- a/frontend-modern/src/features/vmware/VmwarePageSurface.tsx +++ b/frontend-modern/src/features/vmware/VmwarePageSurface.tsx @@ -474,7 +474,6 @@ function VmwareOverview(props: VmwareOverviewProps) { statusModeStorageScope: VMWARE_WORKLOAD_STATUS_STORAGE_SCOPE, columnVisibilityStorageScope: VMWARE_WORKLOAD_COLUMN_VISIBILITY_SCOPE, additionalDefaultHiddenColumnIds: [...VMWARE_WORKLOAD_DEFAULT_HIDDEN_COLUMN_IDS], - compactGroupHeaders: true, inventorySourcesQuery: props.inventorySourcesQuery, groupNodeDrawerMode: 'disabled', metricDisplayMode: props.metricDisplayMode, @@ -545,7 +544,6 @@ function VmwareOverview(props: VmwareOverviewProps) { useWorkloads forcedPlatform={VMWARE_PLATFORM_FILTER} forcedViewMode="vm" - compactGroupHeaders groupNodeDrawerMode="disabled" suppressFilterToolbar emptyStateTitle="No vSphere VMs" diff --git a/frontend-modern/src/utils/__tests__/temperatureDisplayGuardrails.test.ts b/frontend-modern/src/utils/__tests__/temperatureDisplayGuardrails.test.ts index 3f8e8898e8..37c1175902 100644 --- a/frontend-modern/src/utils/__tests__/temperatureDisplayGuardrails.test.ts +++ b/frontend-modern/src/utils/__tests__/temperatureDisplayGuardrails.test.ts @@ -2,15 +2,15 @@ import { describe, expect, it } from 'vitest'; import nodeDrawerOverviewSource from '@/components/Workloads/NodeDrawerOverview.tsx?raw'; import workloadPanelSource from '@/components/Workloads/WorkloadPanel.tsx?raw'; -import nodeGroupHeaderSource from '@/components/shared/NodeGroupHeader.tsx?raw'; import temperatureGaugeSource from '@/components/shared/TemperatureGauge.tsx?raw'; import dockerHostDrawerOverviewSource from '@/features/docker/DockerHostDrawerOverview.tsx?raw'; import agentsMachinesTableSource from '@/features/standalone/AgentsMachinesTable.tsx?raw'; describe('temperature display guardrails', () => { it('keeps colored temperature displays threshold-aware', () => { - expect(nodeGroupHeaderSource).toContain('temperatureThresholds'); - expect(workloadPanelSource).toContain('getNodeTemperatureThresholds(node)'); + expect(workloadPanelSource).toContain( + 'temperatureThresholds={props.getNodeTemperatureThresholds(node()!)}', + ); expect(nodeDrawerOverviewSource).toContain('getTemperatureTextClass(value, thresholds)'); expect(dockerHostDrawerOverviewSource).toContain('temperatureThresholds()'); expect(agentsMachinesTableSource).toContain('getAgentMachineTemperatureMetric'); @@ -19,7 +19,6 @@ describe('temperature display guardrails', () => { }); it('does not reintroduce the previous local temperature severity calls', () => { - expect(nodeGroupHeaderSource).not.toContain('getTemperatureTextClass(cpuTemperature())'); expect(workloadPanelSource).not.toContain('getTemperatureTextClass(temperature)}'); expect(nodeDrawerOverviewSource).not.toContain('getTemperatureTextClass(value)'); expect(dockerHostDrawerOverviewSource).not.toContain('getTemperatureTextClass(temperature)'); From 94b7870855120b2c62fce19102903601b9fc4bfc Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:47:48 +0100 Subject: [PATCH 02/13] Call a disk that is only hot Running Hot, not Replace Now The Physical Disks Health verdict mapped any critical disk risk to "Replace Now". Disk risk also grades temperature (temperature_high at >=70C critical, >=60C warning), so a SMART-passed disk with good life running at 72C told the user to buy a new disk when the fix is airflow. Alerts already keep heat apart: the SMART health and wearout alerts skip temperature_high and raise disk heat as a Temperature metric. The verdict now splits risk reasons into heat and everything else. Heat alone reads "Running Hot" (phone "Hot"), red at the critical tier and amber at warning, with the temperature reason still as its title and drawer summary. FAILED health, any other critical reason, and a critical level no reason explains keep "Replace Now". The more severe class names the verdict and failure evidence wins a tie, so merged reasons that list heat first no longer hide pending sectors. Sorting and the health filter follow the same verdict, so hot disks stay in "Needs attention" at their temperature tier. A reason's code and severity count even when it has no display text. A critical level that no reason explains shows the generic summary rather than a weaker reason's text. Thresholds are unchanged. The tier still comes from the server's flat 60/70C rule, not the per-type alert thresholds that colour the Temp cell. Reconciling those is separate work. --- .../subsystems/frontend-primitives.md | 5 + .../internal/subsystems/storage-recovery.md | 20 ++ .../ad11a59541280777.json | 44 +++ .../Storage/__tests__/DiskList.test.tsx | 105 +++++++ .../__tests__/diskPresentation.test.ts | 290 ++++++++++++++++++ .../storageBackups/diskPresentation.ts | 178 +++++++++-- 6 files changed, 610 insertions(+), 32 deletions(-) create mode 100644 frontend-modern/browser-verification/ad11a59541280777.json diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index b741322660..6fb37dbe06 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -412,6 +412,11 @@ disk header and tooltips. Structured ZFS scan activity supplies a compact badge only during reported rebuild activity; its complete provider summary remains available. Stable disk History catalog/organisation/access ownership is unchanged. +The physical-disk verdict words and their phone forms are `Needs Attention` +(`Attention`), `Running Hot` (`Hot`) and `Replace Now` (`Replace`), all from +`getPhysicalDiskHealthCompactLabel` in `diskPresentation.ts`. A hot disk keeps +the same label at either tier; its tone carries the severity. + Controller expansions carry the supplied kind-specific targets, absolute times, duration and cluster/namespace when the phone row omits those columns. Proxmox coverage distinguishes independent backups from guest-local snapshots; By date diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index cd2ec5bf9e..85d3b643e8 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -6727,6 +6727,26 @@ disk/pool fixture `frontend-modern/browser-tests/history-touch.cjs` and mounted `HistoryChart.test.tsx` regressions verify presentation/input behaviour only, not live appliance collection or recovery success. +### Physical-disk heat is not a replacement verdict + +The Physical Disks Health verdict names the action it asks for. Risk whose only +reason is `temperature_high` reads `Running Hot` (compact `Hot`), red at the +critical tier and amber at warning, with the temperature reason as its summary. +`Replace Now` stays reserved for FAILED health, any other critical reason, and +a critical level that no listed reason explains. `Needs Attention` keeps the +remaining warning evidence. When heat and other evidence both apply, the more +severe class names the verdict and failure evidence wins a tie, whatever order +merged reasons arrive in. Sorting follows the verdict (replace, critical heat, +attention, warning heat), and the health filter places a hot disk at its +temperature tier, so `Needs attention` still lists it. This matches alerting, +which raises disk heat as a Temperature metric and never as disk health. +Thresholds are unchanged: the tier still comes from the server's disk risk +(flat 60/70C), not the per-type alert thresholds that colour the Temp column. +Reconciling those is separate work. `frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts` +and `frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx` cover +the split, merged reason order, the unexplained critical level, sorting and +filtering. + ### Pool-to-physical-disk ownership in Storage details `storagePoolDetailPresentation.ts` must scope inferred ZFS device and UnRAID diff --git a/frontend-modern/browser-verification/ad11a59541280777.json b/frontend-modern/browser-verification/ad11a59541280777.json new file mode 100644 index 0000000000..f0678d07dd --- /dev/null +++ b/frontend-modern/browser-verification/ad11a59541280777.json @@ -0,0 +1,44 @@ +{ + "version": 1, + "base_sha": "8259a74c9d0503df630b09c12a2c1be906fa3d78", + "verified_at": "2026-10-06T22:20:00Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/features/storageBackups/diskPresentation.ts" + ], + "content_sha256": { + "frontend-modern/src/features/storageBackups/diskPresentation.ts": "87469b0d62a31a1657a94a5f645041ad34987cc5af55ca238c6cbf49b0c068ee" + }, + "routes": [ + "/proxmox/storage?tab=disks", + "/proxmox/storage?tab=disks&status=attention", + "/proxmox/storage?tab=disks&status=critical", + "/proxmox/storage?tab=disks&status=warning" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack (8 nodes) whose generator was overlaid locally, never committed, to pin readings from a control file on pve3: Crucial MX500 2TB /dev/nvme2 (SMART PASSED) at 72C, WD Blue SN570 /dev/sd1 at 63C, and Crucial MX500 2TB /dev/nvme0 at 72C with 2 pending sectors. The API returned risk {critical: [temperature_high]}, {warning: [temperature_high]} and {critical: [pending_sectors, temperature_high]} respectively; the backend binary was rebuilt from base 8259a74c9d (includes #2611)", + "Same-session control with the pre-change diskPresentation.ts (3ac5664d95, whose verdict code is unchanged on base 8259a74c9d) swapped in under the running Vite: the 72C temperature-only disk read 'Replace Now' (phone 'Replace') titled 'Disk temperature is 72C' in the row and in its drawer, and the 63C disk read 'Needs Attention' and sorted above the 7%-life disk", + "After the change at 1440 (full layout): rows sort Replace Now (pending sectors, title 'Pending sectors detected (2)', rgb(185,28,28)), Running Hot (title 'Disk temperature is 72C', rgb(185,28,28)), Needs Attention (970 EVO, 'SSD life remaining is 7%', rgb(180,83,9)), Running Hot (title 'Disk temperature is 63C', rgb(180,83,9)), then Healthy/Unknown disks unchanged", + "After the change at 390 (narrow layout): the same rows show the compact words Replace, Hot, Attention, Hot with the same tones and titles", + "Disk drawer header for the 72C temperature-only disk reads 'Running Hot Disk temperature is 72C' at both widths", + "Health filter from the URL at both widths: status=attention lists the two Replace/Hot critical rows plus Attention and the amber Hot row; status=critical lists only Replace Now and the red Running Hot; status=warning lists Needs Attention and the amber Running Hot", + "Known and unchanged: the 63C NVMe reads Running Hot in amber beside a green Temp cell, because the risk tier is the server's flat 60C while the Temp cell uses the NVMe alert threshold; the same disk read Needs Attention before this change. Threshold reconciliation is separate work", + "Amended content (empty-summary reasons keep their code and severity, an unexplained critical level no longer borrows a weaker reason's text): full pass repeated at 1440 and 390 with identical rows, compact words, titles, colours, drawer header and status=attention/critical/warning results" + ], + "interactions": [ + "Loaded the Physical Disks tab unfiltered at 1440x900 and 390x844 and read every row's visible health word, its title, computed colour and the Temp cell", + "Opened the inline drawer for the 72C temperature-only disk by clicking its row at both widths and read the health header", + "Loaded status=attention, status=critical and status=warning at both widths and read the listed rows", + "Swapped the pre-change diskPresentation.ts in under the running Vite, repeated the reads, restored the change and confirmed it byte-identical with cmp; after rebasing onto 8259a74c9d repeated the full after-change pass with a rebuilt backend" + ] +} diff --git a/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx b/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx index 73b9cd330d..887fe416aa 100644 --- a/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx +++ b/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx @@ -343,6 +343,111 @@ describe('DiskList', () => { ); }); + it('says a disk that is only hot is running hot, not due for replacement', () => { + const view = renderDiskList({ + disks: [ + buildDisk('sda', 'pve3', { + model: 'Crucial MX500 2TB', + diskType: 'ssd', + wearout: 95, + temperature: 72, + risk: { + level: 'critical', + reasons: [ + { + code: 'temperature_high', + severity: 'critical', + summary: 'Disk temperature is 72C', + }, + ], + }, + }), + buildDisk('sdb', 'pve3', { + temperature: 72, + smart: { pendingSectors: 2 }, + risk: { + level: 'critical', + reasons: [ + { + code: 'temperature_high', + severity: 'critical', + summary: 'Disk temperature is 72C', + }, + { + code: 'pending_sectors', + severity: 'critical', + summary: 'Pending sectors detected (2)', + }, + ], + }, + }), + ], + nodes: [buildNode('node-pve3', 'pve3')], + selectedNode: null, + searchTerm: '', + }); + + const rows = Array.from(view.container.querySelectorAll('[data-row-id]')); + // The disk that needs replacing sorts above the one that needs cooling. + expect(rows.map((row) => row.getAttribute('data-row-id'))).toEqual(['sdb', 'sda']); + const hotRow = within(rows[1] as HTMLElement); + expect(hotRow.queryByText('Replace Now')).not.toBeInTheDocument(); + expect(hotRow.getByText('Running Hot')).toHaveClass( + 'platform-table-label-full', + 'text-red-700', + ); + expect(hotRow.getByText('Running Hot')).toHaveAttribute('title', 'Disk temperature is 72C'); + expect(hotRow.getByText('Hot')).toHaveClass('platform-table-label-compact', 'text-red-700'); + const failingRow = within(rows[0] as HTMLElement); + expect(failingRow.getByText('Replace Now')).toHaveAttribute( + 'title', + 'Pending sectors detected (2)', + ); + }); + + it('follows a disk from warm to critically hot and back to healthy', async () => { + const heat = (severity: 'warning' | 'critical', celsius: number) => ({ + level: severity, + reasons: [{ code: 'temperature_high', severity, summary: `Disk temperature is ${celsius}C` }], + }); + const [disks, setDisks] = createStore({ + items: [buildDisk('sda', 'pve3', { temperature: 63, risk: heat('warning', 63) })], + }); + const [healthFilter, setHealthFilter] = createSignal<'all' | 'critical'>('all'); + const view = render(() => ( + {}} + /> + )); + const row = () => view.container.querySelector('[data-row-id="sda"]') as HTMLElement | null; + expect(within(row()!).getByText('Running Hot')).toHaveClass('text-amber-700'); + expect(within(row()!).getByText('Hot')).toHaveClass('text-amber-700'); + setHealthFilter('critical'); + await waitFor(() => expect(row()).toBeNull()); + + setDisks( + 'items', + reconcile([buildDisk('sda', 'pve3', { temperature: 72, risk: heat('critical', 72) })]), + ); + await waitFor(() => expect(row()).not.toBeNull()); + expect(within(row()!).getByText('Running Hot')).toHaveClass('text-red-700'); + expect(within(row()!).getByText('Running Hot')).toHaveAttribute( + 'title', + 'Disk temperature is 72C', + ); + + setHealthFilter('all'); + setDisks('items', reconcile([buildDisk('sda', 'pve3', { temperature: 41 })])); + await waitFor(() => expect(within(row()!).getByText('Healthy')).toBeInTheDocument()); + expect(within(row()!).queryByText('Running Hot')).not.toBeInTheDocument(); + }); + it('renders SSD life and falls back to the Proxmox usage string for Belongs', () => { renderDiskList({ disks: [ diff --git a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts index a18d7ccdfe..04ac75dc46 100644 --- a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts +++ b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts @@ -268,10 +268,300 @@ describe('diskPresentation', () => { it('shortens only the health words that cannot fit a phone health column', () => { expect(getPhysicalDiskHealthCompactLabel('Needs Attention')).toBe('Attention'); expect(getPhysicalDiskHealthCompactLabel('Replace Now')).toBe('Replace'); + expect(getPhysicalDiskHealthCompactLabel('Running Hot')).toBe('Hot'); expect(getPhysicalDiskHealthCompactLabel('Healthy')).toBe('Healthy'); expect(getPhysicalDiskHealthCompactLabel('Unknown')).toBe('Unknown'); }); + describe('heat versus replacement evidence', () => { + const buildRiskDisk = ( + id: string, + risk: { level: string; reasons: { code: string; severity: string; summary: string }[] }, + overrides: Record = {}, + ): Resource => + ({ + id, + name: id, + type: 'physical_disk', + status: 'warning', + physicalDisk: { + devPath: `/dev/${id}`, + model: 'Crucial MX500 2TB', + diskType: 'sata', + health: 'PASSED', + wearout: 95, + temperature: 72, + risk, + ...overrides, + }, + identity: { hostname: 'pve3' }, + canonicalIdentity: { hostname: 'pve3' }, + platformType: 'proxmox-pve', + }) as unknown as Resource; + const hot = (severity: string, celsius = 72) => ({ + code: 'temperature_high', + severity, + summary: `Disk temperature is ${celsius}C`, + }); + + it('calls a disk that is only hot Running Hot at the tier of its reading', () => { + const criticalDisk = buildRiskDisk('sda', { level: 'critical', reasons: [hot('critical')] }); + const criticalData = extractPhysicalDiskPresentationData(criticalDisk); + expect(criticalData.riskReasonDetails).toEqual([hot('critical')]); + expect(getPhysicalDiskHealthStatus(criticalData)).toEqual({ + label: 'Running Hot', + summary: 'Disk temperature is 72C', + tone: 'text-red-700 dark:text-red-300', + }); + expect(getPhysicalDiskNormalizedHealth(criticalDisk, criticalData)).toBe('critical'); + + const warmDisk = buildRiskDisk( + 'sdb', + { level: 'warning', reasons: [hot('warning', 63)] }, + { temperature: 63 }, + ); + const warmData = extractPhysicalDiskPresentationData(warmDisk); + expect(getPhysicalDiskHealthStatus(warmData)).toEqual({ + label: 'Running Hot', + summary: 'Disk temperature is 63C', + tone: 'text-amber-700 dark:text-amber-300', + }); + expect(getPhysicalDiskNormalizedHealth(warmDisk, warmData)).toBe('warning'); + }); + + it('keeps Replace Now for failure evidence even when heat is listed first', () => { + // Merged risks keep insertion order, so heat can precede the reason + // that actually calls for a new disk. + const disk = buildRiskDisk('sda', { + level: 'critical', + reasons: [ + hot('critical'), + { + code: 'pending_sectors', + severity: 'critical', + summary: 'Pending sectors detected (2)', + }, + ], + }); + const data = extractPhysicalDiskPresentationData(disk); + expect(getPhysicalDiskHealthStatus(data)).toEqual({ + label: 'Replace Now', + summary: 'Pending sectors detected (2)', + tone: 'text-red-700 dark:text-red-300', + }); + + const failed = extractPhysicalDiskPresentationData( + buildRiskDisk( + 'sdb', + { + level: 'critical', + reasons: [ + hot('critical'), + { + code: 'health_status', + severity: 'critical', + summary: 'Disk reports health status FAILED', + }, + ], + }, + { health: 'FAILED' }, + ), + ); + expect(getPhysicalDiskHealthStatus(failed).label).toBe('Replace Now'); + expect(getPhysicalDiskHealthStatus(failed).summary).toBe('Disk reports health status FAILED'); + }); + + it('lets the more severe of heat and wear evidence name the verdict', () => { + const hotAndWorn = extractPhysicalDiskPresentationData( + buildRiskDisk('sda', { + level: 'critical', + reasons: [ + hot('critical'), + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + ], + }), + ); + expect(getPhysicalDiskHealthStatus(hotAndWorn).label).toBe('Running Hot'); + expect(getPhysicalDiskHealthStatus(hotAndWorn).summary).toBe('Disk temperature is 72C'); + + const warmAndWorn = extractPhysicalDiskPresentationData( + buildRiskDisk( + 'sdb', + { + level: 'warning', + reasons: [ + hot('warning', 63), + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + ], + }, + { temperature: 63 }, + ), + ); + expect(getPhysicalDiskHealthStatus(warmAndWorn)).toEqual({ + label: 'Needs Attention', + summary: 'SSD life remaining is 8%', + tone: 'text-amber-700 dark:text-amber-300', + }); + }); + + it('keeps a critical level that no listed reason explains as Replace Now', () => { + const data = extractPhysicalDiskPresentationData( + buildRiskDisk('sda', { level: 'critical', reasons: [hot('warning', 63)] }), + ); + expect(getPhysicalDiskHealthStatus(data)).toEqual({ + label: 'Replace Now', + summary: 'Disk health has degraded to a critical state.', + tone: 'text-red-700 dark:text-red-300', + }); + }); + + it('decides the verdict from codes and severities even without display text', () => { + const pendingWithoutText = extractPhysicalDiskPresentationData( + buildRiskDisk('sda', { + level: 'critical', + reasons: [ + hot('critical'), + { code: 'pending_sectors', severity: 'critical', summary: '' }, + ], + }), + ); + expect(pendingWithoutText.riskReasons).toEqual(['Disk temperature is 72C']); + expect(getPhysicalDiskHealthStatus(pendingWithoutText)).toEqual({ + label: 'Replace Now', + summary: 'Disk health has degraded to a critical state.', + tone: 'text-red-700 dark:text-red-300', + }); + + const heatWithoutText = extractPhysicalDiskPresentationData( + buildRiskDisk('sdb', { + level: 'critical', + reasons: [{ code: 'temperature_high', severity: 'critical', summary: '' }], + }), + ); + expect(getPhysicalDiskHealthStatus(heatWithoutText).label).toBe('Running Hot'); + }); + + it('does not let a weaker reason stand in for an unexplained critical level', () => { + const data = extractPhysicalDiskPresentationData( + buildRiskDisk('sda', { + level: 'critical', + reasons: [ + { code: 'crc_errors', severity: 'monitor', summary: 'UDMA CRC errors detected (2)' }, + hot('warning', 63), + ], + }), + ); + expect(getPhysicalDiskHealthStatus(data)).toEqual({ + label: 'Replace Now', + summary: 'Disk health has degraded to a critical state.', + tone: 'text-red-700 dark:text-red-300', + }); + }); + + it('reads severities case-insensitively and in any order within a class', () => { + const shouting = extractPhysicalDiskPresentationData( + buildRiskDisk('sda', { + level: 'CRITICAL', + reasons: [{ ...hot('critical'), severity: ' Critical ' }], + }), + ); + expect(getPhysicalDiskHealthStatus(shouting).label).toBe('Running Hot'); + expect(getPhysicalDiskHealthStatus(shouting).tone).toBe('text-red-700 dark:text-red-300'); + + const unsorted = extractPhysicalDiskPresentationData( + buildRiskDisk('sdb', { + level: 'critical', + reasons: [ + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + { + code: 'pending_sectors', + severity: 'critical', + summary: 'Pending sectors detected (2)', + }, + ], + }), + ); + expect(getPhysicalDiskHealthStatus(unsorted).summary).toBe('Pending sectors detected (2)'); + }); + + it('lets SMART counters and low life outrank warning heat', () => { + const warmAndLow = extractPhysicalDiskPresentationData( + buildRiskDisk( + 'sda', + { level: 'warning', reasons: [hot('warning', 63)] }, + { diskType: 'ssd', wearout: 8, temperature: 63 }, + ), + ); + expect(getPhysicalDiskHealthStatus(warmAndLow)).toEqual({ + label: 'Needs Attention', + summary: 'SSD life is running low.', + tone: 'text-amber-700 dark:text-amber-300', + }); + + const warmAndReallocating = extractPhysicalDiskPresentationData( + buildRiskDisk( + 'sdb', + { level: 'warning', reasons: [hot('warning', 63)] }, + { temperature: 63, smart: { reallocatedSectors: 3 } }, + ), + ); + expect(getPhysicalDiskHealthStatus(warmAndReallocating).label).toBe('Needs Attention'); + }); + + it('sorts a FAILED disk with no risk payload ahead of a critically hot one', () => { + const failedDisk = buildRiskDisk( + 'sdb', + { level: 'healthy', reasons: [] }, + { risk: undefined, health: 'FAILED', temperature: 40 }, + ); + const hotDisk = buildRiskDisk('sda', { level: 'critical', reasons: [hot('critical')] }); + const failedData = extractPhysicalDiskPresentationData(failedDisk); + const hotData = extractPhysicalDiskPresentationData(hotDisk); + expect(getPhysicalDiskHealthStatus(failedData).label).toBe('Replace Now'); + expect( + comparePhysicalDiskPresentation(failedDisk, failedData, hotDisk, hotData), + ).toBeLessThan(0); + }); + + it('sorts and filters hot disks by the action their verdict asks for', () => { + const warm = buildRiskDisk('sdc', { level: 'warning', reasons: [hot('warning', 63)] }); + const worn = buildRiskDisk('sdd', { + level: 'warning', + reasons: [ + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + ], + }); + const hotDisk = buildRiskDisk('sda', { level: 'critical', reasons: [hot('critical')] }); + const failing = buildRiskDisk('sdb', { + level: 'critical', + reasons: [ + { code: 'media_errors', severity: 'critical', summary: 'Media errors detected (4)' }, + ], + }); + const healthy = buildRiskDisk( + 'sde', + { level: 'healthy', reasons: [] }, + { risk: undefined, temperature: 38 }, + ); + const disks = [healthy, warm, worn, hotDisk, failing]; + const dataMap = buildPhysicalDiskPresentationDataMap(disks); + const sorted = (healthFilter: 'all' | 'attention' | 'critical' | 'warning') => + filterAndSortPhysicalDisks(disks, { + selectedNode: null, + healthFilter, + searchTerm: '', + getDiskData: (disk) => dataMap.get(disk.id)!, + matchesNode: () => true, + }).map((disk) => disk.id); + + // Device order alone would put each hot disk first. + expect(sorted('all')).toEqual(['sdb', 'sda', 'sdd', 'sdc', 'sde']); + expect(sorted('attention')).toEqual(['sdb', 'sda', 'sdd', 'sdc']); + expect(sorted('critical')).toEqual(['sdb', 'sda']); + expect(sorted('warning')).toEqual(['sdd', 'sdc']); + }); + }); + it('detects SMART warnings from counters', () => { expect( hasPhysicalDiskSmartWarning({ diff --git a/frontend-modern/src/features/storageBackups/diskPresentation.ts b/frontend-modern/src/features/storageBackups/diskPresentation.ts index 2499be5969..3248b29bf6 100644 --- a/frontend-modern/src/features/storageBackups/diskPresentation.ts +++ b/frontend-modern/src/features/storageBackups/diskPresentation.ts @@ -3,6 +3,7 @@ import type { PhysicalDiskCollectionStatus, PhysicalDiskFieldStatus, Resource, + ResourceStorageRiskReason, } from '@/types/resource'; import { getPlatformTableWeightedColumnWidthStyle } from '@/features/platformPage/sharedPlatformPage'; import { @@ -53,6 +54,8 @@ export interface PhysicalDiskPresentationData { health: string; riskLevel?: string; riskReasons: string[]; + /** The same reasons with their codes and severities, as the API sent them. */ + riskReasonDetails?: ResourceStorageRiskReason[]; wearout: number; storageRole?: string; storageGroup?: string; @@ -198,6 +201,7 @@ export const getPhysicalDiskCellPaddingClass = (layout: PhysicalDiskTableLayoutM export const getPhysicalDiskHealthCompactLabel = (label: string): string => { if (label === 'Needs Attention') return 'Attention'; if (label === 'Replace Now') return 'Replace'; + if (label === 'Running Hot') return 'Hot'; return label; }; @@ -402,14 +406,21 @@ export function extractPhysicalDiskPresentationData( ): PhysicalDiskPresentationData { const pd = resource.physicalDisk || ((resource.platformData as any)?.physicalDisk ?? {}); const diskNode = getPhysicalDiskNodeIdentity(resource); - const riskReasons = Array.isArray(pd.risk?.reasons) + // A reason's code and severity decide the verdict even when it carries no + // display text, so only the summary list drops empty summaries. + const riskReasonDetails: ResourceStorageRiskReason[] = Array.isArray(pd.risk?.reasons) ? pd.risk.reasons - .map((reason: { summary?: unknown }) => reason?.summary) - .filter( - (summary: unknown): summary is string => - typeof summary === 'string' && summary.length > 0, - ) + .filter((reason: unknown) => reason !== null && typeof reason === 'object') + .map((reason: { code?: unknown; severity?: unknown; summary?: unknown }) => ({ + code: typeof reason.code === 'string' ? reason.code : '', + severity: typeof reason.severity === 'string' ? reason.severity : '', + summary: typeof reason.summary === 'string' ? reason.summary : '', + })) + .filter((reason: ResourceStorageRiskReason) => reason.code || reason.summary) : []; + const riskReasons = riskReasonDetails + .map((reason) => reason.summary) + .filter((summary) => summary.length > 0); return { node: diskNode.node, @@ -438,6 +449,7 @@ export function extractPhysicalDiskPresentationData( collection: pd.collection, riskLevel: pd.risk?.level, riskReasons, + riskReasonDetails, smartAttributes: pd.smart ? { powerOnHours: pd.smart.powerOnHours, @@ -534,9 +546,24 @@ export function buildPhysicalDiskPresentationDataMap( return map; } +// Disks sort in the order their verdicts ask for action: replacement first, +// then a disk running critically hot, then other warnings, then a warm disk. +const getPhysicalDiskVerdictPriority = (disk: PhysicalDiskPresentationData): number => { + const verdict = getPhysicalDiskHealthVerdict(disk); + switch (verdict.kind) { + case 'replace': + return 400; + case 'hot': + return verdict.critical ? 300 : 100; + case 'attention': + return 200; + default: + return 0; + } +}; + const getPhysicalDiskPriority = (disk: PhysicalDiskPresentationData): number => - (disk.riskLevel === 'critical' ? 300 : disk.riskLevel === 'warning' ? 200 : 0) + - (hasPhysicalDiskSmartWarning(disk) ? 50 : 0); + getPhysicalDiskVerdictPriority(disk) + (hasPhysicalDiskSmartWarning(disk) ? 50 : 0); export function matchesPhysicalDiskSearch( resource: Resource, @@ -685,31 +712,116 @@ export function hasPhysicalDiskSmartWarning(disk: PhysicalDiskPresentationData): ); } +// Heat is the one disk risk whose fix is cooling, not a replacement disk. +// Alerts raise it as a temperature metric rather than as disk health, so the +// verdict names it apart from the evidence that does mean replacing the disk. +const PHYSICAL_DISK_TEMPERATURE_RISK_CODE = 'temperature_high'; + +const PHYSICAL_DISK_RISK_SEVERITY_RANK: Record = { + monitor: 1, + warning: 2, + critical: 3, +}; + +const getPhysicalDiskRiskSeverityRank = (severity: string | undefined): number => + PHYSICAL_DISK_RISK_SEVERITY_RANK[(severity || '').trim().toLowerCase()] ?? 0; + +interface PhysicalDiskRiskEvidence { + rank: number; + summary?: string; +} + +// Splits the disk's risk into heat and everything else, keeping the most +// severe reason of each. A risk level that none of its reasons explains, or +// one sent without reason codes, stays with the disk-health evidence. +function splitPhysicalDiskRisk(disk: PhysicalDiskPresentationData): { + health: PhysicalDiskRiskEvidence; + heat: PhysicalDiskRiskEvidence; +} { + const health: PhysicalDiskRiskEvidence = { rank: 0 }; + const heat: PhysicalDiskRiskEvidence = { rank: 0 }; + const details = + disk.riskReasonDetails ?? + disk.riskReasons.map((summary) => ({ code: '', severity: disk.riskLevel || '', summary })); + for (const reason of details) { + const evidence = reason.code === PHYSICAL_DISK_TEMPERATURE_RISK_CODE ? heat : health; + const rank = getPhysicalDiskRiskSeverityRank(reason.severity); + if (rank > evidence.rank) { + evidence.rank = rank; + evidence.summary = reason.summary || undefined; + } else if (rank === evidence.rank && !evidence.summary) { + evidence.summary = reason.summary || undefined; + } + } + const levelRank = getPhysicalDiskRiskSeverityRank(disk.riskLevel); + if (levelRank > Math.max(health.rank, heat.rank)) { + // A reason with a known, lower severity does not explain this level, so + // its text must not stand in as the verdict's reason. + if (health.rank > 0) health.summary = undefined; + health.rank = levelRank; + } + return { health, heat }; +} + +type PhysicalDiskHealthVerdict = + | { kind: 'replace'; summary: string } + | { kind: 'hot'; critical: boolean; summary: string } + | { kind: 'attention'; summary: string } + | { kind: 'clear' }; + +function getPhysicalDiskHealthVerdict( + disk: PhysicalDiskPresentationData, +): PhysicalDiskHealthVerdict { + const { health, heat } = splitPhysicalDiskRisk(disk); + const critical = PHYSICAL_DISK_RISK_SEVERITY_RANK.critical; + const warning = PHYSICAL_DISK_RISK_SEVERITY_RANK.warning; + const lowLife = isPhysicalDiskWearoutReported(disk) && disk.wearout < 10; + + if (normalizePhysicalDiskHealth(disk.health) === 'FAILED' || health.rank >= critical) { + return { + kind: 'replace', + summary: health.summary || 'Disk health has degraded to a critical state.', + }; + } + if (heat.rank >= critical) { + return { kind: 'hot', critical: true, summary: heat.summary || 'Disk temperature is high.' }; + } + if (health.rank >= warning || hasPhysicalDiskSmartWarning(disk) || lowLife) { + return { + kind: 'attention', + summary: + health.summary || + (lowLife ? 'SSD life is running low.' : 'SMART counters indicate elevated risk.'), + }; + } + if (heat.rank >= warning) { + return { kind: 'hot', critical: false, summary: heat.summary || 'Disk temperature is high.' }; + } + return { kind: 'clear' }; +} + +const PHYSICAL_DISK_CRITICAL_TONE = 'text-red-700 dark:text-red-300'; +const PHYSICAL_DISK_WARNING_TONE = 'text-amber-700 dark:text-amber-300'; + export function getPhysicalDiskHealthStatus( disk: PhysicalDiskPresentationData, ): DiskHealthStatusPresentation { - const normalizedHealth = normalizePhysicalDiskHealth(disk.health); - const criticalRisk = (disk.riskLevel || '').trim().toLowerCase() === 'critical'; - const warningRisk = (disk.riskLevel || '').trim().toLowerCase() === 'warning'; - const smartWarning = hasPhysicalDiskSmartWarning(disk); - const lowLife = isPhysicalDiskWearoutReported(disk) && disk.wearout < 10; + const verdict = getPhysicalDiskHealthVerdict(disk); - if (normalizedHealth === 'FAILED' || criticalRisk) { + if (verdict.kind === 'replace') { + return { label: 'Replace Now', summary: verdict.summary, tone: PHYSICAL_DISK_CRITICAL_TONE }; + } + + if (verdict.kind === 'hot') { return { - label: 'Replace Now', - summary: disk.riskReasons[0] || 'Disk health has degraded to a critical state.', - tone: 'text-red-700 dark:text-red-300', + label: 'Running Hot', + summary: verdict.summary, + tone: verdict.critical ? PHYSICAL_DISK_CRITICAL_TONE : PHYSICAL_DISK_WARNING_TONE, }; } - if (warningRisk || smartWarning || lowLife) { - return { - label: 'Needs Attention', - summary: - disk.riskReasons[0] || - (lowLife ? 'SSD life is running low.' : 'SMART counters indicate elevated risk.'), - tone: 'text-amber-700 dark:text-amber-300', - }; + if (verdict.kind === 'attention') { + return { label: 'Needs Attention', summary: verdict.summary, tone: PHYSICAL_DISK_WARNING_TONE }; } if (isUnraidPhysicalDisk(disk) && !hasUnraidPhysicalDiskFaultSignal(disk)) { @@ -720,7 +832,7 @@ export function getPhysicalDiskHealthStatus( }; } - const isHealthy = PHYSICAL_DISK_HEALTHY_STATES.has(normalizedHealth); + const isHealthy = PHYSICAL_DISK_HEALTHY_STATES.has(normalizePhysicalDiskHealth(disk.health)); return { label: isHealthy ? 'Healthy' : 'Unknown', summary: isHealthy ? 'No active disk-health issues.' : 'Health state is not reported.', @@ -732,13 +844,15 @@ export function getPhysicalDiskNormalizedHealth( resource: Resource, disk: PhysicalDiskPresentationData, ): NormalizedHealth { - const status = getPhysicalDiskHealthStatus(disk).label; - if (status === 'Online') return 'healthy'; + const verdict = getPhysicalDiskHealthVerdict(disk); + const clearLabel = verdict.kind === 'clear' ? getPhysicalDiskHealthStatus(disk).label : null; + if (clearLabel === 'Online') return 'healthy'; if (resource.status === 'offline') return 'offline'; - if (status === 'Replace Now') return 'critical'; - if (status === 'Needs Attention') return 'warning'; - if (status === 'Healthy') return 'healthy'; - return 'unknown'; + if (verdict.kind === 'replace') return 'critical'; + // A disk running hot sits in the health filter at its temperature tier. + if (verdict.kind === 'hot') return verdict.critical ? 'critical' : 'warning'; + if (verdict.kind === 'attention') return 'warning'; + return clearLabel === 'Healthy' ? 'healthy' : 'unknown'; } export function matchesPhysicalDiskHealthFilter( From fb7dd955ed41d43d0fab0cf4a0b09e2d02123028 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:48:26 +0100 Subject: [PATCH 03/13] Keep retained disk temperatures out of AI, Patrol and reports Normalization keeps a disk's last-known temperature when the current observation did not collect one (a disk in standby, a host agent past its reporting lease) and marks it non-available. Several readers still took PhysicalDisk.Temperature at face value. The AI chat context listed a standby disk as needing attention for heat. Patrol raised health issues and triage flags from the retained value. The AI query and disk-health tools handed it to the model as the disk's temperature. Performance reports and the reporting runtime snapshot tabulated it as measured, and the disk temperature charts padded a series out to now with it. Each of them now goes through diskinventory.TemperatureCollected. AI surfaces keep a retained value as context, labelled last known with its reason. Reports and the charts leave it out. monitor.go's private copy of the predicate is replaced by the shared helper. --- .../v6/internal/subsystems/agent-lifecycle.md | 5 + .../v6/internal/subsystems/ai-runtime.md | 18 +++ .../v6/internal/subsystems/api-contracts.md | 8 ++ .../v6/internal/subsystems/monitoring.md | 9 ++ .../internal/subsystems/storage-recovery.md | 3 + internal/ai/patrol_ai.go | 21 ++-- internal/ai/patrol_ai_more_test.go | 66 +++++++++++ internal/ai/patrol_findings.go | 2 +- internal/ai/patrol_triage.go | 2 +- internal/ai/resource_context.go | 8 +- internal/ai/resource_context_test.go | 51 +++++++++ internal/ai/tools/data_types.go | 15 ++- internal/ai/tools/disk_temperature.go | 54 +++++++++ .../ai/tools/physical_disk_evidence_test.go | 104 ++++++++++++++++++ internal/ai/tools/tools_query.go | 9 +- internal/ai/tools/tools_storage.go | 11 +- internal/api/contract_test.go | 58 ++++++++++ internal/api/metrics_reporting_handlers.go | 15 ++- internal/api/reporting_runtime_snapshot.go | 2 +- .../issue1595_collection_trust_test.go | 11 +- internal/monitoring/monitor.go | 20 +--- internal/monitoring/monitor_agents.go | 3 +- internal/monitoring/monitor_metrics.go | 44 +++++--- .../monitoring/monitor_metrics_slo_test.go | 45 ++++++++ .../physical_disk_roundtrip_test.go | 4 +- 25 files changed, 525 insertions(+), 63 deletions(-) create mode 100644 internal/ai/tools/disk_temperature.go diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 99e975cbfa..419c34448e 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -760,6 +760,11 @@ offline the unified-resources adapter reports them `unavailable` with the same reason (`models.HostAgentStoppedReportingReason`) and leaves them out of the disk's risk, and the metrics-history API does not return any such retained disk temperature as a live point. +Downstream readers keep that distinction too. The disk temperature charts do +not pad a series to now with the retained value. The AI chat context, Patrol and +the AI disk tools present it only as a last-known value with its reason. The +performance report and the reporting runtime snapshot leave it out of their +disk tables. An enabled availability target assigned to a host agent creates an agent-lifecycle lease for that exact target/agent pairing. First assignment diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 56b2a47c66..da8d664e67 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -288,6 +288,24 @@ exposing the source-specific ambiguous `wearout` name, and distinguishes it from `smart.percentageUsed` as consumed endurance. The regression proof is `internal/ai/tools/physical_disk_evidence_test.go`. +Disk temperatures reach the model as current readings only when they were +collected now. `tools.SplitDiskTemperature` classifies a physical disk +temperature with `diskinventory.TemperatureCollected`. The physical-disk +projection's `temperature` and the `disk_health` SMART row's `temperature` +carry only a reading the current observation collected. A value normalization +retained instead (a disk in standby, a host agent past its reporting lease) +moves to `last_known_temperature` with `last_known_temperature_reason`. The +same split feeds the AI chat context and Patrol. The "Physical Disks Needing +Attention" list and Patrol's disk issue gate, triage flags and finding +verification judge heat on the collected reading alone. Their text renders a +retained value as "last known 41C (disk is in standby)". Proofs: +`TestDiskToolsReportRetainedTemperatureAsLastKnown` in +`internal/ai/tools/physical_disk_evidence_test.go`, +`TestBuildUnifiedResourceContextJudgesDiskHeatOnCollectedTemperature` in +`internal/ai/resource_context_test.go` and +`TestPatrolPhysicalDiskRowsJudgeHeatOnCollectedTemperature` in +`internal/ai/patrol_ai_more_test.go`. + Retained summaries disclose that point and bucket timestamps describe returned history inside the requested window. Their spacing does not measure collection uptime or explain missing history. Collector lifecycle and retention settings diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index ae4efc4a57..a95035e0fd 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -6332,6 +6332,14 @@ stored one out to `now`, only from a temperature collected now; without one the stored samples are returned as they are. An empty mock range can still fall through to the monitor's generic demo chart series for that disk ID, which is not derived from the disk's reading. +Reports follow the same rule. A performance report's disk table +(`enrichNodeReport` in `internal/api/metrics_reporting_handlers.go`) and the +reporting runtime snapshot's disks (`internal/api/reporting_runtime_snapshot.go`) +take a disk temperature through `reportDiskTemperature`. It keeps a reading only +when `diskinventory.TemperatureCollected` holds, so a retained value is reported +as 0 (no reading) rather than tabulated as measured. Proof: +`TestContract_ReportsOmitRetainedDiskTemperatures` in +`internal/api/contract_test.go`. That same metrics-history contract also owns Kubernetes pod identity normalization. `/api/metrics-store/history` must accept legacy bare pod IDs such as `cluster-1:pod:pod-1`, canonicalize them onto the unified pod metrics diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index a912bfce62..60ac720fba 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -3815,6 +3815,15 @@ consumers outside the history writers: the metrics-history API serves a disk series from the disk's reading or pads a stored one, only from a collected temperature, so a retained standby or silent-agent reading never reappears as a point at the current time. +The SMART history writers in `internal/monitoring/monitor.go` and +`monitor_agents.go` call that helper rather than a private copy of the rule, +and `GetPhysicalDiskTemperatureCharts` pads a short disk temperature series out +to now only with a collected reading: with a retained one the stored samples +stay as they are, and a disk with none gets no series. Mock mode can still +substitute its generic demo series for that disk ID before this step; that +series is not derived from the disk's reading. Proof: +`TestDiskTemperatureChartsPadOnlyWithCollectedReading` in +`internal/monitoring/monitor_metrics_slo_test.go`. Unified-resource physical-disk round trips must retain named `StorageGroup` membership rather than degrading it to the generic `Used` filesystem label. diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index cd2ec5bf9e..207ae4cdd3 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -6779,3 +6779,6 @@ and the Unraid physical-disk adapter marks an inventory temperature kept past the host agent's lease as no longer collected. Both change presentation of values Pulse already holds: no metrics write, retention window, backup, migration or recovery authority is added or moved. +The performance report and reporting runtime snapshot handlers apply the same +test before tabulating a disk temperature. That changes only which held value a +report shows, not any storage or recovery path. diff --git a/internal/ai/patrol_ai.go b/internal/ai/patrol_ai.go index ac6c9072fc..8757ac5d0f 100644 --- a/internal/ai/patrol_ai.go +++ b/internal/ai/patrol_ai.go @@ -1927,8 +1927,11 @@ type patrolPhysicalDiskRow struct { sizeBytes int64 devPath, model string health, status string - wearout, temperature int - smartEvidence []string + wearout int + // temperature.Collected is the only temperature Patrol may judge as + // heat; a retained last-known value is shown as context. + temperature tools.DiskTemperature + smartEvidence []string } type patrolPrecomputeNodeSource struct { @@ -2472,7 +2475,7 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool) health: health, status: status, wearout: r.PhysicalDisk.Wearout, - temperature: r.PhysicalDisk.Temperature, + temperature: tools.SplitDiskTemperature(r.PhysicalDisk.Temperature, r.PhysicalDisk.Collection), smartEvidence: unifiedPhysicalDiskSMARTIssueParts(r.PhysicalDisk.SMART), }) } @@ -2513,7 +2516,7 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool) health: health, status: status, wearout: d.Wearout, - temperature: d.Temperature, + temperature: tools.SplitDiskTemperature(d.Temperature, d.Collection), smartEvidence: modelPhysicalDiskSMARTIssueParts(d.SmartAttributes), }) } @@ -2531,7 +2534,7 @@ func patrolPhysicalDiskHealthIssue(row patrolPhysicalDiskRow) bool { (row.wearout == 0 && (strings.EqualFold(row.diskType, "ssd") || strings.EqualFold(row.diskType, "nvme")))) && row.wearout < 20) || - row.temperature > 55 || + row.temperature.Collected > 55 || len(row.smartEvidence) > 0 } @@ -2959,8 +2962,8 @@ func (p *PatrolService) seedResourceInventoryState(snap patrolRuntimeState, scop wear = fmt.Sprintf("%d%%", row.wearout) } temp := "—" - if row.temperature > 0 { - temp = fmt.Sprintf("%dC", row.temperature) + if text := row.temperature.Format("C"); text != "" { + temp = text } sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s | %s | %s | %s | %s |\n", diskName, node, diskType, size, row.health, wear, temp, row.status)) @@ -3470,8 +3473,8 @@ func (p *PatrolService) seedHealthAndAlertsState(snap patrolRuntimeState, scoped wearout = fmt.Sprintf("%d%%", row.wearout) } temp := "—" - if row.temperature > 0 { - temp = fmt.Sprintf("%d°C", row.temperature) + if text := row.temperature.Format("°C"); text != "" { + temp = text } smartEvidence := "—" if len(row.smartEvidence) > 0 { diff --git a/internal/ai/patrol_ai_more_test.go b/internal/ai/patrol_ai_more_test.go index 7613f10a47..39d6fe8af2 100644 --- a/internal/ai/patrol_ai_more_test.go +++ b/internal/ai/patrol_ai_more_test.go @@ -17,6 +17,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) type noExecutorChatService struct{} @@ -1054,6 +1055,71 @@ func TestPatrolPhysicalDiskHealthIssueDistinguishesKnownZeroWearout(t *testing.T } } +// A retained last-known temperature (a disk in standby, a host agent past its +// reporting lease) is context, not heat. Patrol's disk issue gate, triage flags +// and finding verification judge only a collected reading, and both seed +// tables label the retained value as last known. +func TestPatrolPhysicalDiskRowsJudgeHeatOnCollectedTemperature(t *testing.T) { + standby := &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")} + collected := &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")} + unified := newPatrolRuntimeState(models.StateSnapshot{}) + unified.unifiedResourceProvider = &mockUnifiedResourceProvider{ + getByTypeFunc: func(t unifiedresources.ResourceType) []unifiedresources.Resource { + if t != unifiedresources.ResourceTypePhysicalDisk { + return nil + } + return []unifiedresources.Resource{ + {ID: "disk-standby", Name: "standby", Type: unifiedresources.ResourceTypePhysicalDisk, Status: unifiedresources.StatusOnline, ParentName: "tower", + PhysicalDisk: &unifiedresources.PhysicalDiskMeta{DevPath: "/dev/sda", Model: "WDC WD80EFAX", DiskType: "sata", Health: "PASSED", Wearout: -1, Temperature: 61, Collection: standby}}, + {ID: "disk-hot", Name: "hot", Type: unifiedresources.ResourceTypePhysicalDisk, Status: unifiedresources.StatusOnline, ParentName: "tower", + PhysicalDisk: &unifiedresources.PhysicalDiskMeta{DevPath: "/dev/sdb", Model: "WDC WD80EFAX", DiskType: "sata", Health: "PASSED", Wearout: -1, Temperature: 58, Collection: collected}}, + } + }, + } + legacy := newPatrolRuntimeState(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{ + {ID: "disk-standby", Node: "tower", DevPath: "/dev/sda", Model: "WDC WD80EFAX", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 61, Collection: standby}, + {ID: "disk-hot", Node: "tower", DevPath: "/dev/sdb", Model: "WDC WD80EFAX", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 58, Collection: collected}, + }}) + + for name, snap := range map[string]patrolRuntimeState{"unified": unified, "state": legacy} { + t.Run(name, func(t *testing.T) { + byID := map[string]patrolPhysicalDiskRow{} + for _, row := range patrolPhysicalDiskRows(snap, nil) { + byID[row.id] = row + } + if patrolPhysicalDiskHealthIssue(byID["disk-standby"]) { + t.Fatalf("a standby disk's retained 61C is not a health issue: %+v", byID["disk-standby"]) + } + if !patrolPhysicalDiskHealthIssue(byID["disk-hot"]) { + t.Fatalf("a collected 58C is a health issue: %+v", byID["disk-hot"]) + } + flags := triageDiskHealthChecksState(snap, nil) + if len(flags) != 1 || !strings.Contains(flags[0].Reason, "Disk temperature 58") { + t.Fatalf("triage flags = %+v, want only the collected 58C disk", flags) + } + if verification, ok := patrolLookupPhysicalDiskVerificationState(snap, "disk-standby"); !ok || verification.temperature != 0 { + t.Fatalf("standby disk verification = %+v (found %v), want no temperature", verification, ok) + } + }) + } + + ps := NewPatrolService(nil, nil) + cfg := DefaultPatrolConfig() + now := time.Now() + health := ps.seedHealthAndAlertsState(unified, nil, cfg, now) + for _, part := range []string{"| last known 61°C (disk is in standby) |", "| 58°C |"} { + if !strings.Contains(health, part) { + t.Fatalf("disk health table missing %q:\n%s", part, health) + } + } + inventory := ps.seedResourceInventoryState(unified, nil, cfg, now, false, nil) + for _, part := range []string{"| last known 61C (disk is in standby) |", "| 58C |"} { + if !strings.Contains(inventory, part) { + t.Fatalf("physical disk inventory missing %q:\n%s", part, inventory) + } + } +} + func TestSeedHealthAndAlerts_WithIssues(t *testing.T) { ps := NewPatrolService(nil, nil) cfg := DefaultPatrolConfig() diff --git a/internal/ai/patrol_findings.go b/internal/ai/patrol_findings.go index 8e2a5aaa42..d72a73f0e8 100644 --- a/internal/ai/patrol_findings.go +++ b/internal/ai/patrol_findings.go @@ -2463,7 +2463,7 @@ func patrolVisitPhysicalDiskVerification(snap patrolRuntimeState, visit patrolPh if !visit([]string{disk.id, disk.name, disk.devPath, disk.model}, patrolPhysicalDiskVerification{ health: strings.TrimSpace(disk.health), wearout: disk.wearout, - temperature: disk.temperature, + temperature: disk.temperature.Collected, }) { return true } diff --git a/internal/ai/patrol_triage.go b/internal/ai/patrol_triage.go index 79e8b6c27a..f3d3d635e2 100644 --- a/internal/ai/patrol_triage.go +++ b/internal/ai/patrol_triage.go @@ -345,7 +345,7 @@ func triageDiskHealthChecksState(snap patrolRuntimeState, scopedSet map[string]b flags := make([]TriageFlag, 0) for _, disk := range patrolPhysicalDiskRows(snap, scopedSet) { - flags = append(flags, triagePhysicalDiskFlags(disk.id, disk.name, disk.health, disk.wearout, disk.temperature)...) + flags = append(flags, triagePhysicalDiskFlags(disk.id, disk.name, disk.health, disk.wearout, disk.temperature.Collected)...) } return flags diff --git a/internal/ai/resource_context.go b/internal/ai/resource_context.go index 80c0f4ae8a..1702ac0a05 100644 --- a/internal/ai/resource_context.go +++ b/internal/ai/resource_context.go @@ -5,6 +5,7 @@ import ( "sort" "strings" + "github.com/rcourtman/pulse-go-rewrite/internal/ai/tools" unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" "github.com/rs/zerolog/log" ) @@ -412,7 +413,8 @@ func (s *Service) buildUnifiedResourceContextForModel(destinationModel string) s temperature := 0 if disk.PhysicalDisk != nil { health = strings.ToUpper(strings.TrimSpace(disk.PhysicalDisk.Health)) - temperature = disk.PhysicalDisk.Temperature + // A retained last-known reading is not current heat. + temperature = tools.SplitDiskTemperature(disk.PhysicalDisk.Temperature, disk.PhysicalDisk.Collection).Collected } if disk.Status != unifiedresources.StatusOnline || (health != "" && health != "PASSED" && health != "UNKNOWN") || temperature >= 50 { attention = append(attention, disk) @@ -431,8 +433,8 @@ func (s *Service) buildUnifiedResourceContextForModel(destinationModel string) s if value := strings.TrimSpace(disk.PhysicalDisk.Health); value != "" { health = value } - if disk.PhysicalDisk.Temperature > 0 { - temperature = fmt.Sprintf(", Temp: %dC", disk.PhysicalDisk.Temperature) + if text := tools.SplitDiskTemperature(disk.PhysicalDisk.Temperature, disk.PhysicalDisk.Collection).Format("C"); text != "" { + temperature = ", Temp: " + text } } healthSummary := "" diff --git a/internal/ai/resource_context_test.go b/internal/ai/resource_context_test.go index 80bbc42c37..9ce53f889b 100644 --- a/internal/ai/resource_context_test.go +++ b/internal/ai/resource_context_test.go @@ -8,6 +8,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/agentexec" "github.com/rcourtman/pulse-go-rewrite/internal/truenas" unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) func TestBuildUnifiedResourceContext_NilProvider(t *testing.T) { @@ -741,6 +742,56 @@ func TestBuildUnifiedResourceContextIncludesTrueNASResources(t *testing.T) { } } +// A retained disk temperature (a disk in standby, a host agent past its +// reporting lease) is not current heat: it must not put a disk on the +// attention list, and a disk listed for another reason shows it as last known. +func TestBuildUnifiedResourceContextJudgesDiskHeatOnCollectedTemperature(t *testing.T) { + disk := func(id string, status unifiedresources.ResourceStatus, temperature int, collection diskinventory.FieldStatus) unifiedresources.Resource { + return unifiedresources.Resource{ + ID: id, Name: id, Type: unifiedresources.ResourceTypePhysicalDisk, Status: status, ParentName: "tower", + PhysicalDisk: &unifiedresources.PhysicalDiskMeta{ + Health: "PASSED", Wearout: -1, Temperature: temperature, + Collection: &diskinventory.CollectionStatus{Temperature: collection}, + }, + } + } + disks := []unifiedresources.Resource{ + disk("standby-disk", unifiedresources.StatusOnline, 58, diskinventory.Unavailable("smartctl", "disk is in standby")), + disk("silent-disk", unifiedresources.StatusOffline, 52, diskinventory.Unavailable("smartctl", "host agent stopped reporting")), + disk("hot-disk", unifiedresources.StatusOnline, 55, diskinventory.Available("smartctl")), + } + s := &Service{unifiedResourceProvider: &mockUnifiedResourceProvider{ + getStatsFunc: func() unifiedresources.ResourceStats { + return unifiedresources.ResourceStats{Total: len(disks), ByType: map[unifiedresources.ResourceType]int{unifiedresources.ResourceTypePhysicalDisk: len(disks)}} + }, + getByTypeFunc: func(resourceType unifiedresources.ResourceType) []unifiedresources.Resource { + if resourceType == unifiedresources.ResourceTypePhysicalDisk { + return disks + } + return nil + }, + }} + + got := s.buildUnifiedResourceContext() + _, attention, found := strings.Cut(got, "**Physical Disks Needing Attention:**") + if !found { + t.Fatalf("expected disk attention section, got %q", got) + } + attention, _, _ = strings.Cut(attention, "\n\n") + if entries := strings.Count(attention, "\n- "); entries != 2 { + t.Fatalf("attention entries = %d, want the offline and the collected-hot disk only: %q", entries, attention) + } + if strings.Contains(attention, "58C") { + t.Fatalf("a standby disk's retained 58C must not flag it for heat: %q", attention) + } + if !strings.Contains(attention, "(PASSED), Temp: last known 52C (host agent stopped reporting) [offline]") { + t.Fatalf("expected the offline disk's retained temperature as last known: %q", attention) + } + if !strings.Contains(attention, "(PASSED), Temp: 55C [online]") { + t.Fatalf("expected the collected 55C disk on the attention list: %q", attention) + } +} + func TestBuildUnifiedResourceContext_TruncatesLargeContext(t *testing.T) { largeName := strings.Repeat("a", 60000) diff --git a/internal/ai/tools/data_types.go b/internal/ai/tools/data_types.go index 39c0043c25..89404a52b3 100644 --- a/internal/ai/tools/data_types.go +++ b/internal/ai/tools/data_types.go @@ -1222,7 +1222,12 @@ type SMARTDiskSummary struct { Device string `json:"device"` Model string `json:"model"` Health string `json:"health"` - Temperature int `json:"temperature,omitempty"` + Temperature int `json:"temperature,omitempty"` // Celsius, only when collected now + // LastKnownTemperature is a temperature kept from an earlier observation + // when the current one did not collect it (a disk in standby, a host + // agent past its reporting lease); LastKnownTemperatureReason says why. + LastKnownTemperature int `json:"last_known_temperature,omitempty"` + LastKnownTemperatureReason string `json:"last_known_temperature_reason,omitempty"` } // RAIDArraySummary is a summarized RAID array @@ -2045,10 +2050,16 @@ type PhysicalDiskSummary struct { Risk *unifiedresources.PhysicalDiskRisk `json:"risk,omitempty"` SMART *unifiedresources.SMARTMeta `json:"smart,omitempty"` LifeRemainingPercent *int `json:"life_remaining_percent,omitempty"` // SSD life remaining percent (0-100), nil when unavailable - Temperature *int `json:"temperature,omitempty"` // Celsius, nil when unavailable + Temperature *int `json:"temperature,omitempty"` // Celsius, nil unless collected now RPM *int `json:"rpm,omitempty"` // 0 for SSDs, nil when unavailable Used string `json:"used,omitempty"` LastChecked time.Time `json:"last_checked,omitempty"` + + // LastKnownTemperature is a temperature kept from an earlier observation + // when the current one did not collect it (a disk in standby, a host + // agent past its reporting lease); LastKnownTemperatureReason says why. + LastKnownTemperature *int `json:"last_known_temperature,omitempty"` + LastKnownTemperatureReason string `json:"last_known_temperature_reason,omitempty"` } // ========== Host RAID Types ========== diff --git a/internal/ai/tools/disk_temperature.go b/internal/ai/tools/disk_temperature.go new file mode 100644 index 0000000000..1fdf85ece9 --- /dev/null +++ b/internal/ai/tools/disk_temperature.go @@ -0,0 +1,54 @@ +package tools + +import ( + "fmt" + "strings" + + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" +) + +// DiskTemperature splits a physical disk temperature by its collection state +// for AI context. Collected is a reading the current observation took, and it +// is the only temperature that may be judged as heat. LastKnown is a value +// normalization kept without collecting it now (a disk in standby, a host +// agent past its reporting lease), with the collection state's reason in +// Reason. It is history, never evidence that the disk is hot now. +type DiskTemperature struct { + Collected int + LastKnown int + Reason string +} + +// SplitDiskTemperature classifies a disk temperature with +// diskinventory.TemperatureCollected. +func SplitDiskTemperature(temperature int, collection *diskinventory.CollectionStatus) DiskTemperature { + if diskinventory.TemperatureCollected(temperature, collection) { + return DiskTemperature{Collected: temperature} + } + if temperature <= 0 { + return DiskTemperature{} + } + split := DiskTemperature{LastKnown: temperature} + if collection != nil { + split.Reason = strings.TrimSpace(collection.Temperature.Reason) + } + return split +} + +// Format renders the temperature with unit for AI text: a collected reading as +// the value ("41C"), a retained one as "last known 41C (disk is in standby)", +// and "" when the disk has no temperature. +func (t DiskTemperature) Format(unit string) string { + switch { + case t.Collected > 0: + return fmt.Sprintf("%d%s", t.Collected, unit) + case t.LastKnown > 0: + text := fmt.Sprintf("last known %d%s", t.LastKnown, unit) + if t.Reason != "" { + text += " (" + t.Reason + ")" + } + return text + default: + return "" + } +} diff --git a/internal/ai/tools/physical_disk_evidence_test.go b/internal/ai/tools/physical_disk_evidence_test.go index 320ebff9cb..d7c3d4956c 100644 --- a/internal/ai/tools/physical_disk_evidence_test.go +++ b/internal/ai/tools/physical_disk_evidence_test.go @@ -9,6 +9,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" "github.com/stretchr/testify/require" ) @@ -111,3 +112,106 @@ func TestPhysicalDiskEvidencePreservesCanonicalRisk(t *testing.T) { } } + +// Normalization keeps a disk's last-known temperature when the current +// observation did not collect one (a disk in standby, a host agent past its +// reporting lease). The tools must not hand that value to the model as the +// disk's temperature: it moves to last_known_temperature with its reason. +func TestDiskToolsReportRetainedTemperatureAsLastKnown(t *testing.T) { + standby := &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")} + silent := &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "host agent stopped reporting")} + collected := &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")} + resources := []unifiedresources.Resource{ + {ID: "disk-standby", Name: "Standby disk", Type: unifiedresources.ResourceTypePhysicalDisk, + Status: unifiedresources.StatusOnline, ParentName: "node-one", + PhysicalDisk: &unifiedresources.PhysicalDiskMeta{Health: "PASSED", Wearout: -1, Temperature: 61, Collection: standby}}, + {ID: "disk-live", Name: "Live disk", Type: unifiedresources.ResourceTypePhysicalDisk, + Status: unifiedresources.StatusOnline, ParentName: "node-one", + PhysicalDisk: &unifiedresources.PhysicalDiskMeta{Health: "PASSED", Wearout: -1, Temperature: 38, Collection: collected}}, + } + diskProvider := &mockDiskHealthProvider{} + diskProvider.On("GetHosts").Return([]*unifiedresources.HostView{ + newHostView("host-tower", "tower", "agent-tower", "tower", &unifiedresources.HostSensorMeta{ + SMART: []unifiedresources.HostSMARTMeta{ + {Device: "/dev/sda", Model: "WDC WD80EFAX", Health: "PASSED", Temperature: 58, Collection: silent}, + {Device: "/dev/sdb", Model: "WDC WD80EFAX", Health: "PASSED", Temperature: 36, Collection: collected}, + }, + }, nil, nil), + }) + executor := NewPulseToolExecutor(ExecutorConfig{ + StateProvider: &mockStateProvider{state: models.StateSnapshot{}}, + UnifiedResourceProvider: &stubUnifiedResourceProvider{resources: resources}, + DiskHealthProvider: diskProvider, + ControlLevel: ControlLevelReadOnly, + }) + + for _, tool := range []struct { + name string + args map[string]interface{} + key string + }{ + {"pulse_metrics", map[string]interface{}{"type": "disks"}, "disks"}, + {"pulse_query", map[string]interface{}{"action": "list", "type": "physical-disks"}, "physical_disks"}, + } { + t.Run(tool.name, func(t *testing.T) { + result, err := executor.ExecuteTool(context.Background(), tool.name, tool.args) + require.NoError(t, err) + require.False(t, result.IsError, "%+v", result.Content) + var payload map[string]json.RawMessage + require.NoError(t, json.Unmarshal([]byte(result.Content[0].Text), &payload)) + var disks []PhysicalDiskSummary + require.NoError(t, json.Unmarshal(payload[tool.key], &disks)) + byID := map[string]PhysicalDiskSummary{} + for _, disk := range disks { + byID[disk.ID] = disk + } + retained := byID["disk-standby"] + require.Nil(t, retained.Temperature, "a retained reading is not the disk's temperature") + require.NotNil(t, retained.LastKnownTemperature) + require.Equal(t, 61, *retained.LastKnownTemperature) + require.Equal(t, "disk is in standby", retained.LastKnownTemperatureReason) + live := byID["disk-live"] + require.NotNil(t, live.Temperature) + require.Equal(t, 38, *live.Temperature) + require.Nil(t, live.LastKnownTemperature) + require.NotContains(t, string(payload[tool.key]), `"temperature":61`) + }) + } + + t.Run("pulse_storage disk_health", func(t *testing.T) { + result, err := executor.ExecuteTool(context.Background(), "pulse_storage", map[string]interface{}{"type": "disk_health"}) + require.NoError(t, err) + require.False(t, result.IsError, "%+v", result.Content) + var response DiskHealthResponse + require.NoError(t, json.Unmarshal([]byte(result.Content[0].Text), &response)) + require.Len(t, response.Hosts, 1) + require.Len(t, response.Hosts[0].SMART, 2) + silentDisk, liveDisk := response.Hosts[0].SMART[0], response.Hosts[0].SMART[1] + require.Zero(t, silentDisk.Temperature) + require.Equal(t, 58, silentDisk.LastKnownTemperature) + require.Equal(t, "host agent stopped reporting", silentDisk.LastKnownTemperatureReason) + require.Equal(t, 36, liveDisk.Temperature) + require.Zero(t, liveDisk.LastKnownTemperature) + require.NotContains(t, result.Content[0].Text, `"temperature":58`) + }) +} + +func TestDiskTemperatureFormatsRetainedReadingAsLastKnown(t *testing.T) { + cases := []struct { + temperature int + collection *diskinventory.CollectionStatus + want string + }{ + {41, nil, "41C"}, + {41, &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}, "41C"}, + {41, &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}, "last known 41C (disk is in standby)"}, + {41, &diskinventory.CollectionStatus{Temperature: diskinventory.Missing("smartctl", "")}, "last known 41C"}, + {0, &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}, ""}, + {0, nil, ""}, + } + for _, tc := range cases { + split := SplitDiskTemperature(tc.temperature, tc.collection) + require.Equal(t, tc.want, split.Format("C"), "%d %+v", tc.temperature, tc.collection) + require.Equal(t, diskinventory.TemperatureCollected(tc.temperature, tc.collection), split.Collected > 0) + } +} diff --git a/internal/ai/tools/tools_query.go b/internal/ai/tools/tools_query.go index 525a9e2a12..04762d047b 100644 --- a/internal/ai/tools/tools_query.go +++ b/internal/ai/tools/tools_query.go @@ -3583,9 +3583,12 @@ func physicalDiskSummaryFromResource(resource unifiedresources.Resource) Physica wearout := pd.Wearout summary.LifeRemainingPercent = &wearout } - if pd.Temperature > 0 { - temp := pd.Temperature - summary.Temperature = &temp + temperature := SplitDiskTemperature(pd.Temperature, pd.Collection) + if temperature.Collected > 0 { + summary.Temperature = &temperature.Collected + } else if temperature.LastKnown > 0 { + summary.LastKnownTemperature = &temperature.LastKnown + summary.LastKnownTemperatureReason = temperature.Reason } if pd.RPM > 0 { rpm := pd.RPM diff --git a/internal/ai/tools/tools_storage.go b/internal/ai/tools/tools_storage.go index 988b5d71bf..a84805a494 100644 --- a/internal/ai/tools/tools_storage.go +++ b/internal/ai/tools/tools_storage.go @@ -402,11 +402,14 @@ func (e *PulseToolExecutor) executeGetDiskHealth(_ context.Context, _ map[string // SMART data if sensors := host.Sensors(); sensors != nil { for _, disk := range sensors.SMART { + temperature := SplitDiskTemperature(disk.Temperature, disk.Collection) hostHealth.SMART = append(hostHealth.SMART, SMARTDiskSummary{ - Device: disk.Device, - Model: disk.Model, - Health: disk.Health, - Temperature: disk.Temperature, + Device: disk.Device, + Model: disk.Model, + Health: disk.Health, + Temperature: temperature.Collected, + LastKnownTemperature: temperature.LastKnown, + LastKnownTemperatureReason: temperature.Reason, }) } } diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go index b2cd38ac74..17faf9067b 100644 --- a/internal/api/contract_test.go +++ b/internal/api/contract_test.go @@ -5228,6 +5228,64 @@ func TestContract_PerformanceReportTransportUsesCatalogDefinition(t *testing.T) } } +// Normalization keeps a disk's last-known temperature when the current +// observation did not collect one (a disk in standby, a host agent past its +// reporting lease). Reports present their disk tables as measured, so the +// performance report and the reporting runtime snapshot leave that value out. +func TestContract_ReportsOmitRetainedDiskTemperatures(t *testing.T) { + engine := &stubReportingEngine{data: []byte("report"), contentType: "application/pdf"} + original := reporting.GetEngine() + reporting.SetEngine(engine) + t.Cleanup(func() { reporting.SetEngine(original) }) + + state := models.NewState() + state.Nodes = []models.Node{{ID: "node-1", Name: "node-a", Status: "online"}} + disk := func(id, devPath string, temperature int, collection diskinventory.FieldStatus) unifiedresources.Resource { + return unifiedresources.Resource{ + ID: id, Type: unifiedresources.ResourceTypePhysicalDisk, Name: id, ParentName: "node-a", + Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"node-a"}}, + PhysicalDisk: &unifiedresources.PhysicalDiskMeta{ + DevPath: devPath, DiskType: "hdd", Health: "PASSED", Wearout: -1, Temperature: temperature, + Collection: &diskinventory.CollectionStatus{Temperature: collection}, + }, + } + } + monitor := newReportingMonitorForTest(t, state, []unifiedresources.Resource{ + disk("disk-standby", "/dev/sda", 61, diskinventory.Unavailable("smartctl", "disk is in standby")), + disk("disk-silent", "/dev/sdb", 57, diskinventory.Unavailable("smartctl", "host agent stopped reporting")), + disk("disk-live", "/dev/sdc", 38, diskinventory.Available("smartctl")), + }) + handler := NewReportingHandlers(newReportingMTMForTest(t, monitor), nil) + want := map[string]int{"/dev/sda": 0, "/dev/sdb": 0, "/dev/sdc": 38} + + req := httptest.NewRequest(http.MethodGet, "/api/reporting?format=pdf&resourceType=node&resourceId=node-1", nil) + req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "default")) + rec := httptest.NewRecorder() + handler.HandleGenerateReport(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rec.Code, rec.Body.String()) + } + reported := map[string]int{} + for _, d := range engine.lastReq.Disks { + reported[d.Device] = d.Temperature + } + if !reflect.DeepEqual(reported, want) { + t.Fatalf("performance report disk temperatures = %v, want %v", reported, want) + } + + snapshot, ok := handler.getRuntimeStateSnapshot(context.Background(), "default") + if !ok { + t.Fatal("expected runtime snapshot to be available") + } + snapshotted := map[string]int{} + for _, d := range snapshot.Disks { + snapshotted[d.Device] = d.Temperature + } + if !reflect.DeepEqual(snapshotted, want) { + t.Fatalf("reporting runtime snapshot disk temperatures = %v, want %v", snapshotted, want) + } +} + func TestContract_ReportingRequestCarriesEntitledReportBranding(t *testing.T) { engine := &stubReportingEngine{data: []byte("report"), contentType: "application/pdf"} original := reporting.GetEngine() diff --git a/internal/api/metrics_reporting_handlers.go b/internal/api/metrics_reporting_handlers.go index ebf7c91682..f363c6c0b2 100644 --- a/internal/api/metrics_reporting_handlers.go +++ b/internal/api/metrics_reporting_handlers.go @@ -20,6 +20,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/recovery" recoverymanager "github.com/rcourtman/pulse-go-rewrite/internal/recovery/manager" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" "github.com/rcourtman/pulse-go-rewrite/pkg/reporting" "github.com/rs/zerolog/log" ) @@ -882,13 +883,25 @@ func (h *ReportingHandlers) enrichNodeReport(req *reporting.MetricReportRequest, Type: pd.DiskType, Size: pd.SizeBytes, Health: pd.Health, - Temperature: pd.Temperature, + Temperature: reportDiskTemperature(*pd), WearLevel: pd.Wearout, }) } } } +// reportDiskTemperature returns the disk temperature a report may tabulate: +// one the current observation collected. Normalization may keep a last-known +// temperature it did not collect (a disk in standby, a host agent past its +// reporting lease); a report presents its values as measured, so that one is +// left out (0, rendered as no reading) rather than shown as current. +func reportDiskTemperature(pd unifiedresources.PhysicalDiskMeta) int { + if !diskinventory.TemperatureCollected(pd.Temperature, pd.Collection) { + return 0 + } + return pd.Temperature +} + // enrichVMReport adds VM-specific data to the report request func (h *ReportingHandlers) enrichVMReport(ctx context.Context, orgID string, req *reporting.MetricReportRequest, snapshot reportingEnrichmentSnapshot, start, end time.Time) { // Find the VM diff --git a/internal/api/reporting_runtime_snapshot.go b/internal/api/reporting_runtime_snapshot.go index 62cbc72f9b..e5e36914c5 100644 --- a/internal/api/reporting_runtime_snapshot.go +++ b/internal/api/reporting_runtime_snapshot.go @@ -191,7 +191,7 @@ func (h *ReportingHandlers) getRuntimeStateSnapshot(ctx context.Context, orgID s Type: disk.DiskType, Size: disk.SizeBytes, Health: disk.Health, - Temperature: disk.Temperature, + Temperature: reportDiskTemperature(*disk), WearLevel: disk.Wearout, }) } diff --git a/internal/monitoring/issue1595_collection_trust_test.go b/internal/monitoring/issue1595_collection_trust_test.go index 2c4835d3c9..daa3cf2605 100644 --- a/internal/monitoring/issue1595_collection_trust_test.go +++ b/internal/monitoring/issue1595_collection_trust_test.go @@ -319,8 +319,9 @@ func hasIssue1595Source(sources []unifiedresources.DataSource, want unifiedresou } // An agent report's SMART row may carry a temperature its collector did not -// collect this time (retained across a standby or failed probe). Only a -// collected temperature becomes a history sample. +// collect this time (retained across a standby or failed probe, or kept after +// the agent's reporting lease expired). Only a temperature +// diskinventory.TemperatureCollected accepts becomes a history sample. func TestWriteHostSMARTMetricsRecordsOnlyCollectedTemperatures(t *testing.T) { storeConfig := metrics.DefaultConfig(t.TempDir()) storeConfig.FlushInterval = time.Hour @@ -337,6 +338,10 @@ func TestWriteHostSMARTMetricsRecordsOnlyCollectedTemperatures(t *testing.T) { {Device: "sdb", Serial: "LEGACY1", Temperature: 38}, {Device: "sdc", Serial: "RETAINED1", Temperature: 36, Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}}, + {Device: "sdd", Serial: "SILENT1", Temperature: 44, + Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", models.HostAgentStoppedReportingReason)}}, + {Device: "sde", Serial: "MISSING1", Temperature: 39, + Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Missing("smartctl", "temperature attribute absent")}}, }}} now := time.Now() m.writeHostSMARTMetrics(host, now) @@ -349,6 +354,8 @@ func TestWriteHostSMARTMetricsRecordsOnlyCollectedTemperatures(t *testing.T) { {host.Sensors.SMART[0], 1}, {host.Sensors.SMART[1], 1}, {host.Sensors.SMART[2], 0}, + {host.Sensors.SMART[3], 0}, + {host.Sensors.SMART[4], 0}, } { id := unifiedresources.HostSMARTDiskSourceID(host, tc.disk) points, err := store.Query("disk", id, "smart_temp", now.Add(-time.Minute), now.Add(time.Minute), 0) diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 167d0ce22b..7649b64f28 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -939,7 +939,7 @@ func (m *Monitor) writeSMARTMetrics(disk models.PhysicalDisk, now time.Time) { return } - if diskTemperatureCollected(disk.Temperature, disk.Collection) && m.metricsHistory != nil { + if diskinventory.TemperatureCollected(disk.Temperature, disk.Collection) && m.metricsHistory != nil { m.metricsHistory.AddDiskMetric(resourceID, "smart_temp", float64(disk.Temperature), now) } @@ -953,22 +953,6 @@ func (m *Monitor) writeSMARTMetrics(disk models.PhysicalDisk, now time.Time) { } } -// diskTemperatureCollected reports whether a disk temperature was collected by -// its current observation. Normalization may carry a last-known temperature -// when it was not (standby, an agent past its reporting lease); that value must -// not be recorded as a new history sample. A temperature without collection -// state predates the contract and keeps the old behavior. -func diskTemperatureCollected(temperature int, collection *diskinventory.CollectionStatus) bool { - if temperature <= 0 { - return false - } - if collection == nil { - return true - } - state := collection.Temperature.State - return state == "" || state == diskinventory.FieldAvailable -} - // smartMetricStoreWrites builds the persisted SMART writes for one physical // disk observation. The caller owns batching so the unified sync can anchor the // observation time and drop exact replays (#1966). @@ -989,7 +973,7 @@ func (m *Monitor) smartMetricStoreWrites(disk models.PhysicalDisk, resourceID st }) } - if diskTemperatureCollected(disk.Temperature, disk.Collection) { + if diskinventory.TemperatureCollected(disk.Temperature, disk.Collection) { appendWrite("smart_temp", float64(disk.Temperature)) } diff --git a/internal/monitoring/monitor_agents.go b/internal/monitoring/monitor_agents.go index 376fe3c101..241750c3d5 100644 --- a/internal/monitoring/monitor_agents.go +++ b/internal/monitoring/monitor_agents.go @@ -26,6 +26,7 @@ import ( agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker" "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" "github.com/rcourtman/pulse-go-rewrite/pkg/fsfilters" pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing" "github.com/rs/zerolog" @@ -4086,7 +4087,7 @@ func (m *Monitor) writeHostSMARTMetrics(host models.Host, now time.Time) { continue } - if diskTemperatureCollected(disk.Temperature, disk.Collection) { + if diskinventory.TemperatureCollected(disk.Temperature, disk.Collection) { m.metricsStore.Write("disk", resourceID, "smart_temp", float64(disk.Temperature), now) } diff --git a/internal/monitoring/monitor_metrics.go b/internal/monitoring/monitor_metrics.go index 0e8d37a4d8..1949f205f8 100644 --- a/internal/monitoring/monitor_metrics.go +++ b/internal/monitoring/monitor_metrics.go @@ -8,6 +8,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/mock" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" "github.com/rcourtman/pulse-go-rewrite/pkg/metrics" ) @@ -267,11 +268,13 @@ func (m *Monitor) GetPhysicalDiskTemperatureCharts(duration time.Duration) map[s // Phase 1: Collect disk metadata and resource IDs. type diskMeta struct { - resourceID string - name string - node string - instance string - temperature int + resourceID string + name string + node string + instance string + // collectedTemperature is the reading the current observation took, + // or 0 when the disk only retains a last-known value. + collectedTemperature int } var disks []diskMeta for _, disk := range readState.PhysicalDisks() { @@ -294,12 +297,16 @@ func (m *Monitor) GetPhysicalDiskTemperatureCharts(duration time.Duration) map[s if name == "" { name = strings.TrimSpace(disk.DevPath()) } + collectedTemperature := 0 + if diskinventory.TemperatureCollected(disk.Temperature(), disk.Collection()) { + collectedTemperature = disk.Temperature() + } disks = append(disks, diskMeta{ - resourceID: resourceID, - name: name, - node: strings.TrimSpace(disk.Node()), - instance: strings.TrimSpace(disk.Instance()), - temperature: disk.Temperature(), + resourceID: resourceID, + name: name, + node: strings.TrimSpace(disk.Node()), + instance: strings.TrimSpace(disk.Instance()), + collectedTemperature: collectedTemperature, }) } @@ -349,12 +356,19 @@ func (m *Monitor) GetPhysicalDiskTemperatureCharts(duration time.Duration) map[s // Sparklines require >= 2 points. If the store returned 0 or 1 points // but the disk has a live temperature reading, pad to 2 points so the - // chart can render (flat line at current temperature). + // chart can render (flat line at current temperature). A retained + // last-known temperature (standby, a silent host agent) is not a + // reading taken now, so the stored samples stay as they are and a + // disk with none has no series. if len(tempPoints) < 2 { - now := time.Now() - tempPoints = []MetricPoint{ - {Timestamp: now.Add(-60 * time.Second), Value: float64(d.temperature)}, - {Timestamp: now, Value: float64(d.temperature)}, + if d.collectedTemperature > 0 { + now := time.Now() + tempPoints = []MetricPoint{ + {Timestamp: now.Add(-60 * time.Second), Value: float64(d.collectedTemperature)}, + {Timestamp: now, Value: float64(d.collectedTemperature)}, + } + } else if len(tempPoints) == 0 { + continue } } diff --git a/internal/monitoring/monitor_metrics_slo_test.go b/internal/monitoring/monitor_metrics_slo_test.go index ca24b2cacb..cbd20fb9d6 100644 --- a/internal/monitoring/monitor_metrics_slo_test.go +++ b/internal/monitoring/monitor_metrics_slo_test.go @@ -11,6 +11,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/mock" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" "github.com/rcourtman/pulse-go-rewrite/pkg/metrics" ) @@ -671,6 +672,50 @@ func TestSLO_GetPhysicalDiskTemperatureCharts_WithNativeHistoryFallback(t *testi assertLatencySLO(t, "GetPhysicalDiskTemperatureCharts(native-history fallback)", latencies, target) } +// The disk temperature charts pad a short series to now with the disk's +// current reading. A retained last-known temperature (a disk in standby, a host +// agent past its reporting lease) is not a reading taken now: the samples +// stored while the disk was live stay as they are, and a disk with none gets no +// series rather than a flat line at the retained value. +func TestDiskTemperatureChartsPadOnlyWithCollectedReading(t *testing.T) { + now := time.Now().UTC() + standby := &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")} + silent := &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", models.HostAgentStoppedReportingReason)} + collected := &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")} + registry := unifiedresources.NewRegistry(nil) + registry.IngestSnapshot(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{ + {ID: "disk-standby", Node: "node-a", DevPath: "/dev/sda", Model: "WDC WD80EFAX", Serial: "STANDBY-NO-HISTORY", Temperature: 41, Collection: standby, LastChecked: now}, + {ID: "disk-silent", Node: "node-a", DevPath: "/dev/sdb", Model: "WDC WD80EFAX", Serial: "SILENT-ONE-SAMPLE", Temperature: 43, Collection: silent, LastChecked: now}, + {ID: "disk-live", Node: "node-a", DevPath: "/dev/sdc", Model: "WDC WD80EFAX", Serial: "LIVE-NO-HISTORY", Temperature: 38, Collection: collected, LastChecked: now}, + }}) + m := &Monitor{ + metricsHistory: NewMetricsHistory(1024, 24*time.Hour), + state: models.NewState(), + resourceStore: unifiedresources.NewMonitorAdapter(registry), + } + sampledAt := now.Add(-20 * time.Minute) + m.metricsHistory.AddDiskMetric("SILENT-ONE-SAMPLE", "smart_temp", 44, sampledAt) + + charts := m.GetPhysicalDiskTemperatureCharts(time.Hour) + if entry, ok := charts["STANDBY-NO-HISTORY"]; ok { + t.Fatalf("a standby disk with no stored samples got a series from its retained reading: %+v", entry.Temperature) + } + silentEntry, ok := charts["SILENT-ONE-SAMPLE"] + if !ok || len(silentEntry.Temperature) != 1 || silentEntry.Temperature[0].Value != 44 || + !silentEntry.Temperature[0].Timestamp.Equal(sampledAt) { + t.Fatalf("silent agent disk series = %+v (found %v), want only its stored 44C sample at %s", silentEntry.Temperature, ok, sampledAt) + } + live, ok := charts["LIVE-NO-HISTORY"] + if !ok || len(live.Temperature) != 2 { + t.Fatalf("live disk series = %+v (found %v), want the padded 2-point sparkline", live.Temperature, ok) + } + for _, point := range live.Temperature { + if point.Value != 38 { + t.Fatalf("live disk padded with %.0f, want its collected 38C", point.Value) + } + } +} + func TestSLO_GetDiskMetricsForChart_WithNativeStoreFallback(t *testing.T) { skipMonitoringSLOUnderRace(t) suppressMonitoringTestLogs(t) diff --git a/internal/monitoring/physical_disk_roundtrip_test.go b/internal/monitoring/physical_disk_roundtrip_test.go index 38c45b8963..733e7d092c 100644 --- a/internal/monitoring/physical_disk_roundtrip_test.go +++ b/internal/monitoring/physical_disk_roundtrip_test.go @@ -536,7 +536,7 @@ func TestMergeHostAgentSMARTIntoDisks_LegacyAgentTemperatureFollowsLease(t *test if reporting.Temperature != 38 || reporting.Collection.Temperature != diskinventory.Available(hostAgentLegacySource) { t.Fatalf("legacy agent temperature not recorded as collected: temp=%d collection=%+v", reporting.Temperature, reporting.Collection) } - if !diskTemperatureCollected(reporting.Temperature, reporting.Collection) { + if !diskinventory.TemperatureCollected(reporting.Temperature, reporting.Collection) { t.Fatal("a reporting legacy agent's temperature must reach history") } @@ -549,7 +549,7 @@ func TestMergeHostAgentSMARTIntoDisks_LegacyAgentTemperatureFollowsLease(t *test if got.Temperature != 38 || got.Collection.Temperature.State != diskinventory.FieldUnavailable { t.Fatalf("%s: silent legacy agent temperature presented as collected: temp=%d collection=%+v", name, got.Temperature, got.Collection) } - if diskTemperatureCollected(got.Temperature, got.Collection) { + if diskinventory.TemperatureCollected(got.Temperature, got.Collection) { t.Fatalf("%s: a silent legacy agent's retained temperature must not reach history", name) } } From 7b362b353239a51b61b9d4f4c733ac8633a127e5 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:16:04 +0100 Subject: [PATCH 04/13] Load guest physical disks with the drawer to keep Workloads in budget GuestDrawerOverview statically imported GuestPhysicalDisks, which pulled all of diskPresentation.ts into the WorkloadsSurface table chunk. With the Running Hot verdict that chunk measured 70.64 kB gzip against its 70.23 kB cap, so the Frontend bundle check would fail. The card renders only inside an open drawer for a guest with a Pulse agent, so the overview now lazy-loads it under a local Suspense, the same pattern as the bulk web-link editor. WorkloadsSurface drops to 64.31 kB and the disk presentation builds as its own chunk. The contract test pins the dynamic import. --- .../subsystems/performance-and-scalability.md | 5 +++ .../fb12a674d61da775.json | 35 +++++++++++++++++++ .../Workloads/GuestDrawerOverview.tsx | 16 +++++++-- ...loadsSurface.performance.contract.test.tsx | 6 ++++ 4 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 frontend-modern/browser-verification/fb12a674d61da775.json diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index f78c71e278..5071809ca9 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -1182,6 +1182,11 @@ change may globally weaken the Task 03 lifecycle-state idempotency invariant. Saves run one metadata write per changed link, then dispatch the metadata-changed events together so whole-snapshot listeners coalesce onto one in-flight refetch instead of refetching per link. + The drawer overview lazy-loads `GuestPhysicalDisks.tsx` under a local + `Suspense`, so the physical-disk presentation module + (`diskPresentation.ts`) loads with an open agent guest's drawer instead + of joining the WorkloadsSurface chunk. + `WorkloadsSurface.performance.contract.test.tsx` pins the dynamic import. Drawer history charts belong to `frontend-modern/src/components/Workloads/GuestDrawerHistory.tsx`. A current metric may remain visible in a chart legend only when labelled `current`; it must never be expanded into synthetic timestamps or a flat diff --git a/frontend-modern/browser-verification/fb12a674d61da775.json b/frontend-modern/browser-verification/fb12a674d61da775.json new file mode 100644 index 0000000000..78a9f2ccca --- /dev/null +++ b/frontend-modern/browser-verification/fb12a674d61da775.json @@ -0,0 +1,35 @@ +{ + "version": 1, + "base_sha": "94b7870855120b2c62fce19102903601b9fc4bfc", + "verified_at": "2026-10-06T22:30:00Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx" + ], + "content_sha256": { + "frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx": "6e4003a1508856b944680e35234d9f93ef59a02f870fea6f9f1173f7e641c905" + }, + "routes": [ + "/proxmox/overview" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack (8 nodes, local uncommitted generator overlay pinning pve3 /dev/nvme2 at 72C). Mock guests carry no agent facet and no linked physical disks, so Playwright route interception added agent.agentVersion '6.5.0' to redis-cache-01 (vm-3ae77f2c237ff912) in /api/resources responses and answered the guest's type=physical_disk&parent= query with the stack's real 72C Crucial MX500 resource reparented to that guest", + "Production builds: with this commit (base 8259a74c9d plus the verdict commit 94b7870855) check-bundle-size reports WorkloadsSurface 64.31 kB gzip against a 70.23 kB max, every tracked chunk ok and none missing; origin/main d9ffb39ce8 plus 94b7870855 without this commit measured 70.64 kB (over by 0.40) and d9ffb39ce8 alone 70.20 kB. diskPresentation and GuestPhysicalDisks now build as their own chunks and the WorkloadsSurface chunk no longer contains the disk verdict strings", + "Guest drawer at 1440 and 390: the GuestPhysicalDisks module is requested only after the guest row is clicked open (none before), the parent query is sent once for the guest id, and the card reads 'Physical Disks & SMART (1)', 'Crucial MX500 2TB', '/dev/nvme2', 'Running Hot' (title 'Disk temperature is 72C', rgb(185,28,28)), '72\u00b0C', '1.82 TB'" + ], + "interactions": [ + "Loaded /proxmox/overview at 1440x900 and 390x844, recorded module requests, then clicked the redis-cache-01 guest row to open its drawer and waited for the guest physical-disk card", + "Read the card text, the verdict label, its title and computed colour, and clipped a screenshot of the card at both widths", + "Ran vite build and check-bundle-size on d9ffb39ce8, on d9ffb39ce8 plus 94b7870855, and on this branch" + ] +} diff --git a/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx b/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx index de41cfe2e4..9b30b81b1f 100644 --- a/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx +++ b/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx @@ -1,4 +1,4 @@ -import { Show } from 'solid-js'; +import { Show, Suspense, lazy } from 'solid-js'; import { formatDiscoveryAge } from '@/api/discovery'; import { @@ -29,11 +29,19 @@ import { } from './guestDrawerModel'; import type { NestedWorkloadContext } from './nestedWorkloadContext'; import { WORKLOAD_ACTION_AGENT_LABEL } from './workloadAgentReadiness'; -import { GuestPhysicalDisks } from './GuestPhysicalDisks'; import { RaidCard } from '@/components/shared/cards/RaidCard'; import type { GuestDrawerProps } from './guestDrawerModel'; +// Linked physical disks render only inside an open agent guest's drawer. Load +// them on demand so the disk presentation module stays out of the +// WorkloadsSurface table chunk. +const GuestPhysicalDisks = lazy(() => + import('./GuestPhysicalDisks').then(({ GuestPhysicalDisks }) => ({ + default: GuestPhysicalDisks, + })), +); + interface GuestDrawerOverviewProps { guest: GuestDrawerProps['guest']; guestOsSummary: string; @@ -317,7 +325,9 @@ export function GuestDrawerOverview(props: GuestDrawerOverviewProps) { props.guest.agentRaid?.length } > - + + + diff --git a/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx b/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx index d2aa75fff9..4fa12f172d 100644 --- a/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx @@ -1667,6 +1667,12 @@ describe('Workloads performance contract', () => { expect(guestDrawerManageSource).toContain('ResourceOperatorStateSection'); expect(guestDrawerOverviewSource).toContain('buildWorkloadsDiskPresentation'); expect(guestDrawerOverviewSource).toContain('Filesystems'); + // Linked physical disks load with an open agent guest's drawer, so the + // disk presentation module stays out of the WorkloadsSurface chunk. + expect(guestDrawerOverviewSource).toContain("import('./GuestPhysicalDisks')"); + expect(guestDrawerOverviewSource).not.toMatch( + /^import (?!type )[^;]*from '\.\/GuestPhysicalDisks';/m, + ); // The shared DetailSectionTable gives the earlier of the last two sections // the wider span, so the longer filesystem list must be declared before the // short tag list for mount points to get the readable panel width. From 723f3165a1292268634b6054ab93d3015805d990 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:23:18 +0100 Subject: [PATCH 05/13] Show a silent agent's retained disk temperatures as last known MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a host agent passes its reporting lease, its collected SMART temperatures keep their values with collection.temperature unavailable (#2593), and #2603 taught the Storage disk surfaces to show such a value as last known. The frontend HostDiskSMART type never carried `collection`, so the agent's own surfaces kept presenting the retained readings as current: - The Machines temperature cell fell back to the hottest SMART reading and coloured it against the disk thresholds. On a live mock stack a silent, SMART-only agent read 52°C in threshold green. - Its tooltip, the machine drawer's Thermals rows and the guest drawer's Physical Disks card listed the retained disks as plain readings. The row does not make this obvious. By the time the lease expires the registry has marked a standalone agent's source stale, which outranks offline, so a silent agent reaches the table as `warning` with `agent.stale`, not `offline`. The offline metric fallback never blanks its cells, and the temperature cell has no such gate. Now `HostDiskSMART.collection` is typed and every reader decides through `isPhysicalDiskTemperatureCurrent`: - A retained disk reading never stands for the machine while another disk is current. When no non-standby disk is current and the machine has no positive direct or temperatureCelsius reading, the cell shows the hottest retained non-standby value muted with a dotted underline and screen-reader "last known", without threshold colour. - Tooltip and Thermals rows read "(last known)", the Thermals row with the collection reason as its title. A disk with a zero or negative reading no longer shows a temperature row there. - The guest Physical Disks card uses the Storage table's last-known treatment. The temperature helpers move to diskTemperaturePresentation.ts, which diskPresentation.ts re-exports. On the earlier base, importing diskPresentation from the Machines page and the drawer mapper folded the whole Storage presenter into the shared chunk (+4.9 kB gzip) and renamed its tracked budget entry, failing the bundle check. With the small module, Standalone grows 0.22 kB, the shared chunk 0.20 kB, and Workloads is flat against a build of main. The guest card still loads diskPresentation for extraction and health. Direct and temperatureCelsius readings carry no collection state and still lead the cell. A stale agent's CPU, memory, disk and sensor values still render as current. That is a row-wide decision, left for a separate change. --- .../v6/internal/subsystems/api-contracts.md | 9 + .../subsystems/frontend-primitives.md | 10 ++ .../subsystems/performance-and-scalability.md | 7 + .../internal/subsystems/storage-recovery.md | 15 ++ .../internal/subsystems/unified-resources.md | 14 ++ .../47c60098fc12bf96.json | 54 ++++++ .../__tests__/resourceDetailMappers.test.ts | 85 ++++++++- .../Infrastructure/resourceDetailMappers.ts | 11 +- .../Workloads/GuestPhysicalDisks.tsx | 21 ++- .../__tests__/GuestPhysicalDisks.test.tsx | 96 +++++++++- .../standalone/AgentsMachinesTable.tsx | 28 ++- .../__tests__/AgentsMachinesTable.test.tsx | 148 ++++++++++++++++ .../__tests__/agentMachineTableModel.test.ts | 164 ++++++++++++++++++ .../standalone/agentMachineTableModel.ts | 66 +++++-- .../__tests__/diskPresentation.test.ts | 20 +++ .../storageBackups/diskPresentation.ts | 53 +----- .../diskTemperaturePresentation.ts | 53 ++++++ frontend-modern/src/types/api.ts | 3 + 18 files changed, 787 insertions(+), 70 deletions(-) create mode 100644 frontend-modern/browser-verification/47c60098fc12bf96.json create mode 100644 frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index ae4efc4a57..ecae82c5c7 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -5378,6 +5378,15 @@ Backend API payloads and `frontend-modern/src/types/api.ts` must preserve that optional map without making it a required compatibility field, and clients must keep it as descriptive host telemetry rather than a temperature metric, resource identity, alert metric, or storage/recovery signal. +Host sensor SMART rows (`smart[]`) also carry the optional per-field +`collection` status (`PhysicalDiskCollectionStatus`). Backend API payloads and +`frontend-modern/src/types/api.ts` (`HostDiskSMART.collection`) must preserve it +on the agent facet. When a host agent passes its reporting lease, a SMART +temperature that was current (`available`, or present without provenance) keeps +its value and becomes `unavailable`. A state that was already not available is +kept. Clients must read that state through the shared +`isPhysicalDiskTemperatureCurrent` decision before treating the value as a +current reading. Agent resource-context sections expose host package posture only as bounded operational facts: package manager, pending count, inventory state, inspection freshness, and reboot-required state. Raw package identifiers, versions, and diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 85c281c011..f585ee45a4 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -7803,6 +7803,16 @@ never through the threshold colour classes, so a retained reading cannot look hot or healthy. Table, drawer and pool surfaces share that one decision in `frontend-modern/src/features/storageBackups/diskPresentation.ts` rather than each re-reading `collection.temperature`. +That decision and its class now live in the small +`frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts` +module, which `diskPresentation.ts` re-exports, so the Machines table and the +machine drawer's Thermals rows apply the same treatment without pulling the +Storage presenter into their chunks. Those surfaces and the guest drawer's +Physical Disks card import it directly. +When the cell's value is a retained host-agent SMART temperature, it renders +muted with a dotted underline and screen-reader "last known", as does a +retained value in the guest card. Tooltip and Thermals rows say "(last known)", +and the Thermals row carries the collection reason as its title. The focused browser proofs are `frontend-modern/src/features/patrol/__tests__/patrolRunAcceptance.test.ts`, diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 87517e7399..a7f0088b89 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -2044,6 +2044,13 @@ render those sensor values already present on the selected resource payload, but it must not add host powercap reads, sensor-specific history reads, per-row polling, browser-side command assumptions, or table-wide aggregation work. +Host-agent SMART disk rows in `resourceDetailMappers.ts` mark a retained +temperature "(last known)", with its collection reason as the title, from the +payload's own `collection.temperature` state, through `getPhysicalDiskTemperaturePresentation` +in `frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts`. +That module stays separate from `diskPresentation.ts` so the drawer and +Machines chunks take the decision without the Storage presenter, and the rows +add no reads, polling or aggregation work. The Proxmox node drawer overview should follow the existing guest drawer compact detail-section pattern and expose node-specific context such as platform, kernel, hardware, raw capacity, telemetry, and thermal facts rather than diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index cd2ec5bf9e..2397cd4bb5 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -3814,6 +3814,21 @@ or alert evaluation. Verification: `diskPresentation.test.ts`, `diskDetailPresentation.test.ts`, `storagePoolDetailPresentation.test.ts`, `DiskList.test.tsx` (a row moving between current, retained and legacy states) and `DiskDetail.test.tsx`. +The same decision reaches the disks that only a host agent's own SMART rows +describe. `HostDiskSMART.collection` now crosses into the frontend type, and the +helpers live in `frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts` +(re-exported by `diskPresentation.ts`, one implementation) so the Machines +table and the machine drawer can use them without loading the Storage +presenter. The guest drawer's card already loads it for extraction and health. +A silent agent's retained SMART temperature never becomes the Machines row's +value while another disk has a current reading. When the machine has no +positive direct or `temperatureCelsius` reading and no non-standby disk is +current, the cell shows the hottest retained non-standby value as last known, +without the per-disk-type threshold colour. The guest drawer's Physical Disks card marks a +retained value the way the Physical Disks table does. Still presentation only. +Verification: `diskPresentation.test.ts` (one implementation behind both import +paths), `agentMachineTableModel.test.ts`, `AgentsMachinesTable.test.tsx`, +`resourceDetailMappers.test.ts` and `GuestPhysicalDisks.test.tsx`. Direct SATA, SAS, and NVMe device fallback identities remain compatible. Controller-member fallback identities add controller/target scope only where diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 79650fb3d1..d14cbc296c 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -2269,6 +2269,20 @@ and temperature facts must also compose the frontend-primitives `InfoCardKeyValueRow`. Mobile rows retain their condensed endpoint layout; desktop rows use the shared fixed label track so labels and values remain visually adjacent instead of spanning the full drawer width. +A standalone host agent past its reporting lease reaches the Machines table as +`warning` with `agent.stale`, not `offline`: by the time the lease expires the +registry has marked its only source stale, which outranks offline. The offline +metric fallback therefore does not blank its cells, and the temperature cell +has no such gate. For SMART disk temperatures the provenance travels on +`agent.sensors.smart[].collection`: the Machines temperature cell +(`AgentsMachinesTable.tsx`, `agentMachineTableModel.ts`), its tooltip and the +drawer's Thermals rows (`resourceDetailMappers.ts`) read it through +`isPhysicalDiskTemperatureCurrent`, so a retained disk reading never stands for +the machine while another disk has a current one. A positive direct +`temperature` or `temperatureCelsius` reading carries no collection state and +still leads the cell. Without one, and with no current non-standby disk, the +hottest retained non-standby disk reading shows as last known without +threshold colour. The same boundary applies to availability facts, resource change-history metadata, Docker/PBS/PMG service facts, nested PMG queue/mail breakdowns, and Docker container-update management facts. It also covers action-history facts, diff --git a/frontend-modern/browser-verification/47c60098fc12bf96.json b/frontend-modern/browser-verification/47c60098fc12bf96.json new file mode 100644 index 0000000000..eb0d451e5a --- /dev/null +++ b/frontend-modern/browser-verification/47c60098fc12bf96.json @@ -0,0 +1,54 @@ +{ + "version": 1, + "base_sha": "dcb69cfb020193d05d9e4e668d19b820a0c46d31", + "verified_at": "2026-10-06T22:07:56Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts", + "frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx", + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx", + "frontend-modern/src/features/standalone/agentMachineTableModel.ts", + "frontend-modern/src/features/storageBackups/diskPresentation.ts", + "frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts", + "frontend-modern/src/types/api.ts" + ], + "content_sha256": { + "frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts": "6cdf797534d6ecba8f8007561195f9245899ff881a225f83c2c6c6293b557f8c", + "frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx": "98e9dc568360b5b6c9552433207e0e2fa78b91ba17da53fa513fb42484373ff5", + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx": "a1dfacc661118c77fea482d26ffbc6d1c6aca0d351d77744f7a2dbbd4770ef6f", + "frontend-modern/src/features/standalone/agentMachineTableModel.ts": "1f9be08ed7100a330142eab8900b04474360785e75c1b95a8a42d4bc35fa7233", + "frontend-modern/src/features/storageBackups/diskPresentation.ts": "debb0d30e95b84d010d5b7478d61ce5fa9487c0bfb43b3eb812e16b4a75acf2e", + "frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts": "c62e4d1e74b9278ef34d93e8101c3d41ab67b17885b6810116a2f96bd112301c", + "frontend-modern/src/types/api.ts": "bb2712595745499cc51ed5c6e54ea24998e00d42ce45c8715b1d6c3eed6afc3e" + }, + "routes": [ + "/standalone", + "/proxmox?q=analytics-batch-01" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack built from this tree (base dcb69cfb02) plus a local-only go build -overlay of internal/mock/generator.go: three standalone Linux agents get SMART rows; Apollo-114 (CPU sensors) and Meridian-731 (SMART only) are stamped as State.ExpireHostTelemetry does (status offline, temperatures kept, collection.temperature unavailable 'host agent stopped reporting', source kept), Europa-405 keeps reporting. /api/resources returned agent.sensors.smart[].collection end to end, status warning and agent.stale true for the silent agents.", + "Same-session control with the HEAD runtime files under the same Vite: Meridian-731's Temp cell rendered 52°C in threshold green, the tooltip and the drawer Thermals card listed the retained disks as plain current readings, and the guest Physical Disks card showed plain muted 52°C/41°C with no last-known cue.", + "After: Meridian-731's Temp cell renders '52°C' muted (slate-600 light, slate-400 dark) with a dotted underline, no threshold colour and screen-reader ', last known'; its tooltip lists both disks '(last known)' muted. Apollo-114 keeps its CPU package value (no collection state) while its disk tooltip rows read '(last known)'. Europa-405 is unchanged.", + "Machine drawer (Platform details > Machine details > Thermals) at desktop and phone: 'Disk nvme0n1 52°C (last known)' and 'Disk sda 41°C (last known)' with title 'Last known reading, not current: host agent stopped reporting'; cpu.package unchanged.", + "Guest drawer Physical Disks card at desktop and phone: Playwright routes add the agent version a VM-resident agent would carry to analytics-batch-01 and answer its physical_disk child query with Meridian-731's real registry disks (collection state untouched) re-parented to the VM, because mock mode never links an agent inside a VM. Both disks render muted, dotted, with the reason title and ', last known'.", + "Light and dark schemes on the Machines page. Bundle check: Standalone +0.22 kB, shared chunk +0.20 kB, WorkloadsSurface and InfrastructureWorkspace flat versus a same-machine main build, no tracked chunk renamed.", + "Not covered: a real host agent going silent on hardware, an agent genuinely running inside a VM, or stale-row CPU/memory/disk cells (flagged separately)." + ], + "interactions": [ + "Enabled the Temp column from View > column picker on /standalone and read every row's cell text, class, computed colour and text decoration.", + "Hovered the Temp cell of the SMART-only and the CPU-sensor silent agents and read the tooltip rows and their muted state.", + "Opened the Apollo-114 drawer by row click (desktop) and CPU cell tap (phone), expanded Platform details and the Machine Show details, and read the Thermals rows with their titles.", + "Opened the analytics-batch-01 guest drawer from the Proxmox guest table at desktop and phone and read the Physical Disks card summaries, data-temperature-reading, titles and decoration.", + "Swapped the runtime files to HEAD under the running Vite for the same-session control, then restored and re-ran every pass on the final tree." + ] +} diff --git a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts index 0df9376977..5596569845 100644 --- a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts +++ b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts @@ -10,7 +10,7 @@ import { } from '@/components/Infrastructure/resourceDetailMappers'; import resourceDetailMappersSource from '@/components/Infrastructure/resourceDetailMappers.ts?raw'; import resourceDetailDiscoveryModelSource from '@/components/Infrastructure/resourceDetailDiscoveryModel.ts?raw'; -import type { Resource } from '@/types/resource'; +import type { PhysicalDiskCollectionStatus, Resource } from '@/types/resource'; const createHybridHostResource = (): Resource => ({ @@ -203,6 +203,89 @@ describe('resourceDetailMappers', () => { }, ]); }); + + it('marks a silent agent disk temperature as last known and drops disks without one', () => { + const rows = buildTemperatureRows({ + smart: [ + { + device: 'sda', + temperature: 71, + // A host agent past its reporting lease keeps its last reading. + collection: { + temperature: { + state: 'unavailable', + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }, + }, + { + device: 'sdb', + temperature: 38, + collection: { temperature: { state: 'available', source: 'smartctl' } }, + }, + { + device: 'sdc', + temperature: 0, + collection: { + temperature: { + state: 'unsupported', + source: 'smartctl', + reason: 'device did not expose a temperature reading', + }, + }, + }, + ], + }); + + expect(rows).toEqual([ + { + label: 'Disk sda', + value: '71°C (last known)', + valueTitle: 'Last known reading, not current: host agent stopped reporting', + }, + { label: 'Disk sdb', value: '38°C', valueTitle: '38.0°C' }, + ]); + }); + + it('lists a SMART row only for a positive reading in any state, never a standby disk', () => { + const states: (PhysicalDiskCollectionStatus | undefined)[] = [ + undefined, + { temperature: { state: 'available', source: 'smartctl' } }, + { temperature: { state: 'unavailable', source: 'smartctl' } }, + ]; + for (const collection of states) { + for (const temperature of [0, -3]) { + expect( + buildTemperatureRows({ smart: [{ device: 'sda', temperature, collection }] }), + ).toEqual([]); + } + } + + expect( + buildTemperatureRows({ + smart: [ + { + device: 'sda', + temperature: 44, + standby: true, + collection: { temperature: { state: 'unavailable', source: 'smartctl' } }, + }, + { + device: 'sdb', + temperature: 39, + collection: { temperature: { state: 'missing', source: 'smartctl' } }, + }, + ], + }), + ).toEqual([ + { + label: 'Disk sdb', + value: '39°C (last known)', + valueTitle: 'Last known reading, not current', + }, + ]); + }); }); describe('buildCustomSensorRows', () => { diff --git a/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts b/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts index 3a77a7aded..b6b94a7ea1 100644 --- a/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts +++ b/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts @@ -15,6 +15,7 @@ import type { } from '@/types/api'; import type { Resource, ResourceMetric, ResourceVMwareMeta } from '@/types/resource'; import { formatTemperature } from '@/utils/temperature'; +import { getPhysicalDiskTemperaturePresentation } from '@/features/storageBackups/diskTemperaturePresentation'; import { formatBytes } from '@/utils/format'; import { getActionableAgentIdFromResource } from '@/utils/agentResources'; import { @@ -502,13 +503,17 @@ export const buildTemperatureRows = (sensors?: HostSensorSummary) => { const smart = sensors?.smart; if (smart) { smart - .filter((disk) => !disk.standby && Number.isFinite(disk.temperature)) + .filter((disk) => !disk.standby) .sort((a, b) => a.device.localeCompare(b.device)) .forEach((disk) => { + const temperature = getPhysicalDiskTemperaturePresentation(disk); + if (!temperature) return; + // A retained reading, such as a silent agent's, says so in the value + // because this card has no other cue that it is not current. rows.push({ label: `Disk ${disk.device}`, - value: formatTemperature(disk.temperature), - valueTitle: `${disk.temperature.toFixed(1)}°C`, + value: temperature.current ? temperature.label : `${temperature.label} (last known)`, + valueTitle: temperature.title ?? `${disk.temperature.toFixed(1)}°C`, }); }); } diff --git a/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx b/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx index a9c75d3db7..ceeaa26a50 100644 --- a/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx +++ b/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx @@ -5,9 +5,12 @@ import { extractPhysicalDiskPresentationData, getPhysicalDiskHealthStatus, } from '@/features/storageBackups/diskPresentation'; +import { + PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS, + getPhysicalDiskTemperaturePresentation, +} from '@/features/storageBackups/diskTemperaturePresentation'; import type { Resource } from '@/types/resource'; import { formatBytes } from '@/utils/format'; -import { formatTemperature } from '@/utils/temperature'; // SMART history and chart controls are only needed after a disk row is opened. // Keep their Storage detail module out of the initial Workloads surface. @@ -19,6 +22,7 @@ const GuestPhysicalDiskRow: Component<{ disk: Resource }> = (props) => { const [expanded, setExpanded] = createSignal(false); const data = () => extractPhysicalDiskPresentationData(props.disk); const health = () => getPhysicalDiskHealthStatus(data()); + const temperature = () => getPhysicalDiskTemperaturePresentation(data()); const label = () => data().model || data().devPath || props.disk.name; return ( @@ -35,8 +39,19 @@ const GuestPhysicalDiskRow: Component<{ disk: Resource }> = (props) => { {health().label} - 0}> - {formatTemperature(data().temperature)} + + {(reading) => ( + + {reading().label} + + , last known + + + )} 0}> {formatBytes(data().size)} diff --git a/frontend-modern/src/components/Workloads/__tests__/GuestPhysicalDisks.test.tsx b/frontend-modern/src/components/Workloads/__tests__/GuestPhysicalDisks.test.tsx index 0fa8f2c11a..0b316595db 100644 --- a/frontend-modern/src/components/Workloads/__tests__/GuestPhysicalDisks.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/GuestPhysicalDisks.test.tsx @@ -1,10 +1,13 @@ import { cleanup, fireEvent, render, screen, within } from '@solidjs/testing-library'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { createSignal } from 'solid-js'; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import type { Resource } from '@/types/resource'; import { GuestPhysicalDisks } from '../GuestPhysicalDisks'; const queryState = vi.hoisted(() => ({ resources: [] as Resource[], + // Set to a signal accessor to drive live resource updates. + accessor: undefined as (() => Resource[]) | undefined, error: undefined as unknown, query: '', enabled: false, @@ -14,7 +17,10 @@ vi.mock('@/hooks/useUnifiedResources', () => ({ useUnifiedResources: (options: { query: string; enabled: () => boolean }) => { queryState.query = options.query; queryState.enabled = options.enabled(); - return { resources: () => queryState.resources, error: () => queryState.error }; + return { + resources: () => (queryState.accessor ? queryState.accessor() : queryState.resources), + error: () => queryState.error, + }; }, })); @@ -41,8 +47,15 @@ const disk: Resource = { }; describe('GuestPhysicalDisks', () => { + // The first expansion lazy-loads the Storage disk detail; a cold transform of + // that module can outlast findByText's default wait on a loaded machine. + beforeAll(async () => { + await import('@/components/Storage/DiskDetail'); + }, 30_000); + beforeEach(() => { queryState.resources = []; + queryState.accessor = undefined; queryState.error = undefined; queryState.query = ''; queryState.enabled = false; @@ -84,6 +97,85 @@ describe('GuestPhysicalDisks', () => { expect(screen.queryByTestId('guest-physical-disks')).toBeNull(); }); + it('marks a retained disk temperature as last known', () => { + queryState.resources = [ + disk, + { + ...disk, + id: 'disk-101-b', + displayName: 'Guest NVMe', + physicalDisk: { + ...disk.physicalDisk!, + devPath: '/dev/nvme0', + model: 'Guest NVMe', + temperature: 61, + collection: { + temperature: { + state: 'unavailable', + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }, + }, + }, + ]; + render(() => ); + + const [currentRow, retainedRow] = screen.getAllByTestId('guest-physical-disk'); + const current = within(currentRow).getByText('45°C'); + expect(current).toHaveAttribute('data-temperature-reading', 'current'); + expect(current).not.toHaveAttribute('title'); + + const retained = retainedRow.querySelector('[data-temperature-reading="last-known"]'); + expect(retained).not.toBeNull(); + expect(retained).toHaveTextContent('61°C, last known'); + expect(retained).toHaveClass('underline', 'decoration-dotted', 'text-muted'); + expect(retained).toHaveAttribute( + 'title', + 'Last known reading, not current: host agent stopped reporting', + ); + }); + + it('follows a disk between current and last known as its collection state changes', () => { + const withReading = ( + temperature: number, + collection?: NonNullable['collection'], + ): Resource => ({ + ...disk, + physicalDisk: { ...disk.physicalDisk!, temperature, collection }, + }); + const [resources, setResources] = createSignal([disk]); + queryState.accessor = resources; + render(() => ); + const reading = () => + screen.getByTestId('guest-physical-disk').querySelector('[data-temperature-reading]'); + + expect(reading()).toHaveAttribute('data-temperature-reading', 'current'); + + setResources([ + withReading(61, { + temperature: { + state: 'unavailable', + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }), + ]); + expect(reading()).toHaveAttribute('data-temperature-reading', 'last-known'); + expect(reading()).toHaveTextContent('61°C, last known'); + expect(reading()).toHaveClass('decoration-dotted'); + + setResources([withReading(47, { temperature: { state: 'available', source: 'smartctl' } })]); + expect(reading()).toHaveAttribute('data-temperature-reading', 'current'); + expect(reading()).toHaveTextContent('47°C'); + expect(reading()).not.toHaveTextContent('last known'); + expect(reading()).not.toHaveAttribute('title'); + expect(reading()).not.toHaveClass('decoration-dotted'); + + setResources([withReading(0)]); + expect(reading()).toBeNull(); + }); + it('makes a failed child query visible rather than silently implying no disks', () => { queryState.error = new Error('unavailable'); render(() => ); diff --git a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx index 7e0f98124e..2609a62fa7 100644 --- a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx +++ b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx @@ -57,6 +57,7 @@ import { type PlatformResourceStatusFilter, withPlatformStatusCounts, } from '@/features/platformPage/sharedPlatformPage'; +import { PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS } from '@/features/storageBackups/diskTemperaturePresentation'; import { useColumnVisibility } from '@/hooks/useColumnVisibility'; import type { Disk } from '@/types/api'; import type { Resource, ResourceAvailabilityMeta } from '@/types/resource'; @@ -76,6 +77,7 @@ import { import { asTrimmedString } from '@/utils/stringUtils'; import { getPreferredResourceIP } from '@/utils/resourceIdentity'; import { useRelativeTimeNow } from '@/utils/relativeTimeClock'; +import { formatTemperature } from '@/utils/temperature'; import { RESOURCE_METADATA_CHANGED_EVENT, type ResourceMetadataChangedDetail, @@ -101,6 +103,7 @@ import { getAgentMachineTemperatureTitle, getAgentMachineThermalPressurePresentation, getNextAgentMachineSortState, + isAgentMachineTemperatureLastKnown, sortAgentMachines, type AgentMachineColumn, type AgentMachineColumnId, @@ -183,6 +186,8 @@ const AgentMachineMetricTooltip: Component<{ const AgentMachineTemperatureCell: Component<{ celsius: number | undefined; + /** The value is a retained reading, not a current one. */ + lastKnown?: boolean; metric: AgentMachineTemperatureMetric; sections: AgentMachineTemperatureDetailSection[]; thresholds?: MetricDisplayThresholds | null; @@ -222,7 +227,26 @@ const AgentMachineTemperatureCell: Component<{ } > {(value) => ( - + + {formatTemperature(value())} + , last known + + } + > + + )} } @@ -1565,6 +1589,7 @@ export const AgentsMachinesTable: Component<{ const raidArrays = () => getAgentMachineRaidArrayDetails(machine); const raidSummary = () => getAgentMachineRaidSummary(machine); const temperature = () => getAgentMachineTemperatureCelsius(machine); + const temperatureLastKnown = () => isAgentMachineTemperatureLastKnown(machine); const temperatureMetric = () => getAgentMachineTemperatureMetric(machine); const temperatureThresholds = () => temperatureMetric() === 'diskTemperature' @@ -1791,6 +1816,7 @@ export const AgentsMachinesTable: Component<{ > { expect(screen.getByText('1400 RPM')).toBeInTheDocument(); }); + it('shows a silent agent disk temperature as last known, never as a current reading', async () => { + // A host agent past its reporting lease is stale and keeps its last SMART + // temperatures with a non-available collection state. + const stoppedReporting = { + temperature: { + state: 'unavailable' as const, + source: 'smartctl', + reason: 'host agent stopped reporting', + }, + }; + const { container } = render(() => ( + ({ ...disk, collection: stoppedReporting })), + }, + }, + }), + resource({ + id: 'mixed-host', + name: 'Mixed Host', + agent: { + sensors: { + smart: [ + { + device: '/dev/sda', + model: 'Retained HDD', + temperature: 69, + collection: stoppedReporting, + }, + { device: '/dev/nvme0', model: 'Fast SSD', temperature: 44 }, + ], + }, + }, + }), + ]} + emptyIcon={emptyIcon} + emptyTitle="No machines" + emptyDescription="Install Pulse Agent." + /> + )); + + await openMachineColumnPicker(); + await fireEvent.click(screen.getByLabelText('Temp')); + + const triggerFor = (id: string) => + container.querySelector( + `[data-agents-machine-row="${id}"] [data-agent-machine-temperature-trigger="true"]`, + ) as HTMLElement | null; + + // Nothing is collected now, so the cell keeps the hottest retained value + // without a threshold colour and says it is last known. + const silent = triggerFor('silent-nas'); + expect(silent).not.toBeNull(); + if (!silent) return; + const retained = silent.querySelector('[data-temperature-reading="last-known"]'); + expect(retained).not.toBeNull(); + expect(retained).toHaveTextContent('71°C, last known'); + expect(retained).toHaveClass('underline', 'decoration-dotted', 'text-muted'); + expect(retained?.className).not.toMatch(/text-(green|yellow|red)-/); + + await fireEvent.mouseEnter(silent); + const tooltip = await waitFor(() => { + const element = document.querySelector('[data-agent-machine-temperature-tooltip="true"]'); + expect(element).not.toBeNull(); + return element as HTMLElement; + }); + expect(within(tooltip).getByText('71°C (last known)')).toHaveClass('text-muted'); + expect(within(tooltip).getByText('66°C (last known)')).toHaveClass('text-muted'); + await fireEvent.mouseLeave(silent); + + // A disk collected now leads the cell even when a retained one is hotter. + const mixed = triggerFor('mixed-host'); + expect(mixed).not.toBeNull(); + if (!mixed) return; + expect(mixed).toHaveTextContent('44°C'); + expect(mixed.querySelector('[data-temperature-reading="last-known"]')).toBeNull(); + expect(within(mixed).queryByText('69°C')).toBeNull(); + }); + + it('moves a machine temperature between current and last known as its disk reports change', async () => { + // Fresh objects per update, like parsed API payloads: the store reconciles + // into the objects it was given, so a reused fixture would be rewritten. + const collected = () => ({ temperature: { state: 'available' as const, source: 'smartctl' } }); + const stoppedReporting = () => ({ + temperature: { + state: 'unavailable' as const, + source: 'smartctl', + reason: 'host agent stopped reporting', + }, + }); + const nas = ( + temperature: number, + collection?: ReturnType | ReturnType, + ) => + resource({ + id: 'nas', + name: 'NAS', + agent: { + sensors: { smart: [{ device: '/dev/sda', type: 'sata', temperature, collection }] }, + }, + }); + const [resources, setResources] = createSignal([nas(52, collected())]); + const { container } = render(() => ( + + )); + + await openMachineColumnPicker(); + await fireEvent.click(screen.getByLabelText('Temp')); + const cell = () => + container.querySelector( + '[data-agents-machine-row="nas"] [data-agent-machine-temperature-trigger="true"]', + ) as HTMLElement; + const lastKnown = () => cell().querySelector('[data-temperature-reading="last-known"]'); + + expect(cell()).toHaveTextContent('52°C'); + expect(lastKnown()).toBeNull(); + + setResources([nas(52, stoppedReporting())]); + await waitFor(() => expect(lastKnown()).not.toBeNull()); + expect(cell()).toHaveTextContent('52°C, last known'); + + setResources([nas(47, collected())]); + await waitFor(() => expect(lastKnown()).toBeNull()); + expect(cell()).toHaveTextContent('47°C'); + expect(cell().querySelector('.decoration-dotted')).toBeNull(); + expect(cell()).not.toHaveTextContent('last known'); + + setResources([nas(0)]); + await waitFor(() => expect(cell()).toHaveTextContent('—')); + expect(lastKnown()).toBeNull(); + }); + it('shows thermal pressure when macOS reports no Celsius temperature', async () => { render(() => ( { expect(getAgentMachineTemperatureMetric(machine)).toBe('diskTemperature'); }); + it('shows a retained SMART temperature only as last known', () => { + // A host agent past its reporting lease keeps its last SMART temperature + // with a non-available collection state. + const stoppedReporting = { + temperature: { + state: 'unavailable' as const, + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }; + const mixed = resource({ + agent: { + sensors: { + smart: [ + { + device: '/dev/sda', + model: 'Retained HDD', + type: 'sas', + temperature: 71, + collection: stoppedReporting, + }, + { + device: '/dev/nvme0', + model: 'Fast SSD', + type: 'nvme', + temperature: 44, + collection: { temperature: { state: 'available' as const, source: 'smartctl' } }, + }, + // Agents that predate collection state report no provenance. + { device: '/dev/sdb', temperature: 39 }, + { device: '/dev/sdc', model: 'Cold Standby', temperature: 0, standby: true }, + ], + }, + }, + }); + + // A disk collected now outranks a hotter retained one. + expect(getAgentMachineTemperatureCelsius(mixed)).toBe(44); + expect(isAgentMachineTemperatureLastKnown(mixed)).toBe(false); + expect(getAgentMachineTemperatureMetric(mixed)).toBe('diskTemperature'); + expect(getAgentMachineHottestSmartDiskType(mixed)).toBe('nvme'); + expect(getAgentMachineTemperatureTitle(mixed).split('\n')).toEqual([ + 'Disk Temperatures', + 'Disk /dev/nvme0 Fast SSD: 44°C', + 'Disk /dev/sdb: 39°C', + 'Disk /dev/sda Retained HDD: 71°C (last known)', + 'Disk /dev/sdc Cold Standby: standby', + ]); + expect(getAgentMachineTemperatureDetailSections(mixed)[0].rows[2]).toEqual({ + label: 'Disk /dev/sda Retained HDD', + value: '71°C (last known)', + muted: true, + }); + + // With nothing collected now, the cell keeps the hottest retained value + // and marks it last known. + const onlyRetained = resource({ + agent: { + sensors: { + smart: [ + { device: '/dev/sda', type: 'sata', temperature: 71, collection: stoppedReporting }, + { device: '/dev/nvme0', type: 'nvme', temperature: 58, collection: stoppedReporting }, + ], + }, + }, + }); + expect(getAgentMachineTemperatureCelsius(onlyRetained)).toBe(71); + expect(isAgentMachineTemperatureLastKnown(onlyRetained)).toBe(true); + expect(getAgentMachineTemperatureMetric(onlyRetained)).toBe('diskTemperature'); + expect(getAgentMachineHottestSmartDiskType(onlyRetained)).toBe('sata'); + expect(getAgentMachineTemperatureTitle(onlyRetained)).toBe( + 'Disk Temperatures\nDisk /dev/sda: 71°C (last known)\nDisk /dev/nvme0: 58°C (last known)', + ); + + // CPU sensors carry no collection state, so they still lead the cell. + const withSensors = resource({ + agent: { + sensors: { + temperatureCelsius: { 'cpu.package': 49 }, + smart: [{ device: '/dev/sda', temperature: 71, collection: stoppedReporting }], + }, + }, + }); + expect(getAgentMachineTemperatureCelsius(withSensors)).toBe(49); + expect(isAgentMachineTemperatureLastKnown(withSensors)).toBe(false); + expect(getAgentMachineTemperatureMetric(withSensors)).toBe('temperature'); + }); + + it('offers only positive, non-standby SMART readings to the cell, behind direct and CPU sensors', () => { + const stoppedReporting = { + temperature: { + state: 'unavailable' as const, + source: 'smartctl', + reason: 'host agent stopped reporting', + }, + }; + const retainedDisk = (temperature: number) => ({ + device: '/dev/sda', + temperature, + collection: stoppedReporting, + }); + + // A disk in standby is never a temperature candidate, retained or not. + const standbyOnly = resource({ + agent: { sensors: { smart: [{ ...retainedDisk(52), standby: true }] } }, + }); + expect(getAgentMachineTemperatureCelsius(standbyOnly)).toBeUndefined(); + expect(isAgentMachineTemperatureLastKnown(standbyOnly)).toBe(false); + expect(getAgentMachineTemperatureTitle(standbyOnly)).toBe( + 'Disk Temperatures\nDisk /dev/sda: standby', + ); + + // A direct machine temperature leads a retained disk. + const direct = resource({ + temperature: 47, + agent: { sensors: { smart: [retainedDisk(71)] } }, + }); + expect(getAgentMachineTemperatureCelsius(direct)).toBe(47); + expect(isAgentMachineTemperatureLastKnown(direct)).toBe(false); + expect(getAgentMachineTemperatureMetric(direct)).toBe('temperature'); + + // Additional sensors never lead the cell, so the retained disk is shown. + const additionalOnly = resource({ + agent: { sensors: { additional: { vrm: 49 }, smart: [retainedDisk(71)] } }, + }); + expect(getAgentMachineTemperatureCelsius(additionalOnly)).toBe(71); + expect(isAgentMachineTemperatureLastKnown(additionalOnly)).toBe(true); + + // Sorting follows the value the cell shows: a hotter retained disk does not + // lift a machine whose shown reading is a cooler current one. + const machines = [ + resource({ + id: 'retained', + name: 'Retained', + agent: { sensors: { smart: [retainedDisk(71)] } }, + }), + resource({ + id: 'mixed', + name: 'Mixed', + agent: { + sensors: { smart: [retainedDisk(80), { device: '/dev/sdb', temperature: 35 }] }, + }, + }), + resource({ + id: 'current', + name: 'Current', + agent: { sensors: { smart: [{ device: '/dev/sda', temperature: 50 }] } }, + }), + ]; + const sortedIds = (direction: 'asc' | 'desc') => + sortAgentMachines( + machines, + 'temp', + direction, + () => '', + () => '', + ).map((machine) => machine.id); + expect(sortedIds('desc')).toEqual(['retained', 'current', 'mixed']); + expect(sortedIds('asc')).toEqual(['mixed', 'current', 'retained']); + }); + it('prefers direct and sensor temperatures over SMART fallback temperatures', () => { const direct = resource({ temperature: 55, diff --git a/frontend-modern/src/features/standalone/agentMachineTableModel.ts b/frontend-modern/src/features/standalone/agentMachineTableModel.ts index 5fe81f91f9..661e02c86c 100644 --- a/frontend-modern/src/features/standalone/agentMachineTableModel.ts +++ b/frontend-modern/src/features/standalone/agentMachineTableModel.ts @@ -2,6 +2,7 @@ import type { ColumnDef } from '@/hooks/useColumnVisibility'; import type { HostDiskIO, HostRAIDArray, HostRAIDDevice } from '@/types/api'; import type { Resource } from '@/types/resource'; import { getPlatformTableFiniteMetric } from '@/features/platformPage/sharedPlatformPage'; +import { isPhysicalDiskTemperatureCurrent } from '@/features/storageBackups/diskTemperaturePresentation'; import { formatBytes, normalizeDiskArray } from '@/utils/format'; import { asTrimmedString } from '@/utils/stringUtils'; @@ -155,6 +156,9 @@ type TemperatureReading = { type SmartTemperatureReading = TemperatureReading & { standby?: boolean; diskType?: string; + // False for a retained value, such as the last temperature of a host agent + // that stopped reporting. + current: boolean; }; export type AgentMachineTemperatureDetailRow = { @@ -257,13 +261,21 @@ const getSmartTemperatureReadings = (machine: Resource): SmartTemperatureReading value: temperature ?? 0, standby: disk.standby, diskType: asTrimmedString(disk.type), + current: isPhysicalDiskTemperatureCurrent(disk.collection), }); return readings; }, []); -const getActiveSmartTemperatureReadings = (machine: Resource): TemperatureReading[] => +// Only a temperature collected now can stand for the machine or pick its +// threshold. A retained one stays in the detail rows, marked last known. +const getActiveSmartTemperatureReadings = (machine: Resource): SmartTemperatureReading[] => getSmartTemperatureReadings(machine).filter( - (reading): reading is TemperatureReading => !reading.standby && reading.value > 0, + (reading) => !reading.standby && reading.value > 0 && reading.current, + ); + +const getLastKnownSmartTemperatureReadings = (machine: Resource): SmartTemperatureReading[] => + getSmartTemperatureReadings(machine).filter( + (reading) => !reading.standby && reading.value > 0 && !reading.current, ); const byHighestTemperature = (left: TemperatureReading, right: TemperatureReading): number => @@ -536,22 +548,38 @@ export const getAgentMachineDiskIODetails = (machine: Resource): AgentMachineDis }, []); }; +const hasMachineSensorTemperature = (machine: Resource): boolean => + positiveTemperature(machine.temperature) !== undefined || + getSensorTemperatureReadings(machine).length > 0; + +// The SMART readings the cell falls back to: the ones collected now, or the +// retained ones when no disk has a current reading. +const getCellSmartTemperatureReadings = (machine: Resource): SmartTemperatureReading[] => { + const active = getActiveSmartTemperatureReadings(machine); + return active.length > 0 ? active : getLastKnownSmartTemperatureReadings(machine); +}; + export const getAgentMachineTemperatureCelsius = (machine: Resource): number | undefined => { const direct = positiveTemperature(machine.temperature); if (direct !== undefined) return direct; return ( maxTemperatureReading(getSensorTemperatureReadings(machine)) ?? - maxTemperatureReading(getActiveSmartTemperatureReadings(machine)) + maxTemperatureReading(getCellSmartTemperatureReadings(machine)) ); }; -// Disk type of the hottest active SMART reading — the disk whose temperature -// the cell displays when it falls back to the diskTemperature metric. +// True when the cell's value is a retained SMART reading, such as the last +// disk temperature of a host agent that stopped reporting. +export const isAgentMachineTemperatureLastKnown = (machine: Resource): boolean => + !hasMachineSensorTemperature(machine) && + getActiveSmartTemperatureReadings(machine).length === 0 && + getLastKnownSmartTemperatureReadings(machine).length > 0; + +// Disk type of the hottest SMART reading the cell displays when it falls back +// to the diskTemperature metric. export const getAgentMachineHottestSmartDiskType = (machine: Resource): string | undefined => { - const readings = getSmartTemperatureReadings(machine).filter( - (reading) => !reading.standby && reading.value > 0, - ); + const readings = getCellSmartTemperatureReadings(machine); if (readings.length === 0) return undefined; return readings.reduce((worst, reading) => (reading.value > worst.value ? reading : worst)) .diskType; @@ -560,13 +588,8 @@ export const getAgentMachineHottestSmartDiskType = (machine: Resource): string | export const getAgentMachineTemperatureMetric = ( machine: Resource, ): AgentMachineTemperatureMetric => { - if (positiveTemperature(machine.temperature) !== undefined) return 'temperature'; - if (maxTemperatureReading(getSensorTemperatureReadings(machine)) !== undefined) { - return 'temperature'; - } - if (maxTemperatureReading(getActiveSmartTemperatureReadings(machine)) !== undefined) { - return 'diskTemperature'; - } + if (hasMachineSensorTemperature(machine)) return 'temperature'; + if (getCellSmartTemperatureReadings(machine).length > 0) return 'diskTemperature'; return 'temperature'; }; @@ -585,6 +608,13 @@ export const getAgentMachineTemperatureDetailSections = ( label: `Disk ${reading.label}`, value: formatTemperatureValue(reading), })); + const lastKnownSmartReadings = getLastKnownSmartTemperatureReadings(machine) + .sort(byHighestTemperature) + .map((reading) => ({ + label: `Disk ${reading.label}`, + value: `${formatTemperatureValue(reading)} (last known)`, + muted: true, + })); const standbySmartReadings = getSmartTemperatureReadings(machine) .filter((reading) => reading.standby) .sort(byLabel) @@ -609,7 +639,11 @@ export const getAgentMachineTemperatureDetailSections = ( ...section('Temperatures', capTemperatureRows(sensorReadings)), ...section( 'Disk Temperatures', - capTemperatureRows([...activeSmartReadings, ...standbySmartReadings]), + capTemperatureRows([ + ...activeSmartReadings, + ...lastKnownSmartReadings, + ...standbySmartReadings, + ]), ), ...section('Fan Speeds', capTemperatureRows(fanReadings)), ...section('Other Sensors', capTemperatureRows(additionalReadings)), diff --git a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts index a18d7ccdfe..c3c13377bd 100644 --- a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts +++ b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts @@ -18,6 +18,7 @@ import { PHYSICAL_DISK_SOURCE_BADGE_CLASS, PHYSICAL_DISK_TABLE_CLASS, PHYSICAL_DISK_TABLE_ROW_HOVER_CLASS, + PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS, getPhysicalDiskEmptyStatePresentation, getPhysicalDiskCellPaddingClass, getPhysicalDiskColumnWidthStyle, @@ -27,6 +28,7 @@ import { getPhysicalDiskHealthStatus, getPhysicalDiskHealthSummary, getPhysicalDiskHostLabel, + getPhysicalDiskLastKnownTemperatureTitle, getPhysicalDiskLifeLabel, getPhysicalDiskLifeTextClass, getPhysicalDiskNormalizedHealth, @@ -48,6 +50,7 @@ import { normalizePhysicalDiskFacetFilter, type PhysicalDiskPresentationData, } from '@/features/storageBackups/diskPresentation'; +import * as diskTemperaturePresentation from '@/features/storageBackups/diskTemperaturePresentation'; function makeDiskData( overrides: Partial = {}, @@ -131,6 +134,23 @@ describe('diskPresentation', () => { } }); + it('serves the same temperature decision the Machines table and drawers import directly', () => { + // Tables outside Storage import the small module so they do not pull this + // one into their chunks. Both paths must stay one implementation. + expect(isPhysicalDiskTemperatureCurrent).toBe( + diskTemperaturePresentation.isPhysicalDiskTemperatureCurrent, + ); + expect(getPhysicalDiskTemperaturePresentation).toBe( + diskTemperaturePresentation.getPhysicalDiskTemperaturePresentation, + ); + expect(getPhysicalDiskLastKnownTemperatureTitle).toBe( + diskTemperaturePresentation.getPhysicalDiskLastKnownTemperatureTitle, + ); + expect(PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS).toBe( + diskTemperaturePresentation.PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS, + ); + }); + it('distinguishes unsupported, unavailable, and unexpectedly missing disk evidence', () => { expect( getPhysicalDiskFieldStatusMessage('Disk I/O', { diff --git a/frontend-modern/src/features/storageBackups/diskPresentation.ts b/frontend-modern/src/features/storageBackups/diskPresentation.ts index 2499be5969..fe1ad5ad4e 100644 --- a/frontend-modern/src/features/storageBackups/diskPresentation.ts +++ b/frontend-modern/src/features/storageBackups/diskPresentation.ts @@ -14,7 +14,6 @@ import { getAllFilterOptionLabel } from '@/components/shared/filterOptionPresent import { getPhysicalDiskNodeIdentity } from '@/components/Storage/diskResourceUtils'; import { getInfrastructureSettingsLocationLabel } from '@/utils/infrastructureSettingsPresentation'; import { normalizeStorageSourceKey, storageSourceMatchesFilter } from '@/utils/storageSources'; -import { formatTemperature } from '@/utils/temperature'; import type { NormalizedHealth, StorageHealthFilter } from './models'; import { matchesStorageNodeTerms, parseStorageSearchQuery } from './storageSearchQuery'; @@ -254,11 +253,6 @@ export const PHYSICAL_DISK_HEALTH_WRAP_CLASS = // that would compete with it for the track. export const PHYSICAL_DISK_HEALTH_LABEL_CLASS = 'shrink-0 text-[11px] font-semibold'; export const PHYSICAL_DISK_TEMPERATURE_CLASS = 'text-[11px] font-medium'; -// A retained reading takes no threshold colour, so it cannot read as a disk -// running hot or cool right now. The dotted underline marks the value as one -// its title explains. -export const PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS = - 'text-muted underline decoration-dotted underline-offset-2 cursor-help'; export const PHYSICAL_DISK_SIZE_VALUE_CLASS = 'text-[11px] text-base-content'; const titleize = (value: string | undefined | null): string => @@ -473,44 +467,15 @@ export function getPhysicalDiskFieldStatusMessage( } } -// Normalization may keep the last known temperature when the current -// observation is not available, such as a disk in standby or a host agent that -// stopped reporting. The value is a current reading only when its collection -// state is available, or when the source predates collection state. -export function isPhysicalDiskTemperatureCurrent( - collection: PhysicalDiskCollectionStatus | null | undefined, -): boolean { - const state = collection?.temperature?.state; - return !state || state === 'available'; -} - -export function getPhysicalDiskLastKnownTemperatureTitle( - status: PhysicalDiskFieldStatus | null | undefined, -): string { - const reason = status?.reason?.trim(); - return reason ? `Last known reading, not current: ${reason}` : 'Last known reading, not current'; -} - -export interface PhysicalDiskTemperaturePresentation { - label: string; - current: boolean; - /** Set only for a last-known reading: why the value is not current. */ - title?: string; -} - -export function getPhysicalDiskTemperaturePresentation( - disk: Pick, -): PhysicalDiskTemperaturePresentation | null { - if (!Number.isFinite(disk.temperature) || disk.temperature <= 0) return null; - const current = isPhysicalDiskTemperatureCurrent(disk.collection); - return { - label: formatTemperature(disk.temperature), - current, - title: current - ? undefined - : getPhysicalDiskLastKnownTemperatureTitle(disk.collection?.temperature), - }; -} +// Whether a disk temperature is a current reading lives in its own module so +// tables outside the Storage page can use it without this one. +export { + PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS, + getPhysicalDiskLastKnownTemperatureTitle, + getPhysicalDiskTemperaturePresentation, + isPhysicalDiskTemperatureCurrent, + type PhysicalDiskTemperaturePresentation, +} from './diskTemperaturePresentation'; export function getPhysicalDiskCollectionMessages(disk: PhysicalDiskPresentationData): string[] { const collection = disk.collection; diff --git a/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts b/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts new file mode 100644 index 0000000000..acb63c612a --- /dev/null +++ b/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts @@ -0,0 +1,53 @@ +import type { PhysicalDiskCollectionStatus, PhysicalDiskFieldStatus } from '@/types/resource'; +import { formatTemperature } from '@/utils/temperature'; + +// The canonical decision on whether a disk temperature is a current reading. +// It lives apart from diskPresentation so the Machines table, the machine +// drawer and the guest drawer can use it without pulling the Storage page +// presentation module into their chunks. + +// A retained reading takes no threshold colour, so it cannot read as a disk +// running hot or cool right now. The dotted underline marks the value as one +// its title explains. +export const PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS = + 'text-muted underline decoration-dotted underline-offset-2 cursor-help'; + +// Normalization may keep the last known temperature when the current +// observation is not available, such as a disk in standby or a host agent that +// stopped reporting. The value is a current reading only when its collection +// state is available, or when the source predates collection state. +export function isPhysicalDiskTemperatureCurrent( + collection: PhysicalDiskCollectionStatus | null | undefined, +): boolean { + const state = collection?.temperature?.state; + return !state || state === 'available'; +} + +export function getPhysicalDiskLastKnownTemperatureTitle( + status: PhysicalDiskFieldStatus | null | undefined, +): string { + const reason = status?.reason?.trim(); + return reason ? `Last known reading, not current: ${reason}` : 'Last known reading, not current'; +} + +export interface PhysicalDiskTemperaturePresentation { + label: string; + current: boolean; + /** Set only for a last-known reading: why the value is not current. */ + title?: string; +} + +export function getPhysicalDiskTemperaturePresentation(disk: { + temperature: number; + collection?: PhysicalDiskCollectionStatus | null; +}): PhysicalDiskTemperaturePresentation | null { + if (!Number.isFinite(disk.temperature) || disk.temperature <= 0) return null; + const current = isPhysicalDiskTemperatureCurrent(disk.collection); + return { + label: formatTemperature(disk.temperature), + current, + title: current + ? undefined + : getPhysicalDiskLastKnownTemperatureTitle(disk.collection?.temperature), + }; +} diff --git a/frontend-modern/src/types/api.ts b/frontend-modern/src/types/api.ts index 5b15015d7a..eaa1bfcc3f 100644 --- a/frontend-modern/src/types/api.ts +++ b/frontend-modern/src/types/api.ts @@ -626,6 +626,9 @@ export interface HostDiskSMART { temperature: number; // Temperature in Celsius health?: string; // PASSED, FAILED, UNKNOWN standby?: boolean; // True if disk was in standby + // A non-available temperature state marks the value as last known, such as + // a host agent that stopped reporting. + collection?: import('./resource').PhysicalDiskCollectionStatus; } export interface HostRAIDArray { From 8e93360d10228dd3f4dc7ea9fa781d502de7fd10 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:35:50 +0100 Subject: [PATCH 06/13] Poll only the history each metric table renders useWorkloadTableMetricHistory ran two polled estate-wide reads per instance in Trends mode: the workloads history and the infrastructure summary. Since grouped workload rows lost their host metric cells, the Workloads surface reads only guest series, and the Proxmox nodes table never read anything but node series. Each reader passes its own abort signal, which skips the summary caches' in-flight dedupe, so every instance paid for both routes every 30 seconds. A required series option now names the rows a caller draws. The Workloads state passes 'guests' and the Proxmox nodes table passes 'nodes', so each polls one route. On the demo estate in Trends this takes the vSphere overview from one infrastructure and one workloads read per poll to one workloads read, and the Proxmox overview from two of each to one of each. Node table sparklines are unchanged. The performance contract claimed the two readers deduped their reads; it now states that they do not and that each reader polls only what it renders. The adjacent Bars-mode prose in the performance and frontend-primitives contracts described one active-row request that leaving the row cancels; it now describes the bounded warming window the hook has run since the history lens shipped. --- .../subsystems/frontend-primitives.md | 21 ++++-- .../subsystems/performance-and-scalability.md | 36 +++++++--- .../internal/subsystems/unified-resources.md | 5 +- .../4e061179721ef90a.json | 39 +++++++++++ .../HistoryResponse.contract.test.tsx | 1 + ...loadsSurface.performance.contract.test.tsx | 12 ++++ .../useWorkloadTableMetricHistory.test.tsx | 70 ++++++++++++++++++- .../useWorkloadTableMetricHistory.ts | 14 ++-- .../components/Workloads/useWorkloadsState.ts | 3 + .../features/proxmox/ProxmoxNodesTable.tsx | 7 +- .../__tests__/ProxmoxNodesTable.test.tsx | 39 ++++++++++- 11 files changed, 219 insertions(+), 28 deletions(-) create mode 100644 frontend-modern/browser-verification/4e061179721ef90a.json diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 6c07a49441..1f45a407f9 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -688,11 +688,13 @@ same relative point in the selected history range. Leaving the row clears the cursor and restores the bars and I/O readouts together. The lens mounts with a short reduced-motion-safe fade and must not leave both bar and chart semantics in the accessibility tree simultaneously. -Bar mode resolves history only for that active guest through its canonical -metrics target and the selected compact range; it must not start an -estate-wide chart request merely because the range changes. The active request -key is stable across equivalent live guest snapshots, and leaving the row or -selecting another range aborts superseded browser work. Persistent Trends may +Bar mode resolves history per guest through each guest's canonical metrics +target and the selected compact range, warming only a bounded window of +mounted and adjacent rows with the active guest first; it must not start an +estate-wide chart request merely because the range changes. The request key is +stable across equivalent live guest snapshots, and selecting another range +aborts superseded browser work. Leaving a row does not cancel its read, which +settles into the bounded row cache. Persistent Trends may retain the shared estate reader, but range changes must clear prior-range data unless an exact-key cache entry exists. @@ -7983,6 +7985,15 @@ Rendered table proof belongs in drawer grouping and fallback proof belongs in `frontend-modern/src/components/Infrastructure/__tests__/resourceDetailDrawerMetricsHistoryModel.branchcov0712.test.ts`. +### The Proxmox nodes table reads node history only + +`frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx` draws its Trends +sparklines through `useWorkloadTableMetricHistory` with `series: 'nodes'`, so +it polls the infrastructure summary and no guest history. The embedded +workloads table below it owns guest history and passes `series: 'guests'`. +`ProxmoxNodesTable.test.tsx` pins the option; the per-page polling budget +belongs to performance-and-scalability. + ### Proxmox Storage reuses the shared product-family source scope `frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx` mounts the diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 5231c21704..d970748a0f 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -1920,9 +1920,17 @@ internal persistent signal so the toggle and the hosts-table render stay in lockstep. The history range remains inline in both display modes because the default bars mode uses it for the intent-driven guest-row history lens. Embedded sibling tables that opt into sparklines -re-instantiate `useWorkloadTableMetricHistory`; the cache key matches -the workloads-table reader so both readers dedupe their fetches and -the canonical Workloads hot-path budget is preserved. Standalone +re-instantiate `useWorkloadTableMetricHistory` and name the rows they draw +through its required `series` option. The Proxmox nodes table passes +`'nodes'` and polls only the infrastructure summary; the WorkloadsSurface +reader passes `'guests'` and polls only the workloads history. Two readers +on one page do not share reads: each forwards its own abort signal, which +bypasses the summary caches' in-flight dedupe, and retained query values +seed matching mounts and source changes without deduplicating requests. A +reader therefore must not poll a summary its rows do not render. In Trends +the vSphere overview polls only the workloads history, and the Proxmox +overview polls the workloads history and the infrastructure summary once +each. Standalone WorkloadsSurface callers (no override props) keep the original persistent-signal-backed behavior. Every toolbar that controls a `WorkloadsSurface` consumes @@ -1996,14 +2004,20 @@ The Workloads table metric display mode is part of the protected Workloads hot path. Default bar mode keeps compact current-value bars at rest and swaps only the active fine-pointer or keyboard-focused guest row to the original sparklines; persistent Trends mode keeps sparklines visible for every rendered -row. Bar mode does not hydrate estate history at rest: the selected range is -local state until a row is active, then one centralized query reads only that -guest's canonical metrics target from `/api/metrics-store/history`, capped at -36 points. Equivalent live guest snapshots must not restart that request, and -leaving the row disables it; per-row query owners, fan-out, and polling remain -forbidden. Persistent Trends retains the shared, cache-keyed -`fetchWorkloadsSummaryAndCache` / `fetchInfrastructureSummaryAndCache` reader -for all rendered rows. Range-sensitive readers clear prior-range data when no +row. Bar mode does not hydrate estate history at rest. With the history lens +on, one centralized queue warms a bounded window instead: up to the first six +guests of the mounted table window at rest, then the active guest first plus +its next four and previous one, with at most four reads in flight and 18 cached rows. Each read +targets one guest's canonical metrics target on `/api/metrics-store/history`, +is capped at 36 points, and is never polled. Equivalent live guest snapshots +must not restart a read. A range change or leaving the lens mode cancels +pending and in-flight reads, while leaving a row lets its read settle into the +bounded cache. Per-row query owners, estate-wide fan-out, and polling remain +forbidden. Persistent Trends retains one cache-keyed +`fetchWorkloadsSummaryAndCache` reader for all rendered guest rows. Grouped +host rows render no metric cells, so the WorkloadsSurface reader never polls +`fetchInfrastructureSummaryAndCache`; node series belong to the page's hosts +table reader. Range-sensitive readers clear prior-range data when no exact cache entry exists and forward cancellation so superseded range work does not continue occupying browser connections. Sparkline ranges must stay bounded to the governed compact table windows. Expanded history belongs in the existing guest drawer chart diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index d505a37b64..d60d234213 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -1535,7 +1535,10 @@ When every node shares one cluster and none is standalone, the model drops the `cluster` column (it would repeat one value on every row) and `ProxmoxNodesTable` names that cluster in the table header instead, visible during search and independent of the inventory-count preference, so the -cluster identity is never lost. Workload status buckets read the canonical +cluster identity is never lost. Its Trends sparklines read node history +through `useWorkloadTableMetricHistory` with `series: 'nodes'`; guest history +stays with the embedded workloads table below it, so the nodes table adds no +guest-history poll to the page. Workload status buckets read the canonical health too: a running workload whose unified health names an open `warning_alert` or `critical_alert` reason counts under Attention, while `backup_stale` and other non-alert reasons do not. diff --git a/frontend-modern/browser-verification/4e061179721ef90a.json b/frontend-modern/browser-verification/4e061179721ef90a.json new file mode 100644 index 0000000000..2c87631f4c --- /dev/null +++ b/frontend-modern/browser-verification/4e061179721ef90a.json @@ -0,0 +1,39 @@ +{ + "version": 1, + "base_sha": "1fe9f86d55271345492945a25ab9398892ec6bf0", + "verified_at": "2026-10-06T22:35:29Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Workloads/useWorkloadTableMetricHistory.ts", + "frontend-modern/src/components/Workloads/useWorkloadsState.ts", + "frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx" + ], + "content_sha256": { + "frontend-modern/src/components/Workloads/useWorkloadTableMetricHistory.ts": "1879e91c0eb901261d6497cc0245e2a72b8d0acc72414cd69d3369d187e59fc5", + "frontend-modern/src/components/Workloads/useWorkloadsState.ts": "0f710d2544c964ae36e04f5bdb1bb2e3f2db37deb9b1ad804df6909e22256fc9", + "frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx": "32a8c017790a55dc230585e189bbff5cd3e81ee932a80c4379fc1be3ea61172d" + }, + "routes": ["/proxmox/overview", "/vmware"], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Vite dev server serving this tree (based on PR 2622 head 1fe9f86d55) against a private DEMO_MODE mock backend (50 PVE nodes, 929 guests, 7 ESXi hosts, telemetry off), headless Chromium via Playwright; desktop 1440x900 and phone 390x844.", + "Trends mode, 95 s per page counted per 30 s poll tick, before (base files swapped back under the same server) vs after: /vmware went from 1 /api/charts/infrastructure + 1 /api/charts/workloads read per tick to 0 + 1; /proxmox/overview from 2 + 2 to 1 + 1. Same on both viewports (95 s totals: vmware 4+4 to 0+4, proxmox 8+8 to 4+4).", + "Proxmox nodes table in Trends: every rendered node row draws its CPU, memory and disk sparklines (36 of 36 on desktop, 18 of 18 on phone), same as before; guest table sparklines unchanged (Proxmox 420 desktop / 108 phone, vSphere 54).", + "Bars mode on /proxmox/overview: node table shows no sparklines and the page makes no /api/charts reads.", + "Screenshots of both tables in Trends inspected at both viewports. The only failed request is the read-only demo refusing POST /api/usage/workload-history (403), the Trends activity ping, which this change does not touch." + ], + "interactions": [ + "Desktop: opened the View popover and switched Bars to Trends; phone: opened Filters, then View, and switched Bars to Trends. The Trends button reports aria-pressed=true and the switch triggers exactly one workloads read and one infrastructure read.", + "Navigated between /vmware and /proxmox/overview with Trends persisted and watched three poll cycles on each page.", + "Scrolled the nodes table into view before each capture." + ] +} diff --git a/frontend-modern/src/components/Workloads/__tests__/HistoryResponse.contract.test.tsx b/frontend-modern/src/components/Workloads/__tests__/HistoryResponse.contract.test.tsx index f2a43447a8..85cb5ee0f2 100644 --- a/frontend-modern/src/components/Workloads/__tests__/HistoryResponse.contract.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/HistoryResponse.contract.test.tsx @@ -205,6 +205,7 @@ describe('production History consumers reject unbound response evidence', () => onDemand: () => true, range: () => '24h', selectedNode: () => null, + series: 'guests', }); return ( diff --git a/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx b/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx index 7a272f8b5c..70b013dddc 100644 --- a/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx @@ -452,6 +452,9 @@ describe('Workloads platform-page embed contract', () => { expect(nodesTableSource).toContain('useWorkloadTableMetricHistory'); expect(nodesTableSource).toContain('MetricMiniSparkline'); expect(nodesTableSource).toContain('isSparklineMode'); + // The hosts table draws node rows only: it polls the infrastructure + // summary and leaves guest history to the embedded workloads reader. + expect(nodesTableSource).toContain("series: 'nodes',"); }); it('lets platform pages move grouped host drawers to a dedicated host table owner', async () => { @@ -1432,12 +1435,21 @@ describe('Workloads performance contract', () => { expect(workloadsFilterModelSource).toContain('mobileTrailing?: JSX.Element;'); expect(workloadsStateSource).toContain('useWorkloadRouteState'); expect(workloadsStateSource).toContain('range: workloadMetricHistoryRange'); + // Grouped node rows render no metric cells, so Trends on the Workloads + // surface must not poll the infrastructure summary. + expect(workloadsStateSource).toContain("series: 'guests',"); expect(workloadTableMetricHistoryStateSource).toContain( 'fetchWorkloadsSummaryAndCache(parsed.range', ); expect(workloadTableMetricHistoryStateSource).toContain( 'fetchInfrastructureSummaryAndCache(parsed.range', ); + expect(workloadTableMetricHistoryStateSource).toContain( + "if (options.series !== 'guests' || !options.enabled()) return null;", + ); + expect(workloadTableMetricHistoryStateSource).toContain( + "options.series === 'nodes' && options.enabled() ? buildHistoryScope(options.range()) : null", + ); expect( workloadTableMetricHistoryStateSource.match(/retainPreviousValueOnSourceChange: false/g), ).toHaveLength(2); diff --git a/frontend-modern/src/components/Workloads/__tests__/useWorkloadTableMetricHistory.test.tsx b/frontend-modern/src/components/Workloads/__tests__/useWorkloadTableMetricHistory.test.tsx index 9acd3d5d4c..a176ba845b 100644 --- a/frontend-modern/src/components/Workloads/__tests__/useWorkloadTableMetricHistory.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/useWorkloadTableMetricHistory.test.tsx @@ -4,10 +4,15 @@ import { createSignal, type Accessor } from 'solid-js'; import { ChartsAPI, type AllMetricsHistoryResponse } from '@/api/charts'; import { resetCreateNonSuspendingQueryCacheForTest } from '@/hooks/createNonSuspendingQuery'; +import type { Node } from '@/types/api'; import type { WorkloadGuest } from '@/types/workloads'; +import { __resetInfrastructureSummaryFetchesForTests } from '@/utils/infrastructureSummaryCache'; import { useWorkloadTableMetricHistory } from '../useWorkloadTableMetricHistory'; -import type { WorkloadTableMetricHistoryRange } from '../workloadMetricHistoryModel'; +import { + WORKLOAD_TABLE_HISTORY_POLL_MS, + type WorkloadTableMetricHistoryRange, +} from '../workloadMetricHistoryModel'; const guest = { id: 'cluster-a:pve1:101', @@ -47,6 +52,7 @@ function HistoryProbe(props: { prefetchGuests: props.prefetchGuests, range: props.range, selectedNode: () => null, + series: 'guests', }); const activeHistoryValue = () => { @@ -57,13 +63,75 @@ function HistoryProbe(props: { return
{activeHistoryValue()}
; } +const node = { id: 'cluster-a-pve1', name: 'pve1', instance: 'cluster-a' } as Node; +const point = (value: number) => [{ timestamp: 2, value, min: value, max: value }]; + +function SeriesProbe(props: { series: 'guests' | 'nodes' }) { + const reader = useWorkloadTableMetricHistory({ + enabled: () => true, + range: () => '1h', + series: props.series, + }); + const latest = (series: ReturnType) => + series[0]?.points.at(-1)?.value ?? 'none'; + + return ( +
+ {`guest=${latest(reader.getGuestMetricSeries(guest, 'cpu'))} node=${latest( + reader.getNodeMetricSeries(node, 'cpu'), + )}`} +
+ ); +} + afterEach(() => { cleanup(); resetCreateNonSuspendingQueryCacheForTest(); + __resetInfrastructureSummaryFetchesForTests(); vi.restoreAllMocks(); + vi.useRealTimers(); }); describe('useWorkloadTableMetricHistory', () => { + it.each([ + ['guests', 'guest=11 node=none', 'workloads'], + ['nodes', 'guest=none node=22', 'infrastructure'], + ] as const)( + 'polls only the summary its %s rows read', + async (series, expectedValues, polledRoute) => { + vi.useFakeTimers(); + const workloadSpy = vi.spyOn(ChartsAPI, 'getWorkloadCharts').mockResolvedValue({ + data: { [guest.id]: { cpu: point(11) } }, + dockerData: {}, + guestTypes: {}, + timestamp: 2, + stats: { oldestDataTimestamp: 1 }, + }); + const infrastructureSpy = vi + .spyOn(ChartsAPI, 'getInfrastructureSummaryCharts') + .mockResolvedValue({ + nodeData: { [node.id]: { cpu: point(22) } }, + timestamp: 2, + stats: { oldestDataTimestamp: 1 }, + }); + const [polledSpy, idleSpy] = + polledRoute === 'workloads' + ? [workloadSpy, infrastructureSpy] + : [infrastructureSpy, workloadSpy]; + + render(() => ); + await vi.advanceTimersByTimeAsync(0); + + expect(screen.getByTestId('series-values')).toHaveTextContent(expectedValues); + expect(polledSpy).toHaveBeenCalledTimes(1); + expect(idleSpy).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(WORKLOAD_TABLE_HISTORY_POLL_MS * 2); + expect(polledSpy).toHaveBeenCalledTimes(3); + expect(idleSpy).not.toHaveBeenCalled(); + }, + ); + it('bounds slow-estate warming and prioritizes a distant active guest', async () => { const guests = Array.from({ length: 500 }, (_, index) => ({ ...guest, diff --git a/frontend-modern/src/components/Workloads/useWorkloadTableMetricHistory.ts b/frontend-modern/src/components/Workloads/useWorkloadTableMetricHistory.ts index 9a0969b59f..768fdc4b55 100644 --- a/frontend-modern/src/components/Workloads/useWorkloadTableMetricHistory.ts +++ b/frontend-modern/src/components/Workloads/useWorkloadTableMetricHistory.ts @@ -37,7 +37,13 @@ interface WorkloadTableMetricHistoryOptions { onDemand?: Accessor; prefetchGuests?: Accessor; range: Accessor; - selectedNode: Accessor; + selectedNode?: Accessor; + // The rows this caller draws while `enabled`. Each estate-wide history is a + // polled summary read, so a reader polls only the one its rows consume: + // guest rows read the workloads summary, node rows the infrastructure + // summary. Readers do not share in-flight requests, so a second reader on + // the same page pays for its own reads. + series: 'guests' | 'nodes'; } interface ActiveGuestHistoryQueryKey { @@ -111,13 +117,13 @@ const buildActiveGuestHistoryCacheKey = (key: ActiveGuestHistoryQueryKey): strin export function useWorkloadTableMetricHistory( options: WorkloadTableMetricHistoryOptions, ): WorkloadMetricHistoryReader { - const selectedNodeScope = createMemo(() => normalizeNodeScope(options.selectedNode())); + const selectedNodeScope = createMemo(() => normalizeNodeScope(options.selectedNode?.())); const workloadHistoryScope = createMemo(() => { - if (!options.enabled()) return null; + if (options.series !== 'guests' || !options.enabled()) return null; return buildHistoryScope(options.range(), selectedNodeScope()); }); const infrastructureHistoryScope = createMemo(() => - options.enabled() ? buildHistoryScope(options.range()) : null, + options.series === 'nodes' && options.enabled() ? buildHistoryScope(options.range()) : null, ); const workloadHistory = createNonSuspendingQuery({ diff --git a/frontend-modern/src/components/Workloads/useWorkloadsState.ts b/frontend-modern/src/components/Workloads/useWorkloadsState.ts index 195099789b..8b5b954709 100644 --- a/frontend-modern/src/components/Workloads/useWorkloadsState.ts +++ b/frontend-modern/src/components/Workloads/useWorkloadsState.ts @@ -575,6 +575,9 @@ export function useWorkloadsState(props: WorkloadsSurfaceProps) { prefetchGuests: metricHistoryPrefetchGuests, range: workloadMetricHistoryRange, selectedNode, + // Grouped node rows carry no metric cells, so this reader never polls + // the infrastructure summary. + series: 'guests', }); const [workloadHistoryHintSeen, setWorkloadHistoryHintSeen] = usePersistentSignal( STORAGE_KEYS.WORKLOADS_HISTORY_HINT_SEEN, diff --git a/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx b/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx index 191809f431..469b881f1e 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx @@ -258,12 +258,13 @@ export const ProxmoxNodesTable: Component<{ nodePreview.toggle(); }; - // Use the same canonical history reader the workloads table uses; cache - // keys collide so the two readers dedupe their fetches. + // Use the same canonical history reader the workloads table uses. This + // table draws node rows only, so it polls the infrastructure summary and + // leaves the guest history to the workloads table below. const metricHistory = useWorkloadTableMetricHistory({ enabled: isSparklineMode, range: () => props.metricHistoryRange?.() ?? '1h', - selectedNode: () => '', + series: 'nodes', }); return ( diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx b/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx index fc4e8b9ae9..89b48696e1 100644 --- a/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx +++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx @@ -1,4 +1,5 @@ import { cleanup, fireEvent, render, screen } from '@solidjs/testing-library'; +import { createSignal } from 'solid-js'; import { createStore } from 'solid-js/store'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; @@ -9,6 +10,7 @@ const nodeDrawerMock = vi.hoisted(() => vi.fn()); const activeAlertsMock = vi.hoisted(() => ({ value: {} as Record })); const getMetricThresholdsMock = vi.hoisted(() => vi.fn()); const temperatureGaugeMock = vi.hoisted(() => vi.fn()); +const metricHistoryMock = vi.hoisted(() => vi.fn()); vi.mock('@/hooks/useBreakpoint', () => ({ useBreakpoint: () => ({ @@ -63,9 +65,12 @@ vi.mock('@/components/shared/TemperatureGauge', () => ({ })); vi.mock('@/components/Workloads/useWorkloadTableMetricHistory', () => ({ - useWorkloadTableMetricHistory: () => ({ - getNodeMetricSeries: () => [], - }), + useWorkloadTableMetricHistory: (options: unknown) => { + metricHistoryMock(options); + return { + getNodeMetricSeries: () => [], + }; + }, })); vi.mock('@/components/Workloads/NodeDrawer', () => ({ @@ -578,6 +583,34 @@ describe('ProxmoxNodesTable', () => { expect(screen.queryByTestId('temperature-gauge')).not.toBeInTheDocument(); }); + it('reads node history only, and only while Trends is on', () => { + const [mode, setMode] = createSignal<'bars' | 'sparklines'>('bars'); + render(() => ( + } + emptyTitle="No Proxmox VE nodes" + emptyDescription="No nodes" + /> + )); + + expect(metricHistoryMock).toHaveBeenCalledTimes(1); + const options = metricHistoryMock.mock.calls[0][0] as { + enabled: () => boolean; + selectedNode?: unknown; + series: string; + }; + // Guest history belongs to the workloads table below; a second guest + // reader here would poll the workloads route for rows it never draws. + expect(options.series).toBe('nodes'); + expect(options.selectedNode).toBeUndefined(); + expect(options.enabled()).toBe(false); + setMode('sparklines'); + expect(options.enabled()).toBe(true); + }); + it('updates every availability signal when a live row becomes offline', () => { const [state, setState] = createStore({ nodes: [makeNodeResource({ uptime: 187_200, temperature: 76 })], From 344f1f85310199ec7880c722bd2f5a65718e5244 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:38:25 +0100 Subject: [PATCH 07/13] Raise disk health alerts for mock physical disks checkMockAlerts evaluated guests, agents, nodes, storage, PBS, PMG and Docker against the mock estate but never physical disks. The fixture keeps a FAILED cohort and worn SSDs stable on purpose, so mock and demo stacks showed failing and "Replace Now" disks with no disk-health or disk-wearout alert, and the disk alert lifecycle could not be exercised without real hardware. The pass applies the physical disk poller's boundary: only disks on online nodes, matched by instance and node name, with the linked agent's --disk-exclude patterns forcing a matched device healthy. The per-disk evaluation moves into checkPhysicalDiskAlerts so the poller and the mock pass share one rule. The poller's agent-link settle deferral is not applied, because fixture links and exclusions are complete from the first pass. Like the other loops in checkMockAlerts, the disk loop runs after the pass's single mode check, so the existing race with leaving mock mode now covers disk alerts too. That race also involves other fixture evaluation paths and is left for a monitor-wide fix. On the public demo's estate (8 nodes) this adds one warning-level disk-wearout alert and no disk-health alert: that estate has no FAILED Proxmox disk. --- .../v6/internal/subsystems/monitoring.md | 56 ++++++++ .../monitoring/canonical_guardrails_test.go | 3 +- internal/monitoring/monitor_alerts.go | 43 +++++++ .../monitoring/monitor_mock_alerts_test.go | 121 ++++++++++++++++++ internal/monitoring/monitor_pve.go | 25 ++-- 5 files changed, 238 insertions(+), 10 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index a912bfce62..0c751ad9be 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -1679,6 +1679,62 @@ boundaries: `TestMockKubernetesLoneHeavyPodStaysUnderNodeCeiling` in `internal/mock/generator_test.go`. +Mock alert evaluation covers physical disks as well. Live Proxmox disks are +evaluated by the physical disk poller in `maybePollPhysicalDisksAsync`, which +only polls configured Proxmox instances, so fixture disks had no evaluation +path. The fixture deliberately keeps a FAILED health cohort and worn SSDs +stable across restarts, so the estate showed failing and "Replace Now" disks +with no `disk-health` or `disk-wearout` alert. `checkMockAlerts` now evaluates +fixture disks through `checkMockPhysicalDiskAlerts`. Both paths share the +per-disk rule in `checkPhysicalDiskAlerts` (a device matched by the linked +agent's `--disk-exclude` patterns is evaluated as healthy, otherwise +`CheckDiskHealth` judges the disk), and the mock pass applies the poller's +boundary around it: + +- Only disks on online nodes are evaluated, as the poller only evaluates nodes + it reached. A disk on an offline node keeps whatever alert it had. The mock + pass treats every disk on an online fixture node as collected; the poller + also skips a node whose disk query and SMART fallback both failed. Nodes are + matched by instance and name, as the poller only pairs a disk with its own + instance's nodes. +- Excluding a disk resolves the alerts it already raised, because the excluded + device is evaluated as healthy rather than skipped. +- The poller's wait for host-agent links to settle after a restart (#1674) is + not applied: fixture links and exclusions are complete from the first pass. +- A disk whose node name leaves the estate loses its alerts through + `CleanupAlertsForNodes`, which the same pass already runs. That cleanup + matches node names only, so it relies on fixture node names being unique + across instances, which they are. + +The disk loop relies on the pass's single up-front mode check, like every +loop in `checkMockAlerts` except the host-agent step, which re-checks under its +own lock. A pass already past that check when mock mode is switched off can +reopen its fixture alerts after `SetMockMode(false)` clears them, and +other paths that evaluate fixture data (frontend refresh into unified alert +evaluation, backup rollups) have the same window. That mode-exit race predates +the disk loop and is not closed here; a lock around `checkMockAlerts` alone +would leave the other paths open. + +The disk evaluation runs on the mock alert tick, not the poller's disk +interval (five minutes by default, `PhysicalDiskPollingMinutes` per instance). +`CheckDiskHealth` logs at error level whenever it evaluates a FAILED disk +(models with a known firmware bug skip the health check) and at warn level for +a worn SSD, so an evaluated fixture disk in either state logs on every tick. On +2026-10-06, private stacks with the public demo's estate (8 nodes) carried 12 +Proxmox disks, none FAILED. Baseline against patched, the only new disk alert +was one warning-level `disk-wearout` (a 7% SSD on `pve5`), whose resource ID is +one of the disk resource's canonical aliases, and neither stack had a +`disk-health` alert. `CheckDiskHealth` raises only those two alert types, so +the other differences between the two boots (ZFS and Docker service alerts) are +not from the disk pass; those fixtures draw random state each boot. The default +test estate raises five `disk-health` and one `disk-wearout` alert. The tests +pin the boundary: `TestCheckMockAlertsEvaluatesPhysicalDisks` and +`TestCheckMockPhysicalDiskAlertsFollowsPollerBoundary` in +`internal/monitoring/monitor_mock_alerts_test.go`, with +`TestProxmoxDiskAlertsRunOnMergedDiskState` in +`internal/monitoring/canonical_guardrails_test.go` keeping the poller on the +shared helper. + Host and container-runtime disk collection supports an explicit include list for filesystems hidden by Pulse's automatic virtual/container filtering. The include list is bounded to that automatic filter; explicit disk exclusions diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go index 3c398a8cbf..6234aff22a 100644 --- a/internal/monitoring/canonical_guardrails_test.go +++ b/internal/monitoring/canonical_guardrails_test.go @@ -2234,7 +2234,8 @@ func TestProxmoxDiskAlertsRunOnMergedDiskState(t *testing.T) { file: "monitor_pve.go", snippets: []string{ "allDisks = mergeHostAgentSMARTIntoDisks(allDisks, nodesFromState, hosts)", - "m.alertManager.CheckDiskHealth(inst, disk.Node, proxmoxDiskFromPhysicalDisk(disk))", + "m.checkPhysicalDiskAlerts(inst, disk, diskExcludeByNode[disk.Node])", + "m.alertManager.CheckDiskHealth(instance, disk.Node, proxmoxDiskFromPhysicalDisk(disk))", "func proxmoxDiskFromPhysicalDisk(disk models.PhysicalDisk) proxmox.Disk {", }, }, diff --git a/internal/monitoring/monitor_alerts.go b/internal/monitoring/monitor_alerts.go index 89e870836f..a75e7dd557 100644 --- a/internal/monitoring/monitor_alerts.go +++ b/internal/monitoring/monitor_alerts.go @@ -716,6 +716,14 @@ func (m *Monitor) checkMockAlerts() { m.alertManager.CheckStorageWithCapacityTrend(storage, m.storageCapacityTrend(storage, time.Now())) } + // Check alerts for physical disks. Live disks are evaluated by the + // physical disk poller, which only polls configured Proxmox instances, and + // the fixture deliberately keeps a FAILED cohort and worn SSDs, so without + // this pass the estate shows failing disks with no disk-health or + // disk-wearout alert. + log.Debug().Int("diskCount", len(state.PhysicalDisks)).Msg("checking physical disk alerts") + m.checkMockPhysicalDiskAlerts(state.PhysicalDisks, state.Nodes, state.Hosts) + // Check alerts for PBS instances log.Debug().Int("pbsCount", len(state.PBSInstances)).Msg("checking PBS alerts") for _, pbsInst := range state.PBSInstances { @@ -815,3 +823,38 @@ func (m *Monitor) checkMockDockerHostAlerts(host models.DockerHost) { } m.alertManager.CheckDockerHost(host) } + +// checkMockPhysicalDiskAlerts applies the physical disk poller's alert +// boundary to fixture disks. The poller only evaluates disks on nodes it +// reached, so a disk on a node that is not online keeps whatever alert it +// had, and a device matched by the linked agent's --disk-exclude patterns is +// evaluated as healthy. The poller's wait for host-agent links to settle after +// a restart does not apply: fixture links and exclusions are complete from the +// first pass. +func (m *Monitor) checkMockPhysicalDiskAlerts(disks []models.PhysicalDisk, nodes []models.Node, hosts []models.Host) { + type nodeKey struct{ instance, name string } + + excludeByHost := make(map[string][]string, len(hosts)) + for _, host := range hosts { + if len(host.DiskExclude) > 0 { + excludeByHost[host.ID] = host.DiskExclude + } + } + onlineNodes := make(map[nodeKey]bool, len(nodes)) + excludeByNode := make(map[nodeKey][]string) + for _, node := range nodes { + key := nodeKey{instance: node.Instance, name: node.Name} + onlineNodes[key] = node.Status == "online" + if patterns := excludeByHost[node.LinkedAgentID]; node.LinkedAgentID != "" && len(patterns) > 0 { + excludeByNode[key] = patterns + } + } + + for _, disk := range disks { + key := nodeKey{instance: disk.Instance, name: disk.Node} + if !onlineNodes[key] { + continue + } + m.checkPhysicalDiskAlerts(disk.Instance, disk, excludeByNode[key]) + } +} diff --git a/internal/monitoring/monitor_mock_alerts_test.go b/internal/monitoring/monitor_mock_alerts_test.go index 66e69d616a..3c5c933339 100644 --- a/internal/monitoring/monitor_mock_alerts_test.go +++ b/internal/monitoring/monitor_mock_alerts_test.go @@ -239,3 +239,124 @@ func TestMockHostAgentPassAfterLeavingMockModeRegistersNothing(t *testing.T) { } t.Fatal("stale mock pass registered fixture hostname pve1 and suppressed the real node's memory alert") } + +// diskAlertKeys lists the active disk-health and disk-wearout alerts as +// "type instance/node device" so assertions name the disk, not the alert ID. +func diskAlertKeys(manager *alerts.Manager) map[string]bool { + keys := make(map[string]bool) + for _, alert := range manager.GetActiveAlerts() { + if alert.Type != "disk-health" && alert.Type != "disk-wearout" { + continue + } + devPath, _ := alert.Metadata["disk_path"].(string) + keys[alert.Type+" "+alert.Instance+"/"+alert.Node+" "+devPath] = true + } + return keys +} + +func TestCheckMockAlertsEvaluatesPhysicalDisks(t *testing.T) { + // Pin the default estate: a PULSE_MOCK_* environment can select one with + // no FAILED disk (the public demo's eight nodes have none). + previousEnabled, previousConfig := mock.IsMockEnabled(), mock.GetConfig() + t.Cleanup(func() { + mustSetMockEnabled(t, false) + mock.SetMockConfig(previousConfig) + mustSetMockEnabled(t, previousEnabled) + }) + mustSetMockEnabled(t, false) + cfg := mock.DefaultConfig + cfg.UpdateInterval = 5 * time.Minute + mock.SetMockConfig(cfg) + mustSetMockEnabled(t, true) + manager := newMockHostAlertTestManager(t) + + monitor := &Monitor{alertManager: manager} + monitor.checkMockAlerts() + + // The fixture keeps a FAILED cohort and worn SSDs stable across restarts + // so the disk alert lifecycle has evidence to act on. Each must raise the + // alert the physical disk poller raises for a live disk, and a disk on a + // node the poller would not reach raises nothing. + state := mock.CurrentFixtureGraph().State + onlineNodes := make(map[string]bool, len(state.Nodes)) + for _, node := range state.Nodes { + onlineNodes[node.Instance+"/"+node.Name] = node.Status == "online" + } + want := make(map[string]bool) + failed, worn := 0, 0 + for _, disk := range state.PhysicalDisks { + if !onlineNodes[disk.Instance+"/"+disk.Node] { + continue + } + if strings.EqualFold(disk.Health, "FAILED") { + want["disk-health "+disk.Instance+"/"+disk.Node+" "+disk.DevPath] = true + failed++ + } + if disk.Wearout >= 0 && disk.Wearout < 10 { + want["disk-wearout "+disk.Instance+"/"+disk.Node+" "+disk.DevPath] = true + worn++ + } + } + if failed == 0 || worn == 0 { + t.Fatalf("mock fixture has %d FAILED and %d worn disks on online nodes, want at least one of each", failed, worn) + } + + got := diskAlertKeys(manager) + for key := range got { + if !want[key] { + t.Errorf("mock pass raised unexpected disk alert %q", key) + } + } + for key := range want { + if !got[key] { + t.Errorf("mock pass raised no alert %q", key) + } + } +} + +func TestCheckMockPhysicalDiskAlertsFollowsPollerBoundary(t *testing.T) { + manager := newMockHostAlertTestManager(t) + monitor := &Monitor{alertManager: manager} + + nodes := []models.Node{ + {ID: "west-pve1", Name: "pve1", Instance: "west", Status: "online", LinkedAgentID: "host-west-pve1"}, + {ID: "east-pve1", Name: "pve1", Instance: "east", Status: "offline"}, + } + hosts := []models.Host{{ID: "host-west-pve1", Hostname: "pve1", Status: "online"}} + disks := []models.PhysicalDisk{ + {Instance: "west", Node: "pve1", DevPath: "/dev/sda", Model: "Crucial MX500 2TB", Type: "sata", Health: "FAILED", Wearout: -1}, + {Instance: "west", Node: "pve1", DevPath: "/dev/nvme0n1", Model: "Samsung 980 PRO 2TB", Type: "nvme", Health: "PASSED", Wearout: 6}, + // Same node name on another instance: that node is offline, so the + // poller would not reach it and its disk must not alert. + {Instance: "east", Node: "pve1", DevPath: "/dev/sdb", Model: "WD Red Pro 8TB", Type: "sata", Health: "FAILED", Wearout: -1}, + } + + monitor.checkMockPhysicalDiskAlerts(disks, nodes, hosts) + got := diskAlertKeys(manager) + if len(got) != 2 || !got["disk-health west/pve1 /dev/sda"] || !got["disk-wearout west/pve1 /dev/nvme0n1"] { + t.Fatalf("disk alerts = %v, want only FAILED health on west/pve1 /dev/sda and wearout on west/pve1 /dev/nvme0n1", got) + } + + // The linked agent's --disk-exclude forces matched devices healthy, so + // excluding a disk resolves the alerts it already raised. Endurance + // recovery needs three healthy-looking passes. + hosts[0].DiskExclude = []string{"/dev/sda", "nvme0n1"} + for i := 0; i < 3; i++ { + monitor.checkMockPhysicalDiskAlerts(disks, nodes, hosts) + } + if got := diskAlertKeys(manager); len(got) != 0 { + t.Fatalf("excluded devices kept disk alerts %v", got) + } + + // A node that leaves the estate takes its disk alerts with it through the + // node cleanup the mock pass runs first. + hosts[0].DiskExclude = nil + monitor.checkMockPhysicalDiskAlerts(disks, nodes, hosts) + if got := diskAlertKeys(manager); len(got) != 2 { + t.Fatalf("disk alerts after removing the exclusion = %v, want both west/pve1 alerts back", got) + } + manager.CleanupAlertsForNodes(map[string]bool{"pve2": true}) + if got := diskAlertKeys(manager); len(got) != 0 { + t.Fatalf("disk alerts outlived their departed node: %v", got) + } +} diff --git a/internal/monitoring/monitor_pve.go b/internal/monitoring/monitor_pve.go index 8b03fac7e0..581307f263 100644 --- a/internal/monitoring/monitor_pve.go +++ b/internal/monitoring/monitor_pve.go @@ -1175,15 +1175,7 @@ func (m *Monitor) maybePollPhysicalDisksAsync( Int("wearout", disk.Wearout). Msg("Checking disk health") - if excludePatterns, ok := diskExcludeByNode[disk.Node]; ok && fsfilters.MatchesDeviceExclude(disk.DevPath, excludePatterns) { - healthyDisk := proxmoxDiskFromPhysicalDisk(disk) - healthyDisk.Health = "PASSED" - healthyDisk.Wearout = 100 - m.alertManager.CheckDiskHealth(inst, disk.Node, healthyDisk) - continue - } - - m.alertManager.CheckDiskHealth(inst, disk.Node, proxmoxDiskFromPhysicalDisk(disk)) + m.checkPhysicalDiskAlerts(inst, disk, diskExcludeByNode[disk.Node]) } // Write SMART metrics to persistent store @@ -1380,6 +1372,21 @@ func preserveUnavailablePhysicalDiskEvidence(current, previous models.PhysicalDi return current } +// checkPhysicalDiskAlerts evaluates one Proxmox physical disk's health and +// wearout alerts. A device matched by the linked host agent's --disk-exclude +// patterns is evaluated as healthy, so excluding a disk also resolves the +// alerts it already raised. +func (m *Monitor) checkPhysicalDiskAlerts(instance string, disk models.PhysicalDisk, excludePatterns []string) { + if len(excludePatterns) > 0 && fsfilters.MatchesDeviceExclude(disk.DevPath, excludePatterns) { + healthyDisk := proxmoxDiskFromPhysicalDisk(disk) + healthyDisk.Health = "PASSED" + healthyDisk.Wearout = 100 + m.alertManager.CheckDiskHealth(instance, disk.Node, healthyDisk) + return + } + m.alertManager.CheckDiskHealth(instance, disk.Node, proxmoxDiskFromPhysicalDisk(disk)) +} + func proxmoxDiskFromPhysicalDisk(disk models.PhysicalDisk) proxmox.Disk { return proxmox.Disk{ DevPath: disk.DevPath, From c06af69fcb225f64c7823bcb9536fd2f810714e8 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 00:14:49 +0100 Subject: [PATCH 08/13] Show a host whose agent stopped reporting as offline A host agent past its reporting lease is offline: the monitor marks it so and raises "Host is offline". The registry's stale pass ranked the agent's stale sighting above offline, though, so the machine reached every consumer as a warning. The Machines table showed it amber and rendered its last CPU, memory, disk and temperature as current readings, while its own drawer led with the critical offline alert. A Docker host was offline only until its sighting crossed the 120-second stale threshold, then flipped to warning, and Kubernetes clusters likewise. Each sighting now records when a push reporter (host agent, Docker, Kubernetes) delivered its machine, Docker host or cluster as offline, and status aggregation counts that as offline whatever the sighting's age. The sighting itself still reads stale, and a live source still carries a merged row, so a Proxmox node with a dead agent stays online. A copy that went through JSON re-derives the marker from its stored status, and a manual link joining two resources from one source keeps the fresher sighting, so a retired agent cannot pull a live one down. Offline rows then take the existing danger gate. The Machines Uptime and Temp cells and the Docker hosts Temp cell had no gate and now follow it. The drawer drops a silent agent's uptime and marks its non-disk Thermals rows "(last known)", as retained disk rows already are. --- .../subsystems/frontend-primitives.md | 8 + .../subsystems/performance-and-scalability.md | 5 + .../internal/subsystems/storage-recovery.md | 10 + .../internal/subsystems/unified-resources.md | 38 +- .../3b34c585696e9d41.json | 49 +++ .../Infrastructure/ResourceDetailSummary.tsx | 6 +- .../ResourceDetailSummary.offline.test.tsx | 64 ++++ .../__tests__/resourceDetailMappers.test.ts | 41 +++ .../Infrastructure/resourceDetailMappers.ts | 33 +- .../useResourceDetailDrawerDerivedState.ts | 9 +- .../src/features/docker/DockerHostsTable.tsx | 6 +- .../__tests__/DockerHostsTable.test.tsx | 52 +++ .../standalone/AgentsMachinesTable.tsx | 27 +- .../__tests__/AgentsMachinesTable.test.tsx | 97 ++++- internal/unifiedresources/clone_test.go | 6 + internal/unifiedresources/registry.go | 55 ++- .../registry_merge_policy_test.go | 348 ++++++++++++++++++ internal/unifiedresources/types.go | 6 + 18 files changed, 831 insertions(+), 29 deletions(-) create mode 100644 frontend-modern/browser-verification/3b34c585696e9d41.json create mode 100644 frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.offline.test.tsx diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index f585ee45a4..378d00185c 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -5591,6 +5591,14 @@ Standalone, TrueNAS, and vSphere platform tables and their table-model helpers must compose those helpers instead of declaring local `metricFallback` / `finiteMetric` helpers or inlining centered muted dash fallback markup in metric cells. +A row whose status indicator is `danger` (offline) blanks every reading from +its last report, not only the metric bars. Proxmox node rows already gate +uptime and temperature on online. The Machines table gates its Uptime and +Temperature cells, and the Docker hosts table its temperature cell, on the same +check. A machine another source keeps up stays rendered, since its cell may +carry that source's current reading. Numeric cells keep their own +right-aligned empty dash; the centred `PlatformTableMetricFallback` marker +stays specific to metric-bar cells. Platform table metric severity coloring is alert-backed, not hardcoded. The Docker host and container, Proxmox node, Kubernetes cluster and node, TrueNAS system and app, and vSphere host tables must resolve display thresholds diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index a7f0088b89..5087716ff4 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -2051,6 +2051,11 @@ in `frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts`. That module stays separate from `diskPresentation.ts` so the drawer and Machines chunks take the decision without the Storage presenter, and the rows add no reads, polling or aggregation work. +A silent host agent's other Thermals rows (pressure, limits, GPU, temperatures, +additional sensors, fans, power) take the same "(last known)" suffix and reason +title. `buildTemperatureRows` gets them through its `lastKnownReason` option, +which the drawer sets from the resource's own `agent.stale`, so the drawer +still reads nothing extra. The Proxmox node drawer overview should follow the existing guest drawer compact detail-section pattern and expose node-specific context such as platform, kernel, hardware, raw capacity, telemetry, and thermal facts rather than diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 2397cd4bb5..e96a2af8cf 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -3169,6 +3169,16 @@ pointer isolation are pinned by `TestResourceFromHostPreservesCustomSensorMeta` and `TestCloneResourceIsolatesCustomSensorValues`. +### Expired reporting leases change status only + +`SourceStatus` now carries an unexported marker that a push reporter (host +agent, Docker, Kubernetes) delivered its machine, Docker host or cluster as +offline at lease expiry. Status aggregation reads it, so such a row is +`offline` instead of `warning`. The marker is not serialized, persisted or +exposed on the wire, and the public `status`, `lastSeen` and cadence fields +are unchanged. No storage path, recovery read or protection judgement depends +on it. `TestCloneResource_MutateSourceStatusMap` pins that clones keep it. + ### Shared system-settings boundary dropped dead auto-update schedule fields The shared `internal/api` system-settings surface this subsystem consumes diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index d14cbc296c..f5e20f3eca 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -2270,10 +2270,12 @@ and temperature facts must also compose the frontend-primitives desktop rows use the shared fixed label track so labels and values remain visually adjacent instead of spanning the full drawer width. A standalone host agent past its reporting lease reaches the Machines table as -`warning` with `agent.stale`, not `offline`: by the time the lease expires the -registry has marked its only source stale, which outranks offline. The offline -metric fallback therefore does not blank its cells, and the temperature cell -has no such gate. For SMART disk temperatures the provenance travels on +`offline`, as described in "A push reporter past its lease is offline". The +offline metric fallback blanks its CPU, memory, disk, network, disk I/O, uptime +and temperature cells. The machine drawer drops a silent agent's uptime and +marks its non-disk Thermals rows "(last known)" with the reason as title, +keyed on `agent.stale` because those rows are that agent's own sensors. For SMART disk +temperatures on rows that still render, the provenance travels on `agent.sensors.smart[].collection`: the Machines temperature cell (`AgentsMachinesTable.tsx`, `agentMachineTableModel.ts`), its tooltip and the drawer's Thermals rows (`resourceDetailMappers.ts`) read it through @@ -6030,6 +6032,34 @@ to that boolean. Identity succession continues to rekey the same state row. round-trip persistence, attention suppression, and the retired remediation lock. +### A push reporter past its lease is offline + +The host agent, Docker and Kubernetes collectors push their own reports, and +the monitor marks the machine, Docker host or cluster offline only once its +reporting lease runs out. Ingest records that verdict on the source's +sighting, and `aggregateStatus` counts such a sighting as `offline` whatever +its age. The stale pass used to rank the stale sighting above `offline`, so a +silent standalone agent reached every consumer as `warning`. The Machines +table then showed it amber with its last report rendered as current readings, +while its own drawer led with a critical "Host is offline" alert. A Docker host +was offline only until its sighting crossed the 120-second stale threshold, +then flipped to `warning`. With this rule, such rows take the existing +offline treatment: frontend danger gates, the health verdict's `offline` +reason, offline filters and counts. + +`SourceStatus.Status` keeps describing delivery freshness, so the sighting +still reads `stale`. Health keeps its `telemetry_stale` reason, and +monitored-system reasons are unchanged. A live source still carries a merged +row: a Proxmox node whose linked agent stopped reporting stays `online` through +the PVE poll. Pull sources (Proxmox, PBS, PMG, TrueNAS, vSphere) and the +resources push collectors report about (guests, containers, pods, disks) keep +the stale-to-warning rule. A JSON copy re-derives the unexported marker from +its stored status in `IngestResources`, and a manual link that joins two +resources reported by one source keeps the fresher sighting. The tests in +`registry_merge_policy_test.go` pin the agent, Docker, Kubernetes, +mixed-source, round-trip and manual-link cases, plus the boundaries that stay +unchanged. + ### Canonical object drawer hierarchy The unified resource drawer is the shared operator-facing detail contract for diff --git a/frontend-modern/browser-verification/3b34c585696e9d41.json b/frontend-modern/browser-verification/3b34c585696e9d41.json new file mode 100644 index 0000000000..f9cb8ca29a --- /dev/null +++ b/frontend-modern/browser-verification/3b34c585696e9d41.json @@ -0,0 +1,49 @@ +{ + "version": 1, + "base_sha": "723f3165a1292268634b6054ab93d3015805d990", + "verified_at": "2026-10-06T23:14:41Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx", + "frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts", + "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts", + "frontend-modern/src/features/docker/DockerHostsTable.tsx", + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx" + ], + "content_sha256": { + "frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx": "f141220216a968f5953b7646035108f015a9f7de3c5ae5a9e3251ec6e7046fa5", + "frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts": "07b26eafecbd1ec93fb1dd90d4af6629778cef59a2d9e7450426cb88c90bdfdb", + "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts": "70955e370c5b2611557a5de8ba4df44b8f6dce47c1b08c611a154d232c786501", + "frontend-modern/src/features/docker/DockerHostsTable.tsx": "450cce7a4723ce7fe7b01ffe9d10c7d6439c34a1a6dda4fc0af6689ceb4741b5", + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx": "8d63a04f6a475f39de90cb9813b8bd4315525bf63318cfaf958ac195ae07950f" + }, + "routes": [ + "/standalone/machines", + "/docker" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Two private mock stacks (8-node estate) built from this tree: one with this commit's registry.go/types.go, one with origin/main's, both with a local-only go build -overlay of internal/mock/generator.go that holds standalone Linux agent host-linux-1 (Apollo-114) and Docker host edge-proxy-01 past their reporting leases the way evaluateHostAgents/evaluateDockerAgents leave them: status offline, LastSeen frozen 14m/9m before the first tick, last readings kept. Mock mode skips the host health sweep, hence the overlay.", + "Before (origin/main backend and frontend): /api/resources served Apollo-114 as status warning with agent.stale true and edge-proxy-01 as warning with health stale. The Machines row showed an amber dot, CPU 32%, Memory 74%, Disk 80%, Net and Disk I/O rates, Up 29d and Temp 45\u00b0C in threshold green beside 'Seen 14m ago', while the drawer led with the critical 'Host Apollo-114 is offline' alert and read 'Observed state Degraded, Uptime 29d 12h'.", + "After (this commit): both resources serve status offline, the agent/docker sightings still read stale, and health reads critical with reasons offline then telemetry_stale. The Apollo-114 row shows a red dot and '\u2014' in CPU, Memory, Disk, Net I/O, Disk I/O, Up and Temp at 1440 and 390; the status filter counts it under Offline. Spectrum-709 (reporting) keeps every value, Temp 47\u00b0C included.", + "Machine drawer at 1440 and 390: Runtime context reads 'Observed state Offline' and 'Last seen N mins ago' with no Uptime row; Platform details > Machine 'Show details' drops the System card's Uptime and the Thermals card reads 'Cpu.package 45\u00b0C (last known)' with title 'Last known reading, not current: host agent stopped reporting'.", + "Docker page at 1440 and 390: edge-proxy-01 shows a red dot, '\u2014' for CPU, Memory, Disk and Temp (the hold seeds a retained 51\u00b0C), and the existing 'No report for 16m' marker in Uptime; Edge Apps 01 keeps its values, Temp 38.5\u00b0C included.", + "Bundle check: WorkloadsSurface unchanged (70.20 kB), Standalone +0.04 kB, Docker +0.01 kB, InfrastructureWorkspace flat versus this tree without the frontend diff; all chunks within budget.", + "Not covered: a real agent going silent on hardware, light/dark differences (no new colour classes), and the drawer Disks card, which still threshold-colours a silent agent's retained filesystem usage (left for a follow-up)." + ], + "interactions": [ + "Enabled the Network, Disk I/O, Uptime and Temp columns through the Machines column preferences (localStorage init) and read every cell of the offline and reporting rows plus the status dot class at 1440 and 390.", + "Opened the Apollo-114 drawer by row click at both widths, read the Runtime context rows, expanded Platform details and the Machine 'Show details', and read the System and Thermals cards with their value titles.", + "Opened /docker at both widths and read every cell of edge-proxy-01 and Edge Apps 01.", + "Queried /api/resources on both stacks for status, per-source sighting status and health reasons of the held agent and Docker host." + ] +} diff --git a/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx b/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx index adb1a78581..43975c192f 100644 --- a/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx +++ b/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx @@ -217,9 +217,13 @@ export const InlineResourceSummaryTables: Component { const docker = dockerContainerMeta(props.resource); + // An offline resource has no current uptime: the retained value is from + // its last report, and the Last seen row already says when that was. const uptimeSeconds = docker ? getDockerContainerUptimeSeconds(props.resource) - : props.resource.uptime; + : props.resource.status === 'offline' + ? undefined + : props.resource.uptime; const identityRows = compactDetailRows([ ...props.drawer.primaryIdentityRows().map((row) => makeDetailRow(row.label, row.value)), props.showPlatformId ? makeDetailRow('Platform ID', props.resource.platformId) : null, diff --git a/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.offline.test.tsx b/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.offline.test.tsx new file mode 100644 index 0000000000..530048ca63 --- /dev/null +++ b/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.offline.test.tsx @@ -0,0 +1,64 @@ +import { cleanup, render, screen, within } from '@solidjs/testing-library'; +import { afterEach, describe, expect, it } from 'vitest'; + +import type { Resource } from '@/types/resource'; +import { InlineResourceSummaryTables } from '../ResourceDetailSummary'; +import type { UseResourceDetailDrawerStateResult } from '../useResourceDetailDrawerState'; + +const drawerState = { + primaryIdentityRows: () => [], + identityIpValues: () => [], + identityAliasValues: () => [], + aliasPreviewValues: () => [], + hasAliasOverflow: () => false, + identityCardHasRichData: () => true, + sourceSummary: () => null, + lastSeen: () => '14 mins ago', + lastSeenAbsolute: () => '', +} as unknown as UseResourceDetailDrawerStateResult; + +const machine = (status: Resource['status']): Resource => ({ + id: 'agent-apollo', + name: 'Apollo-114', + displayName: 'Apollo-114', + platformId: 'host-linux-1', + platformType: 'agent', + sourceType: 'agent', + type: 'agent', + status, + uptime: 2_500_000, + lastSeen: Date.parse('2026-10-06T22:00:00Z'), +}); + +afterEach(cleanup); + +describe('resource drawer runtime context for an offline resource', () => { + it('drops the uptime of an offline machine and keeps when it was last seen', () => { + render(() => ( + + )); + + const section = screen.getByTestId('resource-runtime-context-section'); + expect(within(section).getByText('offline')).toBeInTheDocument(); + expect(within(section).queryByText('Uptime')).toBeNull(); + expect(within(section).getByText('Last seen')).toBeInTheDocument(); + expect(within(section).getByText('14 mins ago')).toBeInTheDocument(); + }); + + it('keeps the uptime of a machine that is up', () => { + render(() => ( + + )); + + const section = screen.getByTestId('resource-runtime-context-section'); + expect(within(section).getByText('Uptime')).toBeInTheDocument(); + }); +}); diff --git a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts index 5596569845..9a3f4849f0 100644 --- a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts +++ b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { buildCustomSensorRows, buildTemperatureRows, + HOST_AGENT_STOPPED_REPORTING_REASON, formatInteger, formatSensorName, formatSourceType, @@ -248,6 +249,37 @@ describe('resourceDetailMappers', () => { ]); }); + it("marks a silent agent's sensor readings last known and leaves disks to their own state", () => { + const rows = buildTemperatureRows( + { + thermalState: { pressure: 'serious' }, + temperatureCelsius: { 'cpu.package': 45 }, + fanRpm: { chassis_fan: 1200 }, + smart: [ + { + device: 'sda', + temperature: 38, + collection: { temperature: { state: 'available', source: 'smartctl' } }, + }, + ], + }, + { lastKnownReason: HOST_AGENT_STOPPED_REPORTING_REASON }, + ); + + const lastKnown = 'Last known reading, not current: host agent stopped reporting'; + expect(rows).toEqual([ + { label: 'Thermal pressure', value: 'Serious (last known)', valueTitle: lastKnown }, + { label: 'Cpu.package', value: '45°C (last known)', valueTitle: lastKnown }, + { label: 'Chassis Fan', value: '1,200 RPM (last known)', valueTitle: lastKnown }, + // A disk row follows its own collection state, which the backend + // withdraws when the lease expires. + { label: 'Disk sda', value: '38°C', valueTitle: '38.0°C' }, + ]); + expect(buildTemperatureRows({ temperatureCelsius: { 'cpu.package': 45 } })).toEqual([ + { label: 'Cpu.package', value: '45°C', valueTitle: '45.0°C' }, + ]); + }); + it('lists a SMART row only for a positive reading in any state, never a standby disk', () => { const states: (PhysicalDiskCollectionStatus | undefined)[] = [ undefined, @@ -442,6 +474,15 @@ describe('resourceDetailMappers', () => { expect(agent?.id).not.toBe('resource:host:hash-1'); }); + it('reports no uptime for a silent agent, whose last report is not how long it has been up', () => { + const base = createHybridHostResource(); + const reporting = { ...base, uptime: 2_500_000 } as Resource; + expect(toAgentFromResource(reporting)?.uptimeSeconds).toBe(2_500_000); + + const silent = { ...reporting, agent: { stale: true } } as Resource; + expect(toAgentFromResource(silent)?.uptimeSeconds).toBe(0); + }); + it('preserves the local infrastructure display name for governed resources', () => { const agent = toAgentFromResource({ ...createHybridHostResource(), diff --git a/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts b/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts index b6b94a7ea1..07e3b11301 100644 --- a/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts +++ b/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts @@ -317,7 +317,9 @@ export const toAgentFromResource = ( sensors: agent.sensors, raid: agent.raid, status: resource.status, - uptimeSeconds: agent.uptimeSeconds ?? resource.uptime ?? 0, + // A silent agent's uptime is from its last report, not how long the + // machine has been up. + uptimeSeconds: resource.agent?.stale ? 0 : (agent.uptimeSeconds ?? resource.uptime ?? 0), lastSeen: resource.lastSeen, agentVersion: agent.agentVersion, commandsEnabled: agent.commandsEnabled, @@ -400,7 +402,23 @@ const buildTypedGPUTemperatureKeys = (gpus?: HostSensorSummary['gpu']) => { return keys; }; -export const buildTemperatureRows = (sensors?: HostSensorSummary) => { +// Matches the backend's reason for readings a host agent past its reporting +// lease no longer collects (models.HostAgentStoppedReportingReason). +export const HOST_AGENT_STOPPED_REPORTING_REASON = 'host agent stopped reporting'; + +export type TemperatureRowsOptions = { + /** + * Why the machine's own sensor readings are retained rather than current, + * such as a host agent past its reporting lease. Disk rows keep their own + * collection state instead. + */ + lastKnownReason?: string; +}; + +export const buildTemperatureRows = ( + sensors?: HostSensorSummary, + options: TemperatureRowsOptions = {}, +) => { const rows: { label: string; value: string; valueTitle?: string }[] = []; const thermalState = sensors?.thermalState; const typedGPUTemperatureKeys = buildTypedGPUTemperatureKeys(sensors?.gpu); @@ -500,6 +518,17 @@ export const buildTemperatureRows = (sensors?: HostSensorSummary) => { }); } + // This card has no other cue that a value is not current, so a retained + // sensor reading says so in its value, as a retained disk reading does. + const lastKnownReason = options.lastKnownReason?.trim(); + if (lastKnownReason) { + const title = `Last known reading, not current: ${lastKnownReason}`; + for (const row of rows) { + row.value = `${row.value} (last known)`; + row.valueTitle = title; + } + } + const smart = sensors?.smart; if (smart) { smart diff --git a/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts b/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts index 1b33f7d387..a71b909b29 100644 --- a/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts +++ b/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts @@ -35,6 +35,7 @@ import type { ResourceIntelligence } from '@/types/aiIntelligence'; import { buildCustomSensorRows, buildTemperatureRows, + HOST_AGENT_STOPPED_REPORTING_REASON, toAgentFromResource, toNodeFromProxmox, type AgentPlatformData, @@ -201,7 +202,13 @@ export const useResourceDetailDrawerDerivedState = ( [], ); const agentInfo = createMemo(() => toAgentFromResource(resource(), agentMeta())); - const temperatureRows = createMemo(() => buildTemperatureRows(agentInfo()?.sensors)); + // A silent agent's sensors are its last report, even on a row another + // source keeps online. + const temperatureRows = createMemo(() => + buildTemperatureRows(agentInfo()?.sensors, { + lastKnownReason: resource().agent?.stale ? HOST_AGENT_STOPPED_REPORTING_REASON : undefined, + }), + ); const customSensorRows = createMemo(() => buildCustomSensorRows(agentInfo()?.sensors)); const dockerHostData = createMemo(() => platformData()?.docker as DockerPlatformData | undefined); diff --git a/frontend-modern/src/features/docker/DockerHostsTable.tsx b/frontend-modern/src/features/docker/DockerHostsTable.tsx index fbaa7b0858..0c798eb6f6 100644 --- a/frontend-modern/src/features/docker/DockerHostsTable.tsx +++ b/frontend-modern/src/features/docker/DockerHostsTable.tsx @@ -517,7 +517,11 @@ export const DockerHostsTable: Component<{ class={`${getPlatformTableCellClassForKind('numeric-value')} platform-table-narrow-hidden hidden text-base-content min-[360px]:table-cell md:table-cell`} > diff --git a/frontend-modern/src/features/docker/__tests__/DockerHostsTable.test.tsx b/frontend-modern/src/features/docker/__tests__/DockerHostsTable.test.tsx index f5a4d1b7d3..7e24587e0b 100644 --- a/frontend-modern/src/features/docker/__tests__/DockerHostsTable.test.tsx +++ b/frontend-modern/src/features/docker/__tests__/DockerHostsTable.test.tsx @@ -217,6 +217,58 @@ describe('DockerHostsTable', () => { ); }); + it("drops an offline host's retained temperature with its other readings", () => { + // A Docker host past its reporting lease is offline, and its last + // temperature is no more current than its last CPU, memory and disk. + const { container } = render(() => ( + } + emptyTitle="No Docker hosts" + emptyDescription="No hosts" + showToolbar={false} + /> + )); + + const headers = Array.from(container.querySelectorAll('thead th')).map((th) => + (th.textContent ?? '').trim(), + ); + const tempIndex = headers.findIndex((header) => header.startsWith('Temp')); + expect(tempIndex).toBeGreaterThan(-1); + const temp = (id: string) => + ( + container.querySelectorAll(`[data-docker-host-row="${id}"] td`)[tempIndex]?.textContent ?? + '' + ).trim(); + expect(temp('agent:docker-01')).toBe('—'); + expect(temp('agent:docker-02')).toBe('51.0°C'); + const offlineRow = container.querySelector('[data-docker-host-row="agent:docker-01"]')!; + expect(within(offlineRow as HTMLElement).queryAllByTestId('stacked-memory-bar')).toHaveLength( + 0, + ); + expect(offlineRow.querySelector('[data-docker-host-stale] .sr-only')).toHaveTextContent( + 'No report for 9m', + ); + }); + it('tints a host row for the Docker alerts its drawer lists, not by display name', () => { // Docker alerts are keyed on the Docker host id and carry the hostname // as their node; neither is the row's id or its display name. diff --git a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx index 2609a62fa7..fb4ee7772f 100644 --- a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx +++ b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx @@ -1806,7 +1806,9 @@ export const AgentsMachinesTable: Component<{ class={`${getPlatformTableCellClassForKind('numeric-value')} ${machineColumnWidthClass('uptime')} text-base-content`} > {formatPlatformTableUptimeValue( - machine.uptime ?? machine.agent?.uptimeSeconds, + canRenderMetrics() + ? (machine.uptime ?? machine.agent?.uptimeSeconds) + : undefined, )} @@ -1814,15 +1816,20 @@ export const AgentsMachinesTable: Component<{ - + —} + > + +
diff --git a/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx b/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx index 39b476ff18..3c7f0a3506 100644 --- a/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx +++ b/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx @@ -1160,14 +1160,15 @@ describe('AgentsMachinesTable', () => { expect(screen.getByText('1400 RPM')).toBeInTheDocument(); }); - it('shows a silent agent disk temperature as last known, never as a current reading', async () => { - // A host agent past its reporting lease is stale and keeps its last SMART - // temperatures with a non-available collection state. + it('shows a retained disk temperature as last known, never as a current reading', async () => { + // A reporting agent whose SMART probe returned no temperature keeps the + // last reading with a non-available collection state. (A host agent past + // its lease is offline, and its row shows no temperature at all.) const stoppedReporting = { temperature: { state: 'unavailable' as const, source: 'smartctl', - reason: 'host agent stopped reporting', + reason: 'SMART probe returned no usable temperature data', }, }; const { container } = render(() => ( @@ -1176,9 +1177,7 @@ describe('AgentsMachinesTable', () => { resource({ id: 'silent-nas', name: 'Silent NAS', - status: 'warning', agent: { - stale: true, sensors: { smart: [ { device: '/dev/sda', model: 'Archive HDD', temperature: 71 }, @@ -1249,6 +1248,92 @@ describe('AgentsMachinesTable', () => { expect(within(mixed).queryByText('69°C')).toBeNull(); }); + it("blanks an offline machine's retained readings, temperature and uptime included", async () => { + // A host agent past its reporting lease is offline: CPU, memory, disk and + // the sensors, uptime and disks it last reported are not current. + const stoppedReporting = { + temperature: { + state: 'unavailable' as const, + source: 'smartctl', + reason: 'host agent stopped reporting', + }, + }; + const lastReport: Partial = { + cpu: { current: 32 }, + memory: { total: 100, used: 73, free: 27, current: 73 }, + disk: { total: 100, used: 80, free: 20, current: 80 }, + }; + const sensors = { + temperatureCelsius: { 'cpu.package': 45 }, + thermalState: { pressure: 'serious' }, + smart: [ + { + device: '/dev/sda', + model: 'Archive HDD', + temperature: 41, + collection: stoppedReporting, + }, + ], + }; + const { container } = render(() => ( + + )); + + await openMachineColumnPicker(); + await fireEvent.click(screen.getByLabelText('Temp')); + await fireEvent.click(screen.getByLabelText('Uptime')); + + const cells = (id: string) => { + const row = container.querySelector(`[data-agents-machine-row="${id}"]`) as HTMLElement; + const headers = Array.from(container.querySelectorAll('thead th')).map((th) => + (th.textContent ?? '').replace(/[▲▼]/g, '').trim(), + ); + const values = Array.from(row.querySelectorAll('td')).map((td) => + (td.textContent ?? '').trim(), + ); + return (label: string) => values[headers.indexOf(label)]; + }; + + const silent = cells('silent-host'); + expect(silent('CPU')).toBe('—'); + expect(silent('Up')).toBe('—'); + expect(silent('Temp')).toBe('—'); + expect( + container.querySelector( + '[data-agents-machine-row="silent-host"] [data-agent-machine-temperature-trigger="true"]', + ), + ).toBeNull(); + + const live = cells('live-host'); + expect(live('CPU')).not.toBe('—'); + expect(live('Up')).toBe('28d'); + expect(live('Temp')).toBe('45°C'); + }); + it('moves a machine temperature between current and last known as its disk reports change', async () => { // Fresh objects per update, like parsed API payloads: the store reconciles // into the objects it was given, so a reused fixture would be rewritten. diff --git a/internal/unifiedresources/clone_test.go b/internal/unifiedresources/clone_test.go index 1ad90ad974..151f411769 100644 --- a/internal/unifiedresources/clone_test.go +++ b/internal/unifiedresources/clone_test.go @@ -221,12 +221,18 @@ func TestCloneResource_MutateSourceStatusMap(t *testing.T) { ID: "r-1", SourceStatus: map[DataSource]SourceStatus{ SourceProxmox: {Status: "online", ExpectedUpdateIntervalSeconds: 300}, + SourceAgent: {Status: "stale", leaseExpired: true}, }, } cloned := cloneResource(original) if cloned.SourceStatus[SourceProxmox].ExpectedUpdateIntervalSeconds != 300 { t.Fatal("clone lost collector cadence") } + // A clone that dropped the lease verdict would aggregate a silent agent + // back to warning. + if !cloned.SourceStatus[SourceAgent].leaseExpired { + t.Fatal("clone lost the agent's expired reporting lease") + } cloned.SourceStatus[SourceDocker] = SourceStatus{Status: "online"} if _, exists := original.SourceStatus[SourceDocker]; exists { diff --git a/internal/unifiedresources/registry.go b/internal/unifiedresources/registry.go index cb2f10a48a..c4a31be483 100644 --- a/internal/unifiedresources/registry.go +++ b/internal/unifiedresources/registry.go @@ -809,6 +809,14 @@ func (rr *ResourceRegistry) ingestResources(resources []Resource, thresholds map } } } + // The lease marker is unexported, so a persisted or serialized copy + // arrives without it. Its stored status still carries the verdict. + for source, status := range resource.SourceStatus { + if !status.leaseExpired && reportingLeaseExpired(source, *resource) { + status.leaseExpired = true + resource.SourceStatus[source] = status + } + } rr.mu.Lock() rr.canonicalIdentityIndex = nil @@ -1756,6 +1764,29 @@ func sourceSightingStatus(lastSeen time.Time) string { return "online" } +// reportingLeaseExpired reports whether a source delivered the machine, +// Docker host or Kubernetes cluster it reports for as offline. Those +// collectors push their own reports, and Pulse marks the reporter offline +// only once its reporting lease runs out (evaluateHostAgents, +// evaluateDockerAgents, evaluateKubernetesAgents). The registry's stale +// threshold is a different clock: it can mark the sighting stale before +// the lease ends, or leave it fresh after, so it must not decide the status. +// Resources those sources describe (guests, containers, pods, disks) are +// not lease holders and keep the stale-to-warning rule. +func reportingLeaseExpired(source DataSource, resource Resource) bool { + if resource.Status != StatusOffline { + return false + } + switch source { + case SourceAgent, SourceDocker: + return resource.Type == ResourceTypeAgent + case SourceK8s: + return resource.Type == ResourceTypeK8sCluster + default: + return false + } +} + func (rr *ResourceRegistry) markStaleLocked(now time.Time, thresholds map[DataSource]time.Duration) { thresholds = effectiveStaleThresholds(thresholds) @@ -2848,6 +2879,7 @@ func (rr *ResourceRegistry) ingestRecord(source DataSource, sourceID string, res sighting := resource.SourceStatus[source] sighting.Status = sourceSightingStatus(resource.LastSeen) sighting.LastSeen = resource.LastSeen + sighting.leaseExpired = reportingLeaseExpired(source, resource) resource.SourceStatus = map[DataSource]SourceStatus{source: sighting} resource.parentBySource = make(map[DataSource]string) rr.setSourceParent(&resource, source, resource.ParentID) @@ -3787,6 +3819,7 @@ func (rr *ResourceRegistry) mergeInto(existing *Resource, incoming Resource, sou sighting := incoming.SourceStatus[source] sighting.Status = sourceSightingStatus(incoming.LastSeen) sighting.LastSeen = incoming.LastSeen + sighting.leaseExpired = reportingLeaseExpired(source, incoming) existing.SourceStatus[source] = sighting if incoming.LastSeen.After(existing.LastSeen) { @@ -4425,6 +4458,12 @@ func (rr *ResourceRegistry) mergeResourceData(primary *Resource, other *Resource primary.SourceStatus = make(map[DataSource]SourceStatus) } for source, status := range other.SourceStatus { + // Linked resources can share a source, such as an agent reinstalled + // under a new id. Keep the fresher sighting so a stale or expired one + // cannot override a reporter that is still live. + if current, ok := primary.SourceStatus[source]; ok && current.LastSeen.After(status.LastSeen) { + continue + } primary.SourceStatus[source] = status } if other.LastSeen.After(primary.LastSeen) { @@ -5793,14 +5832,22 @@ func aggregateStatus(resource *Resource) ResourceStatus { best := StatusUnknown bestScore := 0 for _, status := range resource.SourceStatus { - score := statusPriority[strings.ToLower(status.Status)] + state := strings.ToLower(strings.TrimSpace(status.Status)) + // A reporter past its lease is offline, not merely late. Ranking its + // stale sighting above offline would show a silent machine as a + // warning while its retained readings render as current. + if status.leaseExpired { + state = "offline" + } + score := statusPriority[state] if score > bestScore { bestScore = score - if status.Status == "online" { + switch state { + case "online": best = StatusOnline - } else if status.Status == "stale" { + case "stale": best = StatusWarning - } else if status.Status == "offline" { + case "offline": best = StatusOffline } } diff --git a/internal/unifiedresources/registry_merge_policy_test.go b/internal/unifiedresources/registry_merge_policy_test.go index 83c8fda61a..cc78d4b7a1 100644 --- a/internal/unifiedresources/registry_merge_policy_test.go +++ b/internal/unifiedresources/registry_merge_policy_test.go @@ -1,6 +1,7 @@ package unifiedresources import ( + "encoding/json" "strings" "testing" "time" @@ -131,6 +132,353 @@ func TestHostnameIPDoesNotAutoMerge(t *testing.T) { } } +func onlyResourceOfType(t *testing.T, rr *ResourceRegistry, resourceType ResourceType) Resource { + t.Helper() + resources := rr.ListByType(resourceType) + if len(resources) != 1 { + t.Fatalf("expected one %s resource, got %d: %+v", resourceType, len(resources), resources) + } + return resources[0] +} + +// A host agent past its reporting lease is offline. Its source sighting is +// stale too, and the stale pass used to rank that above offline, so the +// machine reached the frontend as a warning with its last report rendered +// as current readings. +func TestLeaseExpiredHostAgentStaysOffline(t *testing.T) { + lastReport := time.Now().UTC().Add(-14 * time.Minute) + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-silent", + MachineID: "machine-silent", + Hostname: "silent", + Status: "offline", + LastSeen: lastReport, + IntervalSeconds: 30, + CPUUsage: 32, + }}, + }) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if resource.Status != StatusOffline { + t.Fatalf("status = %q, want offline for an agent past its lease", resource.Status) + } + if resource.Agent == nil || !resource.Agent.Stale { + t.Fatalf("expected the agent facet to stay flagged stale, got %+v", resource.Agent) + } + // Status describes the resource; the sighting keeps describing delivery + // freshness, which health and monitored-system reasons read. + if got := resource.SourceStatus[SourceAgent].Status; got != "stale" { + t.Fatalf("agent sighting = %q, want stale", got) + } + + // The monitor adapter runs the stale pass again after record sources. + rr.MarkStale(time.Now().UTC(), nil) + if got, _ := rr.Get(resource.ID); got.Status != StatusOffline { + t.Fatalf("second stale pass changed status to %q, want offline", got.Status) + } + + health := EvaluateResourceHealth(resource, nil, time.Now().UTC()) + if health.Verdict != HealthCritical || len(health.Reasons) < 2 || + health.Reasons[0].Code != "offline" || health.Reasons[1].Code != "telemetry_stale" { + t.Fatalf("health = %+v, want critical offline with the stale reason after it", health) + } +} + +// An agent that is late but still inside its lease keeps the warning: the +// monitor has not decided it is gone, so its last readings are only stale. +func TestLateHostAgentInsideLeaseStaysWarning(t *testing.T) { + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-late", + MachineID: "machine-late", + Hostname: "late", + Status: "online", + LastSeen: time.Now().UTC().Add(-90 * time.Second), + IntervalSeconds: 30, + }}, + }) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if resource.Status != StatusWarning { + t.Fatalf("status = %q, want warning for a late agent inside its lease", resource.Status) + } +} + +// A Proxmox node whose linked agent stopped reporting stays online through +// the PVE poll (issue #1515). Only the agent sighting carries the lease. +func TestLeaseExpiredAgentOnLiveProxmoxNodeStaysOnline(t *testing.T) { + now := time.Now().UTC() + const machineID = "machine-pve1" + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Nodes: []models.Node{{ + ID: "homelab-pve1", + Name: "pve1", + Instance: "homelab", + ClusterName: "homelab", + Status: "online", + LastSeen: now, + }}, + Hosts: []models.Host{{ + ID: machineID, + MachineID: machineID, + Hostname: "pve1", + LinkedNodeID: "homelab-pve1", + Status: "offline", + LastSeen: now.Add(-14 * time.Minute), + IntervalSeconds: 30, + }}, + }) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if _, ok := resource.SourceStatus[SourceProxmox]; !ok { + t.Fatalf("expected the node and its agent to share one resource, got sources %+v", resource.SourceStatus) + } + if resource.Status != StatusOnline { + t.Fatalf("status = %q, want online through the live PVE poll", resource.Status) + } + if resource.Agent == nil || !resource.Agent.Stale { + t.Fatalf("expected the dead agent to stay flagged stale, got %+v", resource.Agent) + } +} + +// Docker hosts hold a shorter lease than the registry's Docker stale +// threshold, so a silent Docker host was offline for a while and then +// flipped to warning once its sighting went stale. +func TestLeaseExpiredDockerHostStaysOfflineAfterSightingGoesStale(t *testing.T) { + lastReport := time.Now().UTC().Add(-5 * time.Minute) + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + DockerHosts: []models.DockerHost{{ + ID: "docker-silent", + AgentID: "docker-agent-silent", + Hostname: "docker-silent", + MachineID: "machine-docker-silent", + Status: "offline", + LastSeen: lastReport, + IntervalSeconds: 30, + Containers: []models.DockerContainer{{ + ID: "container-web", + Name: "web", + State: "running", + }}, + }}, + }) + + host := onlyResourceOfType(t, rr, ResourceTypeAgent) + if host.Status != StatusOffline { + t.Fatalf("docker host status = %q, want offline after its sighting went stale", host.Status) + } + if got := host.SourceStatus[SourceDocker].Status; got != "stale" { + t.Fatalf("docker sighting = %q, want stale", got) + } + + // Containers are what the host reports about, not lease holders: a + // running container on a silent host is stale, not known to be down. + container := onlyResourceOfType(t, rr, ResourceTypeAppContainer) + if container.Status != StatusWarning { + t.Fatalf("container status = %q, want warning on a silent host", container.Status) + } +} + +// A machine reporting through both the host agent and Docker is offline once +// both leases expire. The Docker lease ends first, so its sighting can still +// be fresh when the agent's goes stale. +func TestMachineWithExpiredAgentAndDockerLeasesIsOffline(t *testing.T) { + now := time.Now().UTC() + const machineID = "machine-both" + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-both", + MachineID: machineID, + Hostname: "both", + Status: "offline", + LastSeen: now.Add(-3 * time.Minute), + IntervalSeconds: 30, + }}, + DockerHosts: []models.DockerHost{{ + ID: "docker-both", + AgentID: "host-both", + Hostname: "both", + MachineID: machineID, + Status: "offline", + LastSeen: now.Add(-90 * time.Second), + IntervalSeconds: 10, + }}, + }) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if _, ok := resource.SourceStatus[SourceDocker]; !ok { + t.Fatalf("expected agent and Docker to share one resource, got sources %+v", resource.SourceStatus) + } + if got := resource.SourceStatus[SourceDocker].Status; got != "online" { + t.Fatalf("docker sighting = %q, want it still inside the stale threshold", got) + } + if resource.Status != StatusOffline { + t.Fatalf("status = %q, want offline once both leases expired", resource.Status) + } +} + +// A live reporter still carries the machine when the other lease expired. +func TestMachineWithLiveDockerReporterStaysOnline(t *testing.T) { + now := time.Now().UTC() + const machineID = "machine-split" + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-split", + MachineID: machineID, + Hostname: "split", + Status: "offline", + LastSeen: now.Add(-14 * time.Minute), + IntervalSeconds: 30, + }}, + DockerHosts: []models.DockerHost{{ + ID: "docker-split", + AgentID: "docker-agent-split", + Hostname: "split", + MachineID: machineID, + Status: "online", + LastSeen: now, + IntervalSeconds: 10, + }}, + }) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if resource.Status != StatusOnline { + t.Fatalf("status = %q, want online through the live Docker reporter", resource.Status) + } +} + +// The lease marker is unexported, so a resource that went through JSON (a +// persisted or remote copy) must recover it from its stored status. +func TestLeaseExpiredHostAgentStaysOfflineAfterJSONRoundTrip(t *testing.T) { + source := NewRegistry(nil) + source.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-silent", + MachineID: "machine-silent", + Hostname: "silent", + Status: "offline", + LastSeen: time.Now().UTC().Add(-14 * time.Minute), + IntervalSeconds: 30, + }}, + }) + payload, err := json.Marshal(source.List()) + if err != nil { + t.Fatalf("marshal resources: %v", err) + } + var copied []Resource + if err := json.Unmarshal(payload, &copied); err != nil { + t.Fatalf("unmarshal resources: %v", err) + } + + rr := NewRegistry(nil) + rr.IngestResources(copied) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if resource.Status != StatusOffline { + t.Fatalf("status = %q, want offline after a JSON round trip", resource.Status) + } +} + +// A manual link can join two resources reported by the same source, such as +// an agent reinstalled under a new id. The expired sighting of the old one +// must not replace the live sighting of the new one. +func TestManualLinkKeepsTheLiveSightingOfASharedSource(t *testing.T) { + now := time.Now().UTC() + store := NewMemoryStore() + if err := store.AddLink(ResourceLink{ + ResourceA: "agent-reinstalled", + ResourceB: "agent-retired", + PrimaryID: "agent-reinstalled", + }); err != nil { + t.Fatalf("add link: %v", err) + } + rr := NewRegistry(store) + rr.IngestResources([]Resource{ + { + ID: "agent-reinstalled", + Type: ResourceTypeAgent, + Name: "tower", + Status: StatusOnline, + LastSeen: now, + Sources: []DataSource{SourceAgent}, + SourceStatus: map[DataSource]SourceStatus{ + SourceAgent: {Status: "online", LastSeen: now}, + }, + Agent: &AgentData{AgentID: "host-new", Hostname: "tower"}, + }, + { + ID: "agent-retired", + Type: ResourceTypeAgent, + Name: "tower", + Status: StatusOffline, + LastSeen: now.Add(-14 * time.Minute), + Sources: []DataSource{SourceAgent}, + SourceStatus: map[DataSource]SourceStatus{ + SourceAgent: {Status: "stale", LastSeen: now.Add(-14 * time.Minute)}, + }, + Agent: &AgentData{AgentID: "host-old", Hostname: "tower", Stale: true}, + }, + }) + + resource, ok := rr.Get("agent-reinstalled") + if !ok { + t.Fatal("linked resource missing") + } + if resource.Status != StatusOnline { + t.Fatalf("status = %q, want online through the live reinstalled agent", resource.Status) + } + if got := resource.SourceStatus[SourceAgent]; !got.LastSeen.Equal(now) { + t.Fatalf("agent sighting = %+v, want the live sighting", got) + } +} + +func TestLeaseExpiredKubernetesClusterStaysOffline(t *testing.T) { + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + KubernetesClusters: []models.KubernetesCluster{{ + ID: "cluster-silent", + AgentID: "k8s-agent-silent", + Name: "silent", + Status: "offline", + LastSeen: time.Now().UTC().Add(-5 * time.Minute), + IntervalSeconds: 30, + }}, + }) + + cluster := onlyResourceOfType(t, rr, ResourceTypeK8sCluster) + if cluster.Status != StatusOffline { + t.Fatalf("cluster status = %q, want offline after its sighting went stale", cluster.Status) + } +} + +// Pull sources keep the stale-to-warning rule: a stale PVE sighting means +// Pulse lost its poller, not that the node reported itself gone. +func TestStaleProxmoxNodeStaysWarning(t *testing.T) { + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Nodes: []models.Node{{ + ID: "homelab-pve9", + Name: "pve9", + Instance: "homelab", + ClusterName: "homelab", + Status: "online", + LastSeen: time.Now().UTC().Add(-5 * time.Minute), + }}, + }) + + node := onlyResourceOfType(t, rr, ResourceTypeAgent) + if node.Status != StatusWarning { + t.Fatalf("node status = %q, want warning for a stale PVE sighting", node.Status) + } +} + func TestCloneDockerDataPreservesContainerRuntimeMetadata(t *testing.T) { startedAt := time.Date(2026, 6, 11, 13, 15, 30, 0, time.UTC) finishedAt := startedAt.Add(45 * time.Minute) diff --git a/internal/unifiedresources/types.go b/internal/unifiedresources/types.go index 31b01a0e85..d64e288bef 100644 --- a/internal/unifiedresources/types.go +++ b/internal/unifiedresources/types.go @@ -295,6 +295,12 @@ type SourceStatus struct { LastSeen time.Time `json:"lastSeen"` Error string `json:"error,omitempty"` ExpectedUpdateIntervalSeconds int64 `json:"expectedUpdateIntervalSeconds,omitempty"` // Collector-authored cadence, zero uses source default + + // leaseExpired records that this source delivered the machine, Docker + // host or cluster it reports for as offline because its reporting lease + // ran out. Status aggregation counts that as offline whatever the + // sighting's age, while Status keeps describing delivery freshness. + leaseExpired bool } // ResourceIdentity holds identifiers used for matching. From 3fd5411ecc324a02d444c40ea349437c69d08dee Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:50:39 +0100 Subject: [PATCH 09/13] Judge disk heat by the alert disk temperature thresholds Pulse decided whether a physical disk was too hot in three places with three thresholds. Disk risk added temperature_high at a flat 60/70C for every disk type and drove the disk status and the Physical Disks Health verdict. Patrol flagged anything over 55C and the AI chat context listed disks from 50C. Alerts and the Temp column used the per-type thresholds users tune in Alerts (NVMe 70, SAS 65, SATA 55 by default). An NVMe at 63C read Running Hot beside a green Temp cell. A SATA disk at 56C read Healthy beside a red one. A user who raised the NVMe trigger to 75 still got a hot verdict at 70. The alert disk temperature policy now decides disk heat. alerts.Manager.DiskTemperatureThreshold exposes the per-type resolution CheckHost already applies. - Disk risk no longer judges temperature. Every registry that rebuilds risk lacks the alert configuration, so no rule there could honour it. A hot disk keeps an online status. - The Health verdict reads Running Hot when a current reading reaches the disk type's trigger. It uses the thresholds object that colours the Temp cell, so the two cannot disagree. The amber band below the trigger stays a cell colour, not a verdict. - The TrueNAS storage table lost the heat it used to get from risk. It now judges heat the same way: an Attention row with the reason, and a warning dot for phones. - Patrol triage, its disk issue filter, finding recovery and the chat context's attention list judge the collected reading against the same trigger, including scoped runs and a provider set before Patrol starts. - A switched-off agent Disk Temp default resolves to no thresholds, which the displays that read them now treat as off. Per-host Disk Temp overrides do not reach these surfaces yet. TrueNAS disk alerts, the TrueNAS disk drawer and PDF reports keep their own fixed temperatures for now. --- .../v6/internal/subsystems/ai-runtime.md | 16 + .../v6/internal/subsystems/alerts.md | 29 ++ .../subsystems/frontend-primitives.md | 5 +- .../v6/internal/subsystems/monitoring.md | 23 +- .../internal/subsystems/storage-recovery.md | 44 +- .../internal/subsystems/unified-resources.md | 27 +- .../b24ffeda6578c122.json | 60 +++ .../src/components/Storage/DiskList.tsx | 5 +- .../Storage/__tests__/DiskList.test.tsx | 97 ++-- .../components/Storage/useDiskDetailModel.ts | 5 +- .../components/Storage/useDiskListModel.ts | 11 +- .../Workloads/GuestPhysicalDisks.tsx | 4 +- .../__tests__/diskDetailPresentation.test.ts | 2 + .../__tests__/diskPresentation.test.ts | 413 +++++++++--------- .../storageBackups/diskDetailPresentation.ts | 7 +- .../storageBackups/diskPresentation.ts | 101 +++-- .../diskTemperaturePresentation.ts | 27 ++ .../truenas/TrueNASStorageTopologyTable.tsx | 36 +- .../TrueNASStorageTopologyTable.test.tsx | 107 ++++- .../__tests__/truenasPageModel.test.ts | 67 +++ .../src/features/truenas/truenasPageModel.ts | 86 +++- .../utils/__tests__/metricThresholds.test.ts | 23 + .../src/utils/__tests__/temperature.test.ts | 6 + frontend-modern/src/utils/metricThresholds.ts | 10 +- frontend-modern/src/utils/temperature.ts | 5 + internal/ai/alert_threshold_adapter.go | 16 + internal/ai/alert_threshold_adapter_test.go | 25 ++ internal/ai/mock_test.go | 13 + internal/ai/patrol.go | 4 + internal/ai/patrol_ai.go | 19 +- internal/ai/patrol_disk_temperature.go | 50 +++ internal/ai/patrol_disk_temperature_test.go | 228 ++++++++++ internal/ai/patrol_findings.go | 18 +- internal/ai/patrol_run.go | 1 + internal/ai/patrol_state.go | 23 +- internal/ai/patrol_thresholds_test.go | 3 + internal/ai/patrol_triage.go | 13 +- internal/ai/resource_context.go | 8 +- internal/ai/service.go | 9 +- internal/alerts/host.go | 39 +- .../threshold_resolution_shared_test.go | 58 +++ internal/api/ai_handlers_concurrency_test.go | 3 + internal/api/ai_handlers_more_test.go | 3 + internal/storagehealth/risk.go | 40 +- .../storagehealth/risk_branchcov0718_test.go | 48 +- internal/storagehealth/risk_test.go | 110 +---- internal/truenas/provider.go | 7 +- internal/truenas/provider_test.go | 21 +- .../unifiedresources/adapter_coverage_test.go | 12 +- internal/unifiedresources/adapters.go | 18 +- .../unifiedresources/physical_disk_risk.go | 1 - .../physical_disk_risk_test.go | 11 +- internal/unifiedresources/registry_test.go | 156 +++---- .../unifiedresources/storage_risk_test.go | 4 +- 54 files changed, 1514 insertions(+), 663 deletions(-) create mode 100644 frontend-modern/browser-verification/b24ffeda6578c122.json create mode 100644 internal/ai/patrol_disk_temperature.go create mode 100644 internal/ai/patrol_disk_temperature_test.go diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index da8d664e67..216666a720 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -306,6 +306,22 @@ retained value as "last known 41C (disk is in standby)". Proofs: `TestPatrolPhysicalDiskRowsJudgeHeatOnCollectedTemperature` in `internal/ai/patrol_ai_more_test.go`. +They judge that collected reading against the alert disk temperature policy, +not a fixed line. `ThresholdProvider.GetDiskTemperatureThreshold(diskType)` +(the alert manager's per-type trigger and clear) resolves each disk's limits +once in `patrolPhysicalDiskRows`, and the run state carries the provider +(`patrolRuntimeState.thresholdProvider`). With no provider the factory alert +configuration applies. A disk is hot from its trigger: a triage warning, a disk +issue and an AI chat "needing attention" entry, as its temperature alert and +Physical Disks Running Hot verdict are. A disk-high finding recovers at or +below the clear value (under the trigger when there is no band below it), as +the alert does. Scoped runs keep the provider, and a +provider set before Patrol starts is handed to it. Below the trigger Patrol flags nothing, so an NVMe at 63C stays +quiet and a SATA disk at 56C is flagged. Proofs: +`internal/ai/patrol_disk_temperature_test.go` and +`TestAlertThresholdAdapter_DiskTemperatureFollowsAlertPolicy` in +`internal/ai/alert_threshold_adapter_test.go`. + Retained summaries disclose that point and bucket timestamps describe returned history inside the requested window. Their spacing does not measure collection uptime or explain missing history. Collector lifecycle and retention settings diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index c80daddd90..692424c56d 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -1682,6 +1682,35 @@ an alert the next evaluation raises again. `TestConfigSaveKeepsAlertTheEvaluatorKeepsWithoutRecoveryBand` in `internal/alerts/host_unraid_lifecycle_test.go` pin these paths. +### The disk temperature policy decides disk heat + +The policy `CheckHost` applies to a SMART disk (the agent Disk Temp default, +refined by the disk type's `DiskTempByType` entry, and switched off for every +type when that default is off) is the canonical answer to "is this disk too +hot". `diskTemperatureThresholdForType` in `internal/alerts/host.go` holds it, +and `Manager.DiskTemperatureThreshold(diskType)` exposes it, without host +overrides, to Patrol through `AlertThresholdAdapter.GetDiskTemperatureThreshold`. +`DefaultDiskTemperatureThreshold` serves callers with no manager. Disk risk +(`internal/storagehealth`) judges no temperature. The Physical Disks Temp cell +and Health verdict mirror the per-type triggers and a switched-off default in +`resolveDiskTemperatureDisplayThresholds`; host overrides do not reach that +page yet. That resolver returns null when the agent Disk Temp default is off. +The displays that read it treat null as off: no threshold colour +(`getTemperatureTextClass` with `diskTemperature`) and never hot. A +`DiskTempByType` entry cannot be switched off on its own, because +normalization restores a non-positive entry to its default. +A disk is hot from its trigger. Alerts and Patrol findings stay open until the +reading falls to the clear value, or under the trigger when there is no band +below it. The Physical Disks verdict and the TrueNAS Health cell judge only the +current reading against the trigger. Judges that still differ: TrueNAS disk +temperature alerts use `TrueNASDiskDefaults.Temperature` (a flat 55/50), the +TrueNAS disk drawer tones the reading from a fixed 55C, and PDF reports colour +disk temperatures at a fixed 50/60C. +`TestDiskTemperatureThresholdMatchesCheckHostPolicy` in +`internal/alerts/threshold_resolution_shared_test.go` pins per-type resolution, a raised +NVMe trigger that `CheckHost` also honours, the copy, the disabled default and +the nil manager. + ### Agent disk temperature alerts clear when their disk leaves the report `CheckHost` also clears a disk temperature alert, and drops its pending diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 5464d8b393..d58b392ed1 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -414,8 +414,9 @@ available. Stable disk History catalog/organisation/access ownership is unchange The physical-disk verdict words and their phone forms are `Needs Attention` (`Attention`), `Running Hot` (`Hot`) and `Replace Now` (`Replace`), all from -`getPhysicalDiskHealthCompactLabel` in `diskPresentation.ts`. A hot disk keeps -the same label at either tier; its tone carries the severity. +`getPhysicalDiskHealthCompactLabel` in `diskPresentation.ts`. `Running Hot` is +red and starts at the disk's alert trigger, from the same thresholds object +that colours its Temp cell (`getPhysicalDiskTemperatureThresholds`). Controller expansions carry the supplied kind-specific targets, absolute times, duration and cluster/namespace when the phone row omits those columns. Proxmox diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index f5ed572cc3..c233ab1ee2 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -4501,17 +4501,18 @@ emit structured storage topology such as Unraid per-disk state, the shared assessment layer must derive canonical risk and alert severity from that richer disk topology instead of letting coarser aggregate counters override it and flap the operator-facing storage alert surface. -That shared assessment judges only a disk temperature that was collected now. -`storagehealth.CollectedTemperature` passes a reading through when its -`collection.temperature` state is `available`, or when the source predates -collection state, and otherwise drops it before `AssessPhysicalDisk` and -`AssessHostSMARTDisk` build their sample. A last known value retained for a -disk in standby or a host agent past its reporting lease therefore raises no -`temperature_high` reason and cannot turn the disk's verdict or status into -a warning, while the value itself stays on the resource as last-known -history. Proof: `TestDiskAssessmentsIgnoreRetainedTemperature` and -`TestCollectedTemperatureKeepsOnlyCurrentReadings` in -`internal/storagehealth/risk_test.go`. +That shared assessment never judges disk temperature. Heat is a metric owned +by the alert disk temperature policy (`alerts.Manager.DiskTemperatureThreshold`: +the per-type `diskTempByType` trigger, else the agent Disk Temp default), which +users tune per disk type. Disk risk is rebuilt by registries that cannot see +that configuration, so a heat rule here would contradict it: a flat 60/70C rule +once called a healthy 63C NVMe hot and a 56C SATA disk fine. No disk type, +temperature or collection state reaches `storagehealth.Sample`, the TrueNAS +provider and the Unraid inventory assessment included, and a hot disk keeps an +`online` status. Proof: `TestDiskAssessmentsNeverJudgeTemperature` in +`internal/storagehealth/risk_test.go` and +`TestRecordsLeaveDiskHeatToTheAlertPolicy` in +`internal/truenas/provider_test.go`. That same monitoring-owned storage polling boundary also owns cluster-shared Proxmox storage status coherence. `internal/monitoring/monitor_polling_storage.go` must merge shared storage observations across nodes into one cluster-scoped diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 06020140f3..079fabf486 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -6759,23 +6759,33 @@ not live appliance collection or recovery success. ### Physical-disk heat is not a replacement verdict -The Physical Disks Health verdict names the action it asks for. Risk whose only -reason is `temperature_high` reads `Running Hot` (compact `Hot`), red at the -critical tier and amber at warning, with the temperature reason as its summary. -`Replace Now` stays reserved for FAILED health, any other critical reason, and -a critical level that no listed reason explains. `Needs Attention` keeps the -remaining warning evidence. When heat and other evidence both apply, the more -severe class names the verdict and failure evidence wins a tie, whatever order -merged reasons arrive in. Sorting follows the verdict (replace, critical heat, -attention, warning heat), and the health filter places a hot disk at its -temperature tier, so `Needs attention` still lists it. This matches alerting, -which raises disk heat as a Temperature metric and never as disk health. -Thresholds are unchanged: the tier still comes from the server's disk risk -(flat 60/70C), not the per-type alert thresholds that colour the Temp column. -Reconciling those is separate work. `frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts` -and `frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx` cover -the split, merged reason order, the unexplained critical level, sorting and -filtering. +The Physical Disks Health verdict names the action it asks for. A disk whose +current temperature has reached its alert trigger reads `Running Hot` (compact +`Hot`), red, with the reading and the trigger as its summary. `Replace Now` +stays reserved for FAILED health, any critical risk reason, and a critical +level that no listed reason explains. `Needs Attention` keeps the remaining +warning evidence. Heat outranks warning evidence and failure evidence outranks +heat. Sorting follows the verdict (replace, heat, attention), and the health +filter places a hot disk with the critical disks, as its Temp cell reads red. + +Heat has one owner: the alert disk temperature policy (the per-type triggers in +Alerts > Thresholds, `diskTempByType`, else the agent Disk Temp default). +Disk risk never carries heat, because every registry that rebuilds it lacks the +alert configuration. `extractPhysicalDiskPresentationData` takes the alerts +store's `getDiskTemperatureThresholds` resolver, so the Temp cell colour and the +verdict read one thresholds object. The band below the trigger is a Temp cell +colour only, and a retained (not current) reading is never heat. The verdict +judges the current reading, with no memory of an alert. A disk temperature +alert held between its clear value and trigger shows an amber Temp cell, not +`Running Hot`. A switched-off agent Disk Temp default resolves to null, which +leaves the reading uncoloured and never hot, as alerting is off for it. +Per-host Disk Temp overrides do not yet reach Physical Disks. +`frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts` +pins cell-and-verdict agreement for every type from 40C to 80C under factory and +raised triggers, plus retained readings, failure evidence, sorting and +filtering. `frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx` +covers NVMe 63C, SATA 56C and a raised NVMe trigger loaded through the alerts +store. ### Pool-to-physical-disk ownership in Storage details diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 0424cd7e79..6a00c0a3c0 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -1111,6 +1111,15 @@ The first reason renders in the desktop Health column on the row's single line, coloured by bucket, with a count for the rest and the full list in the title and screen-reader text. Phones keep the status dot and reach the reason through the row drawer. Health sorts by severity rank, not label text. +Disk risk carries no heat, so the table judges it itself. A disk whose current +reading has reached its type's alert disk temperature trigger is an Attention +row with the reason "Disk temperature is 57°C, at or above its 55°C alert +threshold.". The trigger comes from the alerts store's +`getDiskTemperatureThresholds`, which the table passes to the row build, +filter, counts, sort and issue. `isPhysicalDiskRunningHot` in +`features/storageBackups/diskTemperaturePresentation.ts` makes the same call +for the Physical Disks verdict. The status dot keeps the source's own state. +`truenasPageModel.test.ts` and `TrueNASStorageTopologyTable.test.tsx` pin it. The vSphere Datastores and Networks tables follow the same exception-first Health column. `getVmwareDatastoreIssue` and `getVmwareNetworkIssue` return nothing for a green row; otherwise the reasons are, in order, an impaired @@ -3564,8 +3573,8 @@ and otherwise `available`. The value is kept as last-known context; it is collected again only when a reporting host sends a positive reading for a disk that is no longer spun down. The Unraid-native disk row carries that state, and so does a SMART row that falls back to the inventory reading because it has -none of its own. The adapter's risk assessment of an Unraid row no longer counts -a withdrawn inventory temperature as heat. +none of its own. The adapter's risk assessment of an Unraid row judges no +temperature at all; heat belongs to the alert disk temperature policy. Cross-source correlation compares normalized serial and WWN values across fields without truncation, allowing a PVE bare-hex array-volume serial to join @@ -4111,13 +4120,13 @@ incidents during cross-source merges. A provider alert such as TrueNAS `truenas_smart` is not presentation-only context; it must become a canonical `physicalDisk.risk.reasons` entry so hybrid agent/API disk resources keep one shared disk-health truth after deduplication. -That shared risk contract judges only a collected temperature. Both the -adapter assessment and the registry's recompute from merged metadata -(`physicalDiskAssessmentFromMeta`) pass the temperature through -`storagehealth.CollectedTemperature`, so a retained `unavailable`, -`unsupported` or `missing` temperature stays visible as last-known evidence -but adds no `temperature_high` reason and leaves the disk `online` rather -than `warning`. Proof: `TestPhysicalDiskRiskIgnoresRetainedTemperature` in +That shared risk contract never judges temperature, collected or retained. +Heat belongs to the alert disk temperature policy, which no registry can see, +so neither the adapter assessment nor the registry's recompute from merged +metadata (`physicalDiskAssessmentFromMeta`) passes a temperature into +`storagehealth`. A 72C disk keeps its reading on the resource, adds no +`temperature_high` reason and stays `online`. Proof: +`TestPhysicalDiskRiskNeverJudgesTemperature` in `internal/unifiedresources/registry_test.go`. That same canonical disk contract now also owns recent aggregate temperature history. When a provider such as TrueNAS can supply `disk.temperature_agg` diff --git a/frontend-modern/browser-verification/b24ffeda6578c122.json b/frontend-modern/browser-verification/b24ffeda6578c122.json new file mode 100644 index 0000000000..8b1f018cc9 --- /dev/null +++ b/frontend-modern/browser-verification/b24ffeda6578c122.json @@ -0,0 +1,60 @@ +{ + "version": 1, + "base_sha": "1b6a9b45d7a85ae2add1ff4213ef80f6b35f7309", + "verified_at": "2026-10-06T22:49:34Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Storage/DiskList.tsx", + "frontend-modern/src/components/Storage/useDiskDetailModel.ts", + "frontend-modern/src/components/Storage/useDiskListModel.ts", + "frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx", + "frontend-modern/src/features/storageBackups/diskDetailPresentation.ts", + "frontend-modern/src/features/storageBackups/diskPresentation.ts", + "frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts", + "frontend-modern/src/features/truenas/TrueNASStorageTopologyTable.tsx", + "frontend-modern/src/features/truenas/truenasPageModel.ts", + "frontend-modern/src/utils/metricThresholds.ts", + "frontend-modern/src/utils/temperature.ts" + ], + "content_sha256": { + "frontend-modern/src/components/Storage/DiskList.tsx": "9fbcdb4659cfdf443f521c9b15969f485876cbdde8d7503110e12325b1899538", + "frontend-modern/src/components/Storage/useDiskDetailModel.ts": "51dea57998a0c0d88f4a8d6410ff16dc0559674cfecf456b61706bac145321af", + "frontend-modern/src/components/Storage/useDiskListModel.ts": "778db34c9def3a921e4bb6f9a36a79dcaa804962e9fb5bc3c03d908c585d8503", + "frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx": "ef649564bd14366aa71121e9c09ac4309bcd299fd1d388ceff5d5969671ae471", + "frontend-modern/src/features/storageBackups/diskDetailPresentation.ts": "dac5b69aa5c603b719946fc3262d810a5a9bdc86469f33c3097542516f73d5cd", + "frontend-modern/src/features/storageBackups/diskPresentation.ts": "4210a0e7d61113f774462571700582d2f32cfe4474f04048e03d728272bd2247", + "frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts": "f983ea771d8f21ab018ca38a802aa7fa9795860c0ff010adcb7507cf9c47678c", + "frontend-modern/src/features/truenas/TrueNASStorageTopologyTable.tsx": "168af9011efa41fc545dfb198f6eff5e9d5dbb003a0cd330c1238a820979c6b9", + "frontend-modern/src/features/truenas/truenasPageModel.ts": "36497cd8a519e3de9564eefb4b1a245fac6b0478c9f9a44a262c24228fc33dc9", + "frontend-modern/src/utils/metricThresholds.ts": "227994b089456c54489d64c90d3a3111421ec0c1b13adddc2744ab2d3749cfc2", + "frontend-modern/src/utils/temperature.ts": "a67d4ec80af39b6057ed155dc8ce5ce4c7e697345f7b9fadb798dcbd31605f9f" + }, + "routes": ["/proxmox/storage?tab=disks&q=West%20Production", "/truenas/storage"], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack built from this exact staged content (embedded UI and backend from one tree). A local-only generator overlay, never committed, pinned Proxmox disks from a control file: Samsung 990 PRO NVMe 63C, Crucial MX500 SATA 56C, WD Black SN850X NVMe 72C, Seagate IronWolf SATA 52C, plus TrueNAS mock disk sda (SATA) at 57C. The API returned no physicalDisk.risk and status online for every pinned disk", + "Same-session control on a second private stack built from the base without this change (main plus #2623): NVMe 63C read 'Running Hot' (amber) beside a green 63C cell with risk warning [temperature_high] and status warning; SATA 56C read 'Healthy' beside a red 56C cell; with the NVMe trigger raised to 75 the NVMe 72C still read 'Running Hot' from the flat 70C rule beside an amber cell", + "After, factory thresholds, 1440: Crucial MX500 56C 'Running Hot' red, title 'Disk temperature is 56\u00b0C, at or above its 55\u00b0C alert threshold.', red 56C cell, sorted first; WD Black 72C 'Running Hot' red with its 70C threshold; Samsung 990 PRO 63C 'Healthy' with a green cell; Crucial MX500 51C 'Healthy' with an amber cell (the band below the trigger stays colour only)", + "After, factory thresholds, 390: same rows with compact 'Hot' in red beside red cells, 'Healthy' beside the green 63C NVMe; no horizontal overflow (scrollWidth 390)", + "After, NVMe trigger raised to 75/70 through PUT /api/alerts/config: WD Black NVMe 72C is no longer Running Hot (its mock SMART health is UNKNOWN so it reads 'Unknown') and its cell is amber; SATA 56C still Running Hot", + "After, agent Disk Temp default switched off (trigger 0) through PUT /api/alerts/config: no disk reads Running Hot and every Temp cell is uncoloured (text-base-content); TrueNAS sda reads healthy with an Online dot", + "After, disk drawer for the 56C SATA disk: header 'Running Hot Disk temperature is 56\u00b0C, at or above its 55\u00b0C alert threshold.' and the Temperature card reads 56\u00b0C", + "After, TrueNAS Storage at 1440: sda (57C) is an Attention row with Health 'Disk temperature is 57\u00b0C, at or above its 55\u00b0C alert threshold.'; the Attention status filter lists it; other disks healthy and unchanged", + "After, TrueNAS Storage at 390: the Health column is phone-hidden, and sda's status dot reads Warning (amber) while cool disks stay Online (green)" + ], + "interactions": [ + "Loaded Physical Disks filtered to West Production at 1440x900 and 390x844 and read each row's health word, title, colour and Temp cell class on both the before and after stacks", + "Changed diskTempByType.nvme to 75/70, then agentDefaults.diskTemperature to 0, through GET/PUT /api/alerts/config and reloaded Physical Disks and TrueNAS Storage after each, then restored the factory config", + "Opened the inline drawer of the 56C SATA disk by clicking its row", + "Loaded TrueNAS Storage at 1440 and 390, clicked the Attention status filter, and read sda's Health cell, title and status dot" + ] +} diff --git a/frontend-modern/src/components/Storage/DiskList.tsx b/frontend-modern/src/components/Storage/DiskList.tsx index 4d463bb5c2..db5c65b4b0 100644 --- a/frontend-modern/src/components/Storage/DiskList.tsx +++ b/frontend-modern/src/components/Storage/DiskList.tsx @@ -13,7 +13,6 @@ import { TableHeader, TableRow, } from '@/components/shared/Table'; -import { useAlertsActivation } from '@/stores/alertsActivation'; import { formatBytes } from '@/utils/format'; import { getTemperatureTextClass } from '@/utils/temperature'; import { @@ -77,6 +76,7 @@ import { getPhysicalDiskEmptyStatePresentation, getPhysicalDiskHealthCompactLabel, getPhysicalDiskHealthStatus, + getPhysicalDiskTemperatureThresholds, getPhysicalDiskHealthSummary, getPhysicalDiskHostLabel, getPhysicalDiskLifeLabel, @@ -132,7 +132,6 @@ export const DiskList: Component = (props) => { }; const columnStyle = (columnId: PhysicalDiskTableColumnId) => getPhysicalDiskColumnWidthStyle(layoutMode(), columnId); - const { getDiskTemperatureThresholds } = useAlertsActivation(); const model = useDiskListModel({ disks: () => props.disks, nodes: () => props.nodes, @@ -523,7 +522,7 @@ export const DiskList: Component = (props) => { reading().current ? getTemperatureTextClass( data().temperature, - getDiskTemperatureThresholds(data().type), + getPhysicalDiskTemperatureThresholds(data()), 'diskTemperature', ) : PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS diff --git a/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx b/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx index 887fe416aa..e91f30ba64 100644 --- a/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx +++ b/frontend-modern/src/components/Storage/__tests__/DiskList.test.tsx @@ -2,8 +2,12 @@ import { cleanup, fireEvent, render, screen, waitFor, within } from '@solidjs/te import { createSignal } from 'solid-js'; import { createStore, reconcile } from 'solid-js/store'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import { AlertsAPI } from '@/api/alerts'; +import type { AlertConfig } from '@/types/alerts'; import type { Resource } from '@/types/resource'; import { DiskList } from '@/components/Storage/DiskList'; +import { useAlertsActivation } from '@/stores/alertsActivation'; +import { eventBus } from '@/stores/events'; vi.mock('@/components/Storage/DiskDetail', () => ({ DiskDetail: (props: { disk: Resource }) =>
{props.disk.id}
, @@ -348,19 +352,9 @@ describe('DiskList', () => { disks: [ buildDisk('sda', 'pve3', { model: 'Crucial MX500 2TB', - diskType: 'ssd', + diskType: 'sata', wearout: 95, - temperature: 72, - risk: { - level: 'critical', - reasons: [ - { - code: 'temperature_high', - severity: 'critical', - summary: 'Disk temperature is 72C', - }, - ], - }, + temperature: 56, }), buildDisk('sdb', 'pve3', { temperature: 72, @@ -368,11 +362,6 @@ describe('DiskList', () => { risk: { level: 'critical', reasons: [ - { - code: 'temperature_high', - severity: 'critical', - summary: 'Disk temperature is 72C', - }, { code: 'pending_sectors', severity: 'critical', @@ -381,6 +370,11 @@ describe('DiskList', () => { ], }, }), + buildDisk('nvme0n1', 'pve3', { + model: 'Samsung 990 PRO', + diskType: 'nvme', + temperature: 63, + }), ], nodes: [buildNode('node-pve3', 'pve3')], selectedNode: null, @@ -389,29 +383,34 @@ describe('DiskList', () => { const rows = Array.from(view.container.querySelectorAll('[data-row-id]')); // The disk that needs replacing sorts above the one that needs cooling. - expect(rows.map((row) => row.getAttribute('data-row-id'))).toEqual(['sdb', 'sda']); + expect(rows.map((row) => row.getAttribute('data-row-id'))).toEqual(['sdb', 'sda', 'nvme0n1']); const hotRow = within(rows[1] as HTMLElement); expect(hotRow.queryByText('Replace Now')).not.toBeInTheDocument(); expect(hotRow.getByText('Running Hot')).toHaveClass( 'platform-table-label-full', 'text-red-700', ); - expect(hotRow.getByText('Running Hot')).toHaveAttribute('title', 'Disk temperature is 72C'); + expect(hotRow.getByText('Running Hot')).toHaveAttribute( + 'title', + 'Disk temperature is 56°C, at or above its 55°C alert threshold.', + ); expect(hotRow.getByText('Hot')).toHaveClass('platform-table-label-compact', 'text-red-700'); + // The Temp cell beside it reads red too: the same SATA alert trigger. + expect(hotRow.getByText('56°C')).toHaveClass('text-red-600'); const failingRow = within(rows[0] as HTMLElement); expect(failingRow.getByText('Replace Now')).toHaveAttribute( 'title', 'Pending sectors detected (2)', ); + // An NVMe at 63C is under its 70C trigger and its 65C warning colour. + const nvmeRow = within(rows[2] as HTMLElement); + expect(nvmeRow.getByText('Healthy')).toBeInTheDocument(); + expect(nvmeRow.getByText('63°C')).toHaveClass('text-green-600'); }); - it('follows a disk from warm to critically hot and back to healthy', async () => { - const heat = (severity: 'warning' | 'critical', celsius: number) => ({ - level: severity, - reasons: [{ code: 'temperature_high', severity, summary: `Disk temperature is ${celsius}C` }], - }); + it('follows a disk from warm to hot and back to healthy', async () => { const [disks, setDisks] = createStore({ - items: [buildDisk('sda', 'pve3', { temperature: 63, risk: heat('warning', 63) })], + items: [buildDisk('sda', 'pve3', { temperature: 52 })], }); const [healthFilter, setHealthFilter] = createSignal<'all' | 'critical'>('all'); const view = render(() => ( @@ -426,21 +425,15 @@ describe('DiskList', () => { /> )); const row = () => view.container.querySelector('[data-row-id="sda"]') as HTMLElement | null; - expect(within(row()!).getByText('Running Hot')).toHaveClass('text-amber-700'); - expect(within(row()!).getByText('Hot')).toHaveClass('text-amber-700'); + // Warm: the Temp cell warns in amber, the verdict stays Healthy. + expect(within(row()!).getByText('52°C')).toHaveClass('text-amber-600'); + expect(within(row()!).getByText('Healthy')).toBeInTheDocument(); setHealthFilter('critical'); await waitFor(() => expect(row()).toBeNull()); - setDisks( - 'items', - reconcile([buildDisk('sda', 'pve3', { temperature: 72, risk: heat('critical', 72) })]), - ); + setDisks('items', reconcile([buildDisk('sda', 'pve3', { temperature: 56 })])); await waitFor(() => expect(row()).not.toBeNull()); expect(within(row()!).getByText('Running Hot')).toHaveClass('text-red-700'); - expect(within(row()!).getByText('Running Hot')).toHaveAttribute( - 'title', - 'Disk temperature is 72C', - ); setHealthFilter('all'); setDisks('items', reconcile([buildDisk('sda', 'pve3', { temperature: 41 })])); @@ -448,6 +441,40 @@ describe('DiskList', () => { expect(within(row()!).queryByText('Running Hot')).not.toBeInTheDocument(); }); + it('judges heat by the NVMe trigger the user raised in Alerts', async () => { + const getConfig = vi.spyOn(AlertsAPI, 'getConfig').mockResolvedValue({ + enabled: true, + activationState: 'active', + agentDefaults: { diskTemperature: { trigger: 55, clear: 50 } }, + diskTempByType: { + nvme: { trigger: 75, clear: 70 }, + sas: { trigger: 65, clear: 60 }, + sata: { trigger: 55, clear: 50 }, + }, + } as unknown as AlertConfig); + try { + const view = renderDiskList({ + disks: [buildDisk('nvme0n1', 'pve3', { diskType: 'nvme', temperature: 72 })], + nodes: [], + selectedNode: null, + searchTerm: '', + }); + const row = () => + within(view.container.querySelector('[data-row-id="nvme0n1"]') as HTMLElement); + // Factory thresholds until the user's configuration loads: 72C passes + // the 70C NVMe trigger. + expect(row().getByText('Running Hot')).toBeInTheDocument(); + expect(row().getByText('72°C')).toHaveClass('text-red-600'); + + await useAlertsActivation().refreshConfig(); + await waitFor(() => expect(row().getByText('Healthy')).toBeInTheDocument()); + expect(row().getByText('72°C')).toHaveClass('text-amber-600'); + } finally { + getConfig.mockRestore(); + eventBus.emit('org_switched', 'default'); + } + }); + it('renders SSD life and falls back to the Proxmox usage string for Belongs', () => { renderDiskList({ disks: [ diff --git a/frontend-modern/src/components/Storage/useDiskDetailModel.ts b/frontend-modern/src/components/Storage/useDiskDetailModel.ts index 2038b38433..e8c31c049b 100644 --- a/frontend-modern/src/components/Storage/useDiskDetailModel.ts +++ b/frontend-modern/src/components/Storage/useDiskDetailModel.ts @@ -3,6 +3,7 @@ import type { HistoryTimeRange } from '@/api/charts'; import { getPhysicalDiskCollectionMessages, extractPhysicalDiskPresentationData, + getPhysicalDiskTemperatureThresholds, type PhysicalDiskPresentationData, } from '@/features/storageBackups/diskPresentation'; import { @@ -23,11 +24,11 @@ export const useDiskDetailModel = (options: UseDiskDetailModelOptions) => { const { getDiskTemperatureThresholds } = useAlertsActivation(); const diskData = createMemo(() => - extractPhysicalDiskPresentationData(options.disk()), + extractPhysicalDiskPresentationData(options.disk(), getDiskTemperatureThresholds), ); const historyResourceId = createMemo(() => resolvePhysicalDiskHistoryResourceId(options.disk())); const attributeCards = createMemo(() => - getDiskDetailAttributeCards(diskData(), getDiskTemperatureThresholds(diskData().type)), + getDiskDetailAttributeCards(diskData(), getPhysicalDiskTemperatureThresholds(diskData())), ); const historyCharts = createMemo(() => getDiskDetailHistoryCharts(diskData())); const healthPresentation = createMemo(() => getDiskDetailHealthPresentation(diskData())); diff --git a/frontend-modern/src/components/Storage/useDiskListModel.ts b/frontend-modern/src/components/Storage/useDiskListModel.ts index a4aec530bf..93428b1a15 100644 --- a/frontend-modern/src/components/Storage/useDiskListModel.ts +++ b/frontend-modern/src/components/Storage/useDiskListModel.ts @@ -9,6 +9,7 @@ import { filterAndSortPhysicalDisks, type PhysicalDiskPresentationData, } from '@/features/storageBackups/diskPresentation'; +import { useAlertsActivation } from '@/stores/alertsActivation'; import { matchesPhysicalDiskNode } from './diskResourceUtils'; type UseDiskListModelOptions = { @@ -26,13 +27,19 @@ type UseDiskListModelOptions = { export const useDiskListModel = (options: UseDiskListModelOptions) => { const hasPVENodes = createMemo(() => options.nodes().length > 0); + const { getDiskTemperatureThresholds } = useAlertsActivation(); - const diskDataById = createMemo(() => buildPhysicalDiskPresentationDataMap(options.disks())); + // Each disk's heat is judged by the user's alert disk temperature thresholds, + // so the presentation data recomputes when the alert configuration loads. + const diskDataById = createMemo(() => + buildPhysicalDiskPresentationDataMap(options.disks(), getDiskTemperatureThresholds), + ); const roleFilterOptions = createMemo(() => buildPhysicalDiskRoleFilterOptions(options.disks())); const groupFilterOptions = createMemo(() => buildPhysicalDiskGroupFilterOptions(options.disks())); const getDiskData = (disk: Resource): PhysicalDiskPresentationData => - diskDataById().get(disk.id) ?? extractPhysicalDiskPresentationData(disk); + diskDataById().get(disk.id) ?? + extractPhysicalDiskPresentationData(disk, getDiskTemperatureThresholds); const selectedNodeResource = createMemo( () => options.nodes().find((node) => node.id === options.selectedNode()) ?? null, diff --git a/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx b/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx index ceeaa26a50..2b057ff711 100644 --- a/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx +++ b/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx @@ -9,6 +9,7 @@ import { PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS, getPhysicalDiskTemperaturePresentation, } from '@/features/storageBackups/diskTemperaturePresentation'; +import { useAlertsActivation } from '@/stores/alertsActivation'; import type { Resource } from '@/types/resource'; import { formatBytes } from '@/utils/format'; @@ -20,7 +21,8 @@ const LazyDiskDetail = lazy(() => const GuestPhysicalDiskRow: Component<{ disk: Resource }> = (props) => { const [expanded, setExpanded] = createSignal(false); - const data = () => extractPhysicalDiskPresentationData(props.disk); + const { getDiskTemperatureThresholds } = useAlertsActivation(); + const data = () => extractPhysicalDiskPresentationData(props.disk, getDiskTemperatureThresholds); const health = () => getPhysicalDiskHealthStatus(data()); const temperature = () => getPhysicalDiskTemperaturePresentation(data()); const label = () => data().model || data().devPath || props.disk.name; diff --git a/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.test.ts b/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.test.ts index 392e467d21..a5f615a718 100644 --- a/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.test.ts +++ b/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.test.ts @@ -69,6 +69,8 @@ describe('diskDetailPresentation', () => { expect(getLinkedDiskTemperatureTextClass(65)).toBe('text-red-500'); expect(getLinkedDiskTemperatureTextClass(52)).toBe('text-yellow-500'); expect(getLinkedDiskTemperatureTextClass(45)).toBe('text-muted'); + // Disk temperature alerting off for the disk: no reading is warm or hot. + expect(getLinkedDiskTemperatureTextClass(80, null)).toBe('text-muted'); // Explicit thresholds (e.g. resolved for an NVMe disk) shift the colors. expect(getLinkedDiskTemperatureTextClass(55, { warning: 65, critical: 70 })).toBe('text-muted'); expect(getLinkedDiskTemperatureTextClass(67, { warning: 65, critical: 70 })).toBe( diff --git a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts index e3ebb4c861..cb94ecd9f7 100644 --- a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts +++ b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts @@ -40,6 +40,7 @@ import { getPhysicalDiskSourceBadgePresentation, getPhysicalDiskTableLayoutModeForContainer, getPhysicalDiskTemperaturePresentation, + getPhysicalDiskTemperatureThresholds, hasUnraidPhysicalDiskFaultSignal, hasPhysicalDiskSmartWarning, isPhysicalDiskWearoutReported, @@ -51,6 +52,11 @@ import { type PhysicalDiskPresentationData, } from '@/features/storageBackups/diskPresentation'; import * as diskTemperaturePresentation from '@/features/storageBackups/diskTemperaturePresentation'; +import { + resolveDiskTemperatureDisplayThresholds, + type MetricDisplayThresholds, +} from '@/utils/metricThresholds'; +import { getTemperatureTextClass } from '@/utils/temperature'; function makeDiskData( overrides: Partial = {}, @@ -293,131 +299,188 @@ describe('diskPresentation', () => { expect(getPhysicalDiskHealthCompactLabel('Unknown')).toBe('Unknown'); }); - describe('heat versus replacement evidence', () => { - const buildRiskDisk = ( - id: string, - risk: { level: string; reasons: { code: string; severity: string; summary: string }[] }, - overrides: Record = {}, - ): Resource => + describe('heat judged by the alert disk temperature policy', () => { + // Disk risk never carries heat. The verdict judges a current reading + // against the same per-type alert thresholds the Temp cell colours by. + const buildDisk = (id: string, overrides: Record = {}): Resource => ({ id, name: id, type: 'physical_disk', - status: 'warning', + status: 'online', physicalDisk: { devPath: `/dev/${id}`, model: 'Crucial MX500 2TB', diskType: 'sata', health: 'PASSED', wearout: 95, - temperature: 72, - risk, + temperature: 40, + collection: { temperature: { state: 'available', source: 'smartctl' } }, ...overrides, }, identity: { hostname: 'pve3' }, canonicalIdentity: { hostname: 'pve3' }, platformType: 'proxmox-pve', }) as unknown as Resource; - const hot = (severity: string, celsius = 72) => ({ - code: 'temperature_high', - severity, - summary: `Disk temperature is ${celsius}C`, - }); + const verdictOf = ( + disk: Resource, + resolve?: (diskType: string) => MetricDisplayThresholds | null, + ) => getPhysicalDiskHealthStatus(extractPhysicalDiskPresentationData(disk, resolve)); + // A user who raised the NVMe trigger to 75C in Alerts; warning colours + // start at its 70C clear value. + const raisedNVMe = (diskType: string) => + diskType === 'nvme' + ? { warning: 70, critical: 75 } + : resolveDiskTemperatureDisplayThresholds(null, diskType); - it('calls a disk that is only hot Running Hot at the tier of its reading', () => { - const criticalDisk = buildRiskDisk('sda', { level: 'critical', reasons: [hot('critical')] }); - const criticalData = extractPhysicalDiskPresentationData(criticalDisk); - expect(criticalData.riskReasonDetails).toEqual([hot('critical')]); - expect(getPhysicalDiskHealthStatus(criticalData)).toEqual({ + it('calls a disk Running Hot from its type alert trigger, not a flat line', () => { + expect(verdictOf(buildDisk('nvme0n1', { diskType: 'nvme', temperature: 63 })).label).toBe( + 'Healthy', + ); + expect(verdictOf(buildDisk('nvme0n1', { diskType: 'nvme', temperature: 72 }))).toEqual({ label: 'Running Hot', - summary: 'Disk temperature is 72C', + summary: 'Disk temperature is 72°C, at or above its 70°C alert threshold.', tone: 'text-red-700 dark:text-red-300', }); - expect(getPhysicalDiskNormalizedHealth(criticalDisk, criticalData)).toBe('critical'); - - const warmDisk = buildRiskDisk( - 'sdb', - { level: 'warning', reasons: [hot('warning', 63)] }, - { temperature: 63 }, - ); - const warmData = extractPhysicalDiskPresentationData(warmDisk); - expect(getPhysicalDiskHealthStatus(warmData)).toEqual({ + expect(verdictOf(buildDisk('sda', { temperature: 56 }))).toEqual({ label: 'Running Hot', - summary: 'Disk temperature is 63C', - tone: 'text-amber-700 dark:text-amber-300', + summary: 'Disk temperature is 56°C, at or above its 55°C alert threshold.', + tone: 'text-red-700 dark:text-red-300', }); - expect(getPhysicalDiskNormalizedHealth(warmDisk, warmData)).toBe('warning'); + // The amber band below the trigger is a Temp cell colour, not a verdict. + expect(verdictOf(buildDisk('sdb', { temperature: 52 })).label).toBe('Healthy'); + + const hotDisk = buildDisk('sdc', { temperature: 56 }); + expect( + getPhysicalDiskNormalizedHealth(hotDisk, extractPhysicalDiskPresentationData(hotDisk)), + ).toBe('critical'); }); - it('keeps Replace Now for failure evidence even when heat is listed first', () => { - // Merged risks keep insertion order, so heat can precede the reason - // that actually calls for a new disk. - const disk = buildRiskDisk('sda', { - level: 'critical', - reasons: [ - hot('critical'), - { - code: 'pending_sectors', - severity: 'critical', - summary: 'Pending sectors detected (2)', - }, - ], + it('follows a raised trigger from the alert configuration', () => { + const nvme72 = buildDisk('nvme0n1', { diskType: 'nvme', temperature: 72 }); + expect(verdictOf(nvme72, raisedNVMe).label).toBe('Healthy'); + expect( + verdictOf(buildDisk('nvme1n1', { diskType: 'nvme', temperature: 76 }), raisedNVMe), + ).toEqual({ + label: 'Running Hot', + summary: 'Disk temperature is 76°C, at or above its 75°C alert threshold.', + tone: 'text-red-700 dark:text-red-300', }); - const data = extractPhysicalDiskPresentationData(disk); - expect(getPhysicalDiskHealthStatus(data)).toEqual({ + expect(verdictOf(buildDisk('sda', { temperature: 56 }), raisedNVMe).label).toBe( + 'Running Hot', + ); + expect(extractPhysicalDiskPresentationData(nvme72, raisedNVMe).temperatureThresholds).toEqual( + { warning: 70, critical: 75 }, + ); + }); + + it('reads red in the Temp cell exactly when the verdict says Running Hot', () => { + const red = getTemperatureTextClass(99, { warning: 1, critical: 2 }, 'diskTemperature'); + // A user who switched disk temperature alerting off resolves to null. + const alertingOff = () => null; + for (const resolve of [undefined, raisedNVMe, alertingOff]) { + for (const diskType of ['nvme', 'sata', 'sas', 'hdd', '']) { + for (let celsius = 40; celsius <= 80; celsius += 1) { + const data = extractPhysicalDiskPresentationData( + buildDisk('sda', { diskType, temperature: celsius }), + resolve, + ); + const cellIsRed = + getTemperatureTextClass( + celsius, + getPhysicalDiskTemperatureThresholds(data), + 'diskTemperature', + ) === red; + expect([ + diskType, + celsius, + getPhysicalDiskHealthStatus(data).label === 'Running Hot', + ]).toEqual([diskType, celsius, cellIsRed]); + } + } + } + }); + + it('calls no disk hot when its temperature alerting is off', () => { + const off = () => null; + const data = extractPhysicalDiskPresentationData(buildDisk('sda', { temperature: 80 }), off); + expect(data.temperatureThresholds).toBeNull(); + expect(getPhysicalDiskHealthStatus(data).label).toBe('Healthy'); + expect( + getTemperatureTextClass(80, getPhysicalDiskTemperatureThresholds(data), 'diskTemperature'), + ).toBe('text-base-content'); + }); + + it('never calls a retained reading hot', () => { + const silent = buildDisk('sda', { + temperature: 80, + collection: { + temperature: { + state: 'unavailable', + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }, + }); + expect(verdictOf(silent).label).toBe('Healthy'); + // A source that predates collection state is current. + expect(verdictOf(buildDisk('sdb', { temperature: 80, collection: undefined })).label).toBe( + 'Running Hot', + ); + }); + + it('keeps Replace Now for failure evidence on a hot disk', () => { + const pending = verdictOf( + buildDisk('sda', { + temperature: 72, + risk: { + level: 'critical', + reasons: [ + { + code: 'pending_sectors', + severity: 'critical', + summary: 'Pending sectors detected (2)', + }, + ], + }, + }), + ); + expect(pending).toEqual({ label: 'Replace Now', summary: 'Pending sectors detected (2)', tone: 'text-red-700 dark:text-red-300', }); - - const failed = extractPhysicalDiskPresentationData( - buildRiskDisk( - 'sdb', - { - level: 'critical', - reasons: [ - hot('critical'), - { - code: 'health_status', - severity: 'critical', - summary: 'Disk reports health status FAILED', - }, - ], - }, - { health: 'FAILED' }, - ), + expect(verdictOf(buildDisk('sdb', { temperature: 72, health: 'FAILED' })).label).toBe( + 'Replace Now', ); - expect(getPhysicalDiskHealthStatus(failed).label).toBe('Replace Now'); - expect(getPhysicalDiskHealthStatus(failed).summary).toBe('Disk reports health status FAILED'); }); - it('lets the more severe of heat and wear evidence name the verdict', () => { - const hotAndWorn = extractPhysicalDiskPresentationData( - buildRiskDisk('sda', { - level: 'critical', - reasons: [ - hot('critical'), - { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, - ], - }), - ); - expect(getPhysicalDiskHealthStatus(hotAndWorn).label).toBe('Running Hot'); - expect(getPhysicalDiskHealthStatus(hotAndWorn).summary).toBe('Disk temperature is 72C'); - - const warmAndWorn = extractPhysicalDiskPresentationData( - buildRiskDisk( - 'sdb', - { + it('lets a hot disk outrank warning-level wear and SMART evidence', () => { + const wornAndHot = verdictOf( + buildDisk('sda', { + temperature: 72, + risk: { level: 'warning', reasons: [ - hot('warning', 63), { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, ], }, - { temperature: 63 }, - ), + }), ); - expect(getPhysicalDiskHealthStatus(warmAndWorn)).toEqual({ + expect(wornAndHot.label).toBe('Running Hot'); + + const wornAndWarm = verdictOf( + buildDisk('sdb', { + temperature: 52, + risk: { + level: 'warning', + reasons: [ + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + ], + }, + }), + ); + expect(wornAndWarm).toEqual({ label: 'Needs Attention', summary: 'SSD life remaining is 8%', tone: 'text-amber-700 dark:text-amber-300', @@ -426,7 +489,15 @@ describe('diskPresentation', () => { it('keeps a critical level that no listed reason explains as Replace Now', () => { const data = extractPhysicalDiskPresentationData( - buildRiskDisk('sda', { level: 'critical', reasons: [hot('warning', 63)] }), + buildDisk('sda', { + risk: { + level: 'critical', + reasons: [ + { code: 'crc_errors', severity: 'monitor', summary: 'UDMA CRC errors detected (2)' }, + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + ], + }, + }), ); expect(getPhysicalDiskHealthStatus(data)).toEqual({ label: 'Replace Now', @@ -435,42 +506,19 @@ describe('diskPresentation', () => { }); }); - it('decides the verdict from codes and severities even without display text', () => { - const pendingWithoutText = extractPhysicalDiskPresentationData( - buildRiskDisk('sda', { - level: 'critical', - reasons: [ - hot('critical'), - { code: 'pending_sectors', severity: 'critical', summary: '' }, - ], - }), - ); - expect(pendingWithoutText.riskReasons).toEqual(['Disk temperature is 72C']); - expect(getPhysicalDiskHealthStatus(pendingWithoutText)).toEqual({ - label: 'Replace Now', - summary: 'Disk health has degraded to a critical state.', - tone: 'text-red-700 dark:text-red-300', - }); - - const heatWithoutText = extractPhysicalDiskPresentationData( - buildRiskDisk('sdb', { - level: 'critical', - reasons: [{ code: 'temperature_high', severity: 'critical', summary: '' }], - }), - ); - expect(getPhysicalDiskHealthStatus(heatWithoutText).label).toBe('Running Hot'); - }); - - it('does not let a weaker reason stand in for an unexplained critical level', () => { + it('decides the verdict from severities even without display text', () => { const data = extractPhysicalDiskPresentationData( - buildRiskDisk('sda', { - level: 'critical', - reasons: [ - { code: 'crc_errors', severity: 'monitor', summary: 'UDMA CRC errors detected (2)' }, - hot('warning', 63), - ], + buildDisk('sda', { + risk: { + level: 'critical', + reasons: [ + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + { code: 'pending_sectors', severity: 'critical', summary: '' }, + ], + }, }), ); + expect(data.riskReasons).toEqual(['SSD life remaining is 8%']); expect(getPhysicalDiskHealthStatus(data)).toEqual({ label: 'Replace Now', summary: 'Disk health has degraded to a critical state.', @@ -479,90 +527,61 @@ describe('diskPresentation', () => { }); it('reads severities case-insensitively and in any order within a class', () => { - const shouting = extractPhysicalDiskPresentationData( - buildRiskDisk('sda', { - level: 'CRITICAL', - reasons: [{ ...hot('critical'), severity: ' Critical ' }], + const shouting = verdictOf( + buildDisk('sda', { + risk: { + level: 'CRITICAL', + reasons: [ + { + code: 'media_errors', + severity: ' Critical ', + summary: 'Media errors detected (4)', + }, + ], + }, }), ); - expect(getPhysicalDiskHealthStatus(shouting).label).toBe('Running Hot'); - expect(getPhysicalDiskHealthStatus(shouting).tone).toBe('text-red-700 dark:text-red-300'); + expect(shouting.label).toBe('Replace Now'); + expect(shouting.summary).toBe('Media errors detected (4)'); - const unsorted = extractPhysicalDiskPresentationData( - buildRiskDisk('sdb', { - level: 'critical', - reasons: [ - { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, - { - code: 'pending_sectors', - severity: 'critical', - summary: 'Pending sectors detected (2)', - }, - ], + const unsorted = verdictOf( + buildDisk('sdb', { + risk: { + level: 'critical', + reasons: [ + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + { + code: 'pending_sectors', + severity: 'critical', + summary: 'Pending sectors detected (2)', + }, + ], + }, }), ); - expect(getPhysicalDiskHealthStatus(unsorted).summary).toBe('Pending sectors detected (2)'); - }); - - it('lets SMART counters and low life outrank warning heat', () => { - const warmAndLow = extractPhysicalDiskPresentationData( - buildRiskDisk( - 'sda', - { level: 'warning', reasons: [hot('warning', 63)] }, - { diskType: 'ssd', wearout: 8, temperature: 63 }, - ), - ); - expect(getPhysicalDiskHealthStatus(warmAndLow)).toEqual({ - label: 'Needs Attention', - summary: 'SSD life is running low.', - tone: 'text-amber-700 dark:text-amber-300', - }); - - const warmAndReallocating = extractPhysicalDiskPresentationData( - buildRiskDisk( - 'sdb', - { level: 'warning', reasons: [hot('warning', 63)] }, - { temperature: 63, smart: { reallocatedSectors: 3 } }, - ), - ); - expect(getPhysicalDiskHealthStatus(warmAndReallocating).label).toBe('Needs Attention'); - }); - - it('sorts a FAILED disk with no risk payload ahead of a critically hot one', () => { - const failedDisk = buildRiskDisk( - 'sdb', - { level: 'healthy', reasons: [] }, - { risk: undefined, health: 'FAILED', temperature: 40 }, - ); - const hotDisk = buildRiskDisk('sda', { level: 'critical', reasons: [hot('critical')] }); - const failedData = extractPhysicalDiskPresentationData(failedDisk); - const hotData = extractPhysicalDiskPresentationData(hotDisk); - expect(getPhysicalDiskHealthStatus(failedData).label).toBe('Replace Now'); - expect( - comparePhysicalDiskPresentation(failedDisk, failedData, hotDisk, hotData), - ).toBeLessThan(0); + expect(unsorted.summary).toBe('Pending sectors detected (2)'); }); it('sorts and filters hot disks by the action their verdict asks for', () => { - const warm = buildRiskDisk('sdc', { level: 'warning', reasons: [hot('warning', 63)] }); - const worn = buildRiskDisk('sdd', { - level: 'warning', - reasons: [ - { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, - ], + const warm = buildDisk('sdc', { temperature: 52 }); + const worn = buildDisk('sdd', { + risk: { + level: 'warning', + reasons: [ + { code: 'wearout_low', severity: 'warning', summary: 'SSD life remaining is 8%' }, + ], + }, }); - const hotDisk = buildRiskDisk('sda', { level: 'critical', reasons: [hot('critical')] }); - const failing = buildRiskDisk('sdb', { - level: 'critical', - reasons: [ - { code: 'media_errors', severity: 'critical', summary: 'Media errors detected (4)' }, - ], + const hotDisk = buildDisk('sda', { temperature: 72 }); + const failing = buildDisk('sdb', { + risk: { + level: 'critical', + reasons: [ + { code: 'media_errors', severity: 'critical', summary: 'Media errors detected (4)' }, + ], + }, }); - const healthy = buildRiskDisk( - 'sde', - { level: 'healthy', reasons: [] }, - { risk: undefined, temperature: 38 }, - ); + const healthy = buildDisk('sde', { temperature: 38 }); const disks = [healthy, warm, worn, hotDisk, failing]; const dataMap = buildPhysicalDiskPresentationDataMap(disks); const sorted = (healthFilter: 'all' | 'attention' | 'critical' | 'warning') => @@ -576,9 +595,9 @@ describe('diskPresentation', () => { // Device order alone would put each hot disk first. expect(sorted('all')).toEqual(['sdb', 'sda', 'sdd', 'sdc', 'sde']); - expect(sorted('attention')).toEqual(['sdb', 'sda', 'sdd', 'sdc']); + expect(sorted('attention')).toEqual(['sdb', 'sda', 'sdd']); expect(sorted('critical')).toEqual(['sdb', 'sda']); - expect(sorted('warning')).toEqual(['sdd', 'sdc']); + expect(sorted('warning')).toEqual(['sdd']); }); }); diff --git a/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts b/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts index 75f66c2cce..4e58c0f28f 100644 --- a/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts +++ b/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts @@ -25,7 +25,8 @@ export function getLinkedDiskTemperatureTextClass( tempCelsius: number, thresholds?: MetricDisplayThresholds | null, ): string { - if (!Number.isFinite(tempCelsius) || tempCelsius <= 0) { + // Null thresholds mean disk temperature alerting is off for this disk. + if (!Number.isFinite(tempCelsius) || tempCelsius <= 0 || thresholds === null) { return 'text-muted'; } const severity = getMetricSeverity(tempCelsius, 'diskTemperature', thresholds); @@ -137,7 +138,9 @@ export function getDiskDetailAttributeCards( cards.push({ label: 'Temperature', value: temperature.label, - ok: getMetricSeverity(disk.temperature, 'diskTemperature', diskTempThresholds) !== 'critical', + ok: + diskTempThresholds === null || + getMetricSeverity(disk.temperature, 'diskTemperature', diskTempThresholds) !== 'critical', }); } else if (temperature) { // A retained reading is neither healthy nor hot now. The collection diff --git a/frontend-modern/src/features/storageBackups/diskPresentation.ts b/frontend-modern/src/features/storageBackups/diskPresentation.ts index 1f25850810..4d8c996f08 100644 --- a/frontend-modern/src/features/storageBackups/diskPresentation.ts +++ b/frontend-modern/src/features/storageBackups/diskPresentation.ts @@ -14,7 +14,15 @@ import { import { getAllFilterOptionLabel } from '@/components/shared/filterOptionPresentation'; import { getPhysicalDiskNodeIdentity } from '@/components/Storage/diskResourceUtils'; import { getInfrastructureSettingsLocationLabel } from '@/utils/infrastructureSettingsPresentation'; +import { + resolveDiskTemperatureDisplayThresholds, + type MetricDisplayThresholds, +} from '@/utils/metricThresholds'; import { normalizeStorageSourceKey, storageSourceMatchesFilter } from '@/utils/storageSources'; +import { + getPhysicalDiskHeatSummary, + isPhysicalDiskRunningHot, +} from './diskTemperaturePresentation'; import type { NormalizedHealth, StorageHealthFilter } from './models'; import { matchesStorageNodeTerms, parseStorageSearchQuery } from './storageSearchQuery'; @@ -67,6 +75,12 @@ export interface PhysicalDiskPresentationData { controller?: string; target?: string; temperature: number; + /** + * The alert disk temperature thresholds for this disk's type. The Temp cell + * colours the reading by them and the Health verdict judges heat by them. + * Absent means the factory thresholds for the type. + */ + temperatureThresholds?: MetricDisplayThresholds | null; rpm: number; used: string; collection?: PhysicalDiskCollectionStatus; @@ -395,13 +409,35 @@ export function getPhysicalDiskHostLabel( return (disk.node || resource.parentName || '').trim(); } +/** + * Resolves the alert disk temperature thresholds for a disk type. Callers with + * the user's alert configuration pass `getDiskTemperatureThresholds` from the + * alerts activation store; the default is the factory configuration. + */ +export type PhysicalDiskTemperatureThresholdResolver = ( + diskType: string, +) => MetricDisplayThresholds | null; + +const resolveFactoryDiskTemperatureThresholds: PhysicalDiskTemperatureThresholdResolver = ( + diskType, +) => resolveDiskTemperatureDisplayThresholds(null, diskType); + +/** The thresholds the Temp cell colours this disk's reading by. */ +export const getPhysicalDiskTemperatureThresholds = ( + disk: Pick, +): MetricDisplayThresholds | null => + disk.temperatureThresholds !== undefined + ? disk.temperatureThresholds + : resolveFactoryDiskTemperatureThresholds(disk.type); + export function extractPhysicalDiskPresentationData( resource: Resource, + resolveTemperatureThresholds: PhysicalDiskTemperatureThresholdResolver = resolveFactoryDiskTemperatureThresholds, ): PhysicalDiskPresentationData { const pd = resource.physicalDisk || ((resource.platformData as any)?.physicalDisk ?? {}); const diskNode = getPhysicalDiskNodeIdentity(resource); - // A reason's code and severity decide the verdict even when it carries no - // display text, so only the summary list drops empty summaries. + // A reason's severity decides the verdict even when it carries no display + // text, so only the summary list drops empty summaries. const riskReasonDetails: ResourceStorageRiskReason[] = Array.isArray(pd.risk?.reasons) ? pd.risk.reasons .filter((reason: unknown) => reason !== null && typeof reason === 'object') @@ -431,6 +467,7 @@ export function extractPhysicalDiskPresentationData( health: pd.health || 'UNKNOWN', wearout: pd.wearout ?? -1, temperature: pd.temperature ?? 0, + temperatureThresholds: resolveTemperatureThresholds(pd.diskType || ''), rpm: pd.rpm ?? 0, used: pd.used || '', storageRole: pd.storageRole, @@ -503,23 +540,24 @@ export function getPhysicalDiskCollectionMessages(disk: PhysicalDiskPresentation export function buildPhysicalDiskPresentationDataMap( disks: Resource[], + resolveTemperatureThresholds?: PhysicalDiskTemperatureThresholdResolver, ): Map { const map = new Map(); for (const disk of disks || []) { - map.set(disk.id, extractPhysicalDiskPresentationData(disk)); + map.set(disk.id, extractPhysicalDiskPresentationData(disk, resolveTemperatureThresholds)); } return map; } // Disks sort in the order their verdicts ask for action: replacement first, -// then a disk running critically hot, then other warnings, then a warm disk. +// then a disk running hot, then other warnings. const getPhysicalDiskVerdictPriority = (disk: PhysicalDiskPresentationData): number => { const verdict = getPhysicalDiskHealthVerdict(disk); switch (verdict.kind) { case 'replace': return 400; case 'hot': - return verdict.critical ? 300 : 100; + return 300; case 'attention': return 200; default: @@ -677,11 +715,6 @@ export function hasPhysicalDiskSmartWarning(disk: PhysicalDiskPresentationData): ); } -// Heat is the one disk risk whose fix is cooling, not a replacement disk. -// Alerts raise it as a temperature metric rather than as disk health, so the -// verdict names it apart from the evidence that does mean replacing the disk. -const PHYSICAL_DISK_TEMPERATURE_RISK_CODE = 'temperature_high'; - const PHYSICAL_DISK_RISK_SEVERITY_RANK: Record = { monitor: 1, warning: 2, @@ -696,20 +729,14 @@ interface PhysicalDiskRiskEvidence { summary?: string; } -// Splits the disk's risk into heat and everything else, keeping the most -// severe reason of each. A risk level that none of its reasons explains, or -// one sent without reason codes, stays with the disk-health evidence. -function splitPhysicalDiskRisk(disk: PhysicalDiskPresentationData): { - health: PhysicalDiskRiskEvidence; - heat: PhysicalDiskRiskEvidence; -} { - const health: PhysicalDiskRiskEvidence = { rank: 0 }; - const heat: PhysicalDiskRiskEvidence = { rank: 0 }; +// Keeps the disk's most severe risk reason. A risk level that none of its +// reasons explains, or one sent without reasons, still sets the rank. +function getPhysicalDiskRiskEvidence(disk: PhysicalDiskPresentationData): PhysicalDiskRiskEvidence { + const evidence: PhysicalDiskRiskEvidence = { rank: 0 }; const details = disk.riskReasonDetails ?? disk.riskReasons.map((summary) => ({ code: '', severity: disk.riskLevel || '', summary })); for (const reason of details) { - const evidence = reason.code === PHYSICAL_DISK_TEMPERATURE_RISK_CODE ? heat : health; const rank = getPhysicalDiskRiskSeverityRank(reason.severity); if (rank > evidence.rank) { evidence.rank = rank; @@ -719,25 +746,25 @@ function splitPhysicalDiskRisk(disk: PhysicalDiskPresentationData): { } } const levelRank = getPhysicalDiskRiskSeverityRank(disk.riskLevel); - if (levelRank > Math.max(health.rank, heat.rank)) { + if (levelRank > evidence.rank) { // A reason with a known, lower severity does not explain this level, so // its text must not stand in as the verdict's reason. - if (health.rank > 0) health.summary = undefined; - health.rank = levelRank; + if (evidence.rank > 0) evidence.summary = undefined; + evidence.rank = levelRank; } - return { health, heat }; + return evidence; } type PhysicalDiskHealthVerdict = | { kind: 'replace'; summary: string } - | { kind: 'hot'; critical: boolean; summary: string } + | { kind: 'hot'; summary: string } | { kind: 'attention'; summary: string } | { kind: 'clear' }; function getPhysicalDiskHealthVerdict( disk: PhysicalDiskPresentationData, ): PhysicalDiskHealthVerdict { - const { health, heat } = splitPhysicalDiskRisk(disk); + const health = getPhysicalDiskRiskEvidence(disk); const critical = PHYSICAL_DISK_RISK_SEVERITY_RANK.critical; const warning = PHYSICAL_DISK_RISK_SEVERITY_RANK.warning; const lowLife = isPhysicalDiskWearoutReported(disk) && disk.wearout < 10; @@ -748,8 +775,12 @@ function getPhysicalDiskHealthVerdict( summary: health.summary || 'Disk health has degraded to a critical state.', }; } - if (heat.rank >= critical) { - return { kind: 'hot', critical: true, summary: heat.summary || 'Disk temperature is high.' }; + const temperatureThresholds = getPhysicalDiskTemperatureThresholds(disk); + if (isPhysicalDiskRunningHot(disk, temperatureThresholds)) { + return { + kind: 'hot', + summary: getPhysicalDiskHeatSummary(disk.temperature, temperatureThresholds), + }; } if (health.rank >= warning || hasPhysicalDiskSmartWarning(disk) || lowLife) { return { @@ -759,9 +790,6 @@ function getPhysicalDiskHealthVerdict( (lowLife ? 'SSD life is running low.' : 'SMART counters indicate elevated risk.'), }; } - if (heat.rank >= warning) { - return { kind: 'hot', critical: false, summary: heat.summary || 'Disk temperature is high.' }; - } return { kind: 'clear' }; } @@ -778,11 +806,7 @@ export function getPhysicalDiskHealthStatus( } if (verdict.kind === 'hot') { - return { - label: 'Running Hot', - summary: verdict.summary, - tone: verdict.critical ? PHYSICAL_DISK_CRITICAL_TONE : PHYSICAL_DISK_WARNING_TONE, - }; + return { label: 'Running Hot', summary: verdict.summary, tone: PHYSICAL_DISK_CRITICAL_TONE }; } if (verdict.kind === 'attention') { @@ -814,8 +838,9 @@ export function getPhysicalDiskNormalizedHealth( if (clearLabel === 'Online') return 'healthy'; if (resource.status === 'offline') return 'offline'; if (verdict.kind === 'replace') return 'critical'; - // A disk running hot sits in the health filter at its temperature tier. - if (verdict.kind === 'hot') return verdict.critical ? 'critical' : 'warning'; + // A disk running hot sits in the health filter with the critical disks, as + // its Temp cell reads red. + if (verdict.kind === 'hot') return 'critical'; if (verdict.kind === 'attention') return 'warning'; return clearLabel === 'Healthy' ? 'healthy' : 'unknown'; } diff --git a/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts b/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts index acb63c612a..b4571083ea 100644 --- a/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts +++ b/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts @@ -1,4 +1,5 @@ import type { PhysicalDiskCollectionStatus, PhysicalDiskFieldStatus } from '@/types/resource'; +import type { MetricDisplayThresholds } from '@/utils/metricThresholds'; import { formatTemperature } from '@/utils/temperature'; // The canonical decision on whether a disk temperature is a current reading. @@ -51,3 +52,29 @@ export function getPhysicalDiskTemperaturePresentation(disk: { : getPhysicalDiskLastKnownTemperatureTitle(disk.collection?.temperature), }; } + +// Disk heat is judged by the alert disk temperature policy, through the +// thresholds the Temp column colours a reading by: a disk runs hot from its +// type's alert trigger (`critical`), the reading its disk temperature alert +// fires at. The band just below the trigger is a colour, not heat. A retained +// reading is not current, and null thresholds mean disk temperature alerting +// is off for the disk, so neither is ever hot. Disk risk never carries heat. +export function isPhysicalDiskRunningHot( + disk: { temperature: number; collection?: PhysicalDiskCollectionStatus | null }, + thresholds: MetricDisplayThresholds | null, +): boolean { + if (!thresholds || !(thresholds.critical > 0)) return false; + if (!Number.isFinite(disk.temperature) || disk.temperature <= 0) return false; + if (!isPhysicalDiskTemperatureCurrent(disk.collection)) return false; + return disk.temperature >= thresholds.critical; +} + +export function getPhysicalDiskHeatSummary( + temperature: number, + thresholds: MetricDisplayThresholds | null, +): string { + const reading = formatTemperature(temperature); + return thresholds && thresholds.critical > 0 + ? `Disk temperature is ${reading}, at or above its ${formatTemperature(thresholds.critical)} alert threshold.` + : `Disk temperature is ${reading}.`; +} diff --git a/frontend-modern/src/features/truenas/TrueNASStorageTopologyTable.tsx b/frontend-modern/src/features/truenas/TrueNASStorageTopologyTable.tsx index 8f3e627337..27c9caf59a 100644 --- a/frontend-modern/src/features/truenas/TrueNASStorageTopologyTable.tsx +++ b/frontend-modern/src/features/truenas/TrueNASStorageTopologyTable.tsx @@ -37,13 +37,15 @@ import { createPlatformResourceLabelResolver, getPlatformResourceDetailRowClass, } from '@/features/platformPage/PlatformResourceDetailTableRow'; +import { useAlertsActivation } from '@/stores/alertsActivation'; import type { Resource } from '@/types/resource'; import { buildTrueNASStorageTopologyRows, filterTrueNASStorageTopologyRows, filterTrueNASStorageTopologyRowsByKind, - getTrueNASResourceDisplayStatus, + getTrueNASStorageDotStatus, getTrueNASStorageIssue, + type TrueNASDiskTemperatureThresholdResolver, type TrueNASStorageIssue, type TrueNASStorageStatusFilter, type TrueNASStorageKindFilter, @@ -232,6 +234,7 @@ type TrueNASStorageSortKey = (typeof TRUENAS_STORAGE_SORT_KEYS)[number]; const getTrueNASStorageSortValue = ( row: TrueNASStorageTopologyRow, key: TrueNASStorageSortKey, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, ): PlatformTableSortValue => { switch (key) { case 'resource': @@ -255,7 +258,9 @@ const getTrueNASStorageSortValue = ( return typeof temperature === 'number' && Number.isFinite(temperature) ? temperature : null; } case 'health': - return HEALTH_SORT_RANK[getTrueNASStorageIssue(row.resource)?.status ?? 'healthy']; + return HEALTH_SORT_RANK[ + getTrueNASStorageIssue(row.resource, resolveDiskTemperatureThresholds)?.status ?? 'healthy' + ]; default: key satisfies never; return null; @@ -271,6 +276,7 @@ const getTrueNASStorageSortValue = ( const sortTrueNASStorageTopologyRows = ( rows: readonly TrueNASStorageTopologyRow[], sort: PlatformTableSortState, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, ): readonly TrueNASStorageTopologyRow[] => { if (!sort.sortKey()) return rows; const present = new Set(rows.map((row) => row.id)); @@ -288,7 +294,9 @@ const sortTrueNASStorageTopologyRows = ( } const ordered: TrueNASStorageTopologyRow[] = []; const emit = (siblings: TrueNASStorageTopologyRow[]) => { - for (const row of sort.sortRows(siblings, getTrueNASStorageSortValue)) { + for (const row of sort.sortRows(siblings, (sibling, key) => + getTrueNASStorageSortValue(sibling, key, resolveDiskTemperatureThresholds), + )) { ordered.push(row); const children = childrenByParent.get(row.id); if (children) emit(children); @@ -308,7 +316,9 @@ export const getTrueNASStorageTopologyIndentClass = (depth: number): string => { const ResourceCell: Component<{ row: TrueNASStorageTopologyRow; detailToggle?: JSX.Element }> = ( props, ) => { - const displayStatus = () => getTrueNASResourceDisplayStatus(props.row.resource); + const { getDiskTemperatureThresholds } = useAlertsActivation(); + const displayStatus = () => + getTrueNASStorageDotStatus(props.row.resource, getDiskTemperatureThresholds); const indicator = () => getSimpleStatusIndicator(displayStatus()); const name = () => resourceName(props.row.resource); return ( @@ -342,7 +352,11 @@ export const TrueNASStorageTopologyTable: Component<{ kindFilter?: TrueNASStorageKindFilter; onKindFilterChange?: (value: TrueNASStorageKindFilter) => void; }> = (props) => { - const rows = createMemo(() => buildTrueNASStorageTopologyRows(props.resources)); + // Disk heat follows the user's alert disk temperature thresholds. + const { getDiskTemperatureThresholds } = useAlertsActivation(); + const rows = createMemo(() => + buildTrueNASStorageTopologyRows(props.resources, getDiskTemperatureThresholds), + ); const [internalKindFilter, setInternalKindFilter] = createSignal('all'); const kindFilter = () => props.kindFilter ?? internalKindFilter(); const setKindFilter = (value: TrueNASStorageKindFilter) => { @@ -356,7 +370,8 @@ export const TrueNASStorageTopologyTable: Component<{ const tableState = createPlatformTableFilterState({ resources: scopedRows, initialStatus: 'all' as TrueNASStorageStatusFilter, - filter: filterTrueNASStorageTopologyRows, + filter: (rows, search, status) => + filterTrueNASStorageTopologyRows(rows, search, status, getDiskTemperatureThresholds), }); const kindFilters = createMemo(() => [ { @@ -373,6 +388,7 @@ export const TrueNASStorageTopologyTable: Component<{ filterTrueNASStorageTopologyRowsByKind(rows(), option.value), tableState.search(), tableState.status(), + getDiskTemperatureThresholds, ).length, })), value: kindFilter, @@ -392,7 +408,9 @@ export const TrueNASStorageTopologyTable: Component<{ sortKeys: TRUENAS_STORAGE_SORT_KEYS, descendingFirst: ['usage', 'disks', 'temp'], }); - const sortedRows = createMemo(() => sortTrueNASStorageTopologyRows(tableState.filtered(), sort)); + const sortedRows = createMemo(() => + sortTrueNASStorageTopologyRows(tableState.filtered(), sort, getDiskTemperatureThresholds), + ); return ( {(row) => { const resource = () => row.resource; - const issue = createMemo(() => getTrueNASStorageIssue(resource())); + const issue = createMemo(() => + getTrueNASStorageIssue(resource(), getDiskTemperatureThresholds), + ); const detailRowId = () => drawer.detailRowId(resource()); const isExpanded = () => drawer.isExpanded(resource()); return ( diff --git a/frontend-modern/src/features/truenas/__tests__/TrueNASStorageTopologyTable.test.tsx b/frontend-modern/src/features/truenas/__tests__/TrueNASStorageTopologyTable.test.tsx index 57982bab5f..07cdc03e00 100644 --- a/frontend-modern/src/features/truenas/__tests__/TrueNASStorageTopologyTable.test.tsx +++ b/frontend-modern/src/features/truenas/__tests__/TrueNASStorageTopologyTable.test.tsx @@ -1,5 +1,10 @@ -import { cleanup, fireEvent, render, screen } from '@solidjs/testing-library'; -import { afterEach, describe, expect, it } from 'vitest'; +import { cleanup, fireEvent, render, screen, waitFor } from '@solidjs/testing-library'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { AlertsAPI } from '@/api/alerts'; +import { useAlertsActivation } from '@/stores/alertsActivation'; +import { eventBus } from '@/stores/events'; +import type { AlertConfig } from '@/types/alerts'; import { TrueNASStorageTopologyTable, @@ -215,4 +220,102 @@ describe('TrueNASStorageTopologyTable', () => { expect(healthyRow?.querySelector('[data-truenas-storage-health]')).toBeNull(); expect(healthyRow).not.toHaveTextContent('Healthy'); }); + + it('flags a disk running hot by its type alert trigger', () => { + // Disk risk carries no heat, so the table judges it with the alerts + // store's per-type thresholds (factory here: SATA 55C, NVMe 70C). + const disk = (id: string, diskType: string, temperature: number) => + makeStorageResource({ + id, + type: 'physical_disk', + name: id, + storage: undefined, + physicalDisk: { devPath: `/dev/${id}`, serial: `serial-${id}`, diskType, temperature }, + }); + const resources = [disk('sda', 'sata', 56), disk('nvme0n1', 'nvme', 63)]; + const { container } = render(() => ( + } + emptyTitle="No storage" + emptyDescription="No storage" + showToolbar={false} + /> + )); + + const hotRow = container.querySelector('[data-truenas-storage-resource="sda"]'); + expect(hotRow?.querySelector('[data-truenas-storage-health="attention"]')).toHaveAttribute( + 'title', + 'Disk temperature is 56°C, at or above its 55°C alert threshold.', + ); + const warmRow = container.querySelector('[data-truenas-storage-resource="nvme0n1"]'); + expect(warmRow?.querySelector('[data-truenas-storage-health]')).toBeNull(); + // Phones hide the Health cell, so the status dot carries the heat too. + expect(hotRow?.querySelector('[title="Warning"]')).not.toBeNull(); + expect(warmRow?.querySelector('[title="Online"]')).not.toBeNull(); + // The hot disk sorts ahead of its cooler sibling. + expect( + [...container.querySelectorAll('[data-truenas-storage-resource]')].map((row) => + row.getAttribute('data-truenas-storage-resource'), + ), + ).toEqual(['sda', 'nvme0n1']); + }); + + it('follows a disk temperature trigger the user raised in Alerts', async () => { + const getConfig = vi.spyOn(AlertsAPI, 'getConfig').mockResolvedValue({ + enabled: true, + activationState: 'active', + agentDefaults: { diskTemperature: { trigger: 55, clear: 50 } }, + diskTempByType: { + nvme: { trigger: 70, clear: 65 }, + sas: { trigger: 65, clear: 60 }, + sata: { trigger: 60, clear: 55 }, + }, + } as unknown as AlertConfig); + try { + const resources = [ + makeStorageResource({ + id: 'sda', + type: 'physical_disk', + name: 'sda', + storage: undefined, + physicalDisk: { + devPath: '/dev/sda', + serial: 'serial-sda', + diskType: 'sata', + temperature: 56, + }, + }), + ]; + const { container } = render(() => ( + } + emptyTitle="No storage" + emptyDescription="No storage" + /> + )); + const health = () => + container.querySelector( + '[data-truenas-storage-resource="sda"] [data-truenas-storage-health]', + ); + expect(health()).toHaveAttribute('data-truenas-storage-health', 'attention'); + expect(screen.getByRole('button', { name: /Attention/ })).toHaveTextContent('1'); + // With the Attention filter on, the storage-type counts judge heat too. + await fireEvent.click(screen.getByRole('button', { name: /Attention/ })); + expect(screen.getByRole('button', { name: 'Physical disks, 1' })).toBeInTheDocument(); + + await useAlertsActivation().refreshConfig(); + await waitFor(() => + expect(screen.getByRole('button', { name: 'Physical disks, 0' })).toBeInTheDocument(), + ); + // The status filter counts follow the same thresholds. + expect(screen.getByRole('button', { name: /Attention/ })).toHaveTextContent('0'); + } finally { + getConfig.mockRestore(); + eventBus.emit('org_switched', 'default'); + } + }); }); diff --git a/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts b/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts index 8f62772d95..1d29985b7c 100644 --- a/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts +++ b/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import type { RecoveryPoint } from '@/types/recovery'; import type { Resource } from '@/types/resource'; +import { resolveDiskTemperatureDisplayThresholds } from '@/utils/metricThresholds'; import { TRUENAS_TAB_SPECS, buildTrueNASPageModel, @@ -630,6 +631,72 @@ describe('truenasPageModel', () => { expect(mapTrueNASStorageStatus(healthyByState)).toBe('healthy'); }); + it('flags a disk whose current reading reaches its alert trigger', () => { + // Disk risk carries no heat, so a hot disk stays `online`; the page judges + // it by the alert disk temperature thresholds, as Physical Disks does. + const resolveFactory = (diskType: string) => + resolveDiskTemperatureDisplayThresholds(null, diskType); + const hotDisk = makeResource({ + id: 'disk-hot', + type: 'physical_disk', + name: 'sda', + status: 'online', + physicalDisk: { + devPath: '/dev/sda', + serial: 'serial-hot', + diskType: 'sata', + health: 'UNKNOWN', + temperature: 56, + }, + }); + expect(mapTrueNASStorageStatus(hotDisk, resolveFactory)).toBe('attention'); + expect(getTrueNASStorageIssue(hotDisk, resolveFactory)).toEqual({ + status: 'attention', + label: 'Attention', + reasons: ['Disk temperature is 56°C, at or above its 55°C alert threshold.'], + }); + // A cool sibling that sorts first by name: heat moves the hot disk ahead + // of it, and the status filter splits them. + const coolDisk = makeResource({ + ...hotDisk, + id: 'disk-cool', + name: 'ada0', + physicalDisk: { ...hotDisk.physicalDisk!, serial: 'serial-cool', temperature: 40 }, + }); + const rows = buildTrueNASStorageTopologyRows([coolDisk, hotDisk], resolveFactory); + expect(rows.map((row) => row.id)).toEqual(['disk:disk-hot', 'disk:disk-cool']); + expect(buildTrueNASStorageTopologyRows([coolDisk, hotDisk]).map((row) => row.id)).toEqual([ + 'disk:disk-cool', + 'disk:disk-hot', + ]); + expect( + filterTrueNASStorageTopologyRows(rows, '', 'attention', resolveFactory).map((row) => row.id), + ).toEqual(['disk:disk-hot']); + expect( + filterTrueNASStorageTopologyRows(rows, '', 'healthy', resolveFactory).map((row) => row.id), + ).toEqual(['disk:disk-cool']); + + // An NVMe at 63C is under its 70C trigger. + const warmNVMe = makeResource({ + ...hotDisk, + id: 'disk-nvme', + physicalDisk: { ...hotDisk.physicalDisk!, diskType: 'nvme', temperature: 63 }, + }); + expect(mapTrueNASStorageStatus(warmNVMe, resolveFactory)).toBe('healthy'); + // A trigger the user raised, alerting switched off, and a retained reading + // are never heat. + expect(mapTrueNASStorageStatus(hotDisk, () => ({ warning: 55, critical: 60 }))).toBe('healthy'); + expect(mapTrueNASStorageStatus(hotDisk, () => null)).toBe('healthy'); + const retained = makeResource({ + ...hotDisk, + physicalDisk: { + ...hotDisk.physicalDisk!, + collection: { temperature: { state: 'unavailable', source: 'truenas', reason: 'stale' } }, + }, + }); + expect(mapTrueNASStorageStatus(retained, resolveFactory)).toBe('healthy'); + }); + it('renders expected receive-side replication readonly posture as healthy', () => { const replicatedDataset = makeResource({ id: 'dataset-replica', diff --git a/frontend-modern/src/features/truenas/truenasPageModel.ts b/frontend-modern/src/features/truenas/truenasPageModel.ts index 8279c78afe..ceca216eae 100644 --- a/frontend-modern/src/features/truenas/truenasPageModel.ts +++ b/frontend-modern/src/features/truenas/truenasPageModel.ts @@ -2,6 +2,11 @@ import { resolveResourcePlatformType } from '@/utils/sourcePlatforms'; import { asTrimmedString } from '@/utils/stringUtils'; import { hasImpairedResourceSource } from '@/utils/resourceSourceHealth'; import { getTrueNASDatasetStateSummary } from '@/utils/truenasDatasetState'; +import type { MetricDisplayThresholds } from '@/utils/metricThresholds'; +import { + getPhysicalDiskHeatSummary, + isPhysicalDiskRunningHot, +} from '@/features/storageBackups/diskTemperaturePresentation'; import type { Resource, ResourceIncident, @@ -402,7 +407,10 @@ export function buildTrueNASStorageChildCounts( export function buildTrueNASStorageTopologyRows( resources: Resource[], + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, ): TrueNASStorageTopologyRow[] { + const compareStorageResources = (left: Resource, right: Resource): number => + compareStorageResourcesByStatus(left, right, resolveDiskTemperatureThresholds); const storageResources = resources.filter( (resource) => isTrueNASPlatform(resource) && @@ -556,8 +564,14 @@ function filterTrueNASStorageTopologyRow( row: TrueNASStorageTopologyRow, search: string, status: TrueNASStorageStatusFilter, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, ): boolean { - if (status !== 'all' && mapTrueNASStorageStatus(row.resource) !== status) return false; + if ( + status !== 'all' && + mapTrueNASStorageStatus(row.resource, resolveDiskTemperatureThresholds) !== status + ) { + return false; + } if (!search.trim()) return true; if (row.kind.includes(search.trim().toLowerCase())) return true; return matchesTrueNASStorageSearch(row.resource, search); @@ -567,9 +581,14 @@ export function filterTrueNASStorageTopologyRows( rows: TrueNASStorageTopologyRow[], search: string, status: TrueNASStorageStatusFilter, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, ): TrueNASStorageTopologyRow[] { const directMatches = new Set( - rows.filter((row) => filterTrueNASStorageTopologyRow(row, search, status)).map((row) => row.id), + rows + .filter((row) => + filterTrueNASStorageTopologyRow(row, search, status, resolveDiskTemperatureThresholds), + ) + .map((row) => row.id), ); if (directMatches.size === rows.length) return rows; @@ -628,8 +647,11 @@ const titleize = (value: string): string => .map((part) => part.charAt(0).toUpperCase() + part.slice(1).toLowerCase()) .join(' '); -const storageStatusRank = (resource: Resource): number => { - switch (mapTrueNASStorageStatus(resource)) { +const storageStatusRank = ( + resource: Resource, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, +): number => { + switch (mapTrueNASStorageStatus(resource, resolveDiskTemperatureThresholds)) { case 'attention': return 0; case 'offline': @@ -641,8 +663,14 @@ const storageStatusRank = (resource: Resource): number => { } }; -const compareStorageResources = (left: Resource, right: Resource): number => { - const rankDelta = storageStatusRank(left) - storageStatusRank(right); +const compareStorageResourcesByStatus = ( + left: Resource, + right: Resource, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, +): number => { + const rankDelta = + storageStatusRank(left, resolveDiskTemperatureThresholds) - + storageStatusRank(right, resolveDiskTemperatureThresholds); if (rankDelta !== 0) return rankDelta; const nameDelta = resourceDisplayName(left).localeCompare(resourceDisplayName(right)); if (nameDelta !== 0) return nameDelta; @@ -699,8 +727,45 @@ export function mapTrueNASIncidentSeverity( return 'info'; } +/** + * Resolves a disk type's alert disk temperature thresholds: the alerts store's + * `getDiskTemperatureThresholds`. Without one, disk heat is not judged. + */ +export type TrueNASDiskTemperatureThresholdResolver = ( + diskType: string, +) => MetricDisplayThresholds | null; + +// Disk risk carries no heat, so a disk whose current reading has reached its +// type's alert trigger is judged here, by the same policy as the Physical +// Disks Running Hot verdict. Returns the heat reason, or null. +function getTrueNASDiskHeatSummary( + resource: Resource, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, +): string | null { + const disk = resource.physicalDisk; + if (!resolveDiskTemperatureThresholds || resource.type !== 'physical_disk' || !disk) return null; + const temperature = disk.temperature ?? 0; + const thresholds = resolveDiskTemperatureThresholds(disk.diskType || ''); + return isPhysicalDiskRunningHot({ temperature, collection: disk.collection }, thresholds) + ? getPhysicalDiskHeatSummary(temperature, thresholds) + : null; +} + +// A hot disk reports `online` (disk risk carries no heat), so its status dot +// takes the warning tone its Health cell gives it. On phones the dot is the +// row's only health signal. +export function getTrueNASStorageDotStatus( + resource: Resource, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, +): string { + const status = getTrueNASResourceDisplayStatus(resource); + if (normalize(status) !== 'online') return status; + return getTrueNASDiskHeatSummary(resource, resolveDiskTemperatureThresholds) ? 'warning' : status; +} + export function mapTrueNASStorageStatus( resource: Resource, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, ): Exclude | 'unknown' { const status = normalize(getTrueNASResourceDisplayStatus(resource)); const zfsState = normalize(resource.storage?.zfsPoolState); @@ -719,6 +784,7 @@ export function mapTrueNASStorageStatus( if (diskHealth && !['passed', 'healthy', 'ok', 'unknown', 'unavailable'].includes(diskHealth)) { return 'attention'; } + if (getTrueNASDiskHeatSummary(resource, resolveDiskTemperatureThresholds)) return 'attention'; if (['online', 'running', 'healthy'].includes(status)) return 'healthy'; return 'unknown'; } @@ -753,8 +819,11 @@ const reasonKey = (text: string): string => // the failing member or test, so it leads. Pulse's derived risk summaries // restate the same pool and disk state, so they only stand in when TrueNAS // raised no alert; the row drawer still lists every one of them. -export function getTrueNASStorageIssue(resource: Resource): TrueNASStorageIssue | null { - const status = mapTrueNASStorageStatus(resource); +export function getTrueNASStorageIssue( + resource: Resource, + resolveDiskTemperatureThresholds?: TrueNASDiskTemperatureThresholdResolver, +): TrueNASStorageIssue | null { + const status = mapTrueNASStorageStatus(resource, resolveDiskTemperatureThresholds); if (status === 'healthy') return null; const reasons: string[] = []; @@ -775,6 +844,7 @@ export function getTrueNASStorageIssue(resource: Resource): TrueNASStorageIssue if (reasons.length === leading) { for (const reason of resource.storage?.risk?.reasons ?? []) push(reason.summary); for (const reason of resource.physicalDisk?.risk?.reasons ?? []) push(reason.summary); + push(getTrueNASDiskHeatSummary(resource, resolveDiskTemperatureThresholds)); } // A read-only dataset raises no incident, so its state tag is the only // reason the provider gives. The drawer reads the same shared mapping. diff --git a/frontend-modern/src/utils/__tests__/metricThresholds.test.ts b/frontend-modern/src/utils/__tests__/metricThresholds.test.ts index 76b8e749d5..59cd271bc2 100644 --- a/frontend-modern/src/utils/__tests__/metricThresholds.test.ts +++ b/frontend-modern/src/utils/__tests__/metricThresholds.test.ts @@ -374,6 +374,29 @@ describe('metricThresholds', () => { }); }); + it('switches every disk type off with the agent disk temperature default', () => { + const config = { + enabled: true, + guestDefaults: {}, + nodeDefaults: {}, + agentDefaults: { diskTemperature: { trigger: 0, clear: 0 } }, + diskTempByType: { + nvme: { trigger: 70, clear: 65 }, + sata: { trigger: 55, clear: 50 }, + }, + storageDefault: { trigger: 85, clear: 80 }, + overrides: { 'host-1': { diskTemperature: { trigger: 80, clear: 75 } } }, + } as AlertConfig; + expect(resolveDiskTemperatureDisplayThresholds(config, 'nvme')).toBeNull(); + expect(resolveDiskTemperatureDisplayThresholds(config, 'sata')).toBeNull(); + expect(resolveDiskTemperatureDisplayThresholds(config, '')).toBeNull(); + // An explicit host override still applies, as CheckHost applies it. + expect(resolveDiskTemperatureDisplayThresholds(config, 'nvme', 'host-1')).toEqual({ + warning: 75, + critical: 80, + }); + }); + it('falls back to seeded per-type disk temperature defaults without config', () => { expect(resolveDiskTemperatureDisplayThresholds(null, 'nvme')).toEqual({ warning: 65, diff --git a/frontend-modern/src/utils/__tests__/temperature.test.ts b/frontend-modern/src/utils/__tests__/temperature.test.ts index 5224735935..6f601bbf0a 100644 --- a/frontend-modern/src/utils/__tests__/temperature.test.ts +++ b/frontend-modern/src/utils/__tests__/temperature.test.ts @@ -130,6 +130,12 @@ describe('temperature', () => { expect(getTemperatureTextClass(80)).toContain('text-red-600'); }); + it('colours no disk reading whose disk temperature alerting is off', () => { + expect(getTemperatureTextClass(80, null, 'diskTemperature')).toBe('text-base-content'); + // Node temperatures keep the factory colours without configured thresholds. + expect(getTemperatureTextClass(80, null, 'temperature')).toContain('text-red-600'); + }); + it('honors configured warning and critical thresholds', () => { const thresholds = { warning: 80, critical: 85 }; diff --git a/frontend-modern/src/utils/metricThresholds.ts b/frontend-modern/src/utils/metricThresholds.ts index 3cc016b000..ee533c19ae 100644 --- a/frontend-modern/src/utils/metricThresholds.ts +++ b/frontend-modern/src/utils/metricThresholds.ts @@ -269,7 +269,9 @@ export const resolveMetricDisplayThresholds = ( * Resolve display thresholds for a physical disk's SMART temperature. * Mirrors the backend precedence: an explicit diskTemperature override on the * host (or inherited linked resource) wins, then the per-type map - * (diskTempByType: nvme/sas/sata), then the global agent default. + * (diskTempByType: nvme/sas/sata), then the global agent default. A + * switched-off agent default switches every type off, as disk temperature + * alerts do. Null means disk temperature alerting is off for this disk. */ export const resolveDiskTemperatureDisplayThresholds = ( config: AlertConfig | null, @@ -285,6 +287,11 @@ export const resolveDiskTemperatureDisplayThresholds = ( return resolveThreshold(overrideValue, FACTORY_AGENT_DEFAULTS.diskTemperature, margin); } + const baseValue = getBaseThresholdValue(config?.agentDefaults, 'diskTemperature'); + if (baseValue !== undefined && resolveThreshold(baseValue, undefined, margin) === null) { + return null; + } + const byTypeFallback = normalizedType ? FACTORY_DISK_TEMP_BY_TYPE[normalizedType] : undefined; if (normalizedType) { const byType = config?.diskTempByType?.[normalizedType]; @@ -293,7 +300,6 @@ export const resolveDiskTemperatureDisplayThresholds = ( } } - const baseValue = getBaseThresholdValue(config?.agentDefaults, 'diskTemperature'); return resolveThreshold( baseValue, byTypeFallback ?? FACTORY_AGENT_DEFAULTS.diskTemperature, diff --git a/frontend-modern/src/utils/temperature.ts b/frontend-modern/src/utils/temperature.ts index 15f608fbac..6a033e47bd 100644 --- a/frontend-modern/src/utils/temperature.ts +++ b/frontend-modern/src/utils/temperature.ts @@ -123,6 +123,11 @@ export const getTemperatureTextClass = ( if (celsius === null || celsius === undefined || !Number.isFinite(celsius)) { return 'text-muted'; } + // Null disk thresholds mean disk temperature alerting is off for that disk, + // so no reading of it is warm or hot. + if (metric === 'diskTemperature' && thresholds === null) { + return 'text-base-content'; + } const readingSeverity = getMetricSeverity( celsius, metric, diff --git a/internal/ai/alert_threshold_adapter.go b/internal/ai/alert_threshold_adapter.go index c10e5e3960..8509f06899 100644 --- a/internal/ai/alert_threshold_adapter.go +++ b/internal/ai/alert_threshold_adapter.go @@ -68,6 +68,22 @@ func (a *AlertThresholdAdapter) GetGuestDiskThreshold() float64 { }, 90) } +// GetDiskTemperatureThreshold returns the disk temperature alert trigger and +// clear values for a disk type, resolved exactly as disk temperature alerts +// resolve them (alerts.Manager.DiskTemperatureThreshold). +func (a *AlertThresholdAdapter) GetDiskTemperatureThreshold(diskType string) (float64, float64) { + var threshold *alerts.HysteresisThreshold + if a.manager != nil { + threshold = a.manager.DiskTemperatureThreshold(diskType) + } else { + threshold = alerts.DefaultDiskTemperatureThreshold(diskType) + } + if threshold == nil || threshold.Trigger <= 0 { + return 0, 0 + } + return threshold.Trigger, threshold.Clear +} + // GetStorageThreshold returns the usage alert trigger threshold for storage (0-100%) func (a *AlertThresholdAdapter) GetStorageThreshold() float64 { return a.getThreshold(func(cfg alerts.AlertConfig) (float64, bool) { diff --git a/internal/ai/alert_threshold_adapter_test.go b/internal/ai/alert_threshold_adapter_test.go index 99a40bd800..f65840ecbd 100644 --- a/internal/ai/alert_threshold_adapter_test.go +++ b/internal/ai/alert_threshold_adapter_test.go @@ -82,3 +82,28 @@ func TestAlertThresholdAdapter_Fallbacks(t *testing.T) { t.Fatalf("GetStorageThreshold default = %v", a.GetStorageThreshold()) } } + +func TestAlertThresholdAdapter_DiskTemperatureFollowsAlertPolicy(t *testing.T) { + if trigger, clear := NewAlertThresholdAdapter(nil).GetDiskTemperatureThreshold("nvme"); trigger != 70 || clear != 65 { + t.Fatalf("no manager: nvme = %v/%v, want the factory 70/65", trigger, clear) + } + + mgr := alerts.NewManager() + cfg := mgr.GetConfig() + cfg.DiskTempByType["nvme"] = alerts.HysteresisThreshold{Trigger: 75, Clear: 70} + mgr.UpdateConfig(cfg) + a := NewAlertThresholdAdapter(mgr) + if trigger, clear := a.GetDiskTemperatureThreshold("nvme"); trigger != 75 || clear != 70 { + t.Fatalf("raised nvme = %v/%v, want 75/70", trigger, clear) + } + if trigger, clear := a.GetDiskTemperatureThreshold("sata"); trigger != 55 || clear != 50 { + t.Fatalf("sata = %v/%v, want 55/50", trigger, clear) + } + + cfg = mgr.GetConfig() + cfg.AgentDefaults.DiskTemperature = &alerts.HysteresisThreshold{Trigger: 0, Clear: 0} + mgr.UpdateConfig(cfg) + if trigger, clear := a.GetDiskTemperatureThreshold("nvme"); trigger != 0 || clear != 0 { + t.Fatalf("disk temperature alerting off: nvme = %v/%v, want 0/0", trigger, clear) + } +} diff --git a/internal/ai/mock_test.go b/internal/ai/mock_test.go index ad2c2836bd..91524141ab 100644 --- a/internal/ai/mock_test.go +++ b/internal/ai/mock_test.go @@ -5,6 +5,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/agentexec" "github.com/rcourtman/pulse-go-rewrite/internal/ai/providers" + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" ) @@ -51,6 +52,9 @@ type mockThresholdProvider struct { guestMem float64 guestDisk float64 storage float64 + // diskTemperature holds trigger/clear by disk type; a type it does not + // list resolves to the factory disk temperature policy. + diskTemperature map[string][2]float64 } func (m *mockThresholdProvider) GetNodeCPUThreshold() float64 { return m.nodeCPU } @@ -59,6 +63,15 @@ func (m *mockThresholdProvider) GetGuestCPUThreshold() float64 { return 0 } func (m *mockThresholdProvider) GetGuestMemoryThreshold() float64 { return m.guestMem } func (m *mockThresholdProvider) GetGuestDiskThreshold() float64 { return m.guestDisk } func (m *mockThresholdProvider) GetStorageThreshold() float64 { return m.storage } +func (m *mockThresholdProvider) GetDiskTemperatureThreshold(diskType string) (float64, float64) { + if limits, ok := m.diskTemperature[diskType]; ok { + return limits[0], limits[1] + } + if threshold := alerts.DefaultDiskTemperatureThreshold(diskType); threshold != nil { + return threshold.Trigger, threshold.Clear + } + return 0, 0 +} type mockUnifiedResourceProvider struct { UnifiedResourceProvider diff --git a/internal/ai/patrol.go b/internal/ai/patrol.go index eb93e20b73..a630554703 100644 --- a/internal/ai/patrol.go +++ b/internal/ai/patrol.go @@ -66,6 +66,10 @@ type ThresholdProvider interface { GetGuestDiskThreshold() float64 // GetStorageThreshold returns the usage alert trigger threshold for storage (0-100%) GetStorageThreshold() float64 + // GetDiskTemperatureThreshold returns the disk temperature alert trigger + // and clear values (°C) for a disk type. A zero trigger means disk + // temperature alerting is off. + GetDiskTemperatureThreshold(diskType string) (trigger, clear float64) } // AlertResolver provides the ability to review and resolve alerts diff --git a/internal/ai/patrol_ai.go b/internal/ai/patrol_ai.go index 8757ac5d0f..a14000ab2e 100644 --- a/internal/ai/patrol_ai.go +++ b/internal/ai/patrol_ai.go @@ -1930,8 +1930,9 @@ type patrolPhysicalDiskRow struct { wearout int // temperature.Collected is the only temperature Patrol may judge as // heat; a retained last-known value is shown as context. - temperature tools.DiskTemperature - smartEvidence []string + temperature tools.DiskTemperature + temperatureLimits diskTemperatureLimits + smartEvidence []string } type patrolPrecomputeNodeSource struct { @@ -2464,7 +2465,7 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool) health = "UNKNOWN" } - rows = append(rows, patrolPhysicalDiskRow{ + row := patrolPhysicalDiskRow{ id: r.ID, name: name, node: strings.TrimSpace(r.ParentName), @@ -2477,7 +2478,9 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool) wearout: r.PhysicalDisk.Wearout, temperature: tools.SplitDiskTemperature(r.PhysicalDisk.Temperature, r.PhysicalDisk.Collection), smartEvidence: unifiedPhysicalDiskSMARTIssueParts(r.PhysicalDisk.SMART), - }) + } + row.temperatureLimits = snap.diskTemperatureLimits(row.diskType) + rows = append(rows, row) } return rows } @@ -2505,7 +2508,7 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool) health = "UNKNOWN" } - rows = append(rows, patrolPhysicalDiskRow{ + row := patrolPhysicalDiskRow{ id: d.ID, name: name, node: strings.TrimSpace(d.Node), @@ -2518,7 +2521,9 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool) wearout: d.Wearout, temperature: tools.SplitDiskTemperature(d.Temperature, d.Collection), smartEvidence: modelPhysicalDiskSMARTIssueParts(d.SmartAttributes), - }) + } + row.temperatureLimits = snap.diskTemperatureLimits(row.diskType) + rows = append(rows, row) } return rows } @@ -2534,7 +2539,7 @@ func patrolPhysicalDiskHealthIssue(row patrolPhysicalDiskRow) bool { (row.wearout == 0 && (strings.EqualFold(row.diskType, "ssd") || strings.EqualFold(row.diskType, "nvme")))) && row.wearout < 20) || - row.temperature.Collected > 55 || + row.temperatureLimits.hot(row.temperature.Collected) || len(row.smartEvidence) > 0 } diff --git a/internal/ai/patrol_disk_temperature.go b/internal/ai/patrol_disk_temperature.go new file mode 100644 index 0000000000..53e58fabe3 --- /dev/null +++ b/internal/ai/patrol_disk_temperature.go @@ -0,0 +1,50 @@ +package ai + +import "github.com/rcourtman/pulse-go-rewrite/internal/alerts" + +// diskTemperatureLimits is the alert disk temperature policy for one disk +// type, the same policy disk temperature alerts and the Physical Disks Health +// verdict judge heat by. A disk is hot at the trigger and stays hot until it +// cools to the clear value. A zero trigger means disk temperature alerting is +// off, so no reading counts as heat. +type diskTemperatureLimits struct { + trigger float64 + clear float64 +} + +// diskTemperatureLimitsFor resolves the policy from the user's alert +// configuration, or from the factory alert configuration when Patrol has no +// threshold provider. +func diskTemperatureLimitsFor(provider ThresholdProvider, diskType string) diskTemperatureLimits { + var trigger, clear float64 + if provider != nil { + trigger, clear = provider.GetDiskTemperatureThreshold(diskType) + } else if threshold := alerts.DefaultDiskTemperatureThreshold(diskType); threshold != nil { + trigger, clear = threshold.Trigger, threshold.Clear + } + if trigger <= 0 { + return diskTemperatureLimits{} + } + if clear <= 0 || clear > trigger { + clear = trigger + } + return diskTemperatureLimits{trigger: trigger, clear: clear} +} + +// hot reports a reading at or above the alert trigger. +func (l diskTemperatureLimits) hot(temperature int) bool { + return l.trigger > 0 && temperature > 0 && float64(temperature) >= l.trigger +} + +// cooled reports a reading at or below the clear value, where a disk +// temperature alert recovers. With no band below the trigger, a reading at +// the trigger still fires, so only a reading under it has cooled. +func (l diskTemperatureLimits) cooled(temperature int) bool { + if l.trigger <= 0 || temperature <= 0 { + return true + } + if l.clear >= l.trigger { + return float64(temperature) < l.trigger + } + return float64(temperature) <= l.clear +} diff --git a/internal/ai/patrol_disk_temperature_test.go b/internal/ai/patrol_disk_temperature_test.go new file mode 100644 index 0000000000..e38358380e --- /dev/null +++ b/internal/ai/patrol_disk_temperature_test.go @@ -0,0 +1,228 @@ +package ai + +import ( + "strings" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +// raisedNVMeThresholds is a user who raised the NVMe disk temperature trigger +// to 75C (clear 70) in Alerts and left every other type at its default. +func raisedNVMeThresholds() *mockThresholdProvider { + return &mockThresholdProvider{diskTemperature: map[string][2]float64{"nvme": {75, 70}}} +} + +func TestDiskTemperatureLimitsFollowTheAlertPolicy(t *testing.T) { + for _, tc := range []struct { + name string + provider ThresholdProvider + diskType string + want diskTemperatureLimits + }{ + {name: "factory nvme", diskType: "nvme", want: diskTemperatureLimits{trigger: 70, clear: 65}}, + {name: "factory sata", diskType: "sata", want: diskTemperatureLimits{trigger: 55, clear: 50}}, + {name: "factory untyped", diskType: "", want: diskTemperatureLimits{trigger: 55, clear: 50}}, + {name: "raised nvme", provider: raisedNVMeThresholds(), diskType: "nvme", want: diskTemperatureLimits{trigger: 75, clear: 70}}, + {name: "raised nvme leaves sata", provider: raisedNVMeThresholds(), diskType: "sata", want: diskTemperatureLimits{trigger: 55, clear: 50}}, + {name: "alerting off", provider: &mockThresholdProvider{diskTemperature: map[string][2]float64{"sata": {0, 0}}}, diskType: "sata", want: diskTemperatureLimits{}}, + {name: "clear above trigger", provider: &mockThresholdProvider{diskTemperature: map[string][2]float64{"sas": {60, 64}}}, diskType: "sas", want: diskTemperatureLimits{trigger: 60, clear: 60}}, + } { + if got := diskTemperatureLimitsFor(tc.provider, tc.diskType); got != tc.want { + t.Errorf("%s: limits = %+v, want %+v", tc.name, got, tc.want) + } + } + + off := diskTemperatureLimits{} + if off.hot(95) || !off.cooled(95) { + t.Fatal("with disk temperature alerting off no reading is heat") + } +} + +func diskTemperatureTriageState(provider ThresholdProvider) patrolRuntimeState { + state := newPatrolRuntimeState(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{ + {ID: "nvme-63", DevPath: "/dev/nvme0n1", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 63}, + {ID: "nvme-66", DevPath: "/dev/nvme1n1", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 66}, + {ID: "nvme-72", DevPath: "/dev/nvme2n1", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 72}, + {ID: "sata-52", DevPath: "/dev/sda", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 52}, + {ID: "sata-56", DevPath: "/dev/sdb", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 56}, + }}) + state.thresholdProvider = provider + return state +} + +// Patrol judges disk heat by the alert disk temperature policy: a warning from +// the per-type trigger and nothing below it, the band under the trigger being +// a Temp cell colour rather than a finding. An NVMe at 63C is not hot; a SATA +// disk at 56C is. +func TestTriageDiskTemperatureFollowsTheAlertPolicy(t *testing.T) { + temperatureFlag := func(flags []TriageFlag, id string) *TriageFlag { + return triageFindFlag(flags, func(f TriageFlag) bool { + return f.ResourceID == id && strings.Contains(f.Reason, "Disk temperature") + }) + } + expect := func(flags []TriageFlag, id, severity string, threshold float64) { + t.Helper() + flag := temperatureFlag(flags, id) + if severity == "" { + if flag != nil { + t.Errorf("%s: unexpected temperature flag %+v", id, *flag) + } + return + } + if flag == nil || flag.Severity != severity || flag.Threshold != threshold { + t.Errorf("%s: temperature flag = %+v, want %s at threshold %.0f", id, flag, severity, threshold) + } + } + + flags := triageDiskHealthChecksState(diskTemperatureTriageState(nil), nil) + expect(flags, "nvme-63", "", 0) + expect(flags, "nvme-66", "", 0) + expect(flags, "nvme-72", "warning", 70) + expect(flags, "sata-52", "", 0) + expect(flags, "sata-56", "warning", 55) + if flag := temperatureFlag(flags, "sata-56"); flag == nil || flag.Reason != "Disk temperature 56°C (alert threshold: 55°C)" { + t.Errorf("sata-56 reason = %+v, want the reading and the alert trigger", flag) + } + + flags = triageDiskHealthChecksState(diskTemperatureTriageState(raisedNVMeThresholds()), nil) + expect(flags, "nvme-72", "", 0) + expect(flags, "sata-56", "warning", 55) + + off := &mockThresholdProvider{diskTemperature: map[string][2]float64{"nvme": {0, 0}, "sata": {0, 0}}} + if flags := triageDiskHealthChecksState(diskTemperatureTriageState(off), nil); len(flags) != 0 { + t.Fatalf("disk temperature alerting off should raise no heat flags, got %+v", flags) + } +} + +// The seed's disk issue list and finding verification share the policy: a +// disk is an issue from the trigger and recovers below the clear value. +func TestPatrolDiskHeatIssueAndRecoveryFollowTheAlertPolicy(t *testing.T) { + issues := map[string]bool{} + for _, row := range patrolPhysicalDiskRows(diskTemperatureTriageState(nil), nil) { + issues[row.id] = patrolPhysicalDiskHealthIssue(row) + } + for id, want := range map[string]bool{"nvme-63": false, "nvme-66": false, "nvme-72": true, "sata-52": false, "sata-56": true} { + if issues[id] != want { + t.Errorf("%s: health issue = %v, want %v", id, issues[id], want) + } + } + for _, row := range patrolPhysicalDiskRows(diskTemperatureTriageState(raisedNVMeThresholds()), nil) { + if row.id == "nvme-72" && patrolPhysicalDiskHealthIssue(row) { + t.Error("nvme-72 under a raised 75C trigger should not be an issue") + } + } + + state := diskTemperatureTriageState(nil) + for id, want := range map[string]bool{"nvme-63": true, "nvme-66": false, "nvme-72": false, "sata-52": false} { + recovered, err := verifyMetricRecoveredState(state, PatrolThresholds{}, "disk-high", id, "physical_disk") + if err != nil { + t.Fatalf("%s: verification error %v", id, err) + } + if recovered != want { + t.Errorf("%s: recovered = %v, want %v", id, recovered, want) + } + } +} + +func TestResourceContextListsDisksHotByTheAlertPolicy(t *testing.T) { + registry := unifiedresources.NewRegistry(nil) + now := time.Now() + registry.IngestSnapshot(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{ + {ID: "pve-nvme", Node: "pve", Instance: "lab", DevPath: "/dev/nvme0n1", Model: "Samsung 980", Serial: "N63", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 63, LastChecked: now}, + {ID: "pve-nvme-72", Node: "pve", Instance: "lab", DevPath: "/dev/nvme1n1", Model: "Samsung 990", Serial: "N72", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 72, LastChecked: now}, + {ID: "pve-sata", Node: "pve", Instance: "lab", DevPath: "/dev/sda", Model: "WDC WD80EFAX", Serial: "S56", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 56, LastChecked: now}, + }}) + + s := &Service{} + s.SetUnifiedResourceProvider(unifiedresources.NewUnifiedAIAdapter(registry)) + attention := func() string { + got := s.buildUnifiedResourceContext() + _, section, found := strings.Cut(got, "Physical Disks Needing Attention") + if !found { + return "" + } + section, _, _ = strings.Cut(section, "\n\n") + return section + } + + // Disk names are redacted in this context, so the readings identify them. + section := attention() + if !strings.Contains(section, "Temp: 56C") || !strings.Contains(section, "Temp: 72C") || strings.Contains(section, "Temp: 63C") { + t.Fatalf("factory policy: attention section = %q, want the SATA 56C and NVMe 72C disks only", section) + } + + s.SetPatrolThresholdProvider(raisedNVMeThresholds()) + section = attention() + if !strings.Contains(section, "Temp: 56C") || strings.Contains(section, "Temp: 72C") || strings.Contains(section, "Temp: 63C") { + t.Fatalf("raised NVMe trigger: attention section = %q, want only the SATA 56C disk", section) + } +} + +// A Patrol run judges disk heat by the thresholds the user configured, through +// both ways a run builds its state. +func TestPatrolRunStateCarriesTheConfiguredDiskTemperaturePolicy(t *testing.T) { + snapshot := models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{ + {ID: "nvme-72", DevPath: "/dev/nvme0n1", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 72}, + }} + ps := NewPatrolService(nil, &mockStateProvider{state: snapshot}) + if got := ps.currentPatrolRuntimeState().diskTemperatureLimits("nvme"); got.trigger != 70 { + t.Fatalf("no provider: nvme trigger = %v, want the factory 70", got.trigger) + } + + ps.SetThresholdProvider(raisedNVMeThresholds()) + for name, state := range map[string]patrolRuntimeState{ + "current": ps.currentPatrolRuntimeState(), + "snapshot": ps.patrolRuntimeStateForSnapshot(snapshot), + } { + if got := state.diskTemperatureLimits("nvme"); got.trigger != 75 || got.clear != 70 { + t.Errorf("%s state: nvme limits = %+v, want the configured 75/70", name, got) + } + if flags := triageDiskHealthChecksState(state, nil); len(flags) != 0 { + t.Errorf("%s state: nvme at 72C under a 75C trigger flagged %+v", name, flags) + } + } +} + +// A scoped Patrol run filters its state; the filter must keep the configured +// thresholds, or a scoped run would judge heat by the factory triggers. +func TestScopedPatrolStateKeepsTheConfiguredDiskTemperaturePolicy(t *testing.T) { + state := diskTemperatureTriageState(raisedNVMeThresholds()) + scoped := filterPatrolStateByScopeState(state, PatrolScope{ResourceIDs: []string{"nvme-72"}}) + if got := scoped.diskTemperatureLimits("nvme"); got.trigger != 75 || got.clear != 70 { + t.Fatalf("scoped state nvme limits = %+v, want the configured 75/70", got) + } + if flags := triageDiskHealthChecksState(scoped, nil); len(flags) != 0 { + t.Fatalf("scoped run flagged nvme at 72C under a 75C trigger: %+v", flags) + } +} + +// The alert threshold provider can arrive before Patrol exists; Patrol must +// still start with it. +func TestPatrolStartsWithAThresholdProviderSetBeforeIt(t *testing.T) { + s := NewService(nil, nil) + s.SetPatrolThresholdProvider(raisedNVMeThresholds()) + s.SetStateProvider(&mockStateProvider{}) + patrol := s.GetPatrolService() + if patrol == nil { + t.Fatal("expected Patrol to start once a state provider exists") + } + if got := patrol.currentPatrolRuntimeState().diskTemperatureLimits("nvme"); got.trigger != 75 { + t.Fatalf("Patrol nvme trigger = %v, want the provider's 75", got.trigger) + } +} + +// A heat finding recovers at the clear value, where its alert recovers. +func TestDiskHeatRecoversAtTheClearValue(t *testing.T) { + limits := diskTemperatureLimitsFor(nil, "nvme") + if !limits.cooled(65) || limits.cooled(66) { + t.Fatalf("nvme limits %+v: want 65C cooled and 66C still warm", limits) + } + // With no band below the trigger, a reading at the trigger is still hot. + noBand := diskTemperatureLimitsFor(&mockThresholdProvider{diskTemperature: map[string][2]float64{"sas": {70, 70}}}, "sas") + if !noBand.hot(70) || noBand.cooled(70) || !noBand.cooled(69) { + t.Fatalf("no-band limits %+v: want 70C hot and not cooled, 69C cooled", noBand) + } +} diff --git a/internal/ai/patrol_findings.go b/internal/ai/patrol_findings.go index d72a73f0e8..1bb5e87af0 100644 --- a/internal/ai/patrol_findings.go +++ b/internal/ai/patrol_findings.go @@ -2291,7 +2291,9 @@ func verifyMetricRecoveredState(snap patrolRuntimeState, thresholds PatrolThresh if disk.wearout >= 0 && disk.wearout < 20 { return false, nil } - if disk.temperature > 55 { + // A hot disk recovers once it cools to the alert clear + // value, as its disk temperature alert does. + if !disk.temperatureLimits.cooled(disk.temperature) { return false, nil } return true, nil @@ -2315,9 +2317,10 @@ func verifyMetricRecoveredState(snap patrolRuntimeState, thresholds PatrolThresh } type patrolPhysicalDiskVerification struct { - health string - wearout int - temperature int + health string + wearout int + temperature int + temperatureLimits diskTemperatureLimits } type patrolPhysicalDiskVisitor func(identifiers []string, verification patrolPhysicalDiskVerification) bool @@ -2461,9 +2464,10 @@ func patrolVisitPhysicalDiskVerification(snap patrolRuntimeState, visit patrolPh rows := patrolPhysicalDiskRows(snap, nil) for _, disk := range rows { if !visit([]string{disk.id, disk.name, disk.devPath, disk.model}, patrolPhysicalDiskVerification{ - health: strings.TrimSpace(disk.health), - wearout: disk.wearout, - temperature: disk.temperature.Collected, + health: strings.TrimSpace(disk.health), + wearout: disk.wearout, + temperature: disk.temperature.Collected, + temperatureLimits: disk.temperatureLimits, }) { return true } diff --git a/internal/ai/patrol_run.go b/internal/ai/patrol_run.go index 67c08b5e60..db7c3decc2 100644 --- a/internal/ai/patrol_run.go +++ b/internal/ai/patrol_run.go @@ -1464,6 +1464,7 @@ func newPatrolScopedFilterState(snap patrolRuntimeState) patrolScopedFilterState filtered: patrolRuntimeState{ readState: snap.readState, unifiedResourceProvider: snap.unifiedResourceProvider, + thresholdProvider: snap.thresholdProvider, }, includedResourceIDs: make(map[string]bool), includedGuestVMIDs: make(map[int]bool), diff --git a/internal/ai/patrol_state.go b/internal/ai/patrol_state.go index 822728ba0a..1d0d7e8493 100644 --- a/internal/ai/patrol_state.go +++ b/internal/ai/patrol_state.go @@ -89,6 +89,15 @@ type patrolRuntimeState struct { ConnectionHealth map[string]bool ActiveAlerts []models.Alert RecentlyResolved []models.ResolvedAlert + // thresholdProvider carries the user's alert thresholds into the run. Nil + // means the factory alert configuration. + thresholdProvider ThresholdProvider +} + +// diskTemperatureLimits resolves the alert disk temperature policy for one +// disk type. +func (s patrolRuntimeState) diskTemperatureLimits(diskType string) diskTemperatureLimits { + return diskTemperatureLimitsFor(s.thresholdProvider, diskType) } func newPatrolRuntimeState(snapshot models.StateSnapshot) patrolRuntimeState { @@ -171,11 +180,16 @@ func (p *PatrolService) currentPatrolRuntimeState() patrolRuntimeState { stateProvider := p.stateProvider readState := p.readState unifiedResourceProvider := p.unifiedResourceProvider + thresholdProvider := p.thresholdProvider p.mu.RUnlock() + var state patrolRuntimeState if stateProvider == nil { - return emptyPatrolRuntimeState(readState, unifiedResourceProvider) + state = emptyPatrolRuntimeState(readState, unifiedResourceProvider) + } else { + state = newPatrolRuntimeStateWithProviders(stateProvider.ReadSnapshot(), readState, unifiedResourceProvider) } - return newPatrolRuntimeStateWithProviders(stateProvider.ReadSnapshot(), readState, unifiedResourceProvider) + state.thresholdProvider = thresholdProvider + return state } func (p *PatrolService) hasPatrolRuntimeInputs() bool { @@ -194,8 +208,11 @@ func (p *PatrolService) patrolRuntimeStateForSnapshot(snapshot models.StateSnaps p.mu.RLock() readState := p.readState unifiedResourceProvider := p.unifiedResourceProvider + thresholdProvider := p.thresholdProvider p.mu.RUnlock() - return newPatrolRuntimeStateWithProviders(snapshot, readState, unifiedResourceProvider) + state := newPatrolRuntimeStateWithProviders(snapshot, readState, unifiedResourceProvider) + state.thresholdProvider = thresholdProvider + return state } func patrolVisitRuntimeResources(s patrolRuntimeState, visit func(patrolRuntimeResourceRecord) bool) { diff --git a/internal/ai/patrol_thresholds_test.go b/internal/ai/patrol_thresholds_test.go index fc92810a6f..53ad2439db 100644 --- a/internal/ai/patrol_thresholds_test.go +++ b/internal/ai/patrol_thresholds_test.go @@ -20,6 +20,9 @@ func (m MockThresholdProvider) GetNodeMemoryThreshold() float64 { return m.Node func (m MockThresholdProvider) GetGuestMemoryThreshold() float64 { return m.GuestMem } func (m MockThresholdProvider) GetGuestDiskThreshold() float64 { return m.GuestDisk } func (m MockThresholdProvider) GetStorageThreshold() float64 { return m.Storage } +func (m MockThresholdProvider) GetDiskTemperatureThreshold(string) (float64, float64) { + return 0, 0 +} func TestCalculatePatrolThresholds_Default(t *testing.T) { // Test default behavior (exact mode) diff --git a/internal/ai/patrol_triage.go b/internal/ai/patrol_triage.go index f3d3d635e2..04822aa4f6 100644 --- a/internal/ai/patrol_triage.go +++ b/internal/ai/patrol_triage.go @@ -345,13 +345,16 @@ func triageDiskHealthChecksState(snap patrolRuntimeState, scopedSet map[string]b flags := make([]TriageFlag, 0) for _, disk := range patrolPhysicalDiskRows(snap, scopedSet) { - flags = append(flags, triagePhysicalDiskFlags(disk.id, disk.name, disk.health, disk.wearout, disk.temperature.Collected)...) + flags = append(flags, triagePhysicalDiskFlags(disk.id, disk.name, disk.health, disk.wearout, disk.temperature.Collected, disk.temperatureLimits)...) } return flags } -func triagePhysicalDiskFlags(resourceID, resourceName, health string, wearout, temperature int) []TriageFlag { +// triagePhysicalDiskFlags judges heat by the alert disk temperature policy: a +// warning from the alert trigger, the reading the disk's temperature alert and +// its Running Hot verdict start at. +func triagePhysicalDiskFlags(resourceID, resourceName, health string, wearout, temperature int, temperatureLimits diskTemperatureLimits) []TriageFlag { flags := make([]TriageFlag, 0, 3) if health != "" && !strings.EqualFold(health, "PASSED") && !strings.EqualFold(health, "UNKNOWN") && !strings.EqualFold(health, "OK") { flags = append(flags, TriageFlag{ @@ -377,17 +380,17 @@ func triagePhysicalDiskFlags(resourceID, resourceName, health string, wearout, t Threshold: 20, }) } - if temperature > 55 { + if temperatureLimits.hot(temperature) { flags = append(flags, TriageFlag{ ResourceID: resourceID, ResourceName: resourceName, ResourceType: "physical_disk", Category: "health", Severity: "warning", - Reason: fmt.Sprintf("Disk temperature %d°C", temperature), + Reason: fmt.Sprintf("Disk temperature %d°C (alert threshold: %.0f°C)", temperature, temperatureLimits.trigger), Metric: "disk", Value: float64(temperature), - Threshold: 55, + Threshold: temperatureLimits.trigger, }) } return flags diff --git a/internal/ai/resource_context.go b/internal/ai/resource_context.go index 1702ac0a05..31218ace81 100644 --- a/internal/ai/resource_context.go +++ b/internal/ai/resource_context.go @@ -52,6 +52,7 @@ func (s *Service) buildUnifiedResourceContextForModel(destinationModel string) s urp := s.unifiedResourceProvider ap := s.alertProvider agentServer := s.agentServer + thresholdProvider := s.thresholdProvider s.mu.RUnlock() if urp != nil { @@ -410,13 +411,14 @@ func (s *Service) buildUnifiedResourceContextForModel(destinationModel string) s attention := make([]unifiedresources.Resource, 0) for _, disk := range physicalDisks { health := "" - temperature := 0 + hot := false if disk.PhysicalDisk != nil { health = strings.ToUpper(strings.TrimSpace(disk.PhysicalDisk.Health)) // A retained last-known reading is not current heat. - temperature = tools.SplitDiskTemperature(disk.PhysicalDisk.Temperature, disk.PhysicalDisk.Collection).Collected + collected := tools.SplitDiskTemperature(disk.PhysicalDisk.Temperature, disk.PhysicalDisk.Collection).Collected + hot = diskTemperatureLimitsFor(thresholdProvider, disk.PhysicalDisk.DiskType).hot(collected) } - if disk.Status != unifiedresources.StatusOnline || (health != "" && health != "PASSED" && health != "UNKNOWN") || temperature >= 50 { + if disk.Status != unifiedresources.StatusOnline || (health != "" && health != "PASSED" && health != "UNKNOWN") || hot { attention = append(attention, disk) } } diff --git a/internal/ai/service.go b/internal/ai/service.go index ddd974b733..9b19fd6cb4 100644 --- a/internal/ai/service.go +++ b/internal/ai/service.go @@ -261,6 +261,7 @@ type Service struct { patrolAutopilotPolicy func() unifiedresources.PatrolAutopilotServerPolicy patrolObjectiveStore *PatrolObjectiveStore patrolService *PatrolService // Background AI monitoring service + thresholdProvider ThresholdProvider // User-configured alert thresholds; nil means factory defaults metadataProvider MetadataProvider // Enables AI to update resource URLs incidentStore *memory.IncidentStore // Alert-scoped investigation memory; not the canonical durable resource history chatService ChatServiceProvider // Chat service for investigation orchestrator @@ -584,6 +585,9 @@ func (s *Service) initPatrolServiceLocked() { s.patrolService = NewPatrolService(s, s.stateProvider) s.patrolService.SetObjectiveStore(s.patrolObjectiveStore) + if s.thresholdProvider != nil { + s.patrolService.SetThresholdProvider(s.thresholdProvider) + } if s.knowledgeStore != nil { s.patrolService.SetKnowledgeStore(s.knowledgeStore) } @@ -831,9 +835,10 @@ func (s *Service) ClearCostHistory() error { // SetPatrolThresholdProvider sets the threshold provider for patrol // This should be called with an AlertThresholdAdapter to connect patrol to user-configured thresholds func (s *Service) SetPatrolThresholdProvider(provider ThresholdProvider) { - s.mu.RLock() + s.mu.Lock() + s.thresholdProvider = provider patrol := s.patrolService - s.mu.RUnlock() + s.mu.Unlock() if patrol != nil { patrol.SetThresholdProvider(provider) diff --git a/internal/alerts/host.go b/internal/alerts/host.go index 157cc4c5bb..9347925507 100644 --- a/internal/alerts/host.go +++ b/internal/alerts/host.go @@ -990,21 +990,50 @@ func hostDiskTemperatureResourceID(hostID, device string) string { } // hostDiskTemperatureThresholdNoLock returns the threshold for one SMART disk -// given the host's resolved disk temperature threshold. An enabled threshold -// gives way to the disk type's DiskTempByType entry unless an explicit host or -// linked-resource override set it. Callers must hold m.mu. +// given the host's resolved disk temperature threshold. Callers must hold m.mu. func (m *Manager) hostDiskTemperatureThresholdNoLock(hostThreshold *HysteresisThreshold, overridden bool, diskType string) *HysteresisThreshold { + return diskTemperatureThresholdForType(m.config.DiskTempByType, hostThreshold, overridden, diskType) +} + +// diskTemperatureThresholdForType is the disk temperature policy. An enabled +// host threshold gives way to the disk type's DiskTempByType entry unless an +// explicit host or linked-resource override set it; a disabled one switches +// disk temperature alerting off for every type. +func diskTemperatureThresholdForType(byType map[string]HysteresisThreshold, hostThreshold *HysteresisThreshold, overridden bool, diskType string) *HysteresisThreshold { if hostThreshold == nil || hostThreshold.Trigger <= 0 || overridden { return hostThreshold } if diskType = strings.ToLower(strings.TrimSpace(diskType)); diskType != "" { - if th, ok := m.config.DiskTempByType[diskType]; ok { + if th, ok := byType[diskType]; ok { return &th } } return hostThreshold } +// DiskTemperatureThreshold returns the threshold that judges a physical disk's +// temperature when no host override applies: the disk type's DiskTempByType +// entry, else the agent default. This is the canonical disk heat policy. Disk +// temperature alerts, Patrol and the Physical Disks Health verdict all judge +// heat against it; disk risk does not. The disk is hot at the trigger, and the +// clear value is where it stops being hot. A nil threshold or a non-positive +// trigger means disk temperature alerting is off. +func (m *Manager) DiskTemperatureThreshold(diskType string) *HysteresisThreshold { + if m == nil { + return DefaultDiskTemperatureThreshold(diskType) + } + m.mu.RLock() + defer m.mu.RUnlock() + return cloneThreshold(m.hostDiskTemperatureThresholdNoLock(m.config.AgentDefaults.DiskTemperature, false, diskType)) +} + +// DefaultDiskTemperatureThreshold is DiskTemperatureThreshold under the +// factory alert configuration, for callers with no alert manager. +func DefaultDiskTemperatureThreshold(diskType string) *HysteresisThreshold { + config := defaultAlertConfig() + return cloneThreshold(diskTemperatureThresholdForType(config.DiskTempByType, config.AgentDefaults.DiskTemperature, false, diskType)) +} + // lowestHostDiskTemperatureThresholdNoLock returns the lowest enabled // threshold any SMART disk of the host can be evaluated against. An alert // whose disk type is unknown is judged against it, so a config save never @@ -1547,8 +1576,6 @@ func splitSMARTAlertReasons(reasons []storagehealth.Reason) ([]storagehealth.Rea switch reason.Code { case "wearout_low", "nvme_available_spare_low", "nvme_percentage_used_high": wearReasons = append(wearReasons, reason) - case "temperature_high": - continue default: healthReasons = append(healthReasons, reason) } diff --git a/internal/alerts/threshold_resolution_shared_test.go b/internal/alerts/threshold_resolution_shared_test.go index bdffd75d22..673a1bab50 100644 --- a/internal/alerts/threshold_resolution_shared_test.go +++ b/internal/alerts/threshold_resolution_shared_test.go @@ -1387,3 +1387,61 @@ func TestCheckMetricResolveRemovesCanonicallyKeyedAlert(t *testing.T) { t.Fatal("expected a recently-resolved entry for the cleared alert") } } + +// DiskTemperatureThreshold is the policy Patrol and the Physical Disks Health +// verdict judge disk heat by, so it must resolve every disk type exactly as +// CheckHost does: the per-type trigger the user set, else the agent default, +// and nothing at all once the agent default is switched off. +func TestDiskTemperatureThresholdMatchesCheckHostPolicy(t *testing.T) { + m := configureDiskTempTypeHostManager(t) + + for diskType, want := range map[string]HysteresisThreshold{ + "nvme": {Trigger: 70, Clear: 65}, + " NVMe": {Trigger: 70, Clear: 65}, + "sas": {Trigger: 65, Clear: 60}, + "sata": {Trigger: 55, Clear: 50}, + "hdd": {Trigger: 55, Clear: 50}, + "": {Trigger: 55, Clear: 50}, + } { + if got := m.DiskTemperatureThreshold(diskType); got == nil || *got != want { + t.Errorf("DiskTemperatureThreshold(%q) = %+v, want %+v", diskType, got, want) + } + if got := DefaultDiskTemperatureThreshold(diskType); got == nil || *got != want { + t.Errorf("DefaultDiskTemperatureThreshold(%q) = %+v, want the factory %+v", diskType, got, want) + } + } + + // A user who raised the NVMe trigger to 75 gets no alert at 72C, and the + // policy says the same disk is not hot. + m.mu.Lock() + m.config.DiskTempByType["nvme"] = HysteresisThreshold{Trigger: 75, Clear: 70} + m.mu.Unlock() + if got := m.DiskTemperatureThreshold("nvme"); got == nil || got.Trigger != 75 || got.Clear != 70 { + t.Fatalf("raised nvme threshold = %+v, want 75/70", got) + } + host := hostWithSMARTDiskTemp("host-temp-raised-nvme", "nvme", 72) + m.CheckHost(host) + if _, exists := testLookupActiveAlert(t, m, hostDiskTempAlertID(host)); exists { + t.Fatalf("expected no alert for nvme at 72C under a raised 75C trigger, active: %v", alertKeys(m)) + } + + // The returned threshold is a copy. + m.DiskTemperatureThreshold("nvme").Trigger = 1 + if got := m.DiskTemperatureThreshold("nvme"); got.Trigger != 75 { + t.Fatalf("caller mutation reached the config: %+v", got) + } + + m.mu.Lock() + m.config.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0, Clear: 0} + m.mu.Unlock() + for _, diskType := range []string{"nvme", "sata", "hdd"} { + if got := m.DiskTemperatureThreshold(diskType); got == nil || got.Trigger > 0 { + t.Errorf("DiskTemperatureThreshold(%q) with disk temperature alerting off = %+v, want a disabled threshold", diskType, got) + } + } + + var nilManager *Manager + if got := nilManager.DiskTemperatureThreshold("nvme"); got == nil || got.Trigger != 70 { + t.Fatalf("nil manager threshold = %+v, want the factory nvme 70", got) + } +} diff --git a/internal/api/ai_handlers_concurrency_test.go b/internal/api/ai_handlers_concurrency_test.go index 7659b34fa0..ca04c61635 100644 --- a/internal/api/ai_handlers_concurrency_test.go +++ b/internal/api/ai_handlers_concurrency_test.go @@ -67,6 +67,9 @@ func (concurrencyThresholdProvider) GetGuestMemoryThreshold() float64 { } func (concurrencyThresholdProvider) GetGuestDiskThreshold() float64 { return 95 } func (concurrencyThresholdProvider) GetStorageThreshold() float64 { return 92 } +func (concurrencyThresholdProvider) GetDiskTemperatureThreshold(string) (float64, float64) { + return 70, 65 +} type concurrencyMetricsHistoryProvider struct{} diff --git a/internal/api/ai_handlers_more_test.go b/internal/api/ai_handlers_more_test.go index f1d8973d81..128483e0c1 100644 --- a/internal/api/ai_handlers_more_test.go +++ b/internal/api/ai_handlers_more_test.go @@ -154,6 +154,9 @@ func (stubThresholdProvider) GetGuestMemoryThreshold() float64 { } func (stubThresholdProvider) GetGuestDiskThreshold() float64 { return 95 } func (stubThresholdProvider) GetStorageThreshold() float64 { return 92 } +func (stubThresholdProvider) GetDiskTemperatureThreshold(string) (float64, float64) { + return 70, 65 +} type stubMetricsHistoryProvider struct{} diff --git a/internal/storagehealth/risk.go b/internal/storagehealth/risk.go index de165bbb5b..990bb6143e 100644 --- a/internal/storagehealth/risk.go +++ b/internal/storagehealth/risk.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/rcourtman/pulse-go-rewrite/internal/models" - "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) type RiskLevel string @@ -31,7 +30,12 @@ type Assessment struct { // SMARTThresholds controls the discrete health evidence that becomes a disk // risk. Counter values at zero disable that rule; the endurance percentages -// use the same convention. Temperature remains a separate metric threshold. +// use the same convention. +// +// Disk risk never judges temperature. Heat is a metric judged by the alert +// disk temperature policy (alerts.Manager.DiskTemperatureThreshold), which +// users tune per disk type and per host. Risk is rebuilt by registries that +// cannot see that configuration, so a heat rule here would disagree with it. type SMARTThresholds struct { HealthFailure bool ReallocatedSectors int64 @@ -61,7 +65,6 @@ func DefaultSMARTThresholds() SMARTThresholds { type Sample struct { Model string Health string - Temperature int Wearout int WearoutKnown bool PowerOnHours int64 @@ -77,29 +80,10 @@ type Sample struct { UnsafeShutdowns int64 } -// CollectedTemperature returns a disk temperature only when its collection -// state marks it as observed now. Normalization may keep the last known value -// when the current observation is not available, such as a disk in standby or -// a host agent past its reporting lease. That retained value is history, not -// evidence that the disk is hot now, so risk must not judge it. A source that -// predates collection state counts as collected. -func CollectedTemperature(temperature int, collection *diskinventory.CollectionStatus) int { - if collection == nil { - return temperature - } - switch collection.Temperature.State { - case "", diskinventory.FieldAvailable: - return temperature - default: - return 0 - } -} - func AssessPhysicalDisk(disk models.PhysicalDisk) Assessment { sample := Sample{ Model: disk.Model, Health: disk.Health, - Temperature: CollectedTemperature(disk.Temperature, disk.Collection), Wearout: disk.Wearout, WearoutKnown: WearoutReported(disk.Wearout, disk.Type), } @@ -113,10 +97,9 @@ func AssessHostSMARTDisk(disk models.HostDiskSMART) Assessment { func AssessHostSMARTDiskWithThresholds(disk models.HostDiskSMART, thresholds SMARTThresholds) Assessment { sample := Sample{ - Model: disk.Model, - Health: disk.Health, - Temperature: CollectedTemperature(disk.Temperature, disk.Collection), - Wearout: -1, + Model: disk.Model, + Health: disk.Health, + Wearout: -1, } applySMARTAttributes(&sample, disk.Attributes) return AssessSampleWithThresholds(sample, thresholds) @@ -216,11 +199,6 @@ func AssessSampleWithThresholds(sample Sample, thresholds SMARTThresholds) Asses } else if percentageWarning { addReason("nvme_percentage_used_high", RiskWarning, fmt.Sprintf("NVMe endurance used is %d%%", sample.PercentageUsed)) } - if sample.Temperature >= 70 { - addReason("temperature_high", RiskCritical, fmt.Sprintf("Disk temperature is %dC", sample.Temperature)) - } else if sample.Temperature >= 60 { - addReason("temperature_high", RiskWarning, fmt.Sprintf("Disk temperature is %dC", sample.Temperature)) - } if thresholds.ReallocatedSectors > 0 && sample.ReallocatedSectors >= thresholds.ReallocatedSectors { addReason("reallocated_sectors", RiskWarning, fmt.Sprintf("Reallocated sectors detected (%d)", sample.ReallocatedSectors)) } diff --git a/internal/storagehealth/risk_branchcov0718_test.go b/internal/storagehealth/risk_branchcov0718_test.go index 52b5eb4165..65200993ae 100644 --- a/internal/storagehealth/risk_branchcov0718_test.go +++ b/internal/storagehealth/risk_branchcov0718_test.go @@ -33,7 +33,7 @@ func TestAssessPhysicalDisk_Branches(t *testing.T) { }{ { // nil SmartAttributes arm: every *int64 pointer is skipped; only - // Model/Health/Temperature/Wearout are sourced from the disk. + // Model/Health/Wearout are sourced from the disk. name: "healthy disk with nil smart attributes stays healthy", disk: models.PhysicalDisk{Model: "Crucial MX500", Health: "PASSED", Temperature: 35, Wearout: 80}, wantLevel: RiskHealthy, @@ -165,18 +165,10 @@ func TestAssessPhysicalDisk_Branches(t *testing.T) { wantReasons: []string{"nvme_percentage_used_high"}, }, { - // Temperature >= 70 -> critical. - name: "temperature critical", - disk: models.PhysicalDisk{Health: "PASSED", Temperature: 72}, - wantLevel: RiskCritical, - wantReasons: []string{"temperature_high"}, - }, - { - // Temperature 60..69 -> warning. - name: "temperature warning", - disk: models.PhysicalDisk{Health: "PASSED", Temperature: 63}, - wantLevel: RiskWarning, - wantReasons: []string{"temperature_high"}, + // Heat is the alert disk temperature policy's to judge, never risk's. + name: "hot disk stays healthy", + disk: models.PhysicalDisk{Health: "PASSED", Temperature: 72}, + wantLevel: RiskHealthy, }, { // Reallocated sectors > 0 via SMART -> warning. @@ -247,20 +239,20 @@ func TestAssessPhysicalDisk_NilSmartAttributesDoesNotPanic(t *testing.T) { // is preserved through the SMART-attribute mapping. func TestAssessPhysicalDisk_MultipleReasonsTakeHighest(t *testing.T) { got := AssessPhysicalDisk(models.PhysicalDisk{ - Health: "PASSED", - Temperature: 63, // warning + Health: "PASSED", SmartAttributes: &models.SMARTAttributes{ - PendingSectors: int64Ptr(1), // critical - UDMACRCErrors: int64Ptr(2), // monitor + ReallocatedSectors: int64Ptr(3), // warning + PendingSectors: int64Ptr(1), // critical + UDMACRCErrors: int64Ptr(2), // monitor }, }) if got.Level != RiskCritical { t.Fatalf("expected critical (highest), got %q; reasons=%+v", got.Level, got.Reasons) } if len(got.Reasons) != 3 { - t.Fatalf("expected 3 reasons (temperature_high, pending_sectors, crc_errors), got %d: %+v", len(got.Reasons), got.Reasons) + t.Fatalf("expected 3 reasons (reallocated_sectors, pending_sectors, crc_errors), got %d: %+v", len(got.Reasons), got.Reasons) } - for _, code := range []string{"pending_sectors", "temperature_high", "crc_errors"} { + for _, code := range []string{"pending_sectors", "reallocated_sectors", "crc_errors"} { if _, ok := reasonSeverity(got, code); !ok { t.Errorf("expected reason %q present, got %+v", code, got.Reasons) } @@ -282,7 +274,7 @@ func TestAssessHostSMARTDisk_Branches(t *testing.T) { wantReasons []string }{ { - // nil Attributes arm: only Model/Health/Temperature are sourced; + // nil Attributes arm: only Model/Health are sourced; // Wearout stays -1 so the wearout branch is skipped. name: "nil attributes stays healthy", disk: models.HostDiskSMART{Model: "WD Blue", Health: "PASSED", Temperature: 35}, @@ -402,18 +394,10 @@ func TestAssessHostSMARTDisk_Branches(t *testing.T) { wantReasons: []string{"nvme_available_spare_low"}, }, { - // Temperature >= 70 -> critical. - name: "temperature critical", - disk: models.HostDiskSMART{Health: "PASSED", Temperature: 75}, - wantLevel: RiskCritical, - wantReasons: []string{"temperature_high"}, - }, - { - // Temperature 60..69 -> warning. - name: "temperature warning", - disk: models.HostDiskSMART{Health: "PASSED", Temperature: 60}, - wantLevel: RiskWarning, - wantReasons: []string{"temperature_high"}, + // Heat is the alert disk temperature policy's to judge, never risk's. + name: "hot disk stays healthy", + disk: models.HostDiskSMART{Health: "PASSED", Temperature: 75}, + wantLevel: RiskHealthy, }, { // Reallocated sectors > 0 via host SMART -> warning. diff --git a/internal/storagehealth/risk_test.go b/internal/storagehealth/risk_test.go index e044189efd..508bfab988 100644 --- a/internal/storagehealth/risk_test.go +++ b/internal/storagehealth/risk_test.go @@ -11,9 +11,8 @@ import ( func TestAssessSample_HealthyDisk(t *testing.T) { assessment := AssessSample(Sample{ - Health: "PASSED", - Temperature: 35, - Wearout: 80, + Health: "PASSED", + Wearout: 80, }) if assessment.Level != RiskHealthy { t.Errorf("expected healthy, got %s", assessment.Level) @@ -308,38 +307,6 @@ func TestAssessSample_NVMePercentageUsedWarning(t *testing.T) { } } -func TestAssessSample_TemperatureCritical(t *testing.T) { - assessment := AssessSample(Sample{ - Health: "PASSED", - Temperature: 72, // >=70 critical - }) - if assessment.Level != RiskCritical { - t.Errorf("expected critical for temp=72, got %s", assessment.Level) - } -} - -func TestAssessSample_TemperatureWarning(t *testing.T) { - assessment := AssessSample(Sample{ - Health: "PASSED", - Temperature: 63, // >=60, <70 warning - }) - if assessment.Level != RiskWarning { - t.Errorf("expected warning for temp=63, got %s", assessment.Level) - } -} - -func TestAssessSample_TemperatureHealthy(t *testing.T) { - assessment := AssessSample(Sample{ - Health: "PASSED", - Temperature: 45, - }) - for _, r := range assessment.Reasons { - if r.Code == "temperature_high" { - t.Error("temp=45 should not trigger temperature_high") - } - } -} - func TestAssessSample_ReallocatedSectors(t *testing.T) { assessment := AssessSample(Sample{ Health: "PASSED", @@ -371,10 +338,10 @@ func TestAssessSample_CRCErrors(t *testing.T) { func TestAssessSample_MultipleIssuesTakesHighest(t *testing.T) { assessment := AssessSample(Sample{ - Health: "PASSED", - Temperature: 63, // warning - PendingSectors: 1, // critical - UDMACRCErrors: 10, // monitor + Health: "PASSED", + ReallocatedSectors: 1, // warning + PendingSectors: 1, // critical + UDMACRCErrors: 10, // monitor }) if assessment.Level != RiskCritical { t.Errorf("expected critical (highest), got %s", assessment.Level) @@ -387,10 +354,11 @@ func TestAssessSample_MultipleIssuesTakesHighest(t *testing.T) { func TestAssessSample_ReasonsSortedBySeverityDescending(t *testing.T) { assessment := AssessSample(Sample{ Health: "PASSED", - Temperature: 63, // warning PendingSectors: 1, // critical UDMACRCErrors: 10, // monitor ReallocatedSectors: 1, // warning + WearoutKnown: true, + Wearout: 8, // warning }) for i := 1; i < len(assessment.Reasons); i++ { prevRank := severityRank(assessment.Reasons[i-1].Severity) @@ -540,58 +508,20 @@ func TestAssessPhysicalDisk_RotationalZeroIsNotRisk(t *testing.T) { } } -func TestCollectedTemperatureKeepsOnlyCurrentReadings(t *testing.T) { - cases := []struct { - name string - collection *diskinventory.CollectionStatus - want int - }{ - {name: "no collection state", collection: nil, want: 72}, - {name: "empty state", collection: &diskinventory.CollectionStatus{}, want: 72}, - {name: "available", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}, want: 72}, - {name: "unavailable", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}, want: 0}, - {name: "unsupported", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "")}, want: 0}, - {name: "missing", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Missing("smartctl", "")}, want: 0}, - } - for _, tc := range cases { - if got := CollectedTemperature(72, tc.collection); got != tc.want { - t.Errorf("%s: CollectedTemperature = %d, want %d", tc.name, got, tc.want) - } - } -} - -func TestDiskAssessmentsIgnoreRetainedTemperature(t *testing.T) { - retained := &diskinventory.CollectionStatus{ - Temperature: diskinventory.Unavailable("host_agent", "host agent stopped reporting"), - } +// Heat belongs to the alert disk temperature policy, which users tune per disk +// type and per host. Disk risk cannot see that configuration, so it must not +// judge temperature at all, however hot and however current the reading. +func TestDiskAssessmentsNeverJudgeTemperature(t *testing.T) { collected := &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")} - - hasTemperatureReason := func(assessment Assessment) bool { - for _, reason := range assessment.Reasons { - if reason.Code == "temperature_high" { - return true - } + for _, temperature := range []int{56, 63, 72, 95} { + disk := models.PhysicalDisk{Model: "WDC WD80EFAX", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: temperature, Collection: collected} + if got := AssessPhysicalDisk(disk); got.Level != RiskHealthy || len(got.Reasons) != 0 { + t.Errorf("%dC physical disk = %+v, want healthy with no reasons", temperature, got) } - return false - } - disk := models.PhysicalDisk{Model: "WDC WD80EFAX", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 72} - disk.Collection = collected - if got := AssessPhysicalDisk(disk); got.Level != RiskCritical || !hasTemperatureReason(got) { - t.Fatalf("collected 72C physical disk = %+v, want critical temperature_high", got) - } - disk.Collection = retained - if got := AssessPhysicalDisk(disk); got.Level != RiskHealthy || hasTemperatureReason(got) { - t.Fatalf("retained 72C physical disk = %+v, want healthy", got) - } - - smart := models.HostDiskSMART{Device: "/dev/sda", Model: "WDC WD80EFAX", Type: "sata", Health: "PASSED", Temperature: 72} - smart.Collection = collected - if got := AssessHostSMARTDisk(smart); got.Level != RiskCritical || !hasTemperatureReason(got) { - t.Fatalf("collected 72C host SMART disk = %+v, want critical temperature_high", got) - } - smart.Collection = retained - if got := AssessHostSMARTDisk(smart); got.Level != RiskHealthy || hasTemperatureReason(got) { - t.Fatalf("retained 72C host SMART disk = %+v, want healthy", got) + smart := models.HostDiskSMART{Device: "/dev/nvme0n1", Model: "Samsung 990 PRO", Type: "nvme", Health: "PASSED", Temperature: temperature, Collection: collected} + if got := AssessHostSMARTDisk(smart); got.Level != RiskHealthy || len(got.Reasons) != 0 { + t.Errorf("%dC host SMART disk = %+v, want healthy with no reasons", temperature, got) + } } } diff --git a/internal/truenas/provider.go b/internal/truenas/provider.go index f24f0efe4b..c20af6a3be 100644 --- a/internal/truenas/provider.go +++ b/internal/truenas/provider.go @@ -1953,10 +1953,9 @@ func zfsPoolRecommendation(assessment storagehealth.Assessment) string { func assessDisk(disk Disk) storagehealth.Assessment { sampleAssessment := storagehealth.AssessSample(storagehealth.Sample{ - Model: strings.TrimSpace(disk.Model), - Health: healthForAssessment(disk), - Temperature: disk.Temperature, - Wearout: -1, + Model: strings.TrimSpace(disk.Model), + Health: healthForAssessment(disk), + Wearout: -1, }) stateUpper := normalizedDiskStatus(disk) diff --git a/internal/truenas/provider_test.go b/internal/truenas/provider_test.go index cd268a2660..34dc1b6451 100644 --- a/internal/truenas/provider_test.go +++ b/internal/truenas/provider_test.go @@ -745,8 +745,9 @@ func TestRecordsIncludeDiskResourcesWithCorrectParentChain(t *testing.T) { foundSmartReason = true } } - if !foundTemperatureReason { - t.Fatalf("expected sdc physical-disk risk to include temperature_high, got %+v", sdc.Resource.PhysicalDisk.Risk.Reasons) + // Heat is the alert disk temperature policy's to judge, not disk risk's. + if foundTemperatureReason { + t.Fatalf("expected sdc physical-disk risk to leave its 63C temperature to the alert policy, got %+v", sdc.Resource.PhysicalDisk.Risk.Reasons) } if !foundSmartReason { t.Fatalf("expected sdc physical-disk risk to include truenas_smart, got %+v", sdc.Resource.PhysicalDisk.Risk.Reasons) @@ -931,7 +932,9 @@ func TestRecordsIncludeTrueNASVMsAsCanonicalWorkloads(t *testing.T) { } } -func TestRecordsElevateOnlineDiskWhenTemperatureCritical(t *testing.T) { +// A hot disk stays online with no disk risk. Its temperature is judged by the +// alert disk temperature policy, which users tune per disk type. +func TestRecordsLeaveDiskHeatToTheAlertPolicy(t *testing.T) { previous := IsFeatureEnabled() SetFeatureEnabled(true) t.Cleanup(func() { @@ -967,14 +970,14 @@ func TestRecordsElevateOnlineDiskWhenTemperatureCritical(t *testing.T) { if diskRecord == nil { t.Fatal("expected physical disk record") } - if diskRecord.Resource.Status != unifiedresources.StatusWarning { - t.Fatalf("expected hot disk status warning, got %s", diskRecord.Resource.Status) + if diskRecord.Resource.Status != unifiedresources.StatusOnline { + t.Fatalf("expected hot disk status online, got %s", diskRecord.Resource.Status) } - if diskRecord.Resource.PhysicalDisk == nil || diskRecord.Resource.PhysicalDisk.Risk == nil { - t.Fatalf("expected hot disk physical risk, got %+v", diskRecord.Resource.PhysicalDisk) + if diskRecord.Resource.PhysicalDisk == nil || diskRecord.Resource.PhysicalDisk.Temperature != 72 { + t.Fatalf("expected the hot disk to keep its 72C reading, got %+v", diskRecord.Resource.PhysicalDisk) } - if diskRecord.Resource.PhysicalDisk.Risk.Level != storagehealth.RiskCritical { - t.Fatalf("expected hot disk critical risk, got %+v", diskRecord.Resource.PhysicalDisk.Risk) + if diskRecord.Resource.PhysicalDisk.Risk != nil { + t.Fatalf("expected no disk risk from temperature, got %+v", diskRecord.Resource.PhysicalDisk.Risk) } } diff --git a/internal/unifiedresources/adapter_coverage_test.go b/internal/unifiedresources/adapter_coverage_test.go index 8953fb4a49..439ae536cf 100644 --- a/internal/unifiedresources/adapter_coverage_test.go +++ b/internal/unifiedresources/adapter_coverage_test.go @@ -1051,7 +1051,8 @@ func TestMetricPercent(t *testing.T) { // derives their state from the host's reporting lease and the disk's spin // state. The native row and a SMART row that borrows the inventory reading // both carry it, and a SMART row with its own reading keeps the agent's state. -// A withdrawn reading never counts as heat in the disk's risk. +// Disk risk never judges heat in any state: the alert disk temperature policy +// owns that. func TestUnraidDiskTemperatureFollowsTheHostLease(t *testing.T) { smartWithout := models.HostDiskSMART{Device: "/dev/sdc", Serial: "UNRAID-1", Collection: &diskinventory.CollectionStatus{ Serial: diskinventory.Available("smartctl"), @@ -1088,7 +1089,6 @@ func TestUnraidDiskTemperatureFollowsTheHostLease(t *testing.T) { Temperature: 72, SpunDown: tc.spunDown} host := models.Host{ID: "agent-tower", Hostname: "tower", Status: tc.status, Unraid: &models.HostUnraidStorage{Disks: []models.HostUnraidDisk{inventory}}} - collected := tc.borrowed.State == diskinventory.FieldAvailable native, _ := resourceFromHostUnraidPhysicalDisk(host, inventory) got := native.PhysicalDisk @@ -1099,8 +1099,8 @@ func TestUnraidDiskTemperatureFollowsTheHostLease(t *testing.T) { if got.Temperature != 72 || nativeStatus != tc.native { t.Fatalf("%s: Unraid row temperature=%d state=%+v, want 72 with %+v", tc.name, got.Temperature, nativeStatus, tc.native) } - if hotRisk(got.Risk) != collected { - t.Fatalf("%s: Unraid row temperature risk = %v, want %v: %+v", tc.name, hotRisk(got.Risk), collected, got.Risk) + if hotRisk(got.Risk) { + t.Fatalf("%s: Unraid row risk judged its 72C temperature: %+v", tc.name, got.Risk) } borrowed, _ := resourceFromHostSMARTDisk(host, smartWithout) @@ -1109,8 +1109,8 @@ func TestUnraidDiskTemperatureFollowsTheHostLease(t *testing.T) { got.Collection.Serial != diskinventory.Available("smartctl") { t.Fatalf("%s: SMART row borrowing the inventory reading: temperature=%d collection=%+v", tc.name, got.Temperature, got.Collection) } - if hotRisk(got.Risk) != collected { - t.Fatalf("%s: borrowed temperature risk = %v, want %v: %+v", tc.name, hotRisk(got.Risk), collected, got.Risk) + if hotRisk(got.Risk) { + t.Fatalf("%s: SMART row risk judged the borrowed 72C temperature: %+v", tc.name, got.Risk) } if smartWithout.Collection.Temperature.State != diskinventory.FieldUnsupported { t.Fatal("the adapter mutated the SMART row's own collection state") diff --git a/internal/unifiedresources/adapters.go b/internal/unifiedresources/adapters.go index da65047d35..e9386c81f0 100644 --- a/internal/unifiedresources/adapters.go +++ b/internal/unifiedresources/adapters.go @@ -804,7 +804,7 @@ func resourceFromHostUnraidPhysicalDisk(host models.Host, disk models.HostUnraid model := strings.TrimSpace(disk.Model) name := firstNonEmpty(model, disk.Name, disk.Device, host.Hostname) health := unraidPhysicalDiskHealth(disk) - assessment := assessUnraidPhysicalDisk(host, disk) + assessment := assessUnraidPhysicalDisk(disk) sizeBytes := unraidDiskSizeBytes(host, disk) var collection *diskinventory.CollectionStatus if status := unraidDiskTemperatureStatus(host, disk); status.State != "" { @@ -939,7 +939,7 @@ func resourceFromHostSMARTDisk(host models.Host, disk models.HostDiskSMART) (Res } assessment := storagehealth.AssessHostSMARTDisk(disk) if unraidDisk != nil { - assessment = storagehealth.SummarizeAssessments(assessment, assessUnraidPhysicalDisk(host, *unraidDisk)) + assessment = storagehealth.SummarizeAssessments(assessment, assessUnraidPhysicalDisk(*unraidDisk)) } wearout := physicalDiskWearoutFromSMARTAttributes(disk.Attributes) @@ -1324,17 +1324,11 @@ func unraidPhysicalDiskHealth(disk models.HostUnraidDisk) string { } } -func assessUnraidPhysicalDisk(host models.Host, disk models.HostUnraidDisk) storagehealth.Assessment { - // A retained inventory temperature is not current evidence of heat. - temperature := 0 - if status := unraidDiskTemperatureStatus(host, disk); status.State == diskinventory.FieldAvailable { - temperature = disk.Temperature - } +func assessUnraidPhysicalDisk(disk models.HostUnraidDisk) storagehealth.Assessment { assessment := storagehealth.AssessSample(storagehealth.Sample{ - Model: disk.Model, - Health: unraidPhysicalDiskHealth(disk), - Temperature: temperature, - Wearout: -1, + Model: disk.Model, + Health: unraidPhysicalDiskHealth(disk), + Wearout: -1, }) addReason := func(code string, severity storagehealth.RiskLevel, summary string) { if strings.TrimSpace(summary) == "" { diff --git a/internal/unifiedresources/physical_disk_risk.go b/internal/unifiedresources/physical_disk_risk.go index 444a62af1d..20ef7fd878 100644 --- a/internal/unifiedresources/physical_disk_risk.go +++ b/internal/unifiedresources/physical_disk_risk.go @@ -82,7 +82,6 @@ func physicalDiskAssessmentFromMeta(meta *PhysicalDiskMeta) storagehealth.Assess sample := storagehealth.Sample{ Model: meta.Model, Health: meta.Health, - Temperature: storagehealth.CollectedTemperature(meta.Temperature, meta.Collection), Wearout: meta.Wearout, WearoutKnown: storagehealth.WearoutReported(meta.Wearout, meta.DiskType), } diff --git a/internal/unifiedresources/physical_disk_risk_test.go b/internal/unifiedresources/physical_disk_risk_test.go index 28087ea2f2..17dd1f2918 100644 --- a/internal/unifiedresources/physical_disk_risk_test.go +++ b/internal/unifiedresources/physical_disk_risk_test.go @@ -41,7 +41,7 @@ func TestPhysicalDiskRiskFromAssessment_ConvertsReasons(t *testing.T) { Level: storagehealth.RiskCritical, Reasons: []storagehealth.Reason{ {Code: "pending_sectors", Severity: storagehealth.RiskCritical, Summary: "Pending sectors detected (5)"}, - {Code: "temperature_high", Severity: storagehealth.RiskWarning, Summary: "Disk temperature is 65C"}, + {Code: "reallocated_sectors", Severity: storagehealth.RiskWarning, Summary: "Reallocated sectors detected (4)"}, }, } result := PhysicalDiskRiskFromAssessment(assessment) @@ -66,7 +66,7 @@ func TestPhysicalDiskRiskFromAssessmentAndIncidents_IncludesDiskHealthIncidents( assessment := storagehealth.Assessment{ Level: storagehealth.RiskWarning, Reasons: []storagehealth.Reason{ - {Code: "temperature_high", Severity: storagehealth.RiskWarning, Summary: "Disk temperature is 63C"}, + {Code: "reallocated_sectors", Severity: storagehealth.RiskWarning, Summary: "Reallocated sectors detected (2)"}, }, } incidents := []ResourceIncident{ @@ -137,14 +137,15 @@ func TestPhysicalDiskAssessmentFromMeta_PendingSectors(t *testing.T) { } } -func TestPhysicalDiskAssessmentFromMeta_HighTemperature(t *testing.T) { +// Heat is the alert disk temperature policy's to judge, never disk risk's. +func TestPhysicalDiskAssessmentFromMeta_HighTemperatureIsNotRisk(t *testing.T) { meta := &PhysicalDiskMeta{ Health: "PASSED", Temperature: 72, } assessment := physicalDiskAssessmentFromMeta(meta) - if assessment.Level != storagehealth.RiskCritical { - t.Errorf("72°C should produce critical assessment, got %s", assessment.Level) + if assessment.Level != storagehealth.RiskHealthy || len(assessment.Reasons) != 0 { + t.Errorf("72°C should leave the assessment healthy, got %+v", assessment) } } diff --git a/internal/unifiedresources/registry_test.go b/internal/unifiedresources/registry_test.go index 0256304351..7c98c19bca 100644 --- a/internal/unifiedresources/registry_test.go +++ b/internal/unifiedresources/registry_test.go @@ -7035,10 +7035,11 @@ func TestRegistryProjectionSnapshotDetachesAndTracksMappings(t *testing.T) { } } -// A retained temperature is last-known history, not evidence that the disk is -// hot now. The risk verdict (and the warning status it drives) must judge only -// a temperature whose collection state says it was observed now. -func TestPhysicalDiskRiskIgnoresRetainedTemperature(t *testing.T) { +// Disk risk (and the warning status it drives) never judges temperature, +// collected or retained. Heat belongs to the alert disk temperature policy, +// which users tune per disk type and per host and which no registry can see. +// The reading itself stays on the resource for that policy to judge. +func TestPhysicalDiskRiskNeverJudgesTemperature(t *testing.T) { // A fresh check time keeps the registry's source-staleness sweep out of // the status this test reads. now := time.Now() @@ -7047,15 +7048,12 @@ func TestPhysicalDiskRiskIgnoresRetainedTemperature(t *testing.T) { cases := []struct { name string collection *diskinventory.CollectionStatus - wantHot bool }{ - {name: "collected now", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("proxmox_node_smart")}, wantHot: true}, - {name: "source predates collection state", collection: nil, wantHot: true}, - {name: "state left empty", collection: &diskinventory.CollectionStatus{}, wantHot: true}, - {name: "retained after the agent went silent", collection: &diskinventory.CollectionStatus{Temperature: retained}, wantHot: false}, - {name: "retained while in standby", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}, wantHot: false}, - {name: "retained after the source stopped exposing it", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature")}, wantHot: false}, - {name: "retained while unexpectedly missing", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Missing("smartctl", "")}, wantHot: false}, + {name: "collected now", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("proxmox_node_smart")}}, + {name: "source predates collection state", collection: nil}, + {name: "state left empty", collection: &diskinventory.CollectionStatus{}}, + {name: "retained after the agent went silent", collection: &diskinventory.CollectionStatus{Temperature: retained}}, + {name: "retained while in standby", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -7089,100 +7087,60 @@ func TestPhysicalDiskRiskIgnoresRetainedTemperature(t *testing.T) { if disk.PhysicalDisk.Temperature != 72 { t.Fatalf("temperature = %d, want the reading kept at 72", disk.PhysicalDisk.Temperature) } - hot := false - if disk.PhysicalDisk.Risk != nil { - for _, reason := range disk.PhysicalDisk.Risk.Reasons { - hot = hot || reason.Code == "temperature_high" - } - } - if hot != tc.wantHot { - t.Fatalf("temperature_high = %v, want %v (risk %+v)", hot, tc.wantHot, disk.PhysicalDisk.Risk) - } - if tc.wantHot && disk.Status != StatusWarning { - t.Fatalf("status = %q, want warning for a collected 72C", disk.Status) - } - if !tc.wantHot && (disk.PhysicalDisk.Risk != nil || disk.Status != StatusOnline) { - t.Fatalf("retained 72C: status = %q risk = %+v, want online with no risk", disk.Status, disk.PhysicalDisk.Risk) + if disk.PhysicalDisk.Risk != nil || disk.Status != StatusOnline { + t.Fatalf("72C disk: status = %q risk = %+v, want online with no risk", disk.Status, disk.PhysicalDisk.Risk) } }) } // A host agent's SMART row merged with the Proxmox inventory row for the - // same disk: the agent's retained reading keeps its unavailable state - // through the registry merge and recompute, so it stays out of risk. - for _, tc := range []struct { - name string - state diskinventory.FieldStatus - wantHot bool - }{ - {name: "agent reading collected", state: diskinventory.Available("host_agent"), wantHot: true}, - {name: "agent reading retained", state: retained, wantHot: false}, - } { - t.Run("merged "+tc.name, func(t *testing.T) { - adapter := NewMonitorAdapter(NewRegistry(nil)) - adapter.PopulateFromSnapshot(models.StateSnapshot{ - Hosts: []models.Host{{ - ID: "host-pve", - Hostname: "pve", - Status: "online", - LastSeen: now, - Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{ - Device: "/dev/sda", - Model: "WDC WD80EFAX", - Serial: "SERIAL-SDA", - Type: "sata", - Temperature: 72, - Health: "PASSED", - Collection: &diskinventory.CollectionStatus{Temperature: tc.state}, - }}}, - }}, - PhysicalDisks: []models.PhysicalDisk{{ - ID: "homelab-pve--dev-sda", - Node: "pve", - Instance: "homelab", - DevPath: "/dev/sda", - Model: "WDC WD80EFAX", - Serial: "SERIAL-SDA", - Type: "sata", - Health: "PASSED", - Wearout: -1, - Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature")}, - LastChecked: now, - }}, - }) - var disks []Resource - for _, resource := range adapter.GetAll() { - if resource.Type == ResourceTypePhysicalDisk { - disks = append(disks, resource) - } - } - if len(disks) != 1 || disks[0].PhysicalDisk == nil { - t.Fatalf("physical disks = %d, want the agent and Proxmox rows merged into one", len(disks)) - } - disk := disks[0] - if disk.PhysicalDisk.Temperature != 72 { - t.Fatalf("merged temperature = %d, want 72", disk.PhysicalDisk.Temperature) - } - hot := false - if disk.PhysicalDisk.Risk != nil { - for _, reason := range disk.PhysicalDisk.Risk.Reasons { - hot = hot || reason.Code == "temperature_high" - } - } - if hot != tc.wantHot { - t.Fatalf("merged temperature_high = %v, want %v (collection %+v, risk %+v)", hot, tc.wantHot, disk.PhysicalDisk.Collection, disk.PhysicalDisk.Risk) - } - }) + // same disk goes through the registry merge and its risk recompute. + adapter := NewMonitorAdapter(NewRegistry(nil)) + adapter.PopulateFromSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-pve", + Hostname: "pve", + Status: "online", + LastSeen: now, + Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{ + Device: "/dev/sda", + Model: "WDC WD80EFAX", + Serial: "SERIAL-SDA", + Type: "sata", + Temperature: 72, + Health: "PASSED", + Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")}, + }}}, + }}, + PhysicalDisks: []models.PhysicalDisk{{ + ID: "homelab-pve--dev-sda", + Node: "pve", + Instance: "homelab", + DevPath: "/dev/sda", + Model: "WDC WD80EFAX", + Serial: "SERIAL-SDA", + Type: "sata", + Health: "PASSED", + Wearout: -1, + Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature")}, + LastChecked: now, + }}, + }) + var merged []Resource + for _, resource := range adapter.GetAll() { + if resource.Type == ResourceTypePhysicalDisk { + merged = append(merged, resource) + } + } + if len(merged) != 1 || merged[0].PhysicalDisk == nil { + t.Fatalf("physical disks = %d, want the agent and Proxmox rows merged into one", len(merged)) + } + if disk := merged[0].PhysicalDisk; disk.Temperature != 72 || disk.Risk != nil { + t.Fatalf("merged disk temperature = %d risk = %+v, want 72 with no risk", disk.Temperature, disk.Risk) } - // The registry recomputes risk from merged metadata after a source merge; - // that path follows the same rule. - meta := &PhysicalDiskMeta{Health: "PASSED", Temperature: 72, Collection: &diskinventory.CollectionStatus{Temperature: retained}} + meta := &PhysicalDiskMeta{Health: "PASSED", Temperature: 72, Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")}} if risk := physicalDiskRiskFromMeta(meta, nil); risk != nil { - t.Fatalf("meta recompute risk = %+v, want none for a retained temperature", risk) - } - meta.Collection = &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")} - if risk := physicalDiskRiskFromMeta(meta, nil); risk == nil || risk.Level != storagehealth.RiskCritical { - t.Fatalf("meta recompute risk = %+v, want critical for a collected 72C", risk) + t.Fatalf("meta recompute risk = %+v, want none for a collected 72C", risk) } } diff --git a/internal/unifiedresources/storage_risk_test.go b/internal/unifiedresources/storage_risk_test.go index d88b2d5b15..06eb5bc6d8 100644 --- a/internal/unifiedresources/storage_risk_test.go +++ b/internal/unifiedresources/storage_risk_test.go @@ -35,7 +35,7 @@ func TestStorageRiskFromAssessment_CriticalConvertsReasons(t *testing.T) { Level: storagehealth.RiskCritical, Reasons: []storagehealth.Reason{ {Code: "pending_sectors", Severity: storagehealth.RiskCritical, Summary: "Bad sectors"}, - {Code: "temperature_high", Severity: storagehealth.RiskWarning, Summary: "Hot disk"}, + {Code: "reallocated_sectors", Severity: storagehealth.RiskWarning, Summary: "Reallocated sectors"}, }, }) if got == nil { @@ -47,7 +47,7 @@ func TestStorageRiskFromAssessment_CriticalConvertsReasons(t *testing.T) { if len(got.Reasons) != 2 { t.Fatalf("expected 2 reasons, got %d", len(got.Reasons)) } - if got.Reasons[0].Code != "pending_sectors" || got.Reasons[1].Code != "temperature_high" { + if got.Reasons[0].Code != "pending_sectors" || got.Reasons[1].Code != "reallocated_sectors" { t.Error("reasons should preserve order and codes") } } From 35bf55af23860aa1cbec348cc40ac07c4e9dc4a1 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 00:09:01 +0100 Subject: [PATCH 10/13] Colour report disk temperatures by the alert thresholds The PDF performance report's disk table coloured a temperature amber from 50C and red from 60C for every disk type, the last fixed disk heat line after disk heat moved to the alert disk temperature policy. An NVMe at 63C, well under its 70C alert trigger, printed red. A user's raised trigger changed nothing. Each reported disk now carries the tenant's alert thresholds for its type (the clear value and the trigger, from alerts.Manager.DiskTemperatureThreshold). The PDF colours the reading amber from the clear value and red from the trigger, as the Physical Disks Temp column does, and leaves it plain when disk temperature alerting is off. --- .../v6/internal/subsystems/agent-lifecycle.md | 3 +- .../v6/internal/subsystems/alerts.md | 8 +-- .../v6/internal/subsystems/api-contracts.md | 9 +++ .../internal/subsystems/storage-recovery.md | 4 +- internal/api/contract_test.go | 64 +++++++++++++++++++ internal/api/metrics_reporting_handlers.go | 40 +++++++++--- pkg/reporting/pdf.go | 22 ++++++- pkg/reporting/pdf_additional_test.go | 23 +++++++ pkg/reporting/reporting.go | 6 ++ 9 files changed, 163 insertions(+), 16 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 419c34448e..a16a714d8b 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -764,7 +764,8 @@ Downstream readers keep that distinction too. The disk temperature charts do not pad a series to now with the retained value. The AI chat context, Patrol and the AI disk tools present it only as a last-known value with its reason. The performance report and the reporting runtime snapshot leave it out of their -disk tables. +disk tables. The performance report colours a collected reading by the +tenant's alert disk temperature thresholds for the disk type, not a fixed line. An enabled availability target assigned to a host agent creates an agent-lifecycle lease for that exact target/agent pairing. First assignment diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 692424c56d..24d5382258 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -1702,10 +1702,10 @@ normalization restores a non-positive entry to its default. A disk is hot from its trigger. Alerts and Patrol findings stay open until the reading falls to the clear value, or under the trigger when there is no band below it. The Physical Disks verdict and the TrueNAS Health cell judge only the -current reading against the trigger. Judges that still differ: TrueNAS disk -temperature alerts use `TrueNASDiskDefaults.Temperature` (a flat 55/50), the -TrueNAS disk drawer tones the reading from a fixed 55C, and PDF reports colour -disk temperatures at a fixed 50/60C. +current reading against the trigger. PDF reports colour disk temperatures by +the same thresholds. Judges that still differ: TrueNAS disk temperature alerts +use `TrueNASDiskDefaults.Temperature` (a flat 55/50), and the TrueNAS disk +drawer tones the reading from a fixed 55C. `TestDiskTemperatureThresholdMatchesCheckHostPolicy` in `internal/alerts/threshold_resolution_shared_test.go` pins per-type resolution, a raised NVMe trigger that `CheckHost` also honours, the copy, the disabled default and diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index e04dff160b..21b9ae0ad6 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -6349,6 +6349,15 @@ when `diskinventory.TemperatureCollected` holds, so a retained value is reported as 0 (no reading) rather than tabulated as measured. Proof: `TestContract_ReportsOmitRetainedDiskTemperatures` in `internal/api/contract_test.go`. +Each reported disk also carries its alert disk temperature thresholds +(`reporting.DiskInfo.TemperatureWarning` / `TemperatureCritical`, the clear +value and the trigger). `reportDiskTemperatureThresholds` resolves them from +the tenant's alert manager (`alerts.Manager.DiskTemperatureThreshold`, the +factory policy when there is none). The PDF disk table colours a reading amber +from the clear value and red from the trigger, as the Physical Disks Temp +column does, and leaves it plain when disk temperature alerting is off. It no +longer uses a fixed 50/60C. Proof: +`TestContract_ReportsCarryDiskTemperatureAlertThresholds`. That same metrics-history contract also owns Kubernetes pod identity normalization. `/api/metrics-store/history` must accept legacy bare pod IDs such as `cluster-1:pod:pod-1`, canonicalize them onto the unified pod metrics diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 079fabf486..6607d9eed8 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -6779,7 +6779,9 @@ judges the current reading, with no memory of an alert. A disk temperature alert held between its clear value and trigger shows an amber Temp cell, not `Running Hot`. A switched-off agent Disk Temp default resolves to null, which leaves the reading uncoloured and never hot, as alerting is off for it. -Per-host Disk Temp overrides do not yet reach Physical Disks. +Per-host Disk Temp overrides do not yet reach Physical Disks. PDF performance +reports colour a disk reading by the same per-type thresholds, which +`reporting.DiskInfo` carries. `frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts` pins cell-and-verdict agreement for every type from 40C to 80C under factory and raised triggers, plus retained readings, failure evidence, sorting and diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go index 17faf9067b..1e0a0292d1 100644 --- a/internal/api/contract_test.go +++ b/internal/api/contract_test.go @@ -5286,6 +5286,70 @@ func TestContract_ReportsOmitRetainedDiskTemperatures(t *testing.T) { } } +// Report disk tables colour a reading by the disk's alert disk temperature +// thresholds (the tenant's per-type policy), not a fixed 50/60C: amber from the +// clear value, red from the trigger, as the Physical Disks Temp column does. +func TestContract_ReportsCarryDiskTemperatureAlertThresholds(t *testing.T) { + engine := &stubReportingEngine{data: []byte("report"), contentType: "application/pdf"} + original := reporting.GetEngine() + reporting.SetEngine(engine) + t.Cleanup(func() { reporting.SetEngine(original) }) + + state := models.NewState() + state.Nodes = []models.Node{{ID: "node-1", Name: "node-a", Status: "online"}} + disk := func(id, devPath, diskType string, temperature int) unifiedresources.Resource { + return unifiedresources.Resource{ + ID: id, Type: unifiedresources.ResourceTypePhysicalDisk, Name: id, ParentName: "node-a", + Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"node-a"}}, + PhysicalDisk: &unifiedresources.PhysicalDiskMeta{ + DevPath: devPath, DiskType: diskType, Health: "PASSED", Wearout: -1, Temperature: temperature, + Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}, + }, + } + } + monitor := newReportingMonitorForTest(t, state, []unifiedresources.Resource{ + disk("disk-nvme", "/dev/nvme0n1", "nvme", 63), + disk("disk-sata", "/dev/sda", "sata", 56), + }) + handler := NewReportingHandlers(newReportingMTMForTest(t, monitor), nil) + thresholdsByDevice := func() map[string][2]float64 { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/api/reporting?format=pdf&resourceType=node&resourceId=node-1", nil) + req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "default")) + rec := httptest.NewRecorder() + handler.HandleGenerateReport(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rec.Code, rec.Body.String()) + } + got := map[string][2]float64{} + for _, d := range engine.lastReq.Disks { + got[d.Device] = [2]float64{d.TemperatureWarning, d.TemperatureCritical} + } + return got + } + + // No alert manager: the factory per-type policy. + if got, want := thresholdsByDevice(), map[string][2]float64{"/dev/nvme0n1": {65, 70}, "/dev/sda": {50, 55}}; !reflect.DeepEqual(got, want) { + t.Fatalf("factory report disk thresholds = %v, want %v", got, want) + } + + manager := alerts.NewManager() + cfg := manager.GetConfig() + cfg.DiskTempByType["nvme"] = alerts.HysteresisThreshold{Trigger: 75, Clear: 70} + manager.UpdateConfig(cfg) + setUnexportedField(t, monitor, "alertManager", manager) + if got, want := thresholdsByDevice(), map[string][2]float64{"/dev/nvme0n1": {70, 75}, "/dev/sda": {50, 55}}; !reflect.DeepEqual(got, want) { + t.Fatalf("configured report disk thresholds = %v, want %v", got, want) + } + + cfg = manager.GetConfig() + cfg.AgentDefaults.DiskTemperature = &alerts.HysteresisThreshold{Trigger: 0, Clear: 0} + manager.UpdateConfig(cfg) + if got, want := thresholdsByDevice(), map[string][2]float64{"/dev/nvme0n1": {0, 0}, "/dev/sda": {0, 0}}; !reflect.DeepEqual(got, want) { + t.Fatalf("report disk thresholds with disk temperature alerting off = %v, want %v", got, want) + } +} + func TestContract_ReportingRequestCarriesEntitledReportBranding(t *testing.T) { engine := &stubReportingEngine{data: []byte("report"), contentType: "application/pdf"} original := reporting.GetEngine() diff --git a/internal/api/metrics_reporting_handlers.go b/internal/api/metrics_reporting_handlers.go index f363c6c0b2..3cf94d3f68 100644 --- a/internal/api/metrics_reporting_handlers.go +++ b/internal/api/metrics_reporting_handlers.go @@ -14,6 +14,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/ai" + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/monitoring" @@ -334,6 +335,9 @@ type reportingEnrichmentSnapshot struct { RecentlyResolved []models.ResolvedAlert LegacyBackups models.PVEBackups Resources []unifiedresources.Resource + // alertManager supplies the tenant's disk temperature thresholds. Nil + // means the factory alert configuration. + alertManager *alerts.Manager } func emptyReportingEnrichmentSnapshot() reportingEnrichmentSnapshot { @@ -405,6 +409,7 @@ func (h *ReportingHandlers) getReportingEnrichmentSnapshot(ctx context.Context, RecentlyResolved: monitor.RecentlyResolvedSnapshot(), LegacyBackups: monitor.PVEBackupsSnapshot(), Resources: unifiedResources, + alertManager: monitor.GetAlertManager(), } snapshot.normalizeCollections() return snapshot, true @@ -876,20 +881,39 @@ func (h *ReportingHandlers) enrichNodeReport(req *reporting.MetricReportRequest, continue } pd := r.PhysicalDisk + temperatureWarning, temperatureCritical := reportDiskTemperatureThresholds(snapshot.alertManager, pd.DiskType) req.Disks = append(req.Disks, reporting.DiskInfo{ - Device: pd.DevPath, - Model: pd.Model, - Serial: pd.Serial, - Type: pd.DiskType, - Size: pd.SizeBytes, - Health: pd.Health, - Temperature: reportDiskTemperature(*pd), - WearLevel: pd.Wearout, + Device: pd.DevPath, + Model: pd.Model, + Serial: pd.Serial, + Type: pd.DiskType, + Size: pd.SizeBytes, + Health: pd.Health, + Temperature: reportDiskTemperature(*pd), + WearLevel: pd.Wearout, + TemperatureWarning: temperatureWarning, + TemperatureCritical: temperatureCritical, }) } } } +// reportDiskTemperatureThresholds returns the alert disk temperature thresholds +// a report colours a disk reading by: the clear value and the trigger, from +// the same policy as disk temperature alerts and the Physical Disks Temp +// column. Zero means disk temperature alerting is off for the disk. +func reportDiskTemperatureThresholds(manager *alerts.Manager, diskType string) (float64, float64) { + threshold := manager.DiskTemperatureThreshold(diskType) + if threshold == nil || threshold.Trigger <= 0 { + return 0, 0 + } + warning := threshold.Clear + if warning <= 0 || warning > threshold.Trigger { + warning = threshold.Trigger + } + return warning, threshold.Trigger +} + // reportDiskTemperature returns the disk temperature a report may tabulate: // one the current observation collected. Normalization may keep a last-known // temperature it did not collect (a disk in standby, a host agent past its diff --git a/pkg/reporting/pdf.go b/pkg/reporting/pdf.go index b1db00fdc1..1cf9f3d575 100644 --- a/pkg/reporting/pdf.go +++ b/pkg/reporting/pdf.go @@ -1670,9 +1670,10 @@ func (g *PDFGenerator) writeDisksSection(pdf *fpdf.Fpdf, data *ReportData) { tempStr := "-" if disk.Temperature > 0 { tempStr = fmt.Sprintf("%dC", disk.Temperature) - if disk.Temperature >= 60 { + switch diskTemperatureSeverity(disk) { + case "critical": pdf.SetTextColor(colorDanger[0], colorDanger[1], colorDanger[2]) - } else if disk.Temperature >= 50 { + case "warning": pdf.SetTextColor(colorWarning[0], colorWarning[1], colorWarning[2]) } } @@ -2808,3 +2809,20 @@ func formatDuration(d time.Duration) string { } return fmt.Sprintf("%d %s", minutes, minWord) } + +// diskTemperatureSeverity colours a disk reading by its alert disk temperature +// thresholds: "critical" from the trigger, "warning" from the clear value, and +// "" below it or when the disk carries no thresholds. +func diskTemperatureSeverity(disk DiskInfo) string { + if disk.Temperature <= 0 || disk.TemperatureCritical <= 0 { + return "" + } + reading := float64(disk.Temperature) + if reading >= disk.TemperatureCritical { + return "critical" + } + if disk.TemperatureWarning > 0 && reading >= disk.TemperatureWarning { + return "warning" + } + return "" +} diff --git a/pkg/reporting/pdf_additional_test.go b/pkg/reporting/pdf_additional_test.go index 49ca2c3fc9..d190c3a4b2 100644 --- a/pkg/reporting/pdf_additional_test.go +++ b/pkg/reporting/pdf_additional_test.go @@ -57,3 +57,26 @@ func TestFormatDuration(t *testing.T) { t.Fatalf("expected 2 days, 1 hour, got %q", got) } } + +// Disk temperatures in a report take the disk's alert thresholds, not a fixed +// 50/60C: an NVMe at 63C under a 70C trigger stays uncoloured, a SATA disk at +// 56C past its 55C trigger is red, and a disk without thresholds is plain. +func TestDiskTemperatureSeverityFollowsAlertThresholds(t *testing.T) { + cases := []struct { + disk DiskInfo + want string + }{ + {DiskInfo{Temperature: 63, TemperatureWarning: 65, TemperatureCritical: 70}, ""}, + {DiskInfo{Temperature: 66, TemperatureWarning: 65, TemperatureCritical: 70}, "warning"}, + {DiskInfo{Temperature: 70, TemperatureWarning: 65, TemperatureCritical: 70}, "critical"}, + {DiskInfo{Temperature: 56, TemperatureWarning: 50, TemperatureCritical: 55}, "critical"}, + {DiskInfo{Temperature: 72, TemperatureWarning: 70, TemperatureCritical: 75}, "warning"}, + {DiskInfo{Temperature: 80}, ""}, + {DiskInfo{Temperature: 0, TemperatureWarning: 50, TemperatureCritical: 55}, ""}, + } + for _, tc := range cases { + if got := diskTemperatureSeverity(tc.disk); got != tc.want { + t.Errorf("diskTemperatureSeverity(%+v) = %q, want %q", tc.disk, got, tc.want) + } + } +} diff --git a/pkg/reporting/reporting.go b/pkg/reporting/reporting.go index cc424e6bc8..66d8e6babf 100644 --- a/pkg/reporting/reporting.go +++ b/pkg/reporting/reporting.go @@ -153,6 +153,12 @@ type DiskInfo struct { Health string // PASSED, FAILED, UNKNOWN Temperature int // Celsius WearLevel int // 0-100, percentage of life REMAINING (100 = healthy, 0 = end of life, -1 = unknown) + // TemperatureWarning and TemperatureCritical are the disk's alert disk + // temperature thresholds in Celsius (the clear value and the trigger). The + // reading is amber from Warning and red from Critical, as in the Physical + // Disks Temp column. A zero Critical leaves the reading uncoloured. + TemperatureWarning float64 + TemperatureCritical float64 } // MultiReportRequest defines the parameters for generating a multi-resource report. From 396bb5d517f14466795a2ccd2ff438fdba0ab077 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 00:17:32 +0100 Subject: [PATCH 11/13] Keep a disk heat finding open without a current reading Patrol verifies a disk-high finding as recovered once the disk's temperature cools to its alert clear value. A disk in standby, or one whose host agent stopped reporting, has no current reading, and a missing reading counted as cooled. A finding raised for a SATA disk at 56C would close as soon as the disk spun down, while its temperature alert stayed open. When a disk has no current reading and its last known reading is still at or above the clear value, verification now reports the recovery as unknown instead of proven. A disk last seen cool still recovers. --- .../v6/internal/subsystems/ai-runtime.md | 3 ++- internal/ai/patrol_disk_temperature_test.go | 21 ++++++++++++++++ internal/ai/patrol_findings.go | 25 +++++++++++++------ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 216666a720..c6402b3694 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -315,7 +315,8 @@ configuration applies. A disk is hot from its trigger: a triage warning, a disk issue and an AI chat "needing attention" entry, as its temperature alert and Physical Disks Running Hot verdict are. A disk-high finding recovers at or below the clear value (under the trigger when there is no band below it), as -the alert does. Scoped runs keep the provider, and a +the alert does. With no current reading, a disk last seen hot leaves that +verification unknown rather than recovered. Scoped runs keep the provider, and a provider set before Patrol starts is handed to it. Below the trigger Patrol flags nothing, so an NVMe at 63C stays quiet and a SATA disk at 56C is flagged. Proofs: `internal/ai/patrol_disk_temperature_test.go` and diff --git a/internal/ai/patrol_disk_temperature_test.go b/internal/ai/patrol_disk_temperature_test.go index e38358380e..d8c433eae4 100644 --- a/internal/ai/patrol_disk_temperature_test.go +++ b/internal/ai/patrol_disk_temperature_test.go @@ -1,12 +1,15 @@ package ai import ( + "errors" "strings" "testing" "time" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) // raisedNVMeThresholds is a user who raised the NVMe disk temperature trigger @@ -226,3 +229,21 @@ func TestDiskHeatRecoversAtTheClearValue(t *testing.T) { t.Fatalf("no-band limits %+v: want 70C hot and not cooled, 69C cooled", noBand) } } + +// With no current reading (standby, a silent agent) a disk last seen hot has +// not shown that it cooled, so heat recovery is unknown rather than proven. +func TestDiskHeatRecoveryNeedsACurrentReading(t *testing.T) { + retained := &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")} + state := newPatrolRuntimeState(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{ + {ID: "sata-retained-hot", DevPath: "/dev/sda", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 56, Collection: retained}, + {ID: "sata-retained-cool", DevPath: "/dev/sdb", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 40, Collection: retained}, + }}) + + if _, err := verifyMetricRecoveredState(state, PatrolThresholds{}, "disk-high", "sata-retained-hot", "physical_disk"); !errors.Is(err, aicontracts.ErrVerificationUnknown) { + t.Fatalf("retained 56C: err = %v, want ErrVerificationUnknown", err) + } + recovered, err := verifyMetricRecoveredState(state, PatrolThresholds{}, "disk-high", "sata-retained-cool", "physical_disk") + if err != nil || !recovered { + t.Fatalf("retained 40C: recovered = %v, err = %v, want recovered", recovered, err) + } +} diff --git a/internal/ai/patrol_findings.go b/internal/ai/patrol_findings.go index 1bb5e87af0..d4f5fbc2b7 100644 --- a/internal/ai/patrol_findings.go +++ b/internal/ai/patrol_findings.go @@ -2296,6 +2296,11 @@ func verifyMetricRecoveredState(snap patrolRuntimeState, thresholds PatrolThresh if !disk.temperatureLimits.cooled(disk.temperature) { return false, nil } + // A disk last seen hot with no current reading has not + // shown that it cooled, so its recovery is unknown. + if disk.temperature <= 0 && !disk.temperatureLimits.cooled(disk.lastKnownTemperature) { + return false, fmt.Errorf("%w: no current temperature reading for %s", aicontracts.ErrVerificationUnknown, rid) + } return true, nil } break @@ -2317,10 +2322,13 @@ func verifyMetricRecoveredState(snap patrolRuntimeState, thresholds PatrolThresh } type patrolPhysicalDiskVerification struct { - health string - wearout int - temperature int - temperatureLimits diskTemperatureLimits + health string + wearout int + temperature int + // lastKnownTemperature is a retained reading the current observation + // did not collect (a disk in standby, a silent agent). + lastKnownTemperature int + temperatureLimits diskTemperatureLimits } type patrolPhysicalDiskVisitor func(identifiers []string, verification patrolPhysicalDiskVerification) bool @@ -2464,10 +2472,11 @@ func patrolVisitPhysicalDiskVerification(snap patrolRuntimeState, visit patrolPh rows := patrolPhysicalDiskRows(snap, nil) for _, disk := range rows { if !visit([]string{disk.id, disk.name, disk.devPath, disk.model}, patrolPhysicalDiskVerification{ - health: strings.TrimSpace(disk.health), - wearout: disk.wearout, - temperature: disk.temperature.Collected, - temperatureLimits: disk.temperatureLimits, + health: strings.TrimSpace(disk.health), + wearout: disk.wearout, + temperature: disk.temperature.Collected, + lastKnownTemperature: disk.temperature.LastKnown, + temperatureLimits: disk.temperatureLimits, }) { return true } From e3e743b076db00da139c04a6daba20fb8e594d82 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 00:21:46 +0100 Subject: [PATCH 12/13] List disk heat beside TrueNAS alert text The TrueNAS storage table's Health reasons put native TrueNAS alert text first and used Pulse's derived summaries only when TrueNAS raised no alert. Disk heat landed in that fallback, so a hot disk with any incident, even an informational one, showed only the incident's text under Attention, and nothing said the disk was hot. Heat is Pulse's own judgement against the alert disk temperature thresholds, not a restatement of TrueNAS state, so it is now always listed after any native alert text. --- .../internal/subsystems/unified-resources.md | 4 ++- .../e3956fd0c9818451.json | 30 +++++++++++++++++++ .../__tests__/truenasPageModel.test.ts | 10 +++++++ .../src/features/truenas/truenasPageModel.ts | 4 ++- 4 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 frontend-modern/browser-verification/e3956fd0c9818451.json diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 6a00c0a3c0..c9fb0f7d27 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -1118,7 +1118,9 @@ threshold.". The trigger comes from the alerts store's `getDiskTemperatureThresholds`, which the table passes to the row build, filter, counts, sort and issue. `isPhysicalDiskRunningHot` in `features/storageBackups/diskTemperaturePresentation.ts` makes the same call -for the Physical Disks verdict. The status dot keeps the source's own state. +for the Physical Disks verdict. The heat reason follows any native TrueNAS +alert text, and a hot disk's status dot turns warning though its source state +stays `online`, because phones show only the dot. `truenasPageModel.test.ts` and `TrueNASStorageTopologyTable.test.tsx` pin it. The vSphere Datastores and Networks tables follow the same exception-first Health column. `getVmwareDatastoreIssue` and `getVmwareNetworkIssue` return diff --git a/frontend-modern/browser-verification/e3956fd0c9818451.json b/frontend-modern/browser-verification/e3956fd0c9818451.json new file mode 100644 index 0000000000..4e012c9ea7 --- /dev/null +++ b/frontend-modern/browser-verification/e3956fd0c9818451.json @@ -0,0 +1,30 @@ +{ + "version": 1, + "base_sha": "396bb5d517f14466795a2ccd2ff438fdba0ab077", + "verified_at": "2026-10-06T23:21:38Z", + "result": "passed", + "changed_paths": ["frontend-modern/src/features/truenas/truenasPageModel.ts"], + "content_sha256": { + "frontend-modern/src/features/truenas/truenasPageModel.ts": "91d7afa5bfb5b47692def18ae0a6e773ca47194d41faf0b57fa5c5c82d57d278" + }, + "routes": ["/truenas/storage"], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack built from this exact content, with a local-only overlay (never committed) pinning TrueNAS mock disks: sdc (SAS, carries the native incident 'Device /dev/sdc has SMART test failures.') at 67C and sda (SATA, no incident) at 57C, factory disk temperature thresholds (SAS 65C, SATA 55C)", + "1440: sdc's Health cell reads 'Device /dev/sdc has SMART test failures.' with '+1', and its title lists that reason then 'Disk temperature is 67\u00b0C, at or above its 65\u00b0C alert threshold.'; before this change the heat reason was dropped whenever an incident had text. sda still reads 'Disk temperature is 57\u00b0C, at or above its 55\u00b0C alert threshold.'", + "390: the Health column stays phone-hidden; sda's status dot reads Warning and the cool sdd reads Online; no horizontal overflow (scrollWidth 390)" + ], + "interactions": [ + "Loaded TrueNAS Storage at 1440x900 and 390x844 and read the sdc and sda Health cells, their titles and visible reason text, and the status dot titles", + "Moved sdc's pinned reading from 57C (under its SAS trigger: incident only, no heat) to 67C and reloaded to confirm the heat reason joins the incident text" + ] +} diff --git a/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts b/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts index 1d29985b7c..3a88ea9308 100644 --- a/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts +++ b/frontend-modern/src/features/truenas/__tests__/truenasPageModel.test.ts @@ -676,6 +676,16 @@ describe('truenasPageModel', () => { filterTrueNASStorageTopologyRows(rows, '', 'healthy', resolveFactory).map((row) => row.id), ).toEqual(['disk:disk-cool']); + // Native TrueNAS alert text leads, and the heat reason stays listed. + const hotWithIncident = makeResource({ + ...hotDisk, + incidents: [{ code: 'truenas_smart', severity: 'info', summary: 'SMART test scheduled' }], + }); + expect(getTrueNASStorageIssue(hotWithIncident, resolveFactory)?.reasons).toEqual([ + 'SMART test scheduled', + 'Disk temperature is 56°C, at or above its 55°C alert threshold.', + ]); + // An NVMe at 63C is under its 70C trigger. const warmNVMe = makeResource({ ...hotDisk, diff --git a/frontend-modern/src/features/truenas/truenasPageModel.ts b/frontend-modern/src/features/truenas/truenasPageModel.ts index ceca216eae..75c13bcf0b 100644 --- a/frontend-modern/src/features/truenas/truenasPageModel.ts +++ b/frontend-modern/src/features/truenas/truenasPageModel.ts @@ -844,8 +844,10 @@ export function getTrueNASStorageIssue( if (reasons.length === leading) { for (const reason of resource.storage?.risk?.reasons ?? []) push(reason.summary); for (const reason of resource.physicalDisk?.risk?.reasons ?? []) push(reason.summary); - push(getTrueNASDiskHeatSummary(resource, resolveDiskTemperatureThresholds)); } + // Heat is Pulse's own judgement, not a restatement of TrueNAS state, so it + // stays listed beside any native alert text. + push(getTrueNASDiskHeatSummary(resource, resolveDiskTemperatureThresholds)); // A read-only dataset raises no incident, so its state tag is the only // reason the provider gives. The drawer reads the same shared mapping. if (reasons.length === leading && resource.storage?.topology === 'dataset') { From 4fd6b483cbb905c7caf132c23396ae83936015d8 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 01:42:54 +0100 Subject: [PATCH 13/13] Use a SMART warning, not heat, in the cluster member disk test TestContract_ConnectionSystemMembersIgnoreOwnedPhysicalDisks proves a disk's health never folds into its cluster node's liveness. Its warning disk came from heat: an NVMe at 63C, which the flat 60C disk risk rule flagged. Disk risk no longer judges temperature, and 63C is under the NVMe alert trigger, so that disk is now online and the precondition failed. Reallocated sectors give the disk the same warning status through real SMART evidence, which keeps the test on its subject. --- internal/api/contract_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go index 1e0a0292d1..fa09df5ea3 100644 --- a/internal/api/contract_test.go +++ b/internal/api/contract_test.go @@ -16889,9 +16889,9 @@ func TestContract_PBSHostAgentComposesIntoOwningSystem(t *testing.T) { // An agent on a Proxmox cluster node reports SMART data for the node's disks, // and the registry stamps each disk with the owning node's identity so it // stays discoverable in the Proxmox workspace. A disk is not a cluster member: -// projecting it as one folded its health into the node row, so a warm NVMe -// drive rendered an actively reporting node as Stale whenever the agent's -// report was newer than the last Proxmox poll. +// projecting it as one folded its health into the node row, so a disk with a +// SMART warning rendered an actively reporting node as Stale whenever the +// agent's report was newer than the last Proxmox poll. func TestContract_ConnectionSystemMembersIgnoreOwnedPhysicalDisks(t *testing.T) { cfg := &config.Config{DataPath: t.TempDir()} monitor, err := monitoring.New(cfg) @@ -16902,6 +16902,7 @@ func TestContract_ConnectionSystemMembersIgnoreOwnedPhysicalDisks(t *testing.T) polledAt := time.Now().UTC().Add(-10 * time.Second) reportedAt := polledAt.Add(6 * time.Second) + reallocatedSectors := int64(3) adapter := unifiedresources.NewMonitorAdapter(nil) adapter.PopulateFromSnapshot(models.StateSnapshot{ Nodes: []models.Node{{ @@ -16930,8 +16931,9 @@ func TestContract_ConnectionSystemMembersIgnoreOwnedPhysicalDisks(t *testing.T) Model: "WD_BLACK SN7100 4TB", Serial: "SN7100-AURORA", Type: "nvme", - Temperature: 63, + Temperature: 41, Health: "PASSED", + Attributes: &models.SMARTAttributes{ReallocatedSectors: &reallocatedSectors}, }}, }, }},