From 723f3165a1292268634b6054ab93d3015805d990 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:23:18 +0100 Subject: [PATCH 1/2] Show a silent agent's retained disk temperatures as last known MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a host agent passes its reporting lease, its collected SMART temperatures keep their values with collection.temperature unavailable (#2593), and #2603 taught the Storage disk surfaces to show such a value as last known. The frontend HostDiskSMART type never carried `collection`, so the agent's own surfaces kept presenting the retained readings as current: - The Machines temperature cell fell back to the hottest SMART reading and coloured it against the disk thresholds. On a live mock stack a silent, SMART-only agent read 52°C in threshold green. - Its tooltip, the machine drawer's Thermals rows and the guest drawer's Physical Disks card listed the retained disks as plain readings. The row does not make this obvious. By the time the lease expires the registry has marked a standalone agent's source stale, which outranks offline, so a silent agent reaches the table as `warning` with `agent.stale`, not `offline`. The offline metric fallback never blanks its cells, and the temperature cell has no such gate. Now `HostDiskSMART.collection` is typed and every reader decides through `isPhysicalDiskTemperatureCurrent`: - A retained disk reading never stands for the machine while another disk is current. When no non-standby disk is current and the machine has no positive direct or temperatureCelsius reading, the cell shows the hottest retained non-standby value muted with a dotted underline and screen-reader "last known", without threshold colour. - Tooltip and Thermals rows read "(last known)", the Thermals row with the collection reason as its title. A disk with a zero or negative reading no longer shows a temperature row there. - The guest Physical Disks card uses the Storage table's last-known treatment. The temperature helpers move to diskTemperaturePresentation.ts, which diskPresentation.ts re-exports. On the earlier base, importing diskPresentation from the Machines page and the drawer mapper folded the whole Storage presenter into the shared chunk (+4.9 kB gzip) and renamed its tracked budget entry, failing the bundle check. With the small module, Standalone grows 0.22 kB, the shared chunk 0.20 kB, and Workloads is flat against a build of main. The guest card still loads diskPresentation for extraction and health. Direct and temperatureCelsius readings carry no collection state and still lead the cell. A stale agent's CPU, memory, disk and sensor values still render as current. That is a row-wide decision, left for a separate change. --- .../v6/internal/subsystems/api-contracts.md | 9 + .../subsystems/frontend-primitives.md | 10 ++ .../subsystems/performance-and-scalability.md | 7 + .../internal/subsystems/storage-recovery.md | 15 ++ .../internal/subsystems/unified-resources.md | 14 ++ .../47c60098fc12bf96.json | 54 ++++++ .../__tests__/resourceDetailMappers.test.ts | 85 ++++++++- .../Infrastructure/resourceDetailMappers.ts | 11 +- .../Workloads/GuestPhysicalDisks.tsx | 21 ++- .../__tests__/GuestPhysicalDisks.test.tsx | 96 +++++++++- .../standalone/AgentsMachinesTable.tsx | 28 ++- .../__tests__/AgentsMachinesTable.test.tsx | 148 ++++++++++++++++ .../__tests__/agentMachineTableModel.test.ts | 164 ++++++++++++++++++ .../standalone/agentMachineTableModel.ts | 66 +++++-- .../__tests__/diskPresentation.test.ts | 20 +++ .../storageBackups/diskPresentation.ts | 53 +----- .../diskTemperaturePresentation.ts | 53 ++++++ frontend-modern/src/types/api.ts | 3 + 18 files changed, 787 insertions(+), 70 deletions(-) create mode 100644 frontend-modern/browser-verification/47c60098fc12bf96.json create mode 100644 frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index ae4efc4a57..ecae82c5c7 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -5378,6 +5378,15 @@ Backend API payloads and `frontend-modern/src/types/api.ts` must preserve that optional map without making it a required compatibility field, and clients must keep it as descriptive host telemetry rather than a temperature metric, resource identity, alert metric, or storage/recovery signal. +Host sensor SMART rows (`smart[]`) also carry the optional per-field +`collection` status (`PhysicalDiskCollectionStatus`). Backend API payloads and +`frontend-modern/src/types/api.ts` (`HostDiskSMART.collection`) must preserve it +on the agent facet. When a host agent passes its reporting lease, a SMART +temperature that was current (`available`, or present without provenance) keeps +its value and becomes `unavailable`. A state that was already not available is +kept. Clients must read that state through the shared +`isPhysicalDiskTemperatureCurrent` decision before treating the value as a +current reading. Agent resource-context sections expose host package posture only as bounded operational facts: package manager, pending count, inventory state, inspection freshness, and reboot-required state. Raw package identifiers, versions, and diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 85c281c011..f585ee45a4 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -7803,6 +7803,16 @@ never through the threshold colour classes, so a retained reading cannot look hot or healthy. Table, drawer and pool surfaces share that one decision in `frontend-modern/src/features/storageBackups/diskPresentation.ts` rather than each re-reading `collection.temperature`. +That decision and its class now live in the small +`frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts` +module, which `diskPresentation.ts` re-exports, so the Machines table and the +machine drawer's Thermals rows apply the same treatment without pulling the +Storage presenter into their chunks. Those surfaces and the guest drawer's +Physical Disks card import it directly. +When the cell's value is a retained host-agent SMART temperature, it renders +muted with a dotted underline and screen-reader "last known", as does a +retained value in the guest card. Tooltip and Thermals rows say "(last known)", +and the Thermals row carries the collection reason as its title. The focused browser proofs are `frontend-modern/src/features/patrol/__tests__/patrolRunAcceptance.test.ts`, diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 87517e7399..a7f0088b89 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -2044,6 +2044,13 @@ render those sensor values already present on the selected resource payload, but it must not add host powercap reads, sensor-specific history reads, per-row polling, browser-side command assumptions, or table-wide aggregation work. +Host-agent SMART disk rows in `resourceDetailMappers.ts` mark a retained +temperature "(last known)", with its collection reason as the title, from the +payload's own `collection.temperature` state, through `getPhysicalDiskTemperaturePresentation` +in `frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts`. +That module stays separate from `diskPresentation.ts` so the drawer and +Machines chunks take the decision without the Storage presenter, and the rows +add no reads, polling or aggregation work. The Proxmox node drawer overview should follow the existing guest drawer compact detail-section pattern and expose node-specific context such as platform, kernel, hardware, raw capacity, telemetry, and thermal facts rather than diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index cd2ec5bf9e..2397cd4bb5 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -3814,6 +3814,21 @@ or alert evaluation. Verification: `diskPresentation.test.ts`, `diskDetailPresentation.test.ts`, `storagePoolDetailPresentation.test.ts`, `DiskList.test.tsx` (a row moving between current, retained and legacy states) and `DiskDetail.test.tsx`. +The same decision reaches the disks that only a host agent's own SMART rows +describe. `HostDiskSMART.collection` now crosses into the frontend type, and the +helpers live in `frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts` +(re-exported by `diskPresentation.ts`, one implementation) so the Machines +table and the machine drawer can use them without loading the Storage +presenter. The guest drawer's card already loads it for extraction and health. +A silent agent's retained SMART temperature never becomes the Machines row's +value while another disk has a current reading. When the machine has no +positive direct or `temperatureCelsius` reading and no non-standby disk is +current, the cell shows the hottest retained non-standby value as last known, +without the per-disk-type threshold colour. The guest drawer's Physical Disks card marks a +retained value the way the Physical Disks table does. Still presentation only. +Verification: `diskPresentation.test.ts` (one implementation behind both import +paths), `agentMachineTableModel.test.ts`, `AgentsMachinesTable.test.tsx`, +`resourceDetailMappers.test.ts` and `GuestPhysicalDisks.test.tsx`. Direct SATA, SAS, and NVMe device fallback identities remain compatible. Controller-member fallback identities add controller/target scope only where diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 79650fb3d1..d14cbc296c 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -2269,6 +2269,20 @@ and temperature facts must also compose the frontend-primitives `InfoCardKeyValueRow`. Mobile rows retain their condensed endpoint layout; desktop rows use the shared fixed label track so labels and values remain visually adjacent instead of spanning the full drawer width. +A standalone host agent past its reporting lease reaches the Machines table as +`warning` with `agent.stale`, not `offline`: by the time the lease expires the +registry has marked its only source stale, which outranks offline. The offline +metric fallback therefore does not blank its cells, and the temperature cell +has no such gate. For SMART disk temperatures the provenance travels on +`agent.sensors.smart[].collection`: the Machines temperature cell +(`AgentsMachinesTable.tsx`, `agentMachineTableModel.ts`), its tooltip and the +drawer's Thermals rows (`resourceDetailMappers.ts`) read it through +`isPhysicalDiskTemperatureCurrent`, so a retained disk reading never stands for +the machine while another disk has a current one. A positive direct +`temperature` or `temperatureCelsius` reading carries no collection state and +still leads the cell. Without one, and with no current non-standby disk, the +hottest retained non-standby disk reading shows as last known without +threshold colour. The same boundary applies to availability facts, resource change-history metadata, Docker/PBS/PMG service facts, nested PMG queue/mail breakdowns, and Docker container-update management facts. It also covers action-history facts, diff --git a/frontend-modern/browser-verification/47c60098fc12bf96.json b/frontend-modern/browser-verification/47c60098fc12bf96.json new file mode 100644 index 0000000000..eb0d451e5a --- /dev/null +++ b/frontend-modern/browser-verification/47c60098fc12bf96.json @@ -0,0 +1,54 @@ +{ + "version": 1, + "base_sha": "dcb69cfb020193d05d9e4e668d19b820a0c46d31", + "verified_at": "2026-10-06T22:07:56Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts", + "frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx", + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx", + "frontend-modern/src/features/standalone/agentMachineTableModel.ts", + "frontend-modern/src/features/storageBackups/diskPresentation.ts", + "frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts", + "frontend-modern/src/types/api.ts" + ], + "content_sha256": { + "frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts": "6cdf797534d6ecba8f8007561195f9245899ff881a225f83c2c6c6293b557f8c", + "frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx": "98e9dc568360b5b6c9552433207e0e2fa78b91ba17da53fa513fb42484373ff5", + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx": "a1dfacc661118c77fea482d26ffbc6d1c6aca0d351d77744f7a2dbbd4770ef6f", + "frontend-modern/src/features/standalone/agentMachineTableModel.ts": "1f9be08ed7100a330142eab8900b04474360785e75c1b95a8a42d4bc35fa7233", + "frontend-modern/src/features/storageBackups/diskPresentation.ts": "debb0d30e95b84d010d5b7478d61ce5fa9487c0bfb43b3eb812e16b4a75acf2e", + "frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts": "c62e4d1e74b9278ef34d93e8101c3d41ab67b17885b6810116a2f96bd112301c", + "frontend-modern/src/types/api.ts": "bb2712595745499cc51ed5c6e54ea24998e00d42ce45c8715b1d6c3eed6afc3e" + }, + "routes": [ + "/standalone", + "/proxmox?q=analytics-batch-01" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack built from this tree (base dcb69cfb02) plus a local-only go build -overlay of internal/mock/generator.go: three standalone Linux agents get SMART rows; Apollo-114 (CPU sensors) and Meridian-731 (SMART only) are stamped as State.ExpireHostTelemetry does (status offline, temperatures kept, collection.temperature unavailable 'host agent stopped reporting', source kept), Europa-405 keeps reporting. /api/resources returned agent.sensors.smart[].collection end to end, status warning and agent.stale true for the silent agents.", + "Same-session control with the HEAD runtime files under the same Vite: Meridian-731's Temp cell rendered 52°C in threshold green, the tooltip and the drawer Thermals card listed the retained disks as plain current readings, and the guest Physical Disks card showed plain muted 52°C/41°C with no last-known cue.", + "After: Meridian-731's Temp cell renders '52°C' muted (slate-600 light, slate-400 dark) with a dotted underline, no threshold colour and screen-reader ', last known'; its tooltip lists both disks '(last known)' muted. Apollo-114 keeps its CPU package value (no collection state) while its disk tooltip rows read '(last known)'. Europa-405 is unchanged.", + "Machine drawer (Platform details > Machine details > Thermals) at desktop and phone: 'Disk nvme0n1 52°C (last known)' and 'Disk sda 41°C (last known)' with title 'Last known reading, not current: host agent stopped reporting'; cpu.package unchanged.", + "Guest drawer Physical Disks card at desktop and phone: Playwright routes add the agent version a VM-resident agent would carry to analytics-batch-01 and answer its physical_disk child query with Meridian-731's real registry disks (collection state untouched) re-parented to the VM, because mock mode never links an agent inside a VM. Both disks render muted, dotted, with the reason title and ', last known'.", + "Light and dark schemes on the Machines page. Bundle check: Standalone +0.22 kB, shared chunk +0.20 kB, WorkloadsSurface and InfrastructureWorkspace flat versus a same-machine main build, no tracked chunk renamed.", + "Not covered: a real host agent going silent on hardware, an agent genuinely running inside a VM, or stale-row CPU/memory/disk cells (flagged separately)." + ], + "interactions": [ + "Enabled the Temp column from View > column picker on /standalone and read every row's cell text, class, computed colour and text decoration.", + "Hovered the Temp cell of the SMART-only and the CPU-sensor silent agents and read the tooltip rows and their muted state.", + "Opened the Apollo-114 drawer by row click (desktop) and CPU cell tap (phone), expanded Platform details and the Machine Show details, and read the Thermals rows with their titles.", + "Opened the analytics-batch-01 guest drawer from the Proxmox guest table at desktop and phone and read the Physical Disks card summaries, data-temperature-reading, titles and decoration.", + "Swapped the runtime files to HEAD under the running Vite for the same-session control, then restored and re-ran every pass on the final tree." + ] +} diff --git a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts index 0df9376977..5596569845 100644 --- a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts +++ b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts @@ -10,7 +10,7 @@ import { } from '@/components/Infrastructure/resourceDetailMappers'; import resourceDetailMappersSource from '@/components/Infrastructure/resourceDetailMappers.ts?raw'; import resourceDetailDiscoveryModelSource from '@/components/Infrastructure/resourceDetailDiscoveryModel.ts?raw'; -import type { Resource } from '@/types/resource'; +import type { PhysicalDiskCollectionStatus, Resource } from '@/types/resource'; const createHybridHostResource = (): Resource => ({ @@ -203,6 +203,89 @@ describe('resourceDetailMappers', () => { }, ]); }); + + it('marks a silent agent disk temperature as last known and drops disks without one', () => { + const rows = buildTemperatureRows({ + smart: [ + { + device: 'sda', + temperature: 71, + // A host agent past its reporting lease keeps its last reading. + collection: { + temperature: { + state: 'unavailable', + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }, + }, + { + device: 'sdb', + temperature: 38, + collection: { temperature: { state: 'available', source: 'smartctl' } }, + }, + { + device: 'sdc', + temperature: 0, + collection: { + temperature: { + state: 'unsupported', + source: 'smartctl', + reason: 'device did not expose a temperature reading', + }, + }, + }, + ], + }); + + expect(rows).toEqual([ + { + label: 'Disk sda', + value: '71°C (last known)', + valueTitle: 'Last known reading, not current: host agent stopped reporting', + }, + { label: 'Disk sdb', value: '38°C', valueTitle: '38.0°C' }, + ]); + }); + + it('lists a SMART row only for a positive reading in any state, never a standby disk', () => { + const states: (PhysicalDiskCollectionStatus | undefined)[] = [ + undefined, + { temperature: { state: 'available', source: 'smartctl' } }, + { temperature: { state: 'unavailable', source: 'smartctl' } }, + ]; + for (const collection of states) { + for (const temperature of [0, -3]) { + expect( + buildTemperatureRows({ smart: [{ device: 'sda', temperature, collection }] }), + ).toEqual([]); + } + } + + expect( + buildTemperatureRows({ + smart: [ + { + device: 'sda', + temperature: 44, + standby: true, + collection: { temperature: { state: 'unavailable', source: 'smartctl' } }, + }, + { + device: 'sdb', + temperature: 39, + collection: { temperature: { state: 'missing', source: 'smartctl' } }, + }, + ], + }), + ).toEqual([ + { + label: 'Disk sdb', + value: '39°C (last known)', + valueTitle: 'Last known reading, not current', + }, + ]); + }); }); describe('buildCustomSensorRows', () => { diff --git a/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts b/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts index 3a77a7aded..b6b94a7ea1 100644 --- a/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts +++ b/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts @@ -15,6 +15,7 @@ import type { } from '@/types/api'; import type { Resource, ResourceMetric, ResourceVMwareMeta } from '@/types/resource'; import { formatTemperature } from '@/utils/temperature'; +import { getPhysicalDiskTemperaturePresentation } from '@/features/storageBackups/diskTemperaturePresentation'; import { formatBytes } from '@/utils/format'; import { getActionableAgentIdFromResource } from '@/utils/agentResources'; import { @@ -502,13 +503,17 @@ export const buildTemperatureRows = (sensors?: HostSensorSummary) => { const smart = sensors?.smart; if (smart) { smart - .filter((disk) => !disk.standby && Number.isFinite(disk.temperature)) + .filter((disk) => !disk.standby) .sort((a, b) => a.device.localeCompare(b.device)) .forEach((disk) => { + const temperature = getPhysicalDiskTemperaturePresentation(disk); + if (!temperature) return; + // A retained reading, such as a silent agent's, says so in the value + // because this card has no other cue that it is not current. rows.push({ label: `Disk ${disk.device}`, - value: formatTemperature(disk.temperature), - valueTitle: `${disk.temperature.toFixed(1)}°C`, + value: temperature.current ? temperature.label : `${temperature.label} (last known)`, + valueTitle: temperature.title ?? `${disk.temperature.toFixed(1)}°C`, }); }); } diff --git a/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx b/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx index a9c75d3db7..ceeaa26a50 100644 --- a/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx +++ b/frontend-modern/src/components/Workloads/GuestPhysicalDisks.tsx @@ -5,9 +5,12 @@ import { extractPhysicalDiskPresentationData, getPhysicalDiskHealthStatus, } from '@/features/storageBackups/diskPresentation'; +import { + PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS, + getPhysicalDiskTemperaturePresentation, +} from '@/features/storageBackups/diskTemperaturePresentation'; import type { Resource } from '@/types/resource'; import { formatBytes } from '@/utils/format'; -import { formatTemperature } from '@/utils/temperature'; // SMART history and chart controls are only needed after a disk row is opened. // Keep their Storage detail module out of the initial Workloads surface. @@ -19,6 +22,7 @@ const GuestPhysicalDiskRow: Component<{ disk: Resource }> = (props) => { const [expanded, setExpanded] = createSignal(false); const data = () => extractPhysicalDiskPresentationData(props.disk); const health = () => getPhysicalDiskHealthStatus(data()); + const temperature = () => getPhysicalDiskTemperaturePresentation(data()); const label = () => data().model || data().devPath || props.disk.name; return ( @@ -35,8 +39,19 @@ const GuestPhysicalDiskRow: Component<{ disk: Resource }> = (props) => { {health().label} - 0}> - {formatTemperature(data().temperature)} + + {(reading) => ( + + {reading().label} + + , last known + + + )} 0}> {formatBytes(data().size)} diff --git a/frontend-modern/src/components/Workloads/__tests__/GuestPhysicalDisks.test.tsx b/frontend-modern/src/components/Workloads/__tests__/GuestPhysicalDisks.test.tsx index 0fa8f2c11a..0b316595db 100644 --- a/frontend-modern/src/components/Workloads/__tests__/GuestPhysicalDisks.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/GuestPhysicalDisks.test.tsx @@ -1,10 +1,13 @@ import { cleanup, fireEvent, render, screen, within } from '@solidjs/testing-library'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { createSignal } from 'solid-js'; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import type { Resource } from '@/types/resource'; import { GuestPhysicalDisks } from '../GuestPhysicalDisks'; const queryState = vi.hoisted(() => ({ resources: [] as Resource[], + // Set to a signal accessor to drive live resource updates. + accessor: undefined as (() => Resource[]) | undefined, error: undefined as unknown, query: '', enabled: false, @@ -14,7 +17,10 @@ vi.mock('@/hooks/useUnifiedResources', () => ({ useUnifiedResources: (options: { query: string; enabled: () => boolean }) => { queryState.query = options.query; queryState.enabled = options.enabled(); - return { resources: () => queryState.resources, error: () => queryState.error }; + return { + resources: () => (queryState.accessor ? queryState.accessor() : queryState.resources), + error: () => queryState.error, + }; }, })); @@ -41,8 +47,15 @@ const disk: Resource = { }; describe('GuestPhysicalDisks', () => { + // The first expansion lazy-loads the Storage disk detail; a cold transform of + // that module can outlast findByText's default wait on a loaded machine. + beforeAll(async () => { + await import('@/components/Storage/DiskDetail'); + }, 30_000); + beforeEach(() => { queryState.resources = []; + queryState.accessor = undefined; queryState.error = undefined; queryState.query = ''; queryState.enabled = false; @@ -84,6 +97,85 @@ describe('GuestPhysicalDisks', () => { expect(screen.queryByTestId('guest-physical-disks')).toBeNull(); }); + it('marks a retained disk temperature as last known', () => { + queryState.resources = [ + disk, + { + ...disk, + id: 'disk-101-b', + displayName: 'Guest NVMe', + physicalDisk: { + ...disk.physicalDisk!, + devPath: '/dev/nvme0', + model: 'Guest NVMe', + temperature: 61, + collection: { + temperature: { + state: 'unavailable', + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }, + }, + }, + ]; + render(() => ); + + const [currentRow, retainedRow] = screen.getAllByTestId('guest-physical-disk'); + const current = within(currentRow).getByText('45°C'); + expect(current).toHaveAttribute('data-temperature-reading', 'current'); + expect(current).not.toHaveAttribute('title'); + + const retained = retainedRow.querySelector('[data-temperature-reading="last-known"]'); + expect(retained).not.toBeNull(); + expect(retained).toHaveTextContent('61°C, last known'); + expect(retained).toHaveClass('underline', 'decoration-dotted', 'text-muted'); + expect(retained).toHaveAttribute( + 'title', + 'Last known reading, not current: host agent stopped reporting', + ); + }); + + it('follows a disk between current and last known as its collection state changes', () => { + const withReading = ( + temperature: number, + collection?: NonNullable['collection'], + ): Resource => ({ + ...disk, + physicalDisk: { ...disk.physicalDisk!, temperature, collection }, + }); + const [resources, setResources] = createSignal([disk]); + queryState.accessor = resources; + render(() => ); + const reading = () => + screen.getByTestId('guest-physical-disk').querySelector('[data-temperature-reading]'); + + expect(reading()).toHaveAttribute('data-temperature-reading', 'current'); + + setResources([ + withReading(61, { + temperature: { + state: 'unavailable', + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }), + ]); + expect(reading()).toHaveAttribute('data-temperature-reading', 'last-known'); + expect(reading()).toHaveTextContent('61°C, last known'); + expect(reading()).toHaveClass('decoration-dotted'); + + setResources([withReading(47, { temperature: { state: 'available', source: 'smartctl' } })]); + expect(reading()).toHaveAttribute('data-temperature-reading', 'current'); + expect(reading()).toHaveTextContent('47°C'); + expect(reading()).not.toHaveTextContent('last known'); + expect(reading()).not.toHaveAttribute('title'); + expect(reading()).not.toHaveClass('decoration-dotted'); + + setResources([withReading(0)]); + expect(reading()).toBeNull(); + }); + it('makes a failed child query visible rather than silently implying no disks', () => { queryState.error = new Error('unavailable'); render(() => ); diff --git a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx index 7e0f98124e..2609a62fa7 100644 --- a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx +++ b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx @@ -57,6 +57,7 @@ import { type PlatformResourceStatusFilter, withPlatformStatusCounts, } from '@/features/platformPage/sharedPlatformPage'; +import { PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS } from '@/features/storageBackups/diskTemperaturePresentation'; import { useColumnVisibility } from '@/hooks/useColumnVisibility'; import type { Disk } from '@/types/api'; import type { Resource, ResourceAvailabilityMeta } from '@/types/resource'; @@ -76,6 +77,7 @@ import { import { asTrimmedString } from '@/utils/stringUtils'; import { getPreferredResourceIP } from '@/utils/resourceIdentity'; import { useRelativeTimeNow } from '@/utils/relativeTimeClock'; +import { formatTemperature } from '@/utils/temperature'; import { RESOURCE_METADATA_CHANGED_EVENT, type ResourceMetadataChangedDetail, @@ -101,6 +103,7 @@ import { getAgentMachineTemperatureTitle, getAgentMachineThermalPressurePresentation, getNextAgentMachineSortState, + isAgentMachineTemperatureLastKnown, sortAgentMachines, type AgentMachineColumn, type AgentMachineColumnId, @@ -183,6 +186,8 @@ const AgentMachineMetricTooltip: Component<{ const AgentMachineTemperatureCell: Component<{ celsius: number | undefined; + /** The value is a retained reading, not a current one. */ + lastKnown?: boolean; metric: AgentMachineTemperatureMetric; sections: AgentMachineTemperatureDetailSection[]; thresholds?: MetricDisplayThresholds | null; @@ -222,7 +227,26 @@ const AgentMachineTemperatureCell: Component<{ } > {(value) => ( - + + {formatTemperature(value())} + , last known + + } + > + + )} } @@ -1565,6 +1589,7 @@ export const AgentsMachinesTable: Component<{ const raidArrays = () => getAgentMachineRaidArrayDetails(machine); const raidSummary = () => getAgentMachineRaidSummary(machine); const temperature = () => getAgentMachineTemperatureCelsius(machine); + const temperatureLastKnown = () => isAgentMachineTemperatureLastKnown(machine); const temperatureMetric = () => getAgentMachineTemperatureMetric(machine); const temperatureThresholds = () => temperatureMetric() === 'diskTemperature' @@ -1791,6 +1816,7 @@ export const AgentsMachinesTable: Component<{ > { expect(screen.getByText('1400 RPM')).toBeInTheDocument(); }); + it('shows a silent agent disk temperature as last known, never as a current reading', async () => { + // A host agent past its reporting lease is stale and keeps its last SMART + // temperatures with a non-available collection state. + const stoppedReporting = { + temperature: { + state: 'unavailable' as const, + source: 'smartctl', + reason: 'host agent stopped reporting', + }, + }; + const { container } = render(() => ( + ({ ...disk, collection: stoppedReporting })), + }, + }, + }), + resource({ + id: 'mixed-host', + name: 'Mixed Host', + agent: { + sensors: { + smart: [ + { + device: '/dev/sda', + model: 'Retained HDD', + temperature: 69, + collection: stoppedReporting, + }, + { device: '/dev/nvme0', model: 'Fast SSD', temperature: 44 }, + ], + }, + }, + }), + ]} + emptyIcon={emptyIcon} + emptyTitle="No machines" + emptyDescription="Install Pulse Agent." + /> + )); + + await openMachineColumnPicker(); + await fireEvent.click(screen.getByLabelText('Temp')); + + const triggerFor = (id: string) => + container.querySelector( + `[data-agents-machine-row="${id}"] [data-agent-machine-temperature-trigger="true"]`, + ) as HTMLElement | null; + + // Nothing is collected now, so the cell keeps the hottest retained value + // without a threshold colour and says it is last known. + const silent = triggerFor('silent-nas'); + expect(silent).not.toBeNull(); + if (!silent) return; + const retained = silent.querySelector('[data-temperature-reading="last-known"]'); + expect(retained).not.toBeNull(); + expect(retained).toHaveTextContent('71°C, last known'); + expect(retained).toHaveClass('underline', 'decoration-dotted', 'text-muted'); + expect(retained?.className).not.toMatch(/text-(green|yellow|red)-/); + + await fireEvent.mouseEnter(silent); + const tooltip = await waitFor(() => { + const element = document.querySelector('[data-agent-machine-temperature-tooltip="true"]'); + expect(element).not.toBeNull(); + return element as HTMLElement; + }); + expect(within(tooltip).getByText('71°C (last known)')).toHaveClass('text-muted'); + expect(within(tooltip).getByText('66°C (last known)')).toHaveClass('text-muted'); + await fireEvent.mouseLeave(silent); + + // A disk collected now leads the cell even when a retained one is hotter. + const mixed = triggerFor('mixed-host'); + expect(mixed).not.toBeNull(); + if (!mixed) return; + expect(mixed).toHaveTextContent('44°C'); + expect(mixed.querySelector('[data-temperature-reading="last-known"]')).toBeNull(); + expect(within(mixed).queryByText('69°C')).toBeNull(); + }); + + it('moves a machine temperature between current and last known as its disk reports change', async () => { + // Fresh objects per update, like parsed API payloads: the store reconciles + // into the objects it was given, so a reused fixture would be rewritten. + const collected = () => ({ temperature: { state: 'available' as const, source: 'smartctl' } }); + const stoppedReporting = () => ({ + temperature: { + state: 'unavailable' as const, + source: 'smartctl', + reason: 'host agent stopped reporting', + }, + }); + const nas = ( + temperature: number, + collection?: ReturnType | ReturnType, + ) => + resource({ + id: 'nas', + name: 'NAS', + agent: { + sensors: { smart: [{ device: '/dev/sda', type: 'sata', temperature, collection }] }, + }, + }); + const [resources, setResources] = createSignal([nas(52, collected())]); + const { container } = render(() => ( + + )); + + await openMachineColumnPicker(); + await fireEvent.click(screen.getByLabelText('Temp')); + const cell = () => + container.querySelector( + '[data-agents-machine-row="nas"] [data-agent-machine-temperature-trigger="true"]', + ) as HTMLElement; + const lastKnown = () => cell().querySelector('[data-temperature-reading="last-known"]'); + + expect(cell()).toHaveTextContent('52°C'); + expect(lastKnown()).toBeNull(); + + setResources([nas(52, stoppedReporting())]); + await waitFor(() => expect(lastKnown()).not.toBeNull()); + expect(cell()).toHaveTextContent('52°C, last known'); + + setResources([nas(47, collected())]); + await waitFor(() => expect(lastKnown()).toBeNull()); + expect(cell()).toHaveTextContent('47°C'); + expect(cell().querySelector('.decoration-dotted')).toBeNull(); + expect(cell()).not.toHaveTextContent('last known'); + + setResources([nas(0)]); + await waitFor(() => expect(cell()).toHaveTextContent('—')); + expect(lastKnown()).toBeNull(); + }); + it('shows thermal pressure when macOS reports no Celsius temperature', async () => { render(() => ( { expect(getAgentMachineTemperatureMetric(machine)).toBe('diskTemperature'); }); + it('shows a retained SMART temperature only as last known', () => { + // A host agent past its reporting lease keeps its last SMART temperature + // with a non-available collection state. + const stoppedReporting = { + temperature: { + state: 'unavailable' as const, + source: 'host_agent', + reason: 'host agent stopped reporting', + }, + }; + const mixed = resource({ + agent: { + sensors: { + smart: [ + { + device: '/dev/sda', + model: 'Retained HDD', + type: 'sas', + temperature: 71, + collection: stoppedReporting, + }, + { + device: '/dev/nvme0', + model: 'Fast SSD', + type: 'nvme', + temperature: 44, + collection: { temperature: { state: 'available' as const, source: 'smartctl' } }, + }, + // Agents that predate collection state report no provenance. + { device: '/dev/sdb', temperature: 39 }, + { device: '/dev/sdc', model: 'Cold Standby', temperature: 0, standby: true }, + ], + }, + }, + }); + + // A disk collected now outranks a hotter retained one. + expect(getAgentMachineTemperatureCelsius(mixed)).toBe(44); + expect(isAgentMachineTemperatureLastKnown(mixed)).toBe(false); + expect(getAgentMachineTemperatureMetric(mixed)).toBe('diskTemperature'); + expect(getAgentMachineHottestSmartDiskType(mixed)).toBe('nvme'); + expect(getAgentMachineTemperatureTitle(mixed).split('\n')).toEqual([ + 'Disk Temperatures', + 'Disk /dev/nvme0 Fast SSD: 44°C', + 'Disk /dev/sdb: 39°C', + 'Disk /dev/sda Retained HDD: 71°C (last known)', + 'Disk /dev/sdc Cold Standby: standby', + ]); + expect(getAgentMachineTemperatureDetailSections(mixed)[0].rows[2]).toEqual({ + label: 'Disk /dev/sda Retained HDD', + value: '71°C (last known)', + muted: true, + }); + + // With nothing collected now, the cell keeps the hottest retained value + // and marks it last known. + const onlyRetained = resource({ + agent: { + sensors: { + smart: [ + { device: '/dev/sda', type: 'sata', temperature: 71, collection: stoppedReporting }, + { device: '/dev/nvme0', type: 'nvme', temperature: 58, collection: stoppedReporting }, + ], + }, + }, + }); + expect(getAgentMachineTemperatureCelsius(onlyRetained)).toBe(71); + expect(isAgentMachineTemperatureLastKnown(onlyRetained)).toBe(true); + expect(getAgentMachineTemperatureMetric(onlyRetained)).toBe('diskTemperature'); + expect(getAgentMachineHottestSmartDiskType(onlyRetained)).toBe('sata'); + expect(getAgentMachineTemperatureTitle(onlyRetained)).toBe( + 'Disk Temperatures\nDisk /dev/sda: 71°C (last known)\nDisk /dev/nvme0: 58°C (last known)', + ); + + // CPU sensors carry no collection state, so they still lead the cell. + const withSensors = resource({ + agent: { + sensors: { + temperatureCelsius: { 'cpu.package': 49 }, + smart: [{ device: '/dev/sda', temperature: 71, collection: stoppedReporting }], + }, + }, + }); + expect(getAgentMachineTemperatureCelsius(withSensors)).toBe(49); + expect(isAgentMachineTemperatureLastKnown(withSensors)).toBe(false); + expect(getAgentMachineTemperatureMetric(withSensors)).toBe('temperature'); + }); + + it('offers only positive, non-standby SMART readings to the cell, behind direct and CPU sensors', () => { + const stoppedReporting = { + temperature: { + state: 'unavailable' as const, + source: 'smartctl', + reason: 'host agent stopped reporting', + }, + }; + const retainedDisk = (temperature: number) => ({ + device: '/dev/sda', + temperature, + collection: stoppedReporting, + }); + + // A disk in standby is never a temperature candidate, retained or not. + const standbyOnly = resource({ + agent: { sensors: { smart: [{ ...retainedDisk(52), standby: true }] } }, + }); + expect(getAgentMachineTemperatureCelsius(standbyOnly)).toBeUndefined(); + expect(isAgentMachineTemperatureLastKnown(standbyOnly)).toBe(false); + expect(getAgentMachineTemperatureTitle(standbyOnly)).toBe( + 'Disk Temperatures\nDisk /dev/sda: standby', + ); + + // A direct machine temperature leads a retained disk. + const direct = resource({ + temperature: 47, + agent: { sensors: { smart: [retainedDisk(71)] } }, + }); + expect(getAgentMachineTemperatureCelsius(direct)).toBe(47); + expect(isAgentMachineTemperatureLastKnown(direct)).toBe(false); + expect(getAgentMachineTemperatureMetric(direct)).toBe('temperature'); + + // Additional sensors never lead the cell, so the retained disk is shown. + const additionalOnly = resource({ + agent: { sensors: { additional: { vrm: 49 }, smart: [retainedDisk(71)] } }, + }); + expect(getAgentMachineTemperatureCelsius(additionalOnly)).toBe(71); + expect(isAgentMachineTemperatureLastKnown(additionalOnly)).toBe(true); + + // Sorting follows the value the cell shows: a hotter retained disk does not + // lift a machine whose shown reading is a cooler current one. + const machines = [ + resource({ + id: 'retained', + name: 'Retained', + agent: { sensors: { smart: [retainedDisk(71)] } }, + }), + resource({ + id: 'mixed', + name: 'Mixed', + agent: { + sensors: { smart: [retainedDisk(80), { device: '/dev/sdb', temperature: 35 }] }, + }, + }), + resource({ + id: 'current', + name: 'Current', + agent: { sensors: { smart: [{ device: '/dev/sda', temperature: 50 }] } }, + }), + ]; + const sortedIds = (direction: 'asc' | 'desc') => + sortAgentMachines( + machines, + 'temp', + direction, + () => '', + () => '', + ).map((machine) => machine.id); + expect(sortedIds('desc')).toEqual(['retained', 'current', 'mixed']); + expect(sortedIds('asc')).toEqual(['mixed', 'current', 'retained']); + }); + it('prefers direct and sensor temperatures over SMART fallback temperatures', () => { const direct = resource({ temperature: 55, diff --git a/frontend-modern/src/features/standalone/agentMachineTableModel.ts b/frontend-modern/src/features/standalone/agentMachineTableModel.ts index 5fe81f91f9..661e02c86c 100644 --- a/frontend-modern/src/features/standalone/agentMachineTableModel.ts +++ b/frontend-modern/src/features/standalone/agentMachineTableModel.ts @@ -2,6 +2,7 @@ import type { ColumnDef } from '@/hooks/useColumnVisibility'; import type { HostDiskIO, HostRAIDArray, HostRAIDDevice } from '@/types/api'; import type { Resource } from '@/types/resource'; import { getPlatformTableFiniteMetric } from '@/features/platformPage/sharedPlatformPage'; +import { isPhysicalDiskTemperatureCurrent } from '@/features/storageBackups/diskTemperaturePresentation'; import { formatBytes, normalizeDiskArray } from '@/utils/format'; import { asTrimmedString } from '@/utils/stringUtils'; @@ -155,6 +156,9 @@ type TemperatureReading = { type SmartTemperatureReading = TemperatureReading & { standby?: boolean; diskType?: string; + // False for a retained value, such as the last temperature of a host agent + // that stopped reporting. + current: boolean; }; export type AgentMachineTemperatureDetailRow = { @@ -257,13 +261,21 @@ const getSmartTemperatureReadings = (machine: Resource): SmartTemperatureReading value: temperature ?? 0, standby: disk.standby, diskType: asTrimmedString(disk.type), + current: isPhysicalDiskTemperatureCurrent(disk.collection), }); return readings; }, []); -const getActiveSmartTemperatureReadings = (machine: Resource): TemperatureReading[] => +// Only a temperature collected now can stand for the machine or pick its +// threshold. A retained one stays in the detail rows, marked last known. +const getActiveSmartTemperatureReadings = (machine: Resource): SmartTemperatureReading[] => getSmartTemperatureReadings(machine).filter( - (reading): reading is TemperatureReading => !reading.standby && reading.value > 0, + (reading) => !reading.standby && reading.value > 0 && reading.current, + ); + +const getLastKnownSmartTemperatureReadings = (machine: Resource): SmartTemperatureReading[] => + getSmartTemperatureReadings(machine).filter( + (reading) => !reading.standby && reading.value > 0 && !reading.current, ); const byHighestTemperature = (left: TemperatureReading, right: TemperatureReading): number => @@ -536,22 +548,38 @@ export const getAgentMachineDiskIODetails = (machine: Resource): AgentMachineDis }, []); }; +const hasMachineSensorTemperature = (machine: Resource): boolean => + positiveTemperature(machine.temperature) !== undefined || + getSensorTemperatureReadings(machine).length > 0; + +// The SMART readings the cell falls back to: the ones collected now, or the +// retained ones when no disk has a current reading. +const getCellSmartTemperatureReadings = (machine: Resource): SmartTemperatureReading[] => { + const active = getActiveSmartTemperatureReadings(machine); + return active.length > 0 ? active : getLastKnownSmartTemperatureReadings(machine); +}; + export const getAgentMachineTemperatureCelsius = (machine: Resource): number | undefined => { const direct = positiveTemperature(machine.temperature); if (direct !== undefined) return direct; return ( maxTemperatureReading(getSensorTemperatureReadings(machine)) ?? - maxTemperatureReading(getActiveSmartTemperatureReadings(machine)) + maxTemperatureReading(getCellSmartTemperatureReadings(machine)) ); }; -// Disk type of the hottest active SMART reading — the disk whose temperature -// the cell displays when it falls back to the diskTemperature metric. +// True when the cell's value is a retained SMART reading, such as the last +// disk temperature of a host agent that stopped reporting. +export const isAgentMachineTemperatureLastKnown = (machine: Resource): boolean => + !hasMachineSensorTemperature(machine) && + getActiveSmartTemperatureReadings(machine).length === 0 && + getLastKnownSmartTemperatureReadings(machine).length > 0; + +// Disk type of the hottest SMART reading the cell displays when it falls back +// to the diskTemperature metric. export const getAgentMachineHottestSmartDiskType = (machine: Resource): string | undefined => { - const readings = getSmartTemperatureReadings(machine).filter( - (reading) => !reading.standby && reading.value > 0, - ); + const readings = getCellSmartTemperatureReadings(machine); if (readings.length === 0) return undefined; return readings.reduce((worst, reading) => (reading.value > worst.value ? reading : worst)) .diskType; @@ -560,13 +588,8 @@ export const getAgentMachineHottestSmartDiskType = (machine: Resource): string | export const getAgentMachineTemperatureMetric = ( machine: Resource, ): AgentMachineTemperatureMetric => { - if (positiveTemperature(machine.temperature) !== undefined) return 'temperature'; - if (maxTemperatureReading(getSensorTemperatureReadings(machine)) !== undefined) { - return 'temperature'; - } - if (maxTemperatureReading(getActiveSmartTemperatureReadings(machine)) !== undefined) { - return 'diskTemperature'; - } + if (hasMachineSensorTemperature(machine)) return 'temperature'; + if (getCellSmartTemperatureReadings(machine).length > 0) return 'diskTemperature'; return 'temperature'; }; @@ -585,6 +608,13 @@ export const getAgentMachineTemperatureDetailSections = ( label: `Disk ${reading.label}`, value: formatTemperatureValue(reading), })); + const lastKnownSmartReadings = getLastKnownSmartTemperatureReadings(machine) + .sort(byHighestTemperature) + .map((reading) => ({ + label: `Disk ${reading.label}`, + value: `${formatTemperatureValue(reading)} (last known)`, + muted: true, + })); const standbySmartReadings = getSmartTemperatureReadings(machine) .filter((reading) => reading.standby) .sort(byLabel) @@ -609,7 +639,11 @@ export const getAgentMachineTemperatureDetailSections = ( ...section('Temperatures', capTemperatureRows(sensorReadings)), ...section( 'Disk Temperatures', - capTemperatureRows([...activeSmartReadings, ...standbySmartReadings]), + capTemperatureRows([ + ...activeSmartReadings, + ...lastKnownSmartReadings, + ...standbySmartReadings, + ]), ), ...section('Fan Speeds', capTemperatureRows(fanReadings)), ...section('Other Sensors', capTemperatureRows(additionalReadings)), diff --git a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts index a18d7ccdfe..c3c13377bd 100644 --- a/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts +++ b/frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts @@ -18,6 +18,7 @@ import { PHYSICAL_DISK_SOURCE_BADGE_CLASS, PHYSICAL_DISK_TABLE_CLASS, PHYSICAL_DISK_TABLE_ROW_HOVER_CLASS, + PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS, getPhysicalDiskEmptyStatePresentation, getPhysicalDiskCellPaddingClass, getPhysicalDiskColumnWidthStyle, @@ -27,6 +28,7 @@ import { getPhysicalDiskHealthStatus, getPhysicalDiskHealthSummary, getPhysicalDiskHostLabel, + getPhysicalDiskLastKnownTemperatureTitle, getPhysicalDiskLifeLabel, getPhysicalDiskLifeTextClass, getPhysicalDiskNormalizedHealth, @@ -48,6 +50,7 @@ import { normalizePhysicalDiskFacetFilter, type PhysicalDiskPresentationData, } from '@/features/storageBackups/diskPresentation'; +import * as diskTemperaturePresentation from '@/features/storageBackups/diskTemperaturePresentation'; function makeDiskData( overrides: Partial = {}, @@ -131,6 +134,23 @@ describe('diskPresentation', () => { } }); + it('serves the same temperature decision the Machines table and drawers import directly', () => { + // Tables outside Storage import the small module so they do not pull this + // one into their chunks. Both paths must stay one implementation. + expect(isPhysicalDiskTemperatureCurrent).toBe( + diskTemperaturePresentation.isPhysicalDiskTemperatureCurrent, + ); + expect(getPhysicalDiskTemperaturePresentation).toBe( + diskTemperaturePresentation.getPhysicalDiskTemperaturePresentation, + ); + expect(getPhysicalDiskLastKnownTemperatureTitle).toBe( + diskTemperaturePresentation.getPhysicalDiskLastKnownTemperatureTitle, + ); + expect(PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS).toBe( + diskTemperaturePresentation.PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS, + ); + }); + it('distinguishes unsupported, unavailable, and unexpectedly missing disk evidence', () => { expect( getPhysicalDiskFieldStatusMessage('Disk I/O', { diff --git a/frontend-modern/src/features/storageBackups/diskPresentation.ts b/frontend-modern/src/features/storageBackups/diskPresentation.ts index 2499be5969..fe1ad5ad4e 100644 --- a/frontend-modern/src/features/storageBackups/diskPresentation.ts +++ b/frontend-modern/src/features/storageBackups/diskPresentation.ts @@ -14,7 +14,6 @@ import { getAllFilterOptionLabel } from '@/components/shared/filterOptionPresent import { getPhysicalDiskNodeIdentity } from '@/components/Storage/diskResourceUtils'; import { getInfrastructureSettingsLocationLabel } from '@/utils/infrastructureSettingsPresentation'; import { normalizeStorageSourceKey, storageSourceMatchesFilter } from '@/utils/storageSources'; -import { formatTemperature } from '@/utils/temperature'; import type { NormalizedHealth, StorageHealthFilter } from './models'; import { matchesStorageNodeTerms, parseStorageSearchQuery } from './storageSearchQuery'; @@ -254,11 +253,6 @@ export const PHYSICAL_DISK_HEALTH_WRAP_CLASS = // that would compete with it for the track. export const PHYSICAL_DISK_HEALTH_LABEL_CLASS = 'shrink-0 text-[11px] font-semibold'; export const PHYSICAL_DISK_TEMPERATURE_CLASS = 'text-[11px] font-medium'; -// A retained reading takes no threshold colour, so it cannot read as a disk -// running hot or cool right now. The dotted underline marks the value as one -// its title explains. -export const PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS = - 'text-muted underline decoration-dotted underline-offset-2 cursor-help'; export const PHYSICAL_DISK_SIZE_VALUE_CLASS = 'text-[11px] text-base-content'; const titleize = (value: string | undefined | null): string => @@ -473,44 +467,15 @@ export function getPhysicalDiskFieldStatusMessage( } } -// Normalization may keep the last known temperature when the current -// observation is not available, such as a disk in standby or a host agent that -// stopped reporting. The value is a current reading only when its collection -// state is available, or when the source predates collection state. -export function isPhysicalDiskTemperatureCurrent( - collection: PhysicalDiskCollectionStatus | null | undefined, -): boolean { - const state = collection?.temperature?.state; - return !state || state === 'available'; -} - -export function getPhysicalDiskLastKnownTemperatureTitle( - status: PhysicalDiskFieldStatus | null | undefined, -): string { - const reason = status?.reason?.trim(); - return reason ? `Last known reading, not current: ${reason}` : 'Last known reading, not current'; -} - -export interface PhysicalDiskTemperaturePresentation { - label: string; - current: boolean; - /** Set only for a last-known reading: why the value is not current. */ - title?: string; -} - -export function getPhysicalDiskTemperaturePresentation( - disk: Pick, -): PhysicalDiskTemperaturePresentation | null { - if (!Number.isFinite(disk.temperature) || disk.temperature <= 0) return null; - const current = isPhysicalDiskTemperatureCurrent(disk.collection); - return { - label: formatTemperature(disk.temperature), - current, - title: current - ? undefined - : getPhysicalDiskLastKnownTemperatureTitle(disk.collection?.temperature), - }; -} +// Whether a disk temperature is a current reading lives in its own module so +// tables outside the Storage page can use it without this one. +export { + PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS, + getPhysicalDiskLastKnownTemperatureTitle, + getPhysicalDiskTemperaturePresentation, + isPhysicalDiskTemperatureCurrent, + type PhysicalDiskTemperaturePresentation, +} from './diskTemperaturePresentation'; export function getPhysicalDiskCollectionMessages(disk: PhysicalDiskPresentationData): string[] { const collection = disk.collection; diff --git a/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts b/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts new file mode 100644 index 0000000000..acb63c612a --- /dev/null +++ b/frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts @@ -0,0 +1,53 @@ +import type { PhysicalDiskCollectionStatus, PhysicalDiskFieldStatus } from '@/types/resource'; +import { formatTemperature } from '@/utils/temperature'; + +// The canonical decision on whether a disk temperature is a current reading. +// It lives apart from diskPresentation so the Machines table, the machine +// drawer and the guest drawer can use it without pulling the Storage page +// presentation module into their chunks. + +// A retained reading takes no threshold colour, so it cannot read as a disk +// running hot or cool right now. The dotted underline marks the value as one +// its title explains. +export const PHYSICAL_DISK_TEMPERATURE_LAST_KNOWN_CLASS = + 'text-muted underline decoration-dotted underline-offset-2 cursor-help'; + +// Normalization may keep the last known temperature when the current +// observation is not available, such as a disk in standby or a host agent that +// stopped reporting. The value is a current reading only when its collection +// state is available, or when the source predates collection state. +export function isPhysicalDiskTemperatureCurrent( + collection: PhysicalDiskCollectionStatus | null | undefined, +): boolean { + const state = collection?.temperature?.state; + return !state || state === 'available'; +} + +export function getPhysicalDiskLastKnownTemperatureTitle( + status: PhysicalDiskFieldStatus | null | undefined, +): string { + const reason = status?.reason?.trim(); + return reason ? `Last known reading, not current: ${reason}` : 'Last known reading, not current'; +} + +export interface PhysicalDiskTemperaturePresentation { + label: string; + current: boolean; + /** Set only for a last-known reading: why the value is not current. */ + title?: string; +} + +export function getPhysicalDiskTemperaturePresentation(disk: { + temperature: number; + collection?: PhysicalDiskCollectionStatus | null; +}): PhysicalDiskTemperaturePresentation | null { + if (!Number.isFinite(disk.temperature) || disk.temperature <= 0) return null; + const current = isPhysicalDiskTemperatureCurrent(disk.collection); + return { + label: formatTemperature(disk.temperature), + current, + title: current + ? undefined + : getPhysicalDiskLastKnownTemperatureTitle(disk.collection?.temperature), + }; +} diff --git a/frontend-modern/src/types/api.ts b/frontend-modern/src/types/api.ts index 5b15015d7a..eaa1bfcc3f 100644 --- a/frontend-modern/src/types/api.ts +++ b/frontend-modern/src/types/api.ts @@ -626,6 +626,9 @@ export interface HostDiskSMART { temperature: number; // Temperature in Celsius health?: string; // PASSED, FAILED, UNKNOWN standby?: boolean; // True if disk was in standby + // A non-available temperature state marks the value as last known, such as + // a host agent that stopped reporting. + collection?: import('./resource').PhysicalDiskCollectionStatus; } export interface HostRAIDArray { From c06af69fcb225f64c7823bcb9536fd2f810714e8 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 7 Oct 2026 00:14:49 +0100 Subject: [PATCH 2/2] Show a host whose agent stopped reporting as offline A host agent past its reporting lease is offline: the monitor marks it so and raises "Host is offline". The registry's stale pass ranked the agent's stale sighting above offline, though, so the machine reached every consumer as a warning. The Machines table showed it amber and rendered its last CPU, memory, disk and temperature as current readings, while its own drawer led with the critical offline alert. A Docker host was offline only until its sighting crossed the 120-second stale threshold, then flipped to warning, and Kubernetes clusters likewise. Each sighting now records when a push reporter (host agent, Docker, Kubernetes) delivered its machine, Docker host or cluster as offline, and status aggregation counts that as offline whatever the sighting's age. The sighting itself still reads stale, and a live source still carries a merged row, so a Proxmox node with a dead agent stays online. A copy that went through JSON re-derives the marker from its stored status, and a manual link joining two resources from one source keeps the fresher sighting, so a retired agent cannot pull a live one down. Offline rows then take the existing danger gate. The Machines Uptime and Temp cells and the Docker hosts Temp cell had no gate and now follow it. The drawer drops a silent agent's uptime and marks its non-disk Thermals rows "(last known)", as retained disk rows already are. --- .../subsystems/frontend-primitives.md | 8 + .../subsystems/performance-and-scalability.md | 5 + .../internal/subsystems/storage-recovery.md | 10 + .../internal/subsystems/unified-resources.md | 38 +- .../3b34c585696e9d41.json | 49 +++ .../Infrastructure/ResourceDetailSummary.tsx | 6 +- .../ResourceDetailSummary.offline.test.tsx | 64 ++++ .../__tests__/resourceDetailMappers.test.ts | 41 +++ .../Infrastructure/resourceDetailMappers.ts | 33 +- .../useResourceDetailDrawerDerivedState.ts | 9 +- .../src/features/docker/DockerHostsTable.tsx | 6 +- .../__tests__/DockerHostsTable.test.tsx | 52 +++ .../standalone/AgentsMachinesTable.tsx | 27 +- .../__tests__/AgentsMachinesTable.test.tsx | 97 ++++- internal/unifiedresources/clone_test.go | 6 + internal/unifiedresources/registry.go | 55 ++- .../registry_merge_policy_test.go | 348 ++++++++++++++++++ internal/unifiedresources/types.go | 6 + 18 files changed, 831 insertions(+), 29 deletions(-) create mode 100644 frontend-modern/browser-verification/3b34c585696e9d41.json create mode 100644 frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.offline.test.tsx diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index f585ee45a4..378d00185c 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -5591,6 +5591,14 @@ Standalone, TrueNAS, and vSphere platform tables and their table-model helpers must compose those helpers instead of declaring local `metricFallback` / `finiteMetric` helpers or inlining centered muted dash fallback markup in metric cells. +A row whose status indicator is `danger` (offline) blanks every reading from +its last report, not only the metric bars. Proxmox node rows already gate +uptime and temperature on online. The Machines table gates its Uptime and +Temperature cells, and the Docker hosts table its temperature cell, on the same +check. A machine another source keeps up stays rendered, since its cell may +carry that source's current reading. Numeric cells keep their own +right-aligned empty dash; the centred `PlatformTableMetricFallback` marker +stays specific to metric-bar cells. Platform table metric severity coloring is alert-backed, not hardcoded. The Docker host and container, Proxmox node, Kubernetes cluster and node, TrueNAS system and app, and vSphere host tables must resolve display thresholds diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index a7f0088b89..5087716ff4 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -2051,6 +2051,11 @@ in `frontend-modern/src/features/storageBackups/diskTemperaturePresentation.ts`. That module stays separate from `diskPresentation.ts` so the drawer and Machines chunks take the decision without the Storage presenter, and the rows add no reads, polling or aggregation work. +A silent host agent's other Thermals rows (pressure, limits, GPU, temperatures, +additional sensors, fans, power) take the same "(last known)" suffix and reason +title. `buildTemperatureRows` gets them through its `lastKnownReason` option, +which the drawer sets from the resource's own `agent.stale`, so the drawer +still reads nothing extra. The Proxmox node drawer overview should follow the existing guest drawer compact detail-section pattern and expose node-specific context such as platform, kernel, hardware, raw capacity, telemetry, and thermal facts rather than diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 2397cd4bb5..e96a2af8cf 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -3169,6 +3169,16 @@ pointer isolation are pinned by `TestResourceFromHostPreservesCustomSensorMeta` and `TestCloneResourceIsolatesCustomSensorValues`. +### Expired reporting leases change status only + +`SourceStatus` now carries an unexported marker that a push reporter (host +agent, Docker, Kubernetes) delivered its machine, Docker host or cluster as +offline at lease expiry. Status aggregation reads it, so such a row is +`offline` instead of `warning`. The marker is not serialized, persisted or +exposed on the wire, and the public `status`, `lastSeen` and cadence fields +are unchanged. No storage path, recovery read or protection judgement depends +on it. `TestCloneResource_MutateSourceStatusMap` pins that clones keep it. + ### Shared system-settings boundary dropped dead auto-update schedule fields The shared `internal/api` system-settings surface this subsystem consumes diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index d14cbc296c..f5e20f3eca 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -2270,10 +2270,12 @@ and temperature facts must also compose the frontend-primitives desktop rows use the shared fixed label track so labels and values remain visually adjacent instead of spanning the full drawer width. A standalone host agent past its reporting lease reaches the Machines table as -`warning` with `agent.stale`, not `offline`: by the time the lease expires the -registry has marked its only source stale, which outranks offline. The offline -metric fallback therefore does not blank its cells, and the temperature cell -has no such gate. For SMART disk temperatures the provenance travels on +`offline`, as described in "A push reporter past its lease is offline". The +offline metric fallback blanks its CPU, memory, disk, network, disk I/O, uptime +and temperature cells. The machine drawer drops a silent agent's uptime and +marks its non-disk Thermals rows "(last known)" with the reason as title, +keyed on `agent.stale` because those rows are that agent's own sensors. For SMART disk +temperatures on rows that still render, the provenance travels on `agent.sensors.smart[].collection`: the Machines temperature cell (`AgentsMachinesTable.tsx`, `agentMachineTableModel.ts`), its tooltip and the drawer's Thermals rows (`resourceDetailMappers.ts`) read it through @@ -6030,6 +6032,34 @@ to that boolean. Identity succession continues to rekey the same state row. round-trip persistence, attention suppression, and the retired remediation lock. +### A push reporter past its lease is offline + +The host agent, Docker and Kubernetes collectors push their own reports, and +the monitor marks the machine, Docker host or cluster offline only once its +reporting lease runs out. Ingest records that verdict on the source's +sighting, and `aggregateStatus` counts such a sighting as `offline` whatever +its age. The stale pass used to rank the stale sighting above `offline`, so a +silent standalone agent reached every consumer as `warning`. The Machines +table then showed it amber with its last report rendered as current readings, +while its own drawer led with a critical "Host is offline" alert. A Docker host +was offline only until its sighting crossed the 120-second stale threshold, +then flipped to `warning`. With this rule, such rows take the existing +offline treatment: frontend danger gates, the health verdict's `offline` +reason, offline filters and counts. + +`SourceStatus.Status` keeps describing delivery freshness, so the sighting +still reads `stale`. Health keeps its `telemetry_stale` reason, and +monitored-system reasons are unchanged. A live source still carries a merged +row: a Proxmox node whose linked agent stopped reporting stays `online` through +the PVE poll. Pull sources (Proxmox, PBS, PMG, TrueNAS, vSphere) and the +resources push collectors report about (guests, containers, pods, disks) keep +the stale-to-warning rule. A JSON copy re-derives the unexported marker from +its stored status in `IngestResources`, and a manual link that joins two +resources reported by one source keeps the fresher sighting. The tests in +`registry_merge_policy_test.go` pin the agent, Docker, Kubernetes, +mixed-source, round-trip and manual-link cases, plus the boundaries that stay +unchanged. + ### Canonical object drawer hierarchy The unified resource drawer is the shared operator-facing detail contract for diff --git a/frontend-modern/browser-verification/3b34c585696e9d41.json b/frontend-modern/browser-verification/3b34c585696e9d41.json new file mode 100644 index 0000000000..f9cb8ca29a --- /dev/null +++ b/frontend-modern/browser-verification/3b34c585696e9d41.json @@ -0,0 +1,49 @@ +{ + "version": 1, + "base_sha": "723f3165a1292268634b6054ab93d3015805d990", + "verified_at": "2026-10-06T23:14:41Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx", + "frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts", + "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts", + "frontend-modern/src/features/docker/DockerHostsTable.tsx", + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx" + ], + "content_sha256": { + "frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx": "f141220216a968f5953b7646035108f015a9f7de3c5ae5a9e3251ec6e7046fa5", + "frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts": "07b26eafecbd1ec93fb1dd90d4af6629778cef59a2d9e7450426cb88c90bdfdb", + "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts": "70955e370c5b2611557a5de8ba4df44b8f6dce47c1b08c611a154d232c786501", + "frontend-modern/src/features/docker/DockerHostsTable.tsx": "450cce7a4723ce7fe7b01ffe9d10c7d6439c34a1a6dda4fc0af6689ceb4741b5", + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx": "8d63a04f6a475f39de90cb9813b8bd4315525bf63318cfaf958ac195ae07950f" + }, + "routes": [ + "/standalone/machines", + "/docker" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Two private mock stacks (8-node estate) built from this tree: one with this commit's registry.go/types.go, one with origin/main's, both with a local-only go build -overlay of internal/mock/generator.go that holds standalone Linux agent host-linux-1 (Apollo-114) and Docker host edge-proxy-01 past their reporting leases the way evaluateHostAgents/evaluateDockerAgents leave them: status offline, LastSeen frozen 14m/9m before the first tick, last readings kept. Mock mode skips the host health sweep, hence the overlay.", + "Before (origin/main backend and frontend): /api/resources served Apollo-114 as status warning with agent.stale true and edge-proxy-01 as warning with health stale. The Machines row showed an amber dot, CPU 32%, Memory 74%, Disk 80%, Net and Disk I/O rates, Up 29d and Temp 45\u00b0C in threshold green beside 'Seen 14m ago', while the drawer led with the critical 'Host Apollo-114 is offline' alert and read 'Observed state Degraded, Uptime 29d 12h'.", + "After (this commit): both resources serve status offline, the agent/docker sightings still read stale, and health reads critical with reasons offline then telemetry_stale. The Apollo-114 row shows a red dot and '\u2014' in CPU, Memory, Disk, Net I/O, Disk I/O, Up and Temp at 1440 and 390; the status filter counts it under Offline. Spectrum-709 (reporting) keeps every value, Temp 47\u00b0C included.", + "Machine drawer at 1440 and 390: Runtime context reads 'Observed state Offline' and 'Last seen N mins ago' with no Uptime row; Platform details > Machine 'Show details' drops the System card's Uptime and the Thermals card reads 'Cpu.package 45\u00b0C (last known)' with title 'Last known reading, not current: host agent stopped reporting'.", + "Docker page at 1440 and 390: edge-proxy-01 shows a red dot, '\u2014' for CPU, Memory, Disk and Temp (the hold seeds a retained 51\u00b0C), and the existing 'No report for 16m' marker in Uptime; Edge Apps 01 keeps its values, Temp 38.5\u00b0C included.", + "Bundle check: WorkloadsSurface unchanged (70.20 kB), Standalone +0.04 kB, Docker +0.01 kB, InfrastructureWorkspace flat versus this tree without the frontend diff; all chunks within budget.", + "Not covered: a real agent going silent on hardware, light/dark differences (no new colour classes), and the drawer Disks card, which still threshold-colours a silent agent's retained filesystem usage (left for a follow-up)." + ], + "interactions": [ + "Enabled the Network, Disk I/O, Uptime and Temp columns through the Machines column preferences (localStorage init) and read every cell of the offline and reporting rows plus the status dot class at 1440 and 390.", + "Opened the Apollo-114 drawer by row click at both widths, read the Runtime context rows, expanded Platform details and the Machine 'Show details', and read the System and Thermals cards with their value titles.", + "Opened /docker at both widths and read every cell of edge-proxy-01 and Edge Apps 01.", + "Queried /api/resources on both stacks for status, per-source sighting status and health reasons of the held agent and Docker host." + ] +} diff --git a/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx b/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx index adb1a78581..43975c192f 100644 --- a/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx +++ b/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx @@ -217,9 +217,13 @@ export const InlineResourceSummaryTables: Component { const docker = dockerContainerMeta(props.resource); + // An offline resource has no current uptime: the retained value is from + // its last report, and the Last seen row already says when that was. const uptimeSeconds = docker ? getDockerContainerUptimeSeconds(props.resource) - : props.resource.uptime; + : props.resource.status === 'offline' + ? undefined + : props.resource.uptime; const identityRows = compactDetailRows([ ...props.drawer.primaryIdentityRows().map((row) => makeDetailRow(row.label, row.value)), props.showPlatformId ? makeDetailRow('Platform ID', props.resource.platformId) : null, diff --git a/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.offline.test.tsx b/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.offline.test.tsx new file mode 100644 index 0000000000..530048ca63 --- /dev/null +++ b/frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.offline.test.tsx @@ -0,0 +1,64 @@ +import { cleanup, render, screen, within } from '@solidjs/testing-library'; +import { afterEach, describe, expect, it } from 'vitest'; + +import type { Resource } from '@/types/resource'; +import { InlineResourceSummaryTables } from '../ResourceDetailSummary'; +import type { UseResourceDetailDrawerStateResult } from '../useResourceDetailDrawerState'; + +const drawerState = { + primaryIdentityRows: () => [], + identityIpValues: () => [], + identityAliasValues: () => [], + aliasPreviewValues: () => [], + hasAliasOverflow: () => false, + identityCardHasRichData: () => true, + sourceSummary: () => null, + lastSeen: () => '14 mins ago', + lastSeenAbsolute: () => '', +} as unknown as UseResourceDetailDrawerStateResult; + +const machine = (status: Resource['status']): Resource => ({ + id: 'agent-apollo', + name: 'Apollo-114', + displayName: 'Apollo-114', + platformId: 'host-linux-1', + platformType: 'agent', + sourceType: 'agent', + type: 'agent', + status, + uptime: 2_500_000, + lastSeen: Date.parse('2026-10-06T22:00:00Z'), +}); + +afterEach(cleanup); + +describe('resource drawer runtime context for an offline resource', () => { + it('drops the uptime of an offline machine and keeps when it was last seen', () => { + render(() => ( + + )); + + const section = screen.getByTestId('resource-runtime-context-section'); + expect(within(section).getByText('offline')).toBeInTheDocument(); + expect(within(section).queryByText('Uptime')).toBeNull(); + expect(within(section).getByText('Last seen')).toBeInTheDocument(); + expect(within(section).getByText('14 mins ago')).toBeInTheDocument(); + }); + + it('keeps the uptime of a machine that is up', () => { + render(() => ( + + )); + + const section = screen.getByTestId('resource-runtime-context-section'); + expect(within(section).getByText('Uptime')).toBeInTheDocument(); + }); +}); diff --git a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts index 5596569845..9a3f4849f0 100644 --- a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts +++ b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { buildCustomSensorRows, buildTemperatureRows, + HOST_AGENT_STOPPED_REPORTING_REASON, formatInteger, formatSensorName, formatSourceType, @@ -248,6 +249,37 @@ describe('resourceDetailMappers', () => { ]); }); + it("marks a silent agent's sensor readings last known and leaves disks to their own state", () => { + const rows = buildTemperatureRows( + { + thermalState: { pressure: 'serious' }, + temperatureCelsius: { 'cpu.package': 45 }, + fanRpm: { chassis_fan: 1200 }, + smart: [ + { + device: 'sda', + temperature: 38, + collection: { temperature: { state: 'available', source: 'smartctl' } }, + }, + ], + }, + { lastKnownReason: HOST_AGENT_STOPPED_REPORTING_REASON }, + ); + + const lastKnown = 'Last known reading, not current: host agent stopped reporting'; + expect(rows).toEqual([ + { label: 'Thermal pressure', value: 'Serious (last known)', valueTitle: lastKnown }, + { label: 'Cpu.package', value: '45°C (last known)', valueTitle: lastKnown }, + { label: 'Chassis Fan', value: '1,200 RPM (last known)', valueTitle: lastKnown }, + // A disk row follows its own collection state, which the backend + // withdraws when the lease expires. + { label: 'Disk sda', value: '38°C', valueTitle: '38.0°C' }, + ]); + expect(buildTemperatureRows({ temperatureCelsius: { 'cpu.package': 45 } })).toEqual([ + { label: 'Cpu.package', value: '45°C', valueTitle: '45.0°C' }, + ]); + }); + it('lists a SMART row only for a positive reading in any state, never a standby disk', () => { const states: (PhysicalDiskCollectionStatus | undefined)[] = [ undefined, @@ -442,6 +474,15 @@ describe('resourceDetailMappers', () => { expect(agent?.id).not.toBe('resource:host:hash-1'); }); + it('reports no uptime for a silent agent, whose last report is not how long it has been up', () => { + const base = createHybridHostResource(); + const reporting = { ...base, uptime: 2_500_000 } as Resource; + expect(toAgentFromResource(reporting)?.uptimeSeconds).toBe(2_500_000); + + const silent = { ...reporting, agent: { stale: true } } as Resource; + expect(toAgentFromResource(silent)?.uptimeSeconds).toBe(0); + }); + it('preserves the local infrastructure display name for governed resources', () => { const agent = toAgentFromResource({ ...createHybridHostResource(), diff --git a/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts b/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts index b6b94a7ea1..07e3b11301 100644 --- a/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts +++ b/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts @@ -317,7 +317,9 @@ export const toAgentFromResource = ( sensors: agent.sensors, raid: agent.raid, status: resource.status, - uptimeSeconds: agent.uptimeSeconds ?? resource.uptime ?? 0, + // A silent agent's uptime is from its last report, not how long the + // machine has been up. + uptimeSeconds: resource.agent?.stale ? 0 : (agent.uptimeSeconds ?? resource.uptime ?? 0), lastSeen: resource.lastSeen, agentVersion: agent.agentVersion, commandsEnabled: agent.commandsEnabled, @@ -400,7 +402,23 @@ const buildTypedGPUTemperatureKeys = (gpus?: HostSensorSummary['gpu']) => { return keys; }; -export const buildTemperatureRows = (sensors?: HostSensorSummary) => { +// Matches the backend's reason for readings a host agent past its reporting +// lease no longer collects (models.HostAgentStoppedReportingReason). +export const HOST_AGENT_STOPPED_REPORTING_REASON = 'host agent stopped reporting'; + +export type TemperatureRowsOptions = { + /** + * Why the machine's own sensor readings are retained rather than current, + * such as a host agent past its reporting lease. Disk rows keep their own + * collection state instead. + */ + lastKnownReason?: string; +}; + +export const buildTemperatureRows = ( + sensors?: HostSensorSummary, + options: TemperatureRowsOptions = {}, +) => { const rows: { label: string; value: string; valueTitle?: string }[] = []; const thermalState = sensors?.thermalState; const typedGPUTemperatureKeys = buildTypedGPUTemperatureKeys(sensors?.gpu); @@ -500,6 +518,17 @@ export const buildTemperatureRows = (sensors?: HostSensorSummary) => { }); } + // This card has no other cue that a value is not current, so a retained + // sensor reading says so in its value, as a retained disk reading does. + const lastKnownReason = options.lastKnownReason?.trim(); + if (lastKnownReason) { + const title = `Last known reading, not current: ${lastKnownReason}`; + for (const row of rows) { + row.value = `${row.value} (last known)`; + row.valueTitle = title; + } + } + const smart = sensors?.smart; if (smart) { smart diff --git a/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts b/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts index 1b33f7d387..a71b909b29 100644 --- a/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts +++ b/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts @@ -35,6 +35,7 @@ import type { ResourceIntelligence } from '@/types/aiIntelligence'; import { buildCustomSensorRows, buildTemperatureRows, + HOST_AGENT_STOPPED_REPORTING_REASON, toAgentFromResource, toNodeFromProxmox, type AgentPlatformData, @@ -201,7 +202,13 @@ export const useResourceDetailDrawerDerivedState = ( [], ); const agentInfo = createMemo(() => toAgentFromResource(resource(), agentMeta())); - const temperatureRows = createMemo(() => buildTemperatureRows(agentInfo()?.sensors)); + // A silent agent's sensors are its last report, even on a row another + // source keeps online. + const temperatureRows = createMemo(() => + buildTemperatureRows(agentInfo()?.sensors, { + lastKnownReason: resource().agent?.stale ? HOST_AGENT_STOPPED_REPORTING_REASON : undefined, + }), + ); const customSensorRows = createMemo(() => buildCustomSensorRows(agentInfo()?.sensors)); const dockerHostData = createMemo(() => platformData()?.docker as DockerPlatformData | undefined); diff --git a/frontend-modern/src/features/docker/DockerHostsTable.tsx b/frontend-modern/src/features/docker/DockerHostsTable.tsx index fbaa7b0858..0c798eb6f6 100644 --- a/frontend-modern/src/features/docker/DockerHostsTable.tsx +++ b/frontend-modern/src/features/docker/DockerHostsTable.tsx @@ -517,7 +517,11 @@ export const DockerHostsTable: Component<{ class={`${getPlatformTableCellClassForKind('numeric-value')} platform-table-narrow-hidden hidden text-base-content min-[360px]:table-cell md:table-cell`} > diff --git a/frontend-modern/src/features/docker/__tests__/DockerHostsTable.test.tsx b/frontend-modern/src/features/docker/__tests__/DockerHostsTable.test.tsx index f5a4d1b7d3..7e24587e0b 100644 --- a/frontend-modern/src/features/docker/__tests__/DockerHostsTable.test.tsx +++ b/frontend-modern/src/features/docker/__tests__/DockerHostsTable.test.tsx @@ -217,6 +217,58 @@ describe('DockerHostsTable', () => { ); }); + it("drops an offline host's retained temperature with its other readings", () => { + // A Docker host past its reporting lease is offline, and its last + // temperature is no more current than its last CPU, memory and disk. + const { container } = render(() => ( + } + emptyTitle="No Docker hosts" + emptyDescription="No hosts" + showToolbar={false} + /> + )); + + const headers = Array.from(container.querySelectorAll('thead th')).map((th) => + (th.textContent ?? '').trim(), + ); + const tempIndex = headers.findIndex((header) => header.startsWith('Temp')); + expect(tempIndex).toBeGreaterThan(-1); + const temp = (id: string) => + ( + container.querySelectorAll(`[data-docker-host-row="${id}"] td`)[tempIndex]?.textContent ?? + '' + ).trim(); + expect(temp('agent:docker-01')).toBe('—'); + expect(temp('agent:docker-02')).toBe('51.0°C'); + const offlineRow = container.querySelector('[data-docker-host-row="agent:docker-01"]')!; + expect(within(offlineRow as HTMLElement).queryAllByTestId('stacked-memory-bar')).toHaveLength( + 0, + ); + expect(offlineRow.querySelector('[data-docker-host-stale] .sr-only')).toHaveTextContent( + 'No report for 9m', + ); + }); + it('tints a host row for the Docker alerts its drawer lists, not by display name', () => { // Docker alerts are keyed on the Docker host id and carry the hostname // as their node; neither is the row's id or its display name. diff --git a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx index 2609a62fa7..fb4ee7772f 100644 --- a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx +++ b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx @@ -1806,7 +1806,9 @@ export const AgentsMachinesTable: Component<{ class={`${getPlatformTableCellClassForKind('numeric-value')} ${machineColumnWidthClass('uptime')} text-base-content`} > {formatPlatformTableUptimeValue( - machine.uptime ?? machine.agent?.uptimeSeconds, + canRenderMetrics() + ? (machine.uptime ?? machine.agent?.uptimeSeconds) + : undefined, )} @@ -1814,15 +1816,20 @@ export const AgentsMachinesTable: Component<{ - + —} + > + + diff --git a/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx b/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx index 39b476ff18..3c7f0a3506 100644 --- a/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx +++ b/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx @@ -1160,14 +1160,15 @@ describe('AgentsMachinesTable', () => { expect(screen.getByText('1400 RPM')).toBeInTheDocument(); }); - it('shows a silent agent disk temperature as last known, never as a current reading', async () => { - // A host agent past its reporting lease is stale and keeps its last SMART - // temperatures with a non-available collection state. + it('shows a retained disk temperature as last known, never as a current reading', async () => { + // A reporting agent whose SMART probe returned no temperature keeps the + // last reading with a non-available collection state. (A host agent past + // its lease is offline, and its row shows no temperature at all.) const stoppedReporting = { temperature: { state: 'unavailable' as const, source: 'smartctl', - reason: 'host agent stopped reporting', + reason: 'SMART probe returned no usable temperature data', }, }; const { container } = render(() => ( @@ -1176,9 +1177,7 @@ describe('AgentsMachinesTable', () => { resource({ id: 'silent-nas', name: 'Silent NAS', - status: 'warning', agent: { - stale: true, sensors: { smart: [ { device: '/dev/sda', model: 'Archive HDD', temperature: 71 }, @@ -1249,6 +1248,92 @@ describe('AgentsMachinesTable', () => { expect(within(mixed).queryByText('69°C')).toBeNull(); }); + it("blanks an offline machine's retained readings, temperature and uptime included", async () => { + // A host agent past its reporting lease is offline: CPU, memory, disk and + // the sensors, uptime and disks it last reported are not current. + const stoppedReporting = { + temperature: { + state: 'unavailable' as const, + source: 'smartctl', + reason: 'host agent stopped reporting', + }, + }; + const lastReport: Partial = { + cpu: { current: 32 }, + memory: { total: 100, used: 73, free: 27, current: 73 }, + disk: { total: 100, used: 80, free: 20, current: 80 }, + }; + const sensors = { + temperatureCelsius: { 'cpu.package': 45 }, + thermalState: { pressure: 'serious' }, + smart: [ + { + device: '/dev/sda', + model: 'Archive HDD', + temperature: 41, + collection: stoppedReporting, + }, + ], + }; + const { container } = render(() => ( + + )); + + await openMachineColumnPicker(); + await fireEvent.click(screen.getByLabelText('Temp')); + await fireEvent.click(screen.getByLabelText('Uptime')); + + const cells = (id: string) => { + const row = container.querySelector(`[data-agents-machine-row="${id}"]`) as HTMLElement; + const headers = Array.from(container.querySelectorAll('thead th')).map((th) => + (th.textContent ?? '').replace(/[▲▼]/g, '').trim(), + ); + const values = Array.from(row.querySelectorAll('td')).map((td) => + (td.textContent ?? '').trim(), + ); + return (label: string) => values[headers.indexOf(label)]; + }; + + const silent = cells('silent-host'); + expect(silent('CPU')).toBe('—'); + expect(silent('Up')).toBe('—'); + expect(silent('Temp')).toBe('—'); + expect( + container.querySelector( + '[data-agents-machine-row="silent-host"] [data-agent-machine-temperature-trigger="true"]', + ), + ).toBeNull(); + + const live = cells('live-host'); + expect(live('CPU')).not.toBe('—'); + expect(live('Up')).toBe('28d'); + expect(live('Temp')).toBe('45°C'); + }); + it('moves a machine temperature between current and last known as its disk reports change', async () => { // Fresh objects per update, like parsed API payloads: the store reconciles // into the objects it was given, so a reused fixture would be rewritten. diff --git a/internal/unifiedresources/clone_test.go b/internal/unifiedresources/clone_test.go index 1ad90ad974..151f411769 100644 --- a/internal/unifiedresources/clone_test.go +++ b/internal/unifiedresources/clone_test.go @@ -221,12 +221,18 @@ func TestCloneResource_MutateSourceStatusMap(t *testing.T) { ID: "r-1", SourceStatus: map[DataSource]SourceStatus{ SourceProxmox: {Status: "online", ExpectedUpdateIntervalSeconds: 300}, + SourceAgent: {Status: "stale", leaseExpired: true}, }, } cloned := cloneResource(original) if cloned.SourceStatus[SourceProxmox].ExpectedUpdateIntervalSeconds != 300 { t.Fatal("clone lost collector cadence") } + // A clone that dropped the lease verdict would aggregate a silent agent + // back to warning. + if !cloned.SourceStatus[SourceAgent].leaseExpired { + t.Fatal("clone lost the agent's expired reporting lease") + } cloned.SourceStatus[SourceDocker] = SourceStatus{Status: "online"} if _, exists := original.SourceStatus[SourceDocker]; exists { diff --git a/internal/unifiedresources/registry.go b/internal/unifiedresources/registry.go index cb2f10a48a..c4a31be483 100644 --- a/internal/unifiedresources/registry.go +++ b/internal/unifiedresources/registry.go @@ -809,6 +809,14 @@ func (rr *ResourceRegistry) ingestResources(resources []Resource, thresholds map } } } + // The lease marker is unexported, so a persisted or serialized copy + // arrives without it. Its stored status still carries the verdict. + for source, status := range resource.SourceStatus { + if !status.leaseExpired && reportingLeaseExpired(source, *resource) { + status.leaseExpired = true + resource.SourceStatus[source] = status + } + } rr.mu.Lock() rr.canonicalIdentityIndex = nil @@ -1756,6 +1764,29 @@ func sourceSightingStatus(lastSeen time.Time) string { return "online" } +// reportingLeaseExpired reports whether a source delivered the machine, +// Docker host or Kubernetes cluster it reports for as offline. Those +// collectors push their own reports, and Pulse marks the reporter offline +// only once its reporting lease runs out (evaluateHostAgents, +// evaluateDockerAgents, evaluateKubernetesAgents). The registry's stale +// threshold is a different clock: it can mark the sighting stale before +// the lease ends, or leave it fresh after, so it must not decide the status. +// Resources those sources describe (guests, containers, pods, disks) are +// not lease holders and keep the stale-to-warning rule. +func reportingLeaseExpired(source DataSource, resource Resource) bool { + if resource.Status != StatusOffline { + return false + } + switch source { + case SourceAgent, SourceDocker: + return resource.Type == ResourceTypeAgent + case SourceK8s: + return resource.Type == ResourceTypeK8sCluster + default: + return false + } +} + func (rr *ResourceRegistry) markStaleLocked(now time.Time, thresholds map[DataSource]time.Duration) { thresholds = effectiveStaleThresholds(thresholds) @@ -2848,6 +2879,7 @@ func (rr *ResourceRegistry) ingestRecord(source DataSource, sourceID string, res sighting := resource.SourceStatus[source] sighting.Status = sourceSightingStatus(resource.LastSeen) sighting.LastSeen = resource.LastSeen + sighting.leaseExpired = reportingLeaseExpired(source, resource) resource.SourceStatus = map[DataSource]SourceStatus{source: sighting} resource.parentBySource = make(map[DataSource]string) rr.setSourceParent(&resource, source, resource.ParentID) @@ -3787,6 +3819,7 @@ func (rr *ResourceRegistry) mergeInto(existing *Resource, incoming Resource, sou sighting := incoming.SourceStatus[source] sighting.Status = sourceSightingStatus(incoming.LastSeen) sighting.LastSeen = incoming.LastSeen + sighting.leaseExpired = reportingLeaseExpired(source, incoming) existing.SourceStatus[source] = sighting if incoming.LastSeen.After(existing.LastSeen) { @@ -4425,6 +4458,12 @@ func (rr *ResourceRegistry) mergeResourceData(primary *Resource, other *Resource primary.SourceStatus = make(map[DataSource]SourceStatus) } for source, status := range other.SourceStatus { + // Linked resources can share a source, such as an agent reinstalled + // under a new id. Keep the fresher sighting so a stale or expired one + // cannot override a reporter that is still live. + if current, ok := primary.SourceStatus[source]; ok && current.LastSeen.After(status.LastSeen) { + continue + } primary.SourceStatus[source] = status } if other.LastSeen.After(primary.LastSeen) { @@ -5793,14 +5832,22 @@ func aggregateStatus(resource *Resource) ResourceStatus { best := StatusUnknown bestScore := 0 for _, status := range resource.SourceStatus { - score := statusPriority[strings.ToLower(status.Status)] + state := strings.ToLower(strings.TrimSpace(status.Status)) + // A reporter past its lease is offline, not merely late. Ranking its + // stale sighting above offline would show a silent machine as a + // warning while its retained readings render as current. + if status.leaseExpired { + state = "offline" + } + score := statusPriority[state] if score > bestScore { bestScore = score - if status.Status == "online" { + switch state { + case "online": best = StatusOnline - } else if status.Status == "stale" { + case "stale": best = StatusWarning - } else if status.Status == "offline" { + case "offline": best = StatusOffline } } diff --git a/internal/unifiedresources/registry_merge_policy_test.go b/internal/unifiedresources/registry_merge_policy_test.go index 83c8fda61a..cc78d4b7a1 100644 --- a/internal/unifiedresources/registry_merge_policy_test.go +++ b/internal/unifiedresources/registry_merge_policy_test.go @@ -1,6 +1,7 @@ package unifiedresources import ( + "encoding/json" "strings" "testing" "time" @@ -131,6 +132,353 @@ func TestHostnameIPDoesNotAutoMerge(t *testing.T) { } } +func onlyResourceOfType(t *testing.T, rr *ResourceRegistry, resourceType ResourceType) Resource { + t.Helper() + resources := rr.ListByType(resourceType) + if len(resources) != 1 { + t.Fatalf("expected one %s resource, got %d: %+v", resourceType, len(resources), resources) + } + return resources[0] +} + +// A host agent past its reporting lease is offline. Its source sighting is +// stale too, and the stale pass used to rank that above offline, so the +// machine reached the frontend as a warning with its last report rendered +// as current readings. +func TestLeaseExpiredHostAgentStaysOffline(t *testing.T) { + lastReport := time.Now().UTC().Add(-14 * time.Minute) + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-silent", + MachineID: "machine-silent", + Hostname: "silent", + Status: "offline", + LastSeen: lastReport, + IntervalSeconds: 30, + CPUUsage: 32, + }}, + }) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if resource.Status != StatusOffline { + t.Fatalf("status = %q, want offline for an agent past its lease", resource.Status) + } + if resource.Agent == nil || !resource.Agent.Stale { + t.Fatalf("expected the agent facet to stay flagged stale, got %+v", resource.Agent) + } + // Status describes the resource; the sighting keeps describing delivery + // freshness, which health and monitored-system reasons read. + if got := resource.SourceStatus[SourceAgent].Status; got != "stale" { + t.Fatalf("agent sighting = %q, want stale", got) + } + + // The monitor adapter runs the stale pass again after record sources. + rr.MarkStale(time.Now().UTC(), nil) + if got, _ := rr.Get(resource.ID); got.Status != StatusOffline { + t.Fatalf("second stale pass changed status to %q, want offline", got.Status) + } + + health := EvaluateResourceHealth(resource, nil, time.Now().UTC()) + if health.Verdict != HealthCritical || len(health.Reasons) < 2 || + health.Reasons[0].Code != "offline" || health.Reasons[1].Code != "telemetry_stale" { + t.Fatalf("health = %+v, want critical offline with the stale reason after it", health) + } +} + +// An agent that is late but still inside its lease keeps the warning: the +// monitor has not decided it is gone, so its last readings are only stale. +func TestLateHostAgentInsideLeaseStaysWarning(t *testing.T) { + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-late", + MachineID: "machine-late", + Hostname: "late", + Status: "online", + LastSeen: time.Now().UTC().Add(-90 * time.Second), + IntervalSeconds: 30, + }}, + }) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if resource.Status != StatusWarning { + t.Fatalf("status = %q, want warning for a late agent inside its lease", resource.Status) + } +} + +// A Proxmox node whose linked agent stopped reporting stays online through +// the PVE poll (issue #1515). Only the agent sighting carries the lease. +func TestLeaseExpiredAgentOnLiveProxmoxNodeStaysOnline(t *testing.T) { + now := time.Now().UTC() + const machineID = "machine-pve1" + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Nodes: []models.Node{{ + ID: "homelab-pve1", + Name: "pve1", + Instance: "homelab", + ClusterName: "homelab", + Status: "online", + LastSeen: now, + }}, + Hosts: []models.Host{{ + ID: machineID, + MachineID: machineID, + Hostname: "pve1", + LinkedNodeID: "homelab-pve1", + Status: "offline", + LastSeen: now.Add(-14 * time.Minute), + IntervalSeconds: 30, + }}, + }) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if _, ok := resource.SourceStatus[SourceProxmox]; !ok { + t.Fatalf("expected the node and its agent to share one resource, got sources %+v", resource.SourceStatus) + } + if resource.Status != StatusOnline { + t.Fatalf("status = %q, want online through the live PVE poll", resource.Status) + } + if resource.Agent == nil || !resource.Agent.Stale { + t.Fatalf("expected the dead agent to stay flagged stale, got %+v", resource.Agent) + } +} + +// Docker hosts hold a shorter lease than the registry's Docker stale +// threshold, so a silent Docker host was offline for a while and then +// flipped to warning once its sighting went stale. +func TestLeaseExpiredDockerHostStaysOfflineAfterSightingGoesStale(t *testing.T) { + lastReport := time.Now().UTC().Add(-5 * time.Minute) + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + DockerHosts: []models.DockerHost{{ + ID: "docker-silent", + AgentID: "docker-agent-silent", + Hostname: "docker-silent", + MachineID: "machine-docker-silent", + Status: "offline", + LastSeen: lastReport, + IntervalSeconds: 30, + Containers: []models.DockerContainer{{ + ID: "container-web", + Name: "web", + State: "running", + }}, + }}, + }) + + host := onlyResourceOfType(t, rr, ResourceTypeAgent) + if host.Status != StatusOffline { + t.Fatalf("docker host status = %q, want offline after its sighting went stale", host.Status) + } + if got := host.SourceStatus[SourceDocker].Status; got != "stale" { + t.Fatalf("docker sighting = %q, want stale", got) + } + + // Containers are what the host reports about, not lease holders: a + // running container on a silent host is stale, not known to be down. + container := onlyResourceOfType(t, rr, ResourceTypeAppContainer) + if container.Status != StatusWarning { + t.Fatalf("container status = %q, want warning on a silent host", container.Status) + } +} + +// A machine reporting through both the host agent and Docker is offline once +// both leases expire. The Docker lease ends first, so its sighting can still +// be fresh when the agent's goes stale. +func TestMachineWithExpiredAgentAndDockerLeasesIsOffline(t *testing.T) { + now := time.Now().UTC() + const machineID = "machine-both" + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-both", + MachineID: machineID, + Hostname: "both", + Status: "offline", + LastSeen: now.Add(-3 * time.Minute), + IntervalSeconds: 30, + }}, + DockerHosts: []models.DockerHost{{ + ID: "docker-both", + AgentID: "host-both", + Hostname: "both", + MachineID: machineID, + Status: "offline", + LastSeen: now.Add(-90 * time.Second), + IntervalSeconds: 10, + }}, + }) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if _, ok := resource.SourceStatus[SourceDocker]; !ok { + t.Fatalf("expected agent and Docker to share one resource, got sources %+v", resource.SourceStatus) + } + if got := resource.SourceStatus[SourceDocker].Status; got != "online" { + t.Fatalf("docker sighting = %q, want it still inside the stale threshold", got) + } + if resource.Status != StatusOffline { + t.Fatalf("status = %q, want offline once both leases expired", resource.Status) + } +} + +// A live reporter still carries the machine when the other lease expired. +func TestMachineWithLiveDockerReporterStaysOnline(t *testing.T) { + now := time.Now().UTC() + const machineID = "machine-split" + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-split", + MachineID: machineID, + Hostname: "split", + Status: "offline", + LastSeen: now.Add(-14 * time.Minute), + IntervalSeconds: 30, + }}, + DockerHosts: []models.DockerHost{{ + ID: "docker-split", + AgentID: "docker-agent-split", + Hostname: "split", + MachineID: machineID, + Status: "online", + LastSeen: now, + IntervalSeconds: 10, + }}, + }) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if resource.Status != StatusOnline { + t.Fatalf("status = %q, want online through the live Docker reporter", resource.Status) + } +} + +// The lease marker is unexported, so a resource that went through JSON (a +// persisted or remote copy) must recover it from its stored status. +func TestLeaseExpiredHostAgentStaysOfflineAfterJSONRoundTrip(t *testing.T) { + source := NewRegistry(nil) + source.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{{ + ID: "host-silent", + MachineID: "machine-silent", + Hostname: "silent", + Status: "offline", + LastSeen: time.Now().UTC().Add(-14 * time.Minute), + IntervalSeconds: 30, + }}, + }) + payload, err := json.Marshal(source.List()) + if err != nil { + t.Fatalf("marshal resources: %v", err) + } + var copied []Resource + if err := json.Unmarshal(payload, &copied); err != nil { + t.Fatalf("unmarshal resources: %v", err) + } + + rr := NewRegistry(nil) + rr.IngestResources(copied) + + resource := onlyResourceOfType(t, rr, ResourceTypeAgent) + if resource.Status != StatusOffline { + t.Fatalf("status = %q, want offline after a JSON round trip", resource.Status) + } +} + +// A manual link can join two resources reported by the same source, such as +// an agent reinstalled under a new id. The expired sighting of the old one +// must not replace the live sighting of the new one. +func TestManualLinkKeepsTheLiveSightingOfASharedSource(t *testing.T) { + now := time.Now().UTC() + store := NewMemoryStore() + if err := store.AddLink(ResourceLink{ + ResourceA: "agent-reinstalled", + ResourceB: "agent-retired", + PrimaryID: "agent-reinstalled", + }); err != nil { + t.Fatalf("add link: %v", err) + } + rr := NewRegistry(store) + rr.IngestResources([]Resource{ + { + ID: "agent-reinstalled", + Type: ResourceTypeAgent, + Name: "tower", + Status: StatusOnline, + LastSeen: now, + Sources: []DataSource{SourceAgent}, + SourceStatus: map[DataSource]SourceStatus{ + SourceAgent: {Status: "online", LastSeen: now}, + }, + Agent: &AgentData{AgentID: "host-new", Hostname: "tower"}, + }, + { + ID: "agent-retired", + Type: ResourceTypeAgent, + Name: "tower", + Status: StatusOffline, + LastSeen: now.Add(-14 * time.Minute), + Sources: []DataSource{SourceAgent}, + SourceStatus: map[DataSource]SourceStatus{ + SourceAgent: {Status: "stale", LastSeen: now.Add(-14 * time.Minute)}, + }, + Agent: &AgentData{AgentID: "host-old", Hostname: "tower", Stale: true}, + }, + }) + + resource, ok := rr.Get("agent-reinstalled") + if !ok { + t.Fatal("linked resource missing") + } + if resource.Status != StatusOnline { + t.Fatalf("status = %q, want online through the live reinstalled agent", resource.Status) + } + if got := resource.SourceStatus[SourceAgent]; !got.LastSeen.Equal(now) { + t.Fatalf("agent sighting = %+v, want the live sighting", got) + } +} + +func TestLeaseExpiredKubernetesClusterStaysOffline(t *testing.T) { + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + KubernetesClusters: []models.KubernetesCluster{{ + ID: "cluster-silent", + AgentID: "k8s-agent-silent", + Name: "silent", + Status: "offline", + LastSeen: time.Now().UTC().Add(-5 * time.Minute), + IntervalSeconds: 30, + }}, + }) + + cluster := onlyResourceOfType(t, rr, ResourceTypeK8sCluster) + if cluster.Status != StatusOffline { + t.Fatalf("cluster status = %q, want offline after its sighting went stale", cluster.Status) + } +} + +// Pull sources keep the stale-to-warning rule: a stale PVE sighting means +// Pulse lost its poller, not that the node reported itself gone. +func TestStaleProxmoxNodeStaysWarning(t *testing.T) { + rr := NewRegistry(nil) + rr.IngestSnapshot(models.StateSnapshot{ + Nodes: []models.Node{{ + ID: "homelab-pve9", + Name: "pve9", + Instance: "homelab", + ClusterName: "homelab", + Status: "online", + LastSeen: time.Now().UTC().Add(-5 * time.Minute), + }}, + }) + + node := onlyResourceOfType(t, rr, ResourceTypeAgent) + if node.Status != StatusWarning { + t.Fatalf("node status = %q, want warning for a stale PVE sighting", node.Status) + } +} + func TestCloneDockerDataPreservesContainerRuntimeMetadata(t *testing.T) { startedAt := time.Date(2026, 6, 11, 13, 15, 30, 0, time.UTC) finishedAt := startedAt.Add(45 * time.Minute) diff --git a/internal/unifiedresources/types.go b/internal/unifiedresources/types.go index 31b01a0e85..d64e288bef 100644 --- a/internal/unifiedresources/types.go +++ b/internal/unifiedresources/types.go @@ -295,6 +295,12 @@ type SourceStatus struct { LastSeen time.Time `json:"lastSeen"` Error string `json:"error,omitempty"` ExpectedUpdateIntervalSeconds int64 `json:"expectedUpdateIntervalSeconds,omitempty"` // Collector-authored cadence, zero uses source default + + // leaseExpired records that this source delivered the machine, Docker + // host or cluster it reports for as offline because its reporting lease + // ran out. Status aggregation counts that as offline whatever the + // sighting's age, while Status keeps describing delivery freshness. + leaseExpired bool } // ResourceIdentity holds identifiers used for matching.