From a1891e5f0ca07a07ad43e4b78cb55adfd07ad6c2 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:00:59 +0100 Subject: [PATCH 01/27] Keep open row drawers spanning the table when columns change An open inline row drawer measured its colspan from the summary row's visible cells only on mount and on resize. Hiding or showing a column from View > Columns, or a column that depends on the data such as the vCenter column, left the drawer with its old span. On the Machines table, hiding Uptime with a drawer open shrank the header and rows to 939px of a 1375px table beside a blank phantom column, and showing System left the drawer 79px short of the actions column. InlineDetailTableRow now re-measures when the summary row's own cells are added, removed or hidden, and when the requested colspan changes. Changes inside a cell's content are ignored, so live metric updates do not re-measure. When the span grows, Chromium keeps a fixed-layout table's old column widths for the widened cell, so the row changes the cell's width for one forced layout and then restores it. The primitive's own test file becomes a registered frontend-primitives proof. --- .../subsystems/frontend-primitives.md | 12 ++++ .../v6/internal/subsystems/registry.json | 1 + .../f374371e7bfb2bfc.json | 47 +++++++++++++ .../shared/InlineDetailTableRow.tsx | 47 ++++++++++++- .../__tests__/InlineDetailTableRow.test.tsx | 70 +++++++++++++++++++ 5 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 frontend-modern/browser-verification/f374371e7bfb2bfc.json diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 308d1470e2..e9bf03ae20 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -5665,6 +5665,18 @@ When focused detail content is removed, `InlineDetailTableRow` restores focus to its current `aria-controls` disclosure with `preventScroll`; live refresh, collapse, and row replacement must not move the surrounding application viewport merely to reveal that control. +The detail cell spans the summary row's visible cells, and `InlineDetailTableRow` +re-measures that span while the row stays open: whenever summary cells are +added, removed, or hidden (a column picker toggle, or a data-driven column +such as the vCenter column that shows only with several vCenters), and +whenever the caller's requested colspan changes. An open drawer must never +keep a phantom column or stop short of a newly shown one. Changes inside a +cell's content do not re-measure, so live metric updates stay cheap. When +the span grows, the row changes the cell's own width for one forced layout +and restores it, because Chromium otherwise keeps a fixed-layout table's old +column widths for the widened cell. Callers keep passing their requested +colspan; they must not re-key or remount the detail row to force a new span, +which would discard drawer state. Inline detail section content is registry-backed separately from the row shell. `DetailSectionTable`, `InlineDetailPanel`, and `detailSectionModel.ts` own detail row compaction, section-table rendering, value-tone classes, and the diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index de73a0f7ab..ea8a0e3c26 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -5972,6 +5972,7 @@ "frontend-modern/src/components/shared/__tests__/Dialog.test.tsx", "frontend-modern/src/components/shared/__tests__/HelpIcon.test.tsx", "frontend-modern/src/components/shared/__tests__/HistoryChart.test.tsx", + "frontend-modern/src/components/shared/__tests__/InlineDetailTableRow.test.tsx", "frontend-modern/src/components/shared/__tests__/MobileNavBar.test.tsx", "frontend-modern/src/components/shared/__tests__/PulseDataGrid.test.tsx", "frontend-modern/src/components/shared/__tests__/SearchField.test.tsx", diff --git a/frontend-modern/browser-verification/f374371e7bfb2bfc.json b/frontend-modern/browser-verification/f374371e7bfb2bfc.json new file mode 100644 index 0000000000..1a02d979cd --- /dev/null +++ b/frontend-modern/browser-verification/f374371e7bfb2bfc.json @@ -0,0 +1,47 @@ +{ + "version": 1, + "base_sha": "d4315cef519bb5729a2b9117cae5d28724c17d84", + "verified_at": "2026-10-06T12:59:58Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/shared/InlineDetailTableRow.tsx" + ], + "content_sha256": { + "frontend-modern/src/components/shared/InlineDetailTableRow.tsx": "27f6c907846d17ef0742386e766719d6e529b17223b1dcc3b586e0caa48cbb96" + }, + "routes": [ + "/standalone", + "/proxmox", + "/vmware/activity" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 1024, + "height": 900 + }, + { + "width": 768, + "height": 900 + } + ], + "states": [ + "Private mock stack (backend + Vite on 7747/5267) built from main 92d946d057 plus this change, compared with the same stack on unchanged main", + "Before, /standalone machines table at 1440: hiding Uptime from View > Columns with a machine drawer open left the drawer at colspan 9 against 8 visible headers, so the header and rows shrank to 939px and a blank 436px phantom column filled the table's right side", + "Before, /standalone at 1440: showing System with the drawer open left the drawer at colspan 7 against 8 headers, stopping 79px short of the table edge under the actions column", + "Before, /proxmox guest table at 1440, 1024 and 768: hiding Type, Uptime or Backup with a guest drawer open left colspan one above the visible header count (the fixed layout collapses the phantom column to about 1px there)", + "After, both tables at 1440 and 768: drawer colSpan equals the visible header and summary cell count after every hide and show, and the drawer, last header cell and last summary cell all end at the table's right edge", + "After, growing span case: the colspan update alone left Chromium laying the widened cell out at its old 7-column width (1241px of 1320px); with the one-off width nudge the cell measures the full table width while keeping the same row node", + "/vmware/activity with the mock's single vCenter: the Activity drawer still spans 6 of 6 header columns at 1440 and 5 of 5 at 768 (the vCenter-count flip itself is covered by the unit test that removes and restores a summary cell)" + ], + "interactions": [ + "Opened a machine drawer on /standalone, then hid and showed Uptime, System, Agent and Last seen through View > Columns while the drawer stayed open, measuring drawer colSpan, header and summary visible cell counts and right edges after each toggle", + "Opened a guest drawer on /proxmox, hid a column first, then showed and hid Type again through View > Columns with the drawer open, measuring the same values", + "Confirmed with a node tag that the drawer row stayed mounted across toggles (no remount, so drawer tab state is kept)", + "Opened a /vmware/activity row drawer at both widths and compared its span with the visible headers", + "Captured clipped screenshots of the /standalone drawer before and after each toggle" + ] +} diff --git a/frontend-modern/src/components/shared/InlineDetailTableRow.tsx b/frontend-modern/src/components/shared/InlineDetailTableRow.tsx index 5aa7776f6f..1b7f479484 100644 --- a/frontend-modern/src/components/shared/InlineDetailTableRow.tsx +++ b/frontend-modern/src/components/shared/InlineDetailTableRow.tsx @@ -1,4 +1,4 @@ -import { createSignal, onCleanup, onMount, splitProps, type JSX } from 'solid-js'; +import { createEffect, createSignal, on, onCleanup, onMount, splitProps, type JSX } from 'solid-js'; import { TableCell, TableRow, type TableRowProps } from './Table'; @@ -88,12 +88,57 @@ export function InlineDetailTableRow(props: InlineDetailTableRowProps) { : undefined; if (tableShell) observer?.observe(tableShell); + // A column picker, or a column that depends on the data (one vCenter + // versus several), can add, remove or hide summary cells while this row + // stays open, without resizing the table shell. Watch the summary row's + // own cells so the detail cell never keeps a stale span. Changes deeper + // inside a cell are ignored: live metric updates must not re-measure. + const summaryRow = detailRow?.previousElementSibling; + let summaryCellsObserver: MutationObserver | undefined; + if (summaryRow instanceof HTMLTableRowElement && typeof MutationObserver === 'function') { + summaryCellsObserver = new MutationObserver((records) => { + const summaryCellsChanged = records.some( + (record) => + record.target === summaryRow || + (record.type === 'attributes' && record.target.parentNode === summaryRow), + ); + if (summaryCellsChanged) syncColspanToVisibleSummaryCells(); + }); + summaryCellsObserver.observe(summaryRow, { + childList: true, + subtree: true, + attributes: true, + attributeFilter: ['class', 'style', 'hidden', 'colspan'], + }); + } + onCleanup(() => { window.removeEventListener('resize', syncColspanToVisibleSummaryCells); observer?.disconnect(); + summaryCellsObserver?.disconnect(); }); }); + // Callers that change the span they ask for (and rows rendered without a + // summary row, which fall back to that span) re-sync on the change. + createEffect(on(requestedColspan, syncColspanToVisibleSummaryCells, { defer: true })); + + // Chromium keeps a fixed-layout table's column widths when a spanning + // cell's colspan grows over a column added since the table last laid out, + // so the detail cell would stop short of the new column. A change to the + // cell's own width makes it recompute them; restore it after one layout. + // The effect is not deferred so its first run records the mounted span. + createEffect( + on(effectiveColspan, (colspan, previousColspan) => { + const detailCell = detailRow?.cells[0]; + if (!detailCell || previousColspan === undefined || colspan <= previousColspan) return; + const width = detailCell.style.width; + detailCell.style.width = '0px'; + void detailCell.offsetWidth; + detailCell.style.width = width; + }), + ); + return ( { }); }); + it('re-spans when a summary column is removed or restored while the row stays open', async () => { + const [showSystem, setShowSystem] = createSignal(true); + render(() => ( + + + + + + + + + + +
Column picker detail
+
+ +
MachineSystemSeen
+ )); + const detailCell = () => screen.getByText('Column picker detail').closest('td'); + + await waitFor(() => expect(detailCell()).toHaveAttribute('colspan', '3')); + setShowSystem(false); + await waitFor(() => expect(detailCell()).toHaveAttribute('colspan', '2')); + setShowSystem(true); + await waitFor(() => expect(detailCell()).toHaveAttribute('colspan', '3')); + // The relayout nudge for a growing span must not leave a width behind. + expect(detailCell()?.style.width).toBe(''); + }); + + it('re-spans when a summary cell is hidden in place while the row stays open', async () => { + const [hideUptime, setHideUptime] = createSignal(false); + render(() => ( + + + + + + + + +
Hidden cell detail
+
+ +
MachineUptimeSeen
+ )); + const detailCell = () => screen.getByText('Hidden cell detail').closest('td'); + + await waitFor(() => expect(detailCell()).toHaveAttribute('colspan', '3')); + setHideUptime(true); + await waitFor(() => expect(detailCell()).toHaveAttribute('colspan', '2')); + }); + + it('follows a changed requested span when there is no summary row to measure', async () => { + const [colspan, setColspan] = createSignal(5); + render(() => ( + + + +
Requested span detail
+
+
+
+ )); + const detailCell = () => screen.getByText('Requested span detail').closest('td'); + + await waitFor(() => expect(detailCell()).toHaveAttribute('colspan', '5')); + setColspan(6); + await waitFor(() => expect(detailCell()).toHaveAttribute('colspan', '6')); + }); + it('restores focus to the controlling disclosure when focused detail content closes', async () => { const Fixture = () => { const [open, setOpen] = createSignal(true); From 18b46a4d147dc9998755cb4eeb52b7e791c267e9 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:23:00 +0100 Subject: [PATCH 02/27] Guard the drawer span nudge and cell-churn filter in tests The width assertion in the first commit would still pass if the Chromium relayout nudge were deleted, and nothing checked that churn inside a summary cell is ignored. jsdom has no layout, so the new test records the detail cell's style writes: one width change when the span grows, none when it shrinks. A second test checks that a summary cell's colspan change re-measures while class and node churn inside a cell does not. Each fails when the nudge is removed or the observer filter is widened to every mutation. --- .../__tests__/InlineDetailTableRow.test.tsx | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/frontend-modern/src/components/shared/__tests__/InlineDetailTableRow.test.tsx b/frontend-modern/src/components/shared/__tests__/InlineDetailTableRow.test.tsx index a6349fe15e..f8aa97bd76 100644 --- a/frontend-modern/src/components/shared/__tests__/InlineDetailTableRow.test.tsx +++ b/frontend-modern/src/components/shared/__tests__/InlineDetailTableRow.test.tsx @@ -115,6 +115,93 @@ describe('InlineDetailTableRow', () => { expect(detailCell()?.style.width).toBe(''); }); + it('nudges the detail cell width once when its span grows, not when it shrinks', async () => { + const [showSystem, setShowSystem] = createSignal(false); + render(() => ( + + + + + + + + + + +
Growing span detail
+
+ +
MachineSystemSeen
+ )); + const detailCell = () => screen.getByText('Growing span detail').closest('td')!; + await waitFor(() => expect(detailCell()).toHaveAttribute('colspan', '2')); + + // jsdom has no layout, so record the style writes Chromium needs to + // recompute fixed-layout column widths for the widened cell. + const previousStyles: string[] = []; + const styleObserver = new MutationObserver((records) => { + for (const record of records) previousStyles.push(record.oldValue ?? ''); + }); + styleObserver.observe(detailCell(), { + attributes: true, + attributeFilter: ['style'], + attributeOldValue: true, + }); + + setShowSystem(true); + await waitFor(() => expect(detailCell()).toHaveAttribute('colspan', '3')); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(previousStyles.some((style) => style.includes('width: 0px'))).toBe(true); + expect(detailCell().style.width).toBe(''); + + previousStyles.length = 0; + setShowSystem(false); + await waitFor(() => expect(detailCell()).toHaveAttribute('colspan', '2')); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(previousStyles).toEqual([]); + styleObserver.disconnect(); + }); + + it('re-measures for a summary cell span change but not for churn inside a cell', async () => { + const [uptimeSpan, setUptimeSpan] = createSignal(1); + const [reading, setReading] = createSignal({ value: '24%', tone: 'text-green-600' }); + const [busy, setBusy] = createSignal(false); + render(() => ( + + + + + + + +
Live metric detail
+
+ +
Machine + {reading().value} + + refreshing + +
+ )); + const detailCell = () => screen.getByText('Live metric detail').closest('td'); + await waitFor(() => expect(detailCell()).toHaveAttribute('colspan', '2')); + + const getComputedStyleSpy = vi.spyOn(window, 'getComputedStyle'); + try { + setReading({ value: '91%', tone: 'text-red-600' }); + setBusy(true); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(getComputedStyleSpy).not.toHaveBeenCalled(); + + setUptimeSpan(2); + await waitFor(() => expect(detailCell()).toHaveAttribute('colspan', '3')); + expect(getComputedStyleSpy).toHaveBeenCalled(); + } finally { + getComputedStyleSpy.mockRestore(); + } + }); + it('re-spans when a summary cell is hidden in place while the row stays open', async () => { const [hideUptime, setHideUptime] = createSignal(false); render(() => ( From 0bd90645132527c38fdc041572ae2cff8c45b0ca Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:39:09 +0100 Subject: [PATCH 03/27] Add a browser check for drawer span after column toggles The unit tests cannot see layout, so a regression in the Chromium width workaround would only show in a real browser. The new spec opens a Machines drawer, hides and restores System through View > Columns, and checks after each step that the drawer spans the visible headers, that the drawer, last header and table share a right edge, and that the drawer row, focus and any inner scroll position survive. It fails on main before the fix (colspan 8 against 7 after hiding) and with only the width workaround removed (drawer 79px short of the table edge after showing), and passed five runs in a row with the fix. --- .../98-inline-drawer-column-span.spec.ts | 262 ++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 tests/integration/tests/98-inline-drawer-column-span.spec.ts diff --git a/tests/integration/tests/98-inline-drawer-column-span.spec.ts b/tests/integration/tests/98-inline-drawer-column-span.spec.ts new file mode 100644 index 0000000000..0aea3a4437 --- /dev/null +++ b/tests/integration/tests/98-inline-drawer-column-span.spec.ts @@ -0,0 +1,262 @@ +import { + expect, + test as base, + type Locator, + type Page, +} from "@playwright/test"; + +import { + ensureAuthenticated, + getMockMode, + setMockMode, + waitForDefaultMockRuntimeReady, +} from "./helpers"; + +let mockModeWasEnabled: boolean | null = null; + +const HTTP_CREDENTIALS = { + username: "admin", + password: "adminadminadmin", +}; + +const test = base; + +test.use({ httpCredentials: HTTP_CREDENTIALS }); + +async function ensureMockModeEnabled(page: Page): Promise { + await ensureAuthenticated(page); + const state = await getMockMode(page); + if (mockModeWasEnabled === null) { + mockModeWasEnabled = state.enabled; + } + if (!state.enabled) { + await setMockMode(page, true); + } else { + await waitForDefaultMockRuntimeReady(page); + } +} + +async function readLayout(drawer: Locator) { + return drawer.evaluate((row: HTMLTableRowElement) => { + const table = row.closest("table")!; + const headers = Array.from( + table.querySelectorAll("thead th"), + ).filter( + (cell) => + getComputedStyle(cell).display !== "none" && + cell.getClientRects().length > 0, + ); + return { + headerSpan: headers.reduce((total, cell) => total + cell.colSpan, 0), + drawerSpan: row.cells[0].colSpan, + drawerRight: row.cells[0].getBoundingClientRect().right, + headerRight: headers[headers.length - 1].getBoundingClientRect().right, + tableRight: table.getBoundingClientRect().right, + }; + }); +} + +async function expectAligned( + drawer: Locator, + headerSpan: number, +): Promise { + await expect(async () => { + const layout = await readLayout(drawer); + expect(layout.headerSpan, "Visible header span").toBe(headerSpan); + expect(layout.drawerSpan, "Drawer colspan matches visible headers").toBe( + layout.headerSpan, + ); + expect( + Math.abs(layout.drawerRight - layout.tableRight), + `Drawer reaches table right edge: ${JSON.stringify(layout)}`, + ).toBeLessThanOrEqual(1); + expect( + Math.abs(layout.headerRight - layout.tableRight), + `Last visible header reaches table right edge: ${JSON.stringify(layout)}`, + ).toBeLessThanOrEqual(1); + expect( + Math.abs(layout.drawerRight - layout.headerRight), + "Drawer and header right edges", + ).toBeLessThanOrEqual(1); + }).toPass({ timeout: 10_000 }); +} + +test.describe.serial("Inline machine drawer column span", () => { + test.setTimeout(240_000); + + test.afterAll(async ({ browser }) => { + if (mockModeWasEnabled === null) return; + + const context = await browser.newContext({ + httpCredentials: HTTP_CREDENTIALS, + }); + const page = await context.newPage(); + try { + const current = await getMockMode(page); + if (current.enabled !== mockModeWasEnabled) { + await setMockMode(page, mockModeWasEnabled); + } + } finally { + await context.close(); + } + }); + + test("resynchronizes a mounted drawer after hiding and restoring System", async ({ + page, + }, testInfo) => { + test.skip( + testInfo.project.name !== "chromium", + "Chromium fixed-table layout regression", + ); + await page.setViewportSize({ width: 1440, height: 900 }); + await ensureMockModeEnabled(page); + await page.goto("/standalone", { waitUntil: "domcontentloaded" }); + + const surface = page.getByTestId("standalone-page"); + const summary = surface + .getByRole("row") + .filter({ + has: page.locator('button[aria-controls^="agents-machine-drawer"]'), + }) + .first(); + const disclosure = summary.locator( + 'button[aria-controls^="agents-machine-drawer"]', + ); + await expect(disclosure).toBeVisible({ timeout: 60_000 }); + await summary.getByRole("cell").nth(1).click(); + + const drawer = surface.locator( + "tr[data-inline-platform-resource-detail-for]", + ); + await expect(drawer).toHaveCount(1); + await expect(drawer).toBeVisible(); + const originalRow = (await drawer.elementHandle())!; + await originalRow.evaluate((row) => + row.setAttribute("data-column-span-test", "original"), + ); + const focusTarget = (await drawer + .getByRole("button", { name: /^Collapse .* details$/ }) + .elementHandle())!; + await focusTarget.evaluate((button: HTMLElement) => + button.focus({ preventScroll: true }), + ); + + const scrollHandle = await drawer.evaluateHandle((row) => { + return ( + Array.from(row.querySelectorAll("*")).find((element) => { + const style = getComputedStyle(element); + return ( + element.getClientRects().length > 0 && + ((/^(auto|scroll)$/.test(style.overflowY) && + element.scrollHeight > element.clientHeight) || + (/^(auto|scroll)$/.test(style.overflowX) && + element.scrollWidth > element.clientWidth)) + ); + }) ?? null + ); + }); + const scrollElement = scrollHandle.asElement(); + const scrollOffset = scrollElement + ? await scrollElement.evaluate((element) => { + element.scrollTop = Math.min( + 32, + element.scrollHeight - element.clientHeight, + ); + element.scrollLeft = Math.min( + 32, + element.scrollWidth - element.clientWidth, + ); + return { top: element.scrollTop, left: element.scrollLeft }; + }) + : null; + + const expectContinuity = async () => { + expect( + await originalRow.evaluate( + (row) => + row.isConnected && + document.querySelector('[data-column-span-test="original"]') === + row, + ), + "Original drawer row stays mounted", + ).toBe(true); + expect( + await focusTarget.evaluate( + (button) => + document.activeElement === button && + button.closest("tr")?.hasAttribute("data-column-span-test"), + ), + "Original interactive element keeps focus inside the drawer", + ).toBe(true); + if (scrollElement && scrollOffset) { + const current = await scrollElement.evaluate((element) => ({ + connected: element.isConnected, + top: element.scrollTop, + left: element.scrollLeft, + })); + expect( + current.connected, + "Original inner scroll container stays mounted", + ).toBe(true); + expect( + current.top, + "Inner vertical scroll survives column changes", + ).toBe(scrollOffset.top); + expect( + current.left, + "Inner horizontal scroll survives column changes", + ).toBe(scrollOffset.left); + } + }; + + const initialSpan = (await readLayout(drawer)).headerSpan; + const view = surface.getByRole("button", { name: "View", exact: true }); + const preferences = surface.getByRole("region", { + name: "View preferences", + exact: true, + }); + + for (const visible of [false, true]) { + await test.step( + visible + ? "Show System after the reduced table has laid out" + : "Hide System with the drawer open", + async () => { + await view.click(); + await preferences.getByRole("button", { name: /^Columns\b/ }).click(); + const system = preferences.getByRole("checkbox", { + name: "System", + exact: true, + }); + await expect(system).toBeChecked({ checked: !visible }); + await focusTarget.evaluate((button: HTMLElement) => + button.focus({ preventScroll: true }), + ); + // Native checkbox activation preserves drawer focus during the column mutation. + await system.evaluate((checkbox: HTMLInputElement) => + checkbox.click(), + ); + await expect(system).toBeChecked({ checked: visible }); + await drawer.evaluate( + (row) => + new Promise((resolve) => { + // Force the reduced table to lay out before restoring the column. + void row.getBoundingClientRect().width; + requestAnimationFrame(() => + requestAnimationFrame(() => resolve()), + ); + }), + ); + await expectAligned(drawer, initialSpan - (visible ? 0 : 1)); + await expectContinuity(); + await view.click(); + await expect(preferences).toBeHidden(); + await focusTarget.evaluate((button: HTMLElement) => + button.focus({ preventScroll: true }), + ); + await expectContinuity(); + }, + ); + } + }); +}); From 75e4ed44a63cafe4c69551322fdaf86406e4dcbe Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:59:02 +0100 Subject: [PATCH 04/27] Keep row, drawer and replication ages moving on the shared clock The shared relative-time clock from the alert Started change reached only PlatformTableRelativeTimeValue cells and the platform alert tables. Other ages were still computed once at render, so they froze while a row or an open drawer stayed mounted and its timestamp did not change: Kubernetes Job and CronJob Detail ages and the drawer's controller section, the drawer's container Created, Started and Finished rows and its Last seen, the Machines identity subtitle and stale-agent tooltip, Proxmox backup ages and their current, aging or stale band, and the Proxmox replication compact ages, expanded Last sync and Next sync countdown. The frozen cases are the ones that matter: a silent agent read "seen just now" for hours, and a stalled replication scheduler's "in 3m" never turned overdue. Each owning component now reads useRelativeTimeNow and passes it as `now`. The backup band measures from the later of the clock and the wall clock, because the clock can trail by one tick and a backup that finished inside it must not band as a future, unknown age. Replication jobs were read once per page mount, so a moving Last sync would have aged a job that keeps syncing and counted it overdue. The Proxmox surface now reads them through createNonSuspendingQuery with a 30-second background poll. A failed read keeps the last jobs and the tab, and a direct replication link holds until the first read resolves. --- .../subsystems/frontend-primitives.md | 9 ++ .../internal/subsystems/storage-recovery.md | 12 ++ .../internal/subsystems/unified-resources.md | 15 ++ .../21de9738e48c4db3.json | 58 +++++++ .../Infrastructure/ResourceDetailSummary.tsx | 17 ++- ...ResourceDetailSummary.relativeAge.test.tsx | 143 ++++++++++++++++++ .../resourceDetailDrawerKubernetesModel.ts | 29 ++-- .../useResourceDetailDrawerDerivedState.ts | 11 +- .../kubernetes/KubernetesControllersTable.tsx | 21 +-- ...netesControllersTable.relativeAge.test.tsx | 94 ++++++++++++ .../features/proxmox/ProxmoxPageSurface.tsx | 42 ++--- .../proxmox/ProxmoxReplicationTable.tsx | 34 +++-- .../ProxmoxPageSurface.contract.test.tsx | 74 ++++++++- .../ProxmoxReplicationTable.test.tsx | 65 ++++++++ .../proxmoxBackupsTableShared.test.tsx | 61 +++++++- .../proxmox/proxmoxBackupsTableShared.tsx | 15 +- .../standalone/AgentsMachinesTable.tsx | 8 +- .../__tests__/AgentsMachinesTable.test.tsx | 47 ++++++ 18 files changed, 686 insertions(+), 69 deletions(-) create mode 100644 frontend-modern/browser-verification/21de9738e48c4db3.json create mode 100644 frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailSummary.relativeAge.test.tsx create mode 100644 frontend-modern/src/features/kubernetes/__tests__/KubernetesControllersTable.relativeAge.test.tsx diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 4772ac4a62..96d34e3b13 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -5190,6 +5190,15 @@ and the platform alert tables pass it as `now` to `formatRelativeTime` / data refreshes, so an age computed only at render froze ("2h ago" three hours later) whenever its timestamp did not change. A relative-age cell must read the shared clock rather than `Date.now()` at render or a table-local interval. +That rule covers every relative age, not only `PlatformTableRelativeTimeValue` +cells: a value formatted through `formatPlatformTableRelativeTimeValue` or +`formatRelativeTime` (compact phone ages, labelled Detail ages, row subtitles +and tooltips, an open drawer's summary rows and annotations) passes `now` from +the owning component's `useRelativeTimeNow`, and a countdown or freshness band +derived from the same time (replication Next sync, backup age bands) reads it +too. The clock can trail the wall clock by up to one tick, so a check that +treats a future time as invalid (the backup age band) measures from the later +of the two. Read-only metadata badges follow the same primitive-owned shell rule. `frontend-modern/src/components/shared/MetadataBadge.tsx` owns filled and outlined appearances, compact sizing, shape, typed tone vocabulary, fit diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index ebbc81311a..a0a1c4fc91 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -983,6 +983,18 @@ Proxmox backup source/state chips are also a frontend-primitives boundary: badge tones, `proxmoxBackupsTableShared.tsx` owns the backup table helper composition, and visible source/state chips must render through `MetadataBadge` instead of restoring local rounded-sm xs badge spans. +Backup ages read the frontend-primitives shared relative-time clock: +`ProxmoxBackupAgeText` measures the full and compact age text and its +current, aging, or stale band from `useRelativeTimeNow`, never earlier than the +wall clock, so a backup row that stays mounted for days moves from current to +aging without new inventory, and a backup that finished inside the clock's last +tick never bands as a future, unknown age. `ProxmoxPageSurface` reads +replication jobs through `createNonSuspendingQuery` with a 30-second background +poll, because the Replication table's Last sync and Next sync move on the same +clock: a failed read reports the error but keeps the last jobs and the tab, and +a direct replication link holds until the first read resolves instead of +falling back to Overview. The poll re-reads the existing read-only +`/api/replication/jobs` snapshot and opens no new storage or recovery path. Proxmox backup-health and recoverable-artifact summary rows follow the same platform-table density contract as every other provider surface. Storage and recovery owns which backup, datastore, workload-type, and VMID facts matter to diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 3804b3df91..5ec26b8341 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -1250,6 +1250,21 @@ platform-details disclosure. The `KubernetesControllersTable` phone projection keeps controller, kind, ready, and issues and demotes Target with `platform-table-phone-hidden`, so kind labels such as `DaemonSet` and `StatefulSet` fit whole instead of clipping in a 15 percent track. +Every relative age a unified-resource consumer derives from a timestamp reads +the frontend-primitives shared relative-time clock (`useRelativeTimeNow`) +rather than `Date.now()` at render, because rows and open drawers stay mounted +while a timestamp that has stopped changing must keep aging: the Kubernetes +controller Detail ages and the drawer's controller section, the drawer's Docker +container Created, Started and Finished rows, its Last seen, and its Docker +update-check and Mail Gateway updated ages, the Machines identity subtitle +(`seen ...` while the Last seen column is hidden) and stale-agent Last report +tooltip, and the Proxmox replication Last sync and Next sync in the row, phone +projection and disclosure. A silent agent or a stalled pvesr scheduler is the +case whose age matters most, so a replication Next sync turns overdue on the +clock without new data. Replication jobs bypass the unified-resource stream, +so `ProxmoxPageSurface` re-reads them in the background every 30 seconds; +moving ages over a snapshot read once at mount would age a job that keeps +syncing and count it overdue. Kubernetes name columns hold the chevron, status dot and name inside one track, so their md widths leave the name room for a typical node or service name at a 768px viewport: Nodes 20 percent (with Capacity at 16 so its diff --git a/frontend-modern/browser-verification/21de9738e48c4db3.json b/frontend-modern/browser-verification/21de9738e48c4db3.json new file mode 100644 index 0000000000..88e0f845d6 --- /dev/null +++ b/frontend-modern/browser-verification/21de9738e48c4db3.json @@ -0,0 +1,58 @@ +{ + "version": 1, + "base_sha": "7aa537abe112e5f0b9c043885c62bc255b700134", + "verified_at": "2026-10-06T13:58:52Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx", + "frontend-modern/src/components/Infrastructure/resourceDetailDrawerKubernetesModel.ts", + "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts", + "frontend-modern/src/features/kubernetes/KubernetesControllersTable.tsx", + "frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx", + "frontend-modern/src/features/proxmox/ProxmoxReplicationTable.tsx", + "frontend-modern/src/features/proxmox/proxmoxBackupsTableShared.tsx", + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx" + ], + "content_sha256": { + "frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx": "7f789e2e38fab255bd080a647b1f9d20185419c336974bc188fa9c2dec3c2e06", + "frontend-modern/src/components/Infrastructure/resourceDetailDrawerKubernetesModel.ts": "34de374717a338b6d7417e0264f3ed1f106c965262a88336a52a3ebe1e0e3208", + "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts": "a0c08531c1b3c9eba9827ddf02ece46e55ca6eece7629b9b005653ac5129b6ef", + "frontend-modern/src/features/kubernetes/KubernetesControllersTable.tsx": "262e641559d38ccadacf45a74a37b3ec95cbb58887c5ab620b966d369cd9a638", + "frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx": "147ce30feded49cbe9bf833daa78edd1a6fbcff6db419d9303b21e62c9fe00e3", + "frontend-modern/src/features/proxmox/ProxmoxReplicationTable.tsx": "9da12964e02d081b80c848db8741f69d701f8c99e73ed01e9c8958c9d6b51930", + "frontend-modern/src/features/proxmox/proxmoxBackupsTableShared.tsx": "4d6c1097737138d1dd6b1ecfc28ee33b0815ffa78279120bcb524cea38053d2a", + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx": "b9c718460ec18d6ef0a72b2fe9d291e0e813feaecfc6d4cd2f8694cda467fbdf" + }, + "routes": [ + "/kubernetes/workloads", + "/proxmox/backups", + "/proxmox/replication", + "/standalone/machines" + ], + "viewports": [ + { + "width": 1600, + "height": 1000 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack built from this tree; Playwright page.clock installed at load, /ws held after first render (server frames dropped) and REST GETs replayed from their last response, then the clock fast-forwarded with no data change", + "Kubernetes workloads 1600px: Job/CronJob Detail \"Completed 48m ago\" / \"Last success 1h ago\" -> \"Completed 3h ago\" / \"Last success 4h ago\" after 3h", + "Kubernetes Job row expansion: controller section \"01:59 PM (48m ago)\" -> \"01:59 PM (3h ago)\" with the absolute time unchanged; Runtime context Last seen \"1s ago\" -> \"3 hours ago\"", + "Proxmox backups 1600px and 390px: tracked artifact \"1d ago\" / \"1d\" in the current (green) band -> \"8d ago\" / \"8d\" in the aging (amber) band, title \"Aging backup age\", after 7 days", + "Proxmox replication 1600px: row and expanded Last sync \"21m ago\" -> \"3h ago\", Next sync \"in 8m\" -> \"2h overdue\" in red after 3h; one background /api/replication/jobs read fired during the jump", + "Proxmox replication 390px compact row: Last sync \"21m\" -> \"3h\", Next sync \"8m\" -> \"-2h\"", + "Machines 1600px with Last seen column hidden: identity title \"| seen just now\" -> \"| seen 3h ago\"; 390px Seen cells \"just now\" -> \"3h ago\"", + "Control on the same stack with the seven runtime files swapped back to the parent: Kubernetes Detail and expansion, replication expanded Last sync / Next sync and phone row, machines subtitle, and phone backup ages stayed frozen; desktop backup text moved but the band stayed green" + ], + "interactions": [ + "Clicked a Kubernetes Job controller row to open its expansion", + "Clicked \"Expand details for replication job 100-0\"", + "Opened View > Choose which columns to display and unchecked Last seen on Machines", + "page.clock.fastForward 3h (Kubernetes, replication, machines) and 7 days (backups) with data held" + ] +} diff --git a/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx b/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx index 86783aec53..adb1a78581 100644 --- a/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx +++ b/frontend-modern/src/components/Infrastructure/ResourceDetailSummary.tsx @@ -11,6 +11,7 @@ import { } from '@/components/shared/detailSectionModel'; import { TagBadges } from '@/components/shared/TagBadges'; import { formatBytes, formatRelativeTime, formatUptime } from '@/utils/format'; +import { useRelativeTimeNow } from '@/utils/relativeTimeClock'; import { getDockerImageRegistryLink } from '@/features/docker/dockerImageReference'; import { getDockerContainerHealthCheckPresentation, @@ -92,7 +93,7 @@ const dockerRuntimeRow = ( }) : null; -const dockerSection = (docker: NonNullable): DetailSection => { +const dockerSection = (docker: NonNullable, now: number): DetailSection => { const labels = docker.labels ?? {}; const labelEntries = Object.entries(labels); const createdAt = dockerTimestampMillis(docker.createdAt); @@ -157,17 +158,17 @@ const dockerSection = (docker: NonNullable): DetailSection = }) : null, createdAt - ? makeDetailRow('Created', formatRelativeTime(createdAt), { + ? makeDetailRow('Created', formatRelativeTime(createdAt, { now }), { title: new Date(createdAt).toLocaleString(), }) : null, startedAt - ? makeDetailRow('Started', formatRelativeTime(startedAt), { + ? makeDetailRow('Started', formatRelativeTime(startedAt, { now }), { title: new Date(startedAt).toLocaleString(), }) : null, finishedAt - ? makeDetailRow('Finished', formatRelativeTime(finishedAt), { + ? makeDetailRow('Finished', formatRelativeTime(finishedAt, { now }), { title: new Date(finishedAt).toLocaleString(), }) : null, @@ -210,6 +211,10 @@ const dockerSection = (docker: NonNullable): DetailSection = }; export const InlineResourceSummaryTables: Component = (props) => { + // An open drawer keeps its rows while the resource's timestamps stay the + // same, so container and controller ages read the shared clock to keep + // moving. + const now = useRelativeTimeNow(); const sections = (): DetailSection[] => { const docker = dockerContainerMeta(props.resource); const uptimeSeconds = docker @@ -317,8 +322,8 @@ export const InlineResourceSummaryTables: Component [], + identityIpValues: () => [], + identityAliasValues: () => [], + aliasPreviewValues: () => [], + hasAliasOverflow: () => false, + identityCardHasRichData: () => true, + sourceSummary: () => null, + lastSeen: () => '', + lastSeenAbsolute: () => '', +} as unknown as UseResourceDetailDrawerStateResult; + +const makeResource = ({ + id, + type, + ...overrides +}: Partial & Pick): Resource => ({ + id, + name: id, + displayName: id, + platformId: 'host-1', + platformType: 'kubernetes', + sourceType: 'agent', + status: 'online', + type, + lastSeen: Date.parse('2026-05-24T13:31:00Z'), + ...overrides, +}); + +const absoluteTime = (timestamp: string): string => + formatPlatformTableDateTimeValue(timestamp, { dateTimeFormat: { year: 'numeric' } }); + +afterEach(() => { + cleanup(); + vi.useRealTimers(); +}); + +describe('resource drawer summary relative ages', () => { + it('keeps an open Kubernetes controller summary age moving without new data', () => { + vi.useFakeTimers({ now: new Date('2026-05-24T13:31:00Z') }); + + render(() => ( + + )); + + const section = () => screen.getByTestId('resource-kubernetes-controller-section'); + expect( + within(section()).getByText(`${absoluteTime('2026-05-24T13:00:00Z')} (31m ago)`), + ).toBeInTheDocument(); + + vi.advanceTimersByTime(2 * 60 * 60 * 1000); + + // The absolute time is unchanged; only the age after it moves. + expect( + within(section()).getByText(`${absoluteTime('2026-05-24T13:00:00Z')} (2h ago)`), + ).toBeInTheDocument(); + }); + + it('keeps an open container summary Created and Started ages moving without new data', () => { + vi.useFakeTimers({ now: new Date('2026-05-24T13:31:00Z') }); + + render(() => ( + + )); + + const row = (label: string) => + within(screen.getByTestId('resource-docker-container-section')) + .getByText(label) + .closest('tr'); + expect(row('Created')).toHaveTextContent('2 hours ago'); + expect(row('Started')).toHaveTextContent('10 mins ago'); + + vi.advanceTimersByTime(3 * 60 * 60 * 1000); + + expect(row('Created')).toHaveTextContent('5 hours ago'); + expect(row('Started')).toHaveTextContent('3 hours ago'); + }); + + it('keeps an open drawer Last seen moving for a resource that stops reporting', () => { + vi.useFakeTimers({ now: new Date('2026-05-24T13:31:00Z') }); + + const { state, dispose } = createRoot((dispose) => ({ + state: useResourceDetailDrawerDerivedState({ + resource: makeResource({ + id: 'node-1', + type: 'agent', + lastSeen: Date.parse('2026-05-24T13:30:00Z'), + }), + debugEnabled: () => false, + discoveryFeatureEnabled: () => false, + resourceIntelligence: () => null, + }), + dispose, + })); + + expect(state.lastSeen()).toBe('1 min ago'); + + // The resource stays silent: its lastSeen never changes again. + vi.advanceTimersByTime(2 * 60 * 60 * 1000); + + expect(state.lastSeen()).toBe('2 hours ago'); + dispose(); + }); +}); diff --git a/frontend-modern/src/components/Infrastructure/resourceDetailDrawerKubernetesModel.ts b/frontend-modern/src/components/Infrastructure/resourceDetailDrawerKubernetesModel.ts index 2d44225f6b..bfd5a21fa0 100644 --- a/frontend-modern/src/components/Infrastructure/resourceDetailDrawerKubernetesModel.ts +++ b/frontend-modern/src/components/Infrastructure/resourceDetailDrawerKubernetesModel.ts @@ -147,8 +147,14 @@ const dangerWhenPositive = (count: number): DetailValueTone => (count > 0 ? 'dan // Kubernetes reports controller timestamps as RFC 3339 strings. The table row // only has room for an age, so the expansion carries the absolute time first // and the age after it: "Oct 2, 2026, 09:13 PM (1h ago)". The value wraps -// because a phone-width detail cell is too narrow for both on one line. -const timestampRow = (label: string, timestamp?: string | null): DetailRow | null => { +// because a phone-width detail cell is too narrow for both on one line. The +// age is measured from `now`, which an open drawer passes from the shared +// relative-time clock so the age keeps moving while the drawer stays open. +const timestampRow = ( + label: string, + timestamp: string | null | undefined, + now: number | undefined, +): DetailRow | null => { const raw = asString(timestamp); if (!raw) return null; const absolute = formatPlatformTableDateTimeValue(raw, { @@ -156,7 +162,7 @@ const timestampRow = (label: string, timestamp?: string | null): DetailRow | nul dateTimeFormat: { year: 'numeric' }, }); if (!absolute) return makeRow(label, raw, { wrap: true }); - const relative = formatPlatformTableRelativeTimeValue(raw, { emptyText: '' }); + const relative = formatPlatformTableRelativeTimeValue(raw, { emptyText: '', now }); return makeRow(label, relative ? `${absolute} (${relative})` : absolute, { wrap: true }); }; @@ -173,7 +179,7 @@ type ControllerRows = { label: string; rows: Array }; // Each kind leads with what its narrow table row drops: the Detail column // (service name, timestamps) below the large layout and Target on a phone. -const controllerRows = (resource: Resource): ControllerRows | null => { +const controllerRows = (resource: Resource, now: number | undefined): ControllerRows | null => { const k = resource.kubernetes; if (!k) return null; switch (resource.type) { @@ -218,8 +224,8 @@ const controllerRows = (resource: Resource): ControllerRows | null => { return { label: 'Job', rows: [ - timestampRow('Started', k.startTime), - timestampRow('Completed', k.completionTime), + timestampRow('Started', k.startTime, now), + timestampRow('Completed', k.completionTime, now), makeRow( 'Duration', formatPlatformTableDurationValue(elapsedSeconds(k.startTime, k.completionTime), { @@ -237,8 +243,8 @@ const controllerRows = (resource: Resource): ControllerRows | null => { label: 'CronJob', rows: [ makeRow('Schedule', k.schedule, { valueClass: 'font-mono' }), - timestampRow('Last run', k.lastScheduleTime), - timestampRow('Last success', k.lastSuccessfulTime), + timestampRow('Last run', k.lastScheduleTime, now), + timestampRow('Last success', k.lastSuccessfulTime, now), makeRow('Suspended', k.suspend === true ? 'Yes' : null, { tone: 'warning' }), countRow('Active', k.active), ], @@ -253,8 +259,11 @@ const controllerRows = (resource: Resource): ControllerRows | null => { // controllers table hides Scope and Detail below its large layout and Target // on a phone, so without this a Job's completion time or a CronJob's last // success is unreachable from a narrow row. -export const buildKubernetesControllerSection = (resource: Resource): DetailSection | null => { - const controller = controllerRows(resource); +export const buildKubernetesControllerSection = ( + resource: Resource, + now?: number, +): DetailSection | null => { + const controller = controllerRows(resource, now); if (!controller) return null; const rows = compactRows([ ...controller.rows, diff --git a/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts b/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts index 66e329b8e3..1b33f7d387 100644 --- a/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts +++ b/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts @@ -7,6 +7,7 @@ import type { HostNetworkInterface } from '@/types/api'; import { requiresGovernedResourceDisplay } from '@/types/resource'; import type { ResourceVMwareMeta } from '@/types/resource'; import { formatAbsoluteTime, formatRelativeTime } from '@/utils/format'; +import { useRelativeTimeNow } from '@/utils/relativeTimeClock'; import { getAgentStatusIndicator } from '@/utils/status'; import { dedupeResourceBadges, @@ -135,7 +136,11 @@ export const useResourceDetailDrawerDerivedState = ( resolveResourceLabelInput?.(resourceId)?.trim() || resourceId; const statusIndicator = createMemo(() => getAgentStatusIndicator({ status: resource().status })); const healthIssue = createMemo(() => getResourceHealthIssuePresentation(resource())); - const lastSeen = createMemo(() => formatRelativeTime(resource().lastSeen)); + // An open drawer keeps its rows while a resource's timestamps stay the same, + // which is exactly when a silent resource's Last seen must keep aging, so + // the drawer's ages read the shared clock. + const now = useRelativeTimeNow(); + const lastSeen = createMemo(() => formatRelativeTime(resource().lastSeen, { now: now() })); const lastSeenAbsolute = createMemo(() => formatAbsoluteTime(resource().lastSeen)); const platformBadge = createMemo(() => getPlatformBadge(resource().platformType)); @@ -210,7 +215,7 @@ export const useResourceDetailDrawerDerivedState = ( if (!raw) return ''; const parsed = Date.parse(raw); if (!Number.isFinite(parsed)) return ''; - return formatRelativeTime(parsed); + return formatRelativeTime(parsed, { now: now() }); }); const dockerHostCommand = createMemo(() => dockerHostData()?.command); const dockerHostCommandActive = createMemo(() => { @@ -278,7 +283,7 @@ export const useResourceDetailDrawerDerivedState = ( if (!raw) return ''; const parsed = Date.parse(raw); if (!Number.isFinite(parsed)) return ''; - return formatRelativeTime(parsed); + return formatRelativeTime(parsed, { now: now() }); }); const pbsVisibleJobBreakdown = createMemo(() => buildPbsVisibleJobBreakdown(pbsData())); const pbsActiveTasks = createMemo(() => buildPbsActiveTasks(pbsData())); diff --git a/frontend-modern/src/features/kubernetes/KubernetesControllersTable.tsx b/frontend-modern/src/features/kubernetes/KubernetesControllersTable.tsx index a70172c5b5..ca11a60e91 100644 --- a/frontend-modern/src/features/kubernetes/KubernetesControllersTable.tsx +++ b/frontend-modern/src/features/kubernetes/KubernetesControllersTable.tsx @@ -30,6 +30,7 @@ import { getPlatformResourceDetailRowClass, } from '@/features/platformPage/PlatformResourceDetailTableRow'; import type { Resource } from '@/types/resource'; +import { useRelativeTimeNow } from '@/utils/relativeTimeClock'; import { compareKubernetesControllers, filterKubernetesResources, @@ -137,9 +138,10 @@ const plainDetail = (text: string): ControllerDetail => ({ text, title: text }); // Kubernetes reports job timestamps as RFC 3339 strings with microseconds, // which never fit the Detail column. The cell says how long ago the event -// happened and keeps the absolute time on hover. -const timeDetail = (label: string, timestamp: string): ControllerDetail => { - const relative = formatPlatformTableRelativeTimeValue(timestamp, { emptyText: '' }); +// happened, measured from the shared clock so it keeps moving while the row +// stays mounted, and keeps the absolute time on hover. +const timeDetail = (label: string, timestamp: string, now: number): ControllerDetail => { + const relative = formatPlatformTableRelativeTimeValue(timestamp, { emptyText: '', now }); if (!relative) return plainDetail(`${label}: ${timestamp}`); const absolute = formatPlatformTableDateTimeValue(timestamp, { dateTimeFormat: { year: 'numeric' }, @@ -147,7 +149,7 @@ const timeDetail = (label: string, timestamp: string): ControllerDetail => { return { text: `${label} ${relative}`, title: `${label}: ${absolute}` }; }; -const apiDetail = (resource: Resource): ControllerDetail => { +const apiDetail = (resource: Resource, now: number): ControllerDetail => { switch (resource.type) { case 'k8s-replicaset': { if (typeof resource.kubernetes?.fullyLabeledReplicas === 'number') { @@ -171,17 +173,17 @@ const apiDetail = (resource: Resource): ControllerDetail => { ); case 'k8s-job': if (resource.kubernetes?.completionTime) { - return timeDetail('Completed', resource.kubernetes.completionTime); + return timeDetail('Completed', resource.kubernetes.completionTime, now); } return resource.kubernetes?.startTime - ? timeDetail('Started', resource.kubernetes.startTime) + ? timeDetail('Started', resource.kubernetes.startTime, now) : plainDetail('—'); case 'k8s-cronjob': if (resource.kubernetes?.lastSuccessfulTime) { - return timeDetail('Last success', resource.kubernetes.lastSuccessfulTime); + return timeDetail('Last success', resource.kubernetes.lastSuccessfulTime, now); } return resource.kubernetes?.lastScheduleTime - ? timeDetail('Last run', resource.kubernetes.lastScheduleTime) + ? timeDetail('Last run', resource.kubernetes.lastScheduleTime, now) : plainDetail('—'); default: return plainDetail('—'); @@ -245,6 +247,7 @@ export const KubernetesControllersTable: Component<{ externalSearch?: () => string; externalStatus?: () => KubernetesResourceStatusFilter; }> = (props) => { + const now = useRelativeTimeNow(); const singleCluster = createMemo(() => isSingleKubernetesCluster(props.resources)); const tableState = createPlatformTableFilterState({ resources: () => props.resources, @@ -402,7 +405,7 @@ export const KubernetesControllersTable: Component<{ const kind = () => controllerKind(resource); const target = () => targetValue(resource); const exceptions = () => exceptionSummary(resource); - const detail = () => apiDetail(resource); + const detail = () => apiDetail(resource, now()); const detailRowId = () => drawer.detailRowId(resource); const isExpanded = () => drawer.isExpanded(resource); diff --git a/frontend-modern/src/features/kubernetes/__tests__/KubernetesControllersTable.relativeAge.test.tsx b/frontend-modern/src/features/kubernetes/__tests__/KubernetesControllersTable.relativeAge.test.tsx new file mode 100644 index 0000000000..7b547bf664 --- /dev/null +++ b/frontend-modern/src/features/kubernetes/__tests__/KubernetesControllersTable.relativeAge.test.tsx @@ -0,0 +1,94 @@ +import { cleanup, render, screen } from '@solidjs/testing-library'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('@/contexts/appRuntime', async (importOriginal) => ({ + ...(await importOriginal()), + useWebSocket: () => ({ activeAlerts: {} }), +})); + +vi.mock('@/api/resources', () => ({ + ResourceAPI: { + getFacetBundle: vi.fn().mockResolvedValue({ + capabilities: [], + relationships: [], + recentChanges: [], + }), + }, +})); + +import type { Resource } from '@/types/resource'; +import { KubernetesControllersTable } from '../KubernetesControllersTable'; + +const makeResource = ({ + id, + type, + ...overrides +}: Partial & Pick): Resource => ({ + id, + name: id, + displayName: id, + platformId: 'cluster-1', + platformType: 'kubernetes', + sourceType: 'agent', + sources: ['kubernetes'], + status: 'online', + type, + lastSeen: 1_700_000_000_000, + ...overrides, +}); + +afterEach(() => { + cleanup(); + vi.useRealTimers(); +}); + +describe('KubernetesControllersTable relative ages', () => { + it('keeps Job and CronJob detail ages moving while the rows stay mounted', () => { + vi.useFakeTimers({ now: new Date('2026-05-24T13:31:00Z') }); + + render(() => ( + } + emptyTitle="No controllers" + emptyDescription="No controllers" + showToolbar={false} + /> + )); + + expect(screen.getByText('Completed 31m ago')).toBeInTheDocument(); + expect(screen.getByText('Last success 36m ago')).toBeInTheDocument(); + + // No data changes: the timestamps stay put and only the clock moves. A + // detail computed once at render would still read "31m ago" here. + vi.advanceTimersByTime(3 * 60 * 60 * 1000); + + expect(screen.getByText('Completed 3h ago')).toBeInTheDocument(); + expect(screen.getByText('Last success 3h ago')).toBeInTheDocument(); + expect(screen.queryByText('Completed 31m ago')).not.toBeInTheDocument(); + }); +}); diff --git a/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx b/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx index d40ce5bf2c..adfd9cc874 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx @@ -1,12 +1,5 @@ import { useLocation } from '@solidjs/router'; -import { - Show, - createEffect, - createMemo, - createResource, - createSignal, - type Accessor, -} from 'solid-js'; +import { Show, createEffect, createMemo, createSignal, type Accessor } from 'solid-js'; import StorageSurface from '@/components/Storage/Storage'; import { WorkloadsFilter } from '@/components/Workloads/WorkloadsFilter'; import { WorkloadsSurface } from '@/components/Workloads/WorkloadsSurface'; @@ -58,7 +51,9 @@ import { ProxmoxCephTable } from './ProxmoxCephTable'; import { ProxmoxMailGatewayTable } from './ProxmoxMailGatewayTable'; import { ProxmoxNodesTable } from './ProxmoxNodesTable'; import { ProxmoxReplicationTable, fetchReplicationJobs } from './ProxmoxReplicationTable'; +import { createNonSuspendingQuery } from '@/hooks/createNonSuspendingQuery'; import { useUnifiedResources } from '@/hooks/useUnifiedResources'; +import type { ReplicationJob } from '@/types/api'; import type { Resource } from '@/types/resource'; import { updateStore } from '@/stores/updates'; import { @@ -85,6 +80,8 @@ const PROXMOX_PLATFORM_FILTER = 'proxmox-all'; const PROXMOX_WORKLOAD_STATUS_STORAGE_SCOPE = 'proxmox'; const PROXMOX_WORKLOAD_EXCLUDED_TYPES = ['app-container'] as const; const PHONE_MOUNTED_TAB_LIMIT = 2; +const REPLICATION_JOBS_POLL_MS = 30_000; +const NO_REPLICATION_JOBS: ReplicationJob[] = []; const VALID_TABS = new Set(PROXMOX_TAB_SPECS.map((tab) => tab.id)); const PROXMOX_WORKLOAD_STATUS_OPTIONS: readonly WorkloadsStatusOption[] = [ { value: 'all', label: 'All' }, @@ -123,13 +120,18 @@ export function ProxmoxPageSurface() { }); // Replication jobs come straight from /api/replication/jobs (they bypass // the unified-resource pipeline), so the surface owns the fetch: the job - // count gates the Replication tab and the same data feeds the table. - // Reading an errored resource throws, hence the `.error` guards. - const [replicationJobs, { refetch: refetchReplicationJobs }] = - createResource(fetchReplicationJobs); - const replicationJobCount = createMemo(() => - replicationJobs.error ? 0 : (replicationJobs() ?? []).length, - ); + // count gates the Replication tab and the same data feeds the table. The + // read polls in the background because the table's Last sync and Next sync + // keep moving on the shared clock: a snapshot read once at mount would age + // a job that keeps syncing and count it overdue. A failed poll keeps the + // last jobs, so the tab does not vanish under the user. + const replicationJobs = createNonSuspendingQuery({ + source: () => 'proxmox-pve', + fetcher: () => fetchReplicationJobs(), + initialValue: NO_REPLICATION_JOBS, + pollMs: REPLICATION_JOBS_POLL_MS, + }); + const replicationJobCount = createMemo(() => replicationJobs.value().length); const visibleTabs = createMemo(() => { // An unknown snapshot is not evidence that every optional integration is // present. Never use estate-wide aggregations here: unrelated VMware VMs @@ -146,7 +148,9 @@ export function ProxmoxPageSurface() { const requested = requestedTab(); // Do not discard a direct link while counts are still unknown: its own // resource query must be allowed to hydrate before deciding it is absent. - return !tabEvidence.facets?.() || visibleTabIds().has(requested) ? requested : 'overview'; + const countsUnknown = + !tabEvidence.facets?.() || (requested === 'replication' && !replicationJobs.resolvedOnce()); + return countsUnknown || visibleTabIds().has(requested) ? requested : 'overview'; }); const shouldHydrateTab = (tab: ProxmoxPageTabId) => activeTab() === tab; const overviewResources = useUnifiedResources({ @@ -403,9 +407,9 @@ export function ProxmoxPageSurface() { void refetchReplicationJobs()} + jobs={replicationJobs.resolvedOnce() ? replicationJobs.value() : undefined} + error={replicationJobs.error() ?? undefined} + onRetry={() => void replicationJobs.refetch()} emptyIcon={} emptyTitle="No replication jobs" emptyDescription="Replication jobs appear here once PVE is configured to replicate guests between nodes." diff --git a/frontend-modern/src/features/proxmox/ProxmoxReplicationTable.tsx b/frontend-modern/src/features/proxmox/ProxmoxReplicationTable.tsx index 2f635487d1..918f4d8486 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxReplicationTable.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxReplicationTable.tsx @@ -6,6 +6,7 @@ import { StatusDot } from '@/components/shared/StatusDot'; import type { StatusIndicatorVariant } from '@/utils/status'; import { TableCell, TableHead, TableRow } from '@/components/shared/Table'; import { apiFetch } from '@/utils/apiClient'; +import { useRelativeTimeNow } from '@/utils/relativeTimeClock'; import { PlatformTableToolbar, PlatformErrorState, @@ -234,8 +235,10 @@ const NEXT_SYNC_TONE_CLASS: Record = { // An overdue next-sync is the one signal that catches a stalled pvesr // scheduler even while the last sync still reports ok, so it gets its own // column instead of folding into the status pill (which mirrors PVE's own -// job state). -function nextSyncFor(job: ReplicationJob): { text: string; tone: NextSyncTone } { +// job state). The countdown is measured from the shared relative-time clock: +// a stalled scheduler leaves nextSync unchanged, so a countdown read once at +// render would sit at "in 3m" and never turn overdue. +function nextSyncFor(job: ReplicationJob, now: number): { text: string; tone: NextSyncTone } { if (!job.enabled) return { text: '—', tone: 'muted' }; let target = 0; if (job.nextSyncUnix && job.nextSyncUnix > 0) { @@ -246,7 +249,7 @@ function nextSyncFor(job: ReplicationJob): { text: string; tone: NextSyncTone } if (Number.isFinite(parsed)) target = parsed; } if (!target) return { text: '—', tone: 'muted' }; - const minutes = Math.floor((target - Date.now()) / 60_000); + const minutes = Math.floor((target - now) / 60_000); if (minutes < 0) { const overdue = Math.abs(minutes); const text = overdue < 60 ? `${overdue}m overdue` : `${Math.floor(overdue / 60)}h overdue`; @@ -283,6 +286,9 @@ export const ProxmoxReplicationTable: Component<{ const [search, setSearch] = createSignal(''); const [status, setStatus] = createSignal('all'); const [expandedJobKey, setExpandedJobKey] = createSignal(null); + // Rows stay mounted between job refreshes, so Last sync and Next sync read + // the shared clock to keep moving while a job's timestamps do not change. + const now = useRelativeTimeNow(); const filterByStatus = (want: ReplicationStatusFilter) => { const split = splitSearchExclusions(search()); @@ -502,7 +508,7 @@ export const ProxmoxReplicationTable: Component<{ {(job, index) => { const classification = classifyJob(job); const ind = indicatorFor(classification); - const next = nextSyncFor(job); + const next = () => nextSyncFor(job, now()); const sourceNode = (job.sourceNode ?? '').trim() || '—'; const targetNode = (job.targetNode ?? '').trim() || '—'; const guestLabel = formatGuestLabel(job); @@ -607,9 +613,9 @@ export const ProxmoxReplicationTable: Component<{ } > - {formatPlatformTableRelativeTimeValue( - syncTimeValue(job), - ).replace(/ ago$/, '')} + {formatPlatformTableRelativeTimeValue(syncTimeValue(job), { + now: now(), + }).replace(/ ago$/, '')}
@@ -618,11 +624,11 @@ export const ProxmoxReplicationTable: Component<{ class={`${getPlatformTableCellClassForKind('numeric-value')} text-base-content ${mobilePaddingClass()}`} > {isCompact() - ? compactReplicationNextSyncText(next.text, next.tone) - : next.text} + ? compactReplicationNextSyncText(next().text, next().tone) + : next().text} @@ -701,13 +707,15 @@ export const ProxmoxReplicationTable: Component<{
{job.schedule || '—'}
Last sync
- {formatPlatformTableRelativeTimeValue(syncTimeValue(job))} + {formatPlatformTableRelativeTimeValue(syncTimeValue(job), { + now: now(), + })}
Next sync
- {next.text} + {next().text}
Duration
diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx b/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx index 73a1dee144..f387a48ebb 100644 --- a/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx +++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxPageSurface.contract.test.tsx @@ -21,6 +21,7 @@ const mockWorkloadSearch = vi.hoisted(() => vi.fn(() => '')); const mockSelectedNode = vi.hoisted(() => vi.fn<() => string | null>(() => null)); const mockHandleNodeSelect = vi.hoisted(() => vi.fn()); const mockWorkloadsOptions = vi.hoisted(() => vi.fn()); +const mockFetchReplicationJobs = vi.hoisted(() => vi.fn(() => Promise.resolve([] as unknown[]))); const makeResource = (resource: Partial & Pick): Resource => ({ @@ -152,8 +153,14 @@ vi.mock('../ProxmoxNodesTable', () => ({ })); vi.mock('../ProxmoxReplicationTable', () => ({ - ProxmoxReplicationTable: () =>
, - fetchReplicationJobs: () => Promise.resolve([]), + ProxmoxReplicationTable: (props: { jobs?: unknown[]; error?: unknown }) => ( +
+ ), + fetchReplicationJobs: () => mockFetchReplicationJobs(), })); const renderSurface = () => @@ -630,6 +637,69 @@ describe('ProxmoxPageSurface contract', () => { }); }); + it('re-reads replication jobs in the background and keeps the tab through a failed read', async () => { + vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] }); + mockPathname.mockReturnValue('/proxmox/replication'); + const node = makeResource({ id: 'node:pve1', type: 'agent', name: 'pve1' }); + mockUseUnifiedResources.mockReturnValue({ + resources: () => [node], + facets: () => ({ incidentCount: 0, byType: {} }), + loading: () => false, + error: () => null, + refetch: vi.fn(async () => []), + }); + let resolveFirstRead: (jobs: unknown[]) => void = () => undefined; + mockFetchReplicationJobs.mockReset(); + mockFetchReplicationJobs.mockReturnValueOnce( + new Promise((resolve) => { + resolveFirstRead = resolve; + }), + ); + + try { + renderSurface(); + const tabs = screen.getByTestId('platform-section-tabs'); + // Counts are known but the first read is pending: a direct link stays put. + expect(tabs).toHaveAttribute('data-active', 'replication'); + expect(screen.getByTestId('replication-table')).toHaveAttribute('data-jobs', 'loading'); + + resolveFirstRead([{ id: 'job-1' }]); + await waitFor(() => + expect(screen.getByTestId('replication-table')).toHaveAttribute('data-jobs', '1'), + ); + expect(tabs).toHaveAttribute('data-tabs', 'overview,replication'); + + // The table's ages move on the shared clock, so its jobs are re-read in + // the background rather than aging a snapshot taken at mount. + mockFetchReplicationJobs.mockResolvedValueOnce([{ id: 'job-1' }, { id: 'job-2' }]); + vi.advanceTimersByTime(30_000); + await waitFor(() => + expect(screen.getByTestId('replication-table')).toHaveAttribute('data-jobs', '2'), + ); + + // A failed read reports the error but keeps the tab and the user on it. + mockFetchReplicationJobs.mockRejectedValueOnce(new Error('replication read failed')); + vi.advanceTimersByTime(30_000); + await waitFor(() => + expect(screen.getByTestId('replication-table')).toHaveAttribute('data-error', 'yes'), + ); + expect(tabs).toHaveAttribute('data-tabs', 'overview,replication'); + expect(tabs).toHaveAttribute('data-active', 'replication'); + + mockFetchReplicationJobs.mockResolvedValueOnce([{ id: 'job-1' }]); + vi.advanceTimersByTime(30_000); + await waitFor(() => + expect(screen.getByTestId('replication-table')).toHaveAttribute('data-error', 'no'), + ); + expect(screen.getByTestId('replication-table')).toHaveAttribute('data-jobs', '1'); + expect(mockFetchReplicationJobs).toHaveBeenCalledTimes(4); + } finally { + vi.useRealTimers(); + mockFetchReplicationJobs.mockReset(); + mockFetchReplicationJobs.mockImplementation(() => Promise.resolve([])); + } + }); + it('does not surface stale-agent notices for development builds without an agent target', () => { mockVersionInfo.mockReturnValue({ version: '6.0.0-rc.6+git.172.g2c360f779.dirty', diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxReplicationTable.test.tsx b/frontend-modern/src/features/proxmox/__tests__/ProxmoxReplicationTable.test.tsx index b302446ed7..8e9f45cdec 100644 --- a/frontend-modern/src/features/proxmox/__tests__/ProxmoxReplicationTable.test.tsx +++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxReplicationTable.test.tsx @@ -1,6 +1,7 @@ import { cleanup, fireEvent, render, screen } from '@solidjs/testing-library'; import { afterEach, describe, expect, it, vi } from 'vitest'; import type { ReplicationJob } from '@/types/api'; +import { RELATIVE_TIME_TICK_MS } from '@/utils/relativeTimeClock'; import { ProxmoxReplicationTable, REPLICATION_MOBILE_COLUMNS, @@ -132,3 +133,67 @@ describe('ProxmoxReplicationTable', () => { expect(replicationTableSource).toMatch(/job\.lastSyncStatus,\s*job\.error,\s*job\.comment,/); }); }); + +describe('ProxmoxReplicationTable relative times', () => { + const start = Date.parse('2026-10-04T12:00:00Z'); + // Last synced 5 minutes before the table opens, next sync due 10 minutes after. + const job = () => + replicationJob({ + lastSyncUnix: (start - 5 * 60_000) / 1000, + nextSyncUnix: (start + 10 * 60_000) / 1000, + }); + const renderTable = () => + render(() => ( + undefined} + emptyIcon={} + emptyTitle="No replication jobs" + emptyDescription="Replication jobs appear here." + /> + )); + // Two hours pass with no new job data; the next clock tick re-reads them. + const twoHoursLater = () => { + vi.setSystemTime(start + 2 * 60 * 60_000); + vi.advanceTimersByTime(RELATIVE_TIME_TICK_MS); + }; + + afterEach(() => vi.useRealTimers()); + + it('keeps the compact Last sync and Next sync moving while the job does not change', () => { + vi.useFakeTimers({ now: start }); + vi.spyOn(HTMLElement.prototype, 'clientWidth', 'get').mockReturnValue(400); + renderTable(); + + expect(screen.getByText('5m')).toBeInTheDocument(); + expect(screen.getByText('10m')).toBeInTheDocument(); + + twoHoursLater(); + + // A stalled scheduler leaves nextSync unchanged, so the countdown must + // turn overdue on its own. + expect(screen.getByText('2h')).toBeInTheDocument(); + expect(screen.getByText('-1h')).toHaveClass('text-red-600'); + expect(screen.queryByText('10m')).not.toBeInTheDocument(); + }); + + it('keeps the expanded Last sync and Next sync moving while the job does not change', async () => { + vi.useFakeTimers({ now: start }); + renderTable(); + await fireEvent.click( + screen.getByRole('button', { name: 'Expand details for replication job 100-0' }), + ); + const detail = (label: string) => + screen.getByText(label, { selector: 'dt' }).nextElementSibling as HTMLElement; + + expect(detail('Last sync')).toHaveTextContent('5m ago'); + expect(detail('Next sync')).toHaveTextContent('in 10m'); + + twoHoursLater(); + + expect(detail('Last sync')).toHaveTextContent('2h ago'); + expect(detail('Next sync')).toHaveTextContent('1h overdue'); + expect(detail('Next sync')).toHaveClass('text-red-600'); + }); +}); diff --git a/frontend-modern/src/features/proxmox/__tests__/proxmoxBackupsTableShared.test.tsx b/frontend-modern/src/features/proxmox/__tests__/proxmoxBackupsTableShared.test.tsx index 594501f26a..2d6ec3ca25 100644 --- a/frontend-modern/src/features/proxmox/__tests__/proxmoxBackupsTableShared.test.tsx +++ b/frontend-modern/src/features/proxmox/__tests__/proxmoxBackupsTableShared.test.tsx @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { cleanup, render, screen } from '@solidjs/testing-library'; +import { cleanup, render, screen, within } from '@solidjs/testing-library'; import { WorkloadTypeBadge } from '@/components/shared/WorkloadTypeBadge'; import proxmoxBackupServersTableSource from '../ProxmoxBackupServersTable.tsx?raw'; import proxmoxCoverageTableSource from '../ProxmoxCoverageTable.tsx?raw'; @@ -13,6 +13,7 @@ import { SortableHead, formatCompactBackupAge, } from '../proxmoxBackupsTableShared'; +import { RELATIVE_TIME_TICK_MS } from '@/utils/relativeTimeClock'; import type { RecoverableArtifact } from '../proxmoxBackupRecoveryModel'; afterEach(cleanup); @@ -205,3 +206,61 @@ describe('backup-date-evidence age cell', () => { expect(value.title).toContain(createdAt); }); }); + +describe('backup age cell on the shared clock', () => { + afterEach(() => vi.useRealTimers()); + + it('keeps the age and its freshness band moving while the artifact does not change', () => { + vi.useFakeTimers({ now: new Date('2026-10-04T12:00:00Z') }); + const createdAt = '2026-10-04T06:00:00Z'; + const backup = artifact({ createdAt, createdMs: Date.parse(createdAt) }); + + render(() => ( + <> + + + + + + + + )); + + expect(screen.getByTestId('full')).toHaveTextContent('6h ago'); + expect(screen.getByText('6h')).toHaveClass('text-emerald-600'); + + // Eight days pass with no new backup and no data refresh. The next clock + // tick moves both ages and drops the backup out of the current band. + vi.setSystemTime(new Date('2026-10-12T12:00:00Z')); + vi.advanceTimersByTime(RELATIVE_TIME_TICK_MS); + + expect(screen.getByTestId('full')).toHaveTextContent('8d ago'); + const compact = screen.getByText('8d'); + expect(compact).toHaveClass('text-amber-600'); + expect(compact.closest('[title]')?.getAttribute('title')).toBe( + `Aging backup age · ${createdAt}`, + ); + }); + + it('does not band a backup that finished after the last clock tick as unknown', () => { + vi.useFakeTimers({ now: new Date('2026-10-04T12:00:00Z') }); + // Another mounted cell starts the shared clock at 12:00:00. + render(() => ); + + // A backup that finished 20s later arrives before the next tick. + vi.setSystemTime(new Date('2026-10-04T12:00:25Z')); + const createdAt = '2026-10-04T12:00:20Z'; + render(() => ( + + + + )); + + const fresh = within(screen.getByTestId('fresh')).getByText('now'); + expect(fresh).toHaveClass('text-emerald-600'); + expect(screen.queryByText('Unknown')).not.toBeInTheDocument(); + }); +}); diff --git a/frontend-modern/src/features/proxmox/proxmoxBackupsTableShared.tsx b/frontend-modern/src/features/proxmox/proxmoxBackupsTableShared.tsx index a82b9c27eb..4828f3f754 100644 --- a/frontend-modern/src/features/proxmox/proxmoxBackupsTableShared.tsx +++ b/frontend-modern/src/features/proxmox/proxmoxBackupsTableShared.tsx @@ -10,6 +10,7 @@ import { PlatformTableRelativeTimeValue, formatPlatformTableRelativeTimeValue, } from '@/features/platformPage/sharedPlatformPage'; +import { useRelativeTimeNow } from '@/utils/relativeTimeClock'; import { getRecoveryAgeBand, @@ -224,14 +225,20 @@ export function RowMetricBar(props: { // Phone columns are too narrow for "18h ago", so the compact form drops the // suffix ("18h", "now") and keeps the full age band and timestamp on hover. -export function formatCompactBackupAge(createdAt: string): string { - const relative = formatPlatformTableRelativeTimeValue(createdAt); +export function formatCompactBackupAge(createdAt: string, now?: number): string { + const relative = formatPlatformTableRelativeTimeValue(createdAt, { now }); if (relative === 'just now') return 'now'; return relative.replace(/ ago$/, ''); } export function ProxmoxBackupAgeText(props: { artifact: RecoverableArtifact; compact?: boolean }) { - const band = () => getRecoveryAgeBand(props.artifact.createdMs); + // Backup rows stay mounted while an artifact's creation time never changes, + // so the age and its freshness band both read the shared clock. The clock + // can trail the wall clock by up to one tick, and a backup that finished + // inside that window must not band as a future (unknown) age. + const tickNow = useRelativeTimeNow(); + const now = () => Math.max(tickNow(), Date.now()); + const band = () => getRecoveryAgeBand(props.artifact.createdMs, now()); const title = () => { if (band() === 'unknown') return recoveryAgeTitleByBand.unknown; const parts = [recoveryAgeTitleByBand[band()], props.artifact.createdAt].filter(Boolean); @@ -250,7 +257,7 @@ export function ProxmoxBackupAgeText(props: { artifact: RecoverableArtifact; com when={props.compact} fallback={} > - {formatCompactBackupAge(props.artifact.createdAt)} + {formatCompactBackupAge(props.artifact.createdAt, now())} diff --git a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx index 4b99ca84a0..2015ac6092 100644 --- a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx +++ b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx @@ -73,6 +73,7 @@ import { } from '@/utils/raidPresentation'; import { asTrimmedString } from '@/utils/stringUtils'; import { getPreferredResourceIP } from '@/utils/resourceIdentity'; +import { useRelativeTimeNow } from '@/utils/relativeTimeClock'; import { RESOURCE_METADATA_CHANGED_EVENT, type ResourceMetadataChangedDetail, @@ -1174,6 +1175,9 @@ export const AgentsMachinesTable: Component<{ onExternalStatusChange?: (value: PlatformResourceStatusFilter) => void; onResetFilters?: () => void; }> = (props) => { + // A machine that stops reporting keeps the same last-seen time, which is + // exactly when its age must keep moving, so row ages read the shared clock. + const now = useRelativeTimeNow(); const [locallyRemovedResourceIds, setLocallyRemovedResourceIds] = createSignal< Record >({}); @@ -1482,7 +1486,7 @@ export const AgentsMachinesTable: Component<{ const agentStaleTitle = () => { const last = asTrimmedString(machine.agent?.lastReportAt); const when = last - ? ` Last report ${formatPlatformTableRelativeTimeValue(last)}.` + ? ` Last report ${formatPlatformTableRelativeTimeValue(last, { now: now() })}.` : ''; return `Agent has stopped reporting.${when} Re-run the install command from the Pulse UI to refresh its token.`; }; @@ -1534,7 +1538,7 @@ export const AgentsMachinesTable: Component<{ ? availabilityFor(machine)?.lastChecked : machine.lastSeen; const lastSeenLabel = () => - formatPlatformTableRelativeTimeValue(lastSeenValue()); + formatPlatformTableRelativeTimeValue(lastSeenValue(), { now: now() }); const machineSubtitle = () => machineRowSubtitleFor( name(), diff --git a/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx b/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx index 723f0ff3c0..b5c4916578 100644 --- a/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx +++ b/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx @@ -4,6 +4,7 @@ import { AgentMetadataAPI } from '@/api/agentMetadata'; import { MonitoringAPI } from '@/api/monitoring'; import type { Resource } from '@/types/resource'; import { STORAGE_KEYS } from '@/utils/localStorage'; +import { RELATIVE_TIME_TICK_MS } from '@/utils/relativeTimeClock'; import { RESOURCE_METADATA_CHANGED_EVENT } from '@/utils/resourceMetadataEvents'; import { AgentsMachinesTable } from '../AgentsMachinesTable'; @@ -1121,4 +1122,50 @@ describe('AgentsMachinesTable', () => { 'Thermal pressure nominal via pmset', ); }); + + it("keeps a silent machine's seen and last-report ages moving while its data does not change", async () => { + const start = Date.parse('2026-07-08T09:05:00Z'); + vi.useFakeTimers({ toFake: ['Date', 'setInterval', 'clearInterval'], now: start }); + + try { + render(() => ( + + )); + await openMachineColumnPicker(); + await fireEvent.click(screen.getByLabelText('Last seen')); + + expect(screen.getByTitle('omv · 192.168.0.21 | seen 5m ago')).toBeInTheDocument(); + expect( + screen.getByTitle(/Agent has stopped reporting\. Last report 5m ago\./), + ).toBeInTheDocument(); + + // The agent stays silent: no new data arrives and only the clock moves. + vi.setSystemTime(start + 3 * 60 * 60 * 1000); + vi.advanceTimersByTime(RELATIVE_TIME_TICK_MS); + + expect(screen.getByTitle('omv · 192.168.0.21 | seen 3h ago')).toBeInTheDocument(); + expect( + screen.getByTitle(/Agent has stopped reporting\. Last report 3h ago\./), + ).toBeInTheDocument(); + } finally { + vi.useRealTimers(); + } + }); }); From 9206486d56e25d2181c77fe3d50fb79916d23c73 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:07:08 +0100 Subject: [PATCH 05/27] Hand a node only the usage metrics its agent evaluates CheckHost registered a host agent's node link before its disabled returns. With agent alerts switched off, or a host override disabling the agent, the node still released its CPU, memory and disk alerts to an agent that evaluated nothing, so nothing alerted on that machine's usage. The same happened per metric: an agent CPU threshold set to off with the node CPU threshold on left the machine with no CPU alert. The link now records which usage metrics the agent actually evaluated on its last report: a live CPU threshold, a live memory threshold with a known reading, and at least one filesystem with a live disk threshold. CheckHost registers it after evaluation and removes it on the disabled returns. CheckNode releases only the metrics the agent owns and evaluates the rest itself, so deduplication can only remove a duplicate, never the last alert for a machine. The link also carries the agent's display name, which the handover notification work builds on. --- .../v6/internal/subsystems/alerts.md | 18 ++- internal/alerts/host.go | 22 ++- internal/alerts/host_dedup_test.go | 13 +- internal/alerts/manager.go | 8 +- internal/alerts/node.go | 138 +++++++++++------- .../threshold_resolution_shared_test.go | 61 ++++++++ 6 files changed, 190 insertions(+), 70 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index c385b63579..aca39a8060 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -1803,11 +1803,16 @@ host-agent deduplication bookkeeping, and instance-scoped node display-name cache updates; future Proxmox node alert behavior should extend that resource checker owner rather than expanding the central Manager file. When a reporting host agent is linked to a node (`LinkedNodeID`, the monitor's -automatic or operator-set identity decision), the agent resource owns that -machine's CPU, memory and disk usage alerts, and `CheckNode` releases its own -copies through the disabled-threshold path every cycle: the pending run is -dropped and any node alert still open from before the link is resolved, never -left frozen until the agent goes offline. Deduplication keys on that link, +automatic or operator-set identity decision), the agent resource owns each +CPU, memory or disk usage metric it actually evaluates, and `CheckNode` releases +its own copy of those metrics through the disabled-threshold path every cycle: +the pending run is dropped and any node alert still open from before the link is +resolved, never left frozen until the agent goes offline. `CheckHost` registers +the link only after evaluation, recording which metrics had a live threshold +(memory also needs a known reading, disk at least one evaluated filesystem), and +removes it while agent alerts are disabled; a metric the agent does not evaluate +stays with the node, so deduplication never leaves a machine unmonitored. +Deduplication keys on that link, never on a hostname match, so a same-named node in another instance keeps its alerts, an agent reporting an FQDN still dedups its node, and an operator unlink hands the alerts back. CPU temperature has no host-agent metric, so the node @@ -1826,7 +1831,8 @@ instance name in `Instance`. `TestCheckNodeReleasesOpenMetricAlertWhenHostAgentRegisters`, `TestCheckNodeMissingTemperatureDoesNotResolveOpenAlert`, `TestCheckNodeMissingTemperatureInterruptsTimingRuns` and -`TestConfigSaveKeepsNodeTemperatureAlertOverTrigger` in +`TestConfigSaveKeepsNodeTemperatureAlertOverTrigger` and +`TestCheckNodeKeepsUsageMetricsTheAgentDoesNotEvaluate` in `internal/alerts/threshold_resolution_shared_test.go`, and `TestHostAgentDeduplicationFollowsNodeLink` in `internal/alerts/host_dedup_test.go`, pin these rules. diff --git a/internal/alerts/host.go b/internal/alerts/host.go index 8e0e8c5516..718cc75f9a 100644 --- a/internal/alerts/host.go +++ b/internal/alerts/host.go @@ -220,10 +220,6 @@ func (m *Manager) CheckHost(host models.Host) { return } - // Record the Proxmox node this agent is linked to, so the node releases the - // usage alerts the agent now owns instead of alerting twice for one machine. - m.registerHostAgentNodeLink(host) - // Cache display name so host alerts show the user-configured name. m.UpdateNodeDisplayName("", host.Hostname, host.DisplayName) @@ -242,11 +238,15 @@ func (m *Manager) CheckHost(host models.Host) { } m.mu.RUnlock() + // While this agent evaluates nothing, its linked node keeps its own usage + // alerts; the link is registered below once the evaluated metrics are known. if !alertsEnabled { + m.unregisterHostAgentNodeLink(host.ID) return } if disableAllAgents { + m.unregisterHostAgentNodeLink(host.ID) // Clear any existing host alerts when all host alerts are disabled m.clearHostMetricAlerts(host.ID) m.clearHostDiskAlerts(host.ID) @@ -257,6 +257,7 @@ func (m *Manager) CheckHost(host models.Host) { } if thresholds.Disabled { + m.unregisterHostAgentNodeLink(host.ID) m.clearHostMetricAlerts(host.ID) m.clearHostDiskAlerts(host.ID) m.clearHostRAIDAlerts(host.ID) @@ -386,6 +387,7 @@ func (m *Manager) CheckHost(host models.Host) { } seenDisks := make(map[string]struct{}, len(host.Disks)) + evaluatesDiskUsage := false if len(host.Sensors.SMART) > 0 { for _, disk := range host.Sensors.SMART { diskResourceID, diskName := hostSMARTDiskResourceID(host, disk) @@ -443,6 +445,7 @@ func (m *Manager) CheckHost(host models.Host) { if effectiveDiskThreshold == nil { continue } + evaluatesDiskUsage = evaluatesDiskUsage || effectiveDiskThreshold.Trigger > 0 diskMetadata := cloneMetadata(baseMetadata) diskMetadata["metric"] = "disk" @@ -490,6 +493,17 @@ func (m *Manager) CheckHost(host models.Host) { m.cleanupHostDiskAlerts(host, seenDisks) + // The linked node releases exactly the usage metrics evaluated above and + // keeps the rest, so deduplication never leaves the machine unmonitored. + m.registerHostAgentNodeLink(hostAgentNodeLink{ + agentID: host.ID, + agentName: resourceName, + nodeID: host.LinkedNodeID, + cpu: thresholds.CPU != nil && thresholds.CPU.Trigger > 0, + memory: thresholds.Memory != nil && thresholds.Memory.Trigger > 0 && host.Memory.HasKnownUsage(), + disk: evaluatesDiskUsage, + }) + if host.Unraid != nil { m.syncHostUnraidStorageAlert(host, nodeName, instanceName, resourceName, baseMetadata) } else { diff --git a/internal/alerts/host_dedup_test.go b/internal/alerts/host_dedup_test.go index a0b4ac3751..aee90c38fa 100644 --- a/internal/alerts/host_dedup_test.go +++ b/internal/alerts/host_dedup_test.go @@ -7,7 +7,7 @@ import ( ) func TestHostAgentDeduplicatesNodeAlerts(t *testing.T) { - agent := models.Host{ID: "host-pi", Hostname: "pi", LinkedNodeID: "node/pi"} + agent := hostAgentNodeLink{agentID: "host-pi", nodeID: "node/pi", cpu: true, memory: true, disk: true} // Test 1: Without a linked host agent, node metrics ARE checked t.Run("node_metrics_checked_without_agent", func(t *testing.T) { @@ -81,7 +81,7 @@ func TestHostAgentDeduplicatesNodeAlerts(t *testing.T) { // Register and then unregister m.registerHostAgentNodeLink(agent) - m.unregisterHostAgentNodeLink(agent.ID) + m.unregisterHostAgentNodeLink(agent.agentID) // Verify host agent is NOT linked if m.hasHostAgentForNode("node/pi") { @@ -112,8 +112,9 @@ func TestHostAgentDeduplicatesNodeAlerts(t *testing.T) { // instances can both have a node named "pve", and an agent can report an FQDN. func TestHostAgentDeduplicationFollowsNodeLink(t *testing.T) { m := NewManager() + m.config.Enabled = true - m.registerHostAgentNodeLink(models.Host{ID: "host-a", Hostname: "pve.example.com", LinkedNodeID: "site-a-pve"}) + m.CheckHost(models.Host{ID: "host-a", Hostname: "pve.example.com", LinkedNodeID: "site-a-pve", CPUUsage: 10}) if !m.hasHostAgentForNode("site-a-pve") { t.Error("Expected the linked node to dedup even though the agent hostname is an FQDN") } @@ -121,13 +122,13 @@ func TestHostAgentDeduplicationFollowsNodeLink(t *testing.T) { t.Error("Expected a same-named node in another instance to keep its own alerts") } - m.registerHostAgentNodeLink(models.Host{ID: "host-b", Hostname: "pve"}) + m.CheckHost(models.Host{ID: "host-b", Hostname: "pve", CPUUsage: 10}) if m.hasHostAgentForNode("site-b-pve") { t.Error("Expected an unlinked agent with a matching hostname not to suppress node alerts") } // An operator unlinking the agent hands the node's alerts back. - m.registerHostAgentNodeLink(models.Host{ID: "host-a", Hostname: "pve.example.com"}) + m.CheckHost(models.Host{ID: "host-a", Hostname: "pve.example.com", CPUUsage: 10}) if m.hasHostAgentForNode("site-a-pve") { t.Error("Expected an unlinked agent to stop deduplicating its former node") } @@ -168,7 +169,7 @@ func TestHandleHostOfflineUnregistersNodeLink(t *testing.T) { } // Register the node link - m.registerHostAgentNodeLink(host) + m.registerHostAgentNodeLink(hostAgentNodeLink{agentID: host.ID, nodeID: host.LinkedNodeID, cpu: true}) if !m.hasHostAgentForNode("pve-offlinehost") { t.Error("Expected host agent registered") diff --git a/internal/alerts/manager.go b/internal/alerts/manager.go index a511bc3691..603dbfcb76 100644 --- a/internal/alerts/manager.go +++ b/internal/alerts/manager.go @@ -102,9 +102,9 @@ type Manager struct { // Cleanup control cleanupStop chan struct{} // Signal to stop cleanup goroutine // Host agent deduplication: the Proxmox node each reporting host agent is - // linked to. While a link holds, the agent resource owns that machine's - // CPU, memory and disk usage alerts and the node releases its own copies. - hostAgentNodeLinks map[string]string // Host agent ID -> linked node ID + // linked to and the usage metrics the agent evaluates for it. The node + // releases only those metrics and keeps evaluating the rest itself. + hostAgentNodeLinks map[string]hostAgentNodeLink // Host agent ID -> node link // Node display name caches. Proxmox nodes can share the same raw node name // across multiple configured instances, so keep instance-scoped entries in // addition to the legacy raw-name cache used by instance-less resources. @@ -242,7 +242,7 @@ func NewManagerWithDataDir(dataDir string, options ...ManagerOption) *Manager { flappingHistory: make(map[string][]time.Time), flappingActive: make(map[string]bool), cleanupStop: make(chan struct{}), - hostAgentNodeLinks: make(map[string]string), + hostAgentNodeLinks: make(map[string]hostAgentNodeLink), nodeDisplayNames: make(map[string]string), instanceNodeDisplayNames: make(map[string]string), now: time.Now, diff --git a/internal/alerts/node.go b/internal/alerts/node.go index 5f9e0500b2..2c310e3d83 100644 --- a/internal/alerts/node.go +++ b/internal/alerts/node.go @@ -93,27 +93,37 @@ func (m *Manager) CheckNode(node models.Node) { m.clearNodeOfflineAlert(node) // Check each metric (only if node is online and reachable) - // Check for host agent deduplication: if a host agent is linked to this node, - // CPU, memory and disk usage alerts belong to the agent resource (CheckHost - // evaluates them), so the node releases its own copies instead of alerting twice. - if m.hasHostAgentForNode(node.ID) { - m.releaseNodeMetricAlerts(node, "cpu", "memory", "disk") - } else { - var memoryMetric *UnifiedResourceMetric - if node.Memory.HasKnownUsage() { - memoryMetric = &UnifiedResourceMetric{Percent: node.Memory.Usage} - } - m.evaluateUnifiedMetrics(&UnifiedResourceInput{ - ID: node.ID, - Type: "node", - Name: node.Name, - Node: node.Name, - Instance: node.Instance, - CPU: &UnifiedResourceMetric{Percent: node.CPU * 100}, - Memory: memoryMetric, - Disk: &UnifiedResourceMetric{Percent: node.Disk.Usage}, - }, thresholds, nil) + // Check for host agent deduplication: a usage metric that a linked host + // agent evaluates belongs to the agent resource (CheckHost raises it), so + // the node releases its own copy instead of alerting twice. A metric the + // agent does not evaluate (agent alerts disabled, threshold off, usage + // unknown) stays with the node, so the machine is never left unmonitored. + link, _ := m.linkedHostAgentForNode(node.ID) + input := &UnifiedResourceInput{ + ID: node.ID, + Type: "node", + Name: node.Name, + Node: node.Name, + Instance: node.Instance, } + released := make([]string, 0, 3) + if link.cpu { + released = append(released, "cpu") + } else { + input.CPU = &UnifiedResourceMetric{Percent: node.CPU * 100} + } + if link.memory { + released = append(released, "memory") + } else if node.Memory.HasKnownUsage() { + input.Memory = &UnifiedResourceMetric{Percent: node.Memory.Usage} + } + if link.disk { + released = append(released, "disk") + } else { + input.Disk = &UnifiedResourceMetric{Percent: node.Disk.Usage} + } + m.releaseNodeMetricAlerts(node, released...) + m.evaluateUnifiedMetrics(input, thresholds, nil) // CPU temperature stays with the node even when a host agent runs on it: // CheckHost has no CPU temperature metric, and the node poll already @@ -189,32 +199,46 @@ func (m *Manager) releaseNodeMetricAlerts(node models.Node, metrics ...string) { } } -// registerHostAgentNodeLink records which Proxmox node a reporting host agent -// is linked to. While the link holds, the agent resource owns that machine's -// CPU, memory and disk usage alerts. The link is the monitor's identity decision -// (automatic or operator-set), so an agent whose hostname merely matches a node -// name in another instance, or one an operator unlinked, never makes a node -// release its alerts, and an agent reporting an FQDN still dedups its node. -func (m *Manager) registerHostAgentNodeLink(host models.Host) { - hostID := strings.TrimSpace(host.ID) - if hostID == "" { +// hostAgentNodeLink is a reporting host agent's link to a Proxmox node and the +// usage metrics the agent evaluates for that machine. The link is the monitor's +// identity decision (automatic or operator-set), so an agent whose hostname +// merely matches a node name in another instance, or one an operator unlinked, +// never makes a node release its alerts, and an agent reporting an FQDN still +// dedups its node. +type hostAgentNodeLink struct { + agentID string + agentName string + nodeID string + cpu bool + memory bool + disk bool +} + +// registerHostAgentNodeLink records a host agent's node link after CheckHost +// has decided which usage metrics it evaluates. A link without a node or +// without any evaluated metric is removed, so the node keeps all its alerts. +func (m *Manager) registerHostAgentNodeLink(link hostAgentNodeLink) { + link.agentID = strings.TrimSpace(link.agentID) + link.nodeID = strings.TrimSpace(link.nodeID) + if link.agentID == "" { return } - linkedNodeID := strings.TrimSpace(host.LinkedNodeID) - m.mu.Lock() - previous, existed := m.hostAgentNodeLinks[hostID] - if linkedNodeID == "" { - delete(m.hostAgentNodeLinks, hostID) - } else { - m.hostAgentNodeLinks[hostID] = linkedNodeID + if link.nodeID == "" || !(link.cpu || link.memory || link.disk) { + m.unregisterHostAgentNodeLink(link.agentID) + return } + m.mu.Lock() + previous, existed := m.hostAgentNodeLinks[link.agentID] + m.hostAgentNodeLinks[link.agentID] = link m.mu.Unlock() - if !existed || previous != linkedNodeID { + if !existed || previous != link { log.Debug(). - Str("hostID", hostID). - Str("hostname", host.Hostname). - Str("linkedNodeID", linkedNodeID). + Str("hostID", link.agentID). + Str("linkedNodeID", link.nodeID). + Bool("cpu", link.cpu). + Bool("memory", link.memory). + Bool("disk", link.disk). Msg("Updated host agent node link for deduplication") } } @@ -227,33 +251,47 @@ func (m *Manager) unregisterHostAgentNodeLink(hostID string) { return } m.mu.Lock() - linkedNodeID, existed := m.hostAgentNodeLinks[hostID] + link, existed := m.hostAgentNodeLinks[hostID] delete(m.hostAgentNodeLinks, hostID) m.mu.Unlock() if existed { log.Debug(). Str("hostID", hostID). - Str("linkedNodeID", linkedNodeID). + Str("linkedNodeID", link.nodeID). Msg("Removed host agent node link from deduplication") } } -// hasHostAgentForNode reports whether a reporting host agent is linked to the -// given Proxmox node, in which case the node's usage alerts belong to the agent. -func (m *Manager) hasHostAgentForNode(nodeID string) bool { +// linkedHostAgentForNode returns the host agent link that owns usage metrics +// for the given Proxmox node. If several agents are linked to one node, the +// lowest agent ID wins so the choice is stable. +func (m *Manager) linkedHostAgentForNode(nodeID string) (hostAgentNodeLink, bool) { nodeID = strings.TrimSpace(nodeID) if nodeID == "" { - return false + return hostAgentNodeLink{}, false } m.mu.RLock() defer m.mu.RUnlock() - for _, linkedNodeID := range m.hostAgentNodeLinks { - if linkedNodeID == nodeID { - return true + var found hostAgentNodeLink + ok := false + for _, link := range m.hostAgentNodeLinks { + if link.nodeID != nodeID { + continue + } + if !ok || link.agentID < found.agentID { + found = link + ok = true } } - return false + return found, ok +} + +// hasHostAgentForNode reports whether a reporting host agent is linked to the +// given Proxmox node and owns at least one of its usage metrics. +func (m *Manager) hasHostAgentForNode(nodeID string) bool { + _, ok := m.linkedHostAgentForNode(nodeID) + return ok } // UpdateNodeDisplayName caches the display name for a node/host so alerts diff --git a/internal/alerts/threshold_resolution_shared_test.go b/internal/alerts/threshold_resolution_shared_test.go index 540ab6aedd..3b3b3f1b44 100644 --- a/internal/alerts/threshold_resolution_shared_test.go +++ b/internal/alerts/threshold_resolution_shared_test.go @@ -359,6 +359,67 @@ func TestCheckNodeMissingTemperatureDoesNotResolveOpenAlert(t *testing.T) { } } +// Deduplication only hands the agent the usage metrics it evaluates. With agent +// alerts switched off, or one agent threshold off, the node keeps those alerts +// so the machine is never left unmonitored. +func TestCheckNodeKeepsUsageMetricsTheAgentDoesNotEvaluate(t *testing.T) { + setup := func(t *testing.T) (*Manager, models.Node, models.Host) { + m := newTestManager(t) + m.mu.Lock() + m.config.Enabled = true + m.config.TimeThresholds = map[string]int{} + m.config.NodeDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75} + m.config.NodeDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80} + m.config.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75} + m.config.AgentDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80} + m.mu.Unlock() + node, host := testNodeWithHostAgent() + node.CPU = 0.95 + node.Memory = models.Memory{Total: 100, Used: 95, Free: 5, Usage: 95} + host.CPUUsage = 95 + host.Memory = node.Memory + return m, node, host + } + nodeCPU := func(node models.Node) string { return canonicalMetricStateID(node.ID, "cpu") } + nodeMemory := func(node models.Node) string { return canonicalMetricStateID(node.ID, "memory") } + + t.Run("agent_alerts_disabled", func(t *testing.T) { + m, node, host := setup(t) + m.CheckHost(host) + m.CheckNode(node) + if testHasActiveAlert(t, m, nodeCPU(node)) { + t.Fatalf("expected the agent to own CPU while its alerts are enabled") + } + + m.mu.Lock() + m.config.DisableAllAgents = true + m.mu.Unlock() + m.CheckHost(host) + m.CheckNode(node) + if !testHasActiveAlert(t, m, nodeCPU(node)) || !testHasActiveAlert(t, m, nodeMemory(node)) { + t.Fatalf("expected the node to alert on CPU and memory while agent alerts are disabled") + } + }) + + t.Run("agent_cpu_threshold_off", func(t *testing.T) { + m, node, host := setup(t) + m.mu.Lock() + m.config.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 0, Clear: 0} + m.mu.Unlock() + m.CheckHost(host) + m.CheckNode(node) + if !testHasActiveAlert(t, m, nodeCPU(node)) { + t.Fatalf("expected the node to keep CPU when the agent CPU threshold is off") + } + if testHasActiveAlert(t, m, nodeMemory(node)) { + t.Fatalf("expected the agent to keep owning memory") + } + if got := testActiveAlertIDsOfType(m, "memory"); len(got) != 1 { + t.Fatalf("expected exactly one memory alert for the machine, got %v", got) + } + }) +} + // A missing reading keeps the incident but must not let a sustained-for or // recovery delay complete across the gap in evidence. func TestCheckNodeMissingTemperatureInterruptsTimingRuns(t *testing.T) { From 123821b02a8dc4a4203406672cf6df628eb89914 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:04:51 +0100 Subject: [PATCH 06/27] Carry a typed reason when an alert closes without recovering When a Pulse agent is linked to a Proxmox node, the node releases its CPU, memory and disk alerts to the agent, and any node alert still open closes. The close has to reach integrations so a PagerDuty or webhook incident opened from the node alert does not stay open forever, but it went out as a recovery: "pve1 on pve1 is now healthy" for a node still at 95% memory, with synthesized "recovery projection" evidence behind it. #1682 is the same complaint from another trigger: a close announced as "now healthy" when nothing recovered. A resolved alert can now carry an AlertResolution saying why it closed. The first reason is moved_to_agent, with the agent's alert resource ID and name. releaseCanonicalMetricAlert, the disabled-threshold release pulled out of evaluateCanonicalMetricAlert, takes the resolution, and clearAlertWithResolution stamps a copy on the alert before it resolves. It rides the Alert JSON into the resolved callback, the delivery queue, the history row and the event log. The synthesized closing evidence and the resolve transition say what happened instead of claiming a recovery projection. Notifications still send the close and change only the words: "Memory alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading." for webhook messages, a MOVED title without the green check on ntfy, and "Pulse alert moved" on email and Apprise. A nil resolution, which every caller passes today, leaves ordinary recoveries and the disabled-threshold path unchanged. The node release adopts the reason once the host-agent node link records which metrics the agent evaluates. --- .../v6/internal/subsystems/alerts.md | 19 ++ .../v6/internal/subsystems/notifications.md | 26 +++ internal/alerts/active_lifecycle.go | 7 + internal/alerts/callbacks.go | 8 +- internal/alerts/canonical_metric.go | 38 ++-- internal/alerts/history.go | 1 + internal/alerts/model.go | 3 + internal/alerts/operational_contract.go | 15 +- internal/alerts/operational_contract_test.go | 157 +++++++++++++++ internal/alerts/resolution.go | 79 ++++++++ internal/notifications/notifications.go | 68 +++++-- .../notifications/resolved_handover_test.go | 183 ++++++++++++++++++ 12 files changed, 573 insertions(+), 31 deletions(-) create mode 100644 internal/alerts/resolution.go create mode 100644 internal/notifications/resolved_handover_test.go diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index c270dc70d3..4a1cc835d1 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -780,6 +780,25 @@ must migrate through `internal/alerts/operational_contract.go` and name their limited provenance honestly rather than inventing confirmed provider evidence. Acknowledgement remains distinct from resolution, and every resolution transition references recovery evidence separate from its trigger evidence. +A resolved alert may carry a typed `AlertResolution` +(`internal/alerts/resolution.go`) when it closed without its condition being +observed to clear. The one reason so far, `moved_to_agent`, describes a metric +alert whose resource's linked Pulse agent now owns the metric. +`releaseCanonicalMetricAlert` is the disabled-threshold release extracted from +`evaluateCanonicalMetricAlert`: a caller releasing a metric for any other +reason passes a resolution, and `clearAlertWithResolution` stamps a copy on the +removed alert before it is resolved. The resolved callback snapshot, the +history row, the resolved event-log entry (the reason code, with `Summary()` as +its message) and the Alert JSON then carry it. Resolved consumers still receive +the close, because integrations close their incident by alert ID. The +synthesized closing evidence carries the resolution reason and summary instead +of `legacy_alert_recovery_projection`, and the resolve transition carries the +summary as its `reason`. The cause stays `recovery_evidence`, the only cause +operational trust lets enter `resolved`. A nil resolution leaves the +disabled-threshold path and every recovery unchanged. +`TestReleasedMetricAlertCarriesResolutionInsteadOfRecovery` and +`TestResolvedHistoryRowKeepsResolutionWithoutEventLog` in +`internal/alerts/operational_contract_test.go` pin this. ## Extension Points diff --git a/docs/release-control/v6/internal/subsystems/notifications.md b/docs/release-control/v6/internal/subsystems/notifications.md index 637c689ced..bb8bba2518 100644 --- a/docs/release-control/v6/internal/subsystems/notifications.md +++ b/docs/release-control/v6/internal/subsystems/notifications.md @@ -1216,3 +1216,29 @@ The live monitor does not interpret pending-only cancellation counts as recipient acceptance. Per-occurrence/per-destination receipts remain the sole recovery admission proof, including partial delivery and lost RAM markers after restart; an unannounced destination receives no recovery. + +### A close that was not a recovery says what happened + +A resolved alert can carry an `alerts.AlertResolution` when it closed without +recovering. The one reason so far, `moved_to_agent`, means the metric now +belongs to the resource's linked Pulse agent. Delivery is unchanged: the close +is still queued, gated on the firing receipt and sent, and PagerDuty still +receives `resolve` for the alert's dedup key. Only the wording changes. +`resolvedAlertMessage` gives the resolution's summary with the alert named +("Memory alert moved to pve1 (Host Agent). This is not a recovery: check the +agent for the current reading.") in place of " on is now +healthy" for every templated webhook `.Message`. Grouped lists append the +short outcome ("moved to pve1 (Host Agent)") to the moved entry. ntfy titles a +single moved alert `MOVED: ` and swaps the green check tag for +`arrow_right`. Email and Apprise title it `Pulse alert moved: ` and +add the summary after the breach message. The resolution rides the Alert JSON, +so it survives the delivery queue round trip and appears in the default JSON +payload's `alerts` array. Built-in template titles still say Resolved, which is +accurate for the closed alert, and a custom template that hard-codes recovery +text keeps it. Issue #1682 is the precedent: a recipient called "is now +healthy" a false factual claim when an alert closed without recovering. + +Verification: `resolved_handover_test.go` drives the firing, its receipt and +the queued close through the normal entry points for ntfy, and renders Discord, +PagerDuty, a grouped Slack list and email/Apprise content, plus the unchanged +ordinary recovery wording. diff --git a/internal/alerts/active_lifecycle.go b/internal/alerts/active_lifecycle.go index 3771f98205..0357ce5d66 100644 --- a/internal/alerts/active_lifecycle.go +++ b/internal/alerts/active_lifecycle.go @@ -98,6 +98,12 @@ func (m *Manager) addRecentlyResolvedWithPrimaryLock(resolved *ResolvedAlert) { // clearAlert removes an alert if it exists. func (m *Manager) clearAlert(alertID string) { + m.clearAlertWithResolution(alertID, nil) +} + +// clearAlertWithResolution removes an alert if it exists. A non-nil +// resolution marks the close as something other than a recovery. +func (m *Manager) clearAlertWithResolution(alertID string, resolution *AlertResolution) { m.mu.Lock() alert, exists := m.getActiveAlertNoLock(alertID) if exists { @@ -111,6 +117,7 @@ func (m *Manager) clearAlert(alertID string) { } publicID := effectiveAlertID(alert, alertID) + alert.Resolution = resolution.Clone() resolvedAlert := m.newResolvedAlert(alert, time.Now(), nil) m.addRecentlyResolvedUnlocked(resolvedAlert) diff --git a/internal/alerts/callbacks.go b/internal/alerts/callbacks.go index 4a6a7a9d05..ccf0a8b64c 100644 --- a/internal/alerts/callbacks.go +++ b/internal/alerts/callbacks.go @@ -430,7 +430,13 @@ func (m *Manager) safeCallResolvedAlertCallback(resolved *ResolvedAlert, fallbac } // Record the resolution before the callback guard: this funnel is the // resolve seam for every lifecycle path, with or without subscribers. - m.recordAlertEvent(eventlog.TypeResolved, alert, fallbackID, "", "Alert resolved.", nil) + eventReason, eventMessage := "", "Alert resolved." + if alert != nil { + if summary := alert.Resolution.Summary(); summary != "" { + eventReason, eventMessage = string(alert.Resolution.Reason), summary + } + } + m.recordAlertEvent(eventlog.TypeResolved, alert, fallbackID, eventReason, eventMessage, nil) callbacks := m.getResolvedCallbacks() resolvedCallback := m.callbacks.resolvedAlertCallback() diff --git a/internal/alerts/canonical_metric.go b/internal/alerts/canonical_metric.go index fcfa75baa5..4a57560cfc 100644 --- a/internal/alerts/canonical_metric.go +++ b/internal/alerts/canonical_metric.go @@ -76,6 +76,30 @@ func formatMetricValue(value float64, suffix string) string { return fmt.Sprintf("%.1f%s", value, suffix) } +// releaseCanonicalMetricAlert stops evaluating a metric spec the way a +// disabled threshold does: any pending run is dropped and any open alert is +// resolved. A non-nil resolution says why the close was not a recovery, and +// travels with the resolved alert to every consumer. +func (m *Manager) releaseCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec, resourceName, node, instance, resourceType string, value float64, resolution *AlertResolution) { + if spec.MetricThreshold == nil { + return + } + storageKey := canonicalTrackingKeyForSpec(spec, spec.ID) + m.mu.Lock() + m.core.ApplyMetric(reducer.MetricSignal{ + ResourceID: spec.ResourceID, + Key: spec.ID, + Metric: spec.MetricThreshold.Metric, + Value: value, + ObservedAt: time.Now(), + }, reducer.MetricRule{}) + m.mu.Unlock() + // A guest that moved nodes may hold this alert under its old node-scoped + // identity; re-home it first so the clear below can resolve it. + m.rehomeStrandedGuestAlert(storageKey, spec.ID, string(spec.Kind), spec.ResourceID, resourceName, node, instance, resourceType) + m.clearAlertWithResolution(storageKey, resolution) +} + func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec, resourceName, node, instance, resourceType string, value float64, threshold *HysteresisThreshold, opts *metricOptions) { if spec.MetricThreshold == nil { return @@ -86,19 +110,7 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec trackingKey := storageKey metricType := spec.MetricThreshold.Metric if spec.Disabled || spec.MetricThreshold.Trigger <= 0 { - m.mu.Lock() - m.core.ApplyMetric(reducer.MetricSignal{ - ResourceID: spec.ResourceID, - Key: spec.ID, - Metric: metricType, - Value: value, - ObservedAt: time.Now(), - }, reducer.MetricRule{}) - m.mu.Unlock() - // A guest that moved nodes may hold this alert under its old node-scoped - // identity; re-home it first so the clear below can resolve it. - m.rehomeStrandedGuestAlert(storageKey, spec.ID, string(spec.Kind), spec.ResourceID, resourceName, node, instance, resourceType) - m.clearAlert(storageKey) + m.releaseCanonicalMetricAlert(spec, resourceName, node, instance, resourceType, value, nil) return } diff --git a/internal/alerts/history.go b/internal/alerts/history.go index 10d38cd244..efefa385cb 100644 --- a/internal/alerts/history.go +++ b/internal/alerts/history.go @@ -394,6 +394,7 @@ func (hm *HistoryManager) UpdateAlertOperationalContractForAlert(alert *Alert) { hm.history[i].Alert.LatestTransition = snapshot.LatestTransition hm.history[i].Alert.Transitions = snapshot.Transitions hm.history[i].Alert.Evidence = snapshot.Evidence + hm.history[i].Alert.Resolution = snapshot.Resolution // The live alert is authoritative for acknowledgement; a row // that keeps its fire-time ack state shows an acknowledged // alert as unacknowledged for its whole active life. diff --git a/internal/alerts/model.go b/internal/alerts/model.go index 4b6324a2fd..cce963c444 100644 --- a/internal/alerts/model.go +++ b/internal/alerts/model.go @@ -136,6 +136,7 @@ type Alert struct { AckTime *time.Time `json:"ackTime,omitempty"` AckUser string `json:"ackUser,omitempty"` Correlation *AlertCorrelation `json:"correlation,omitempty"` + Resolution *AlertResolution `json:"resolution,omitempty"` // set only when a close was not a recovery Metadata map[string]interface{} `json:"metadata,omitempty"` LastNotified *time.Time `json:"lastNotified,omitempty"` LastEscalation int `json:"lastEscalation,omitempty"` @@ -168,6 +169,8 @@ func (a *Alert) Clone() *Alert { clone.Correlation = cloneAlertCorrelation(a.Correlation) } + clone.Resolution = a.Resolution.Clone() + if len(a.EscalationTimes) > 0 { clone.EscalationTimes = append([]time.Time(nil), a.EscalationTimes...) } diff --git a/internal/alerts/operational_contract.go b/internal/alerts/operational_contract.go index 0923383a8d..f59fe89a1b 100644 --- a/internal/alerts/operational_contract.go +++ b/internal/alerts/operational_contract.go @@ -311,6 +311,7 @@ func markOperationalResolved( Cause: operationaltrust.TransitionRecoveryEvidence, CauseKey: record.CauseKey, EvidenceIDs: recoveryEvidenceIDs, + Reason: alert.Resolution.Summary(), } alert.Transitions = appendOperationalTransition( alert.Transitions, @@ -497,6 +498,16 @@ func legacyAlertRecoveryEvidenceEnvelope( if err != nil { return operationaltrust.EvidenceEnvelope{}, false } + reason := &operationaltrust.EvidenceReason{ + Code: legacyAlertRecoveryEvidenceReason, + } + // A close that was not a recovery must not be recorded as one. + if summary := alert.Resolution.Summary(); summary != "" { + reason = &operationaltrust.EvidenceReason{ + Code: string(alert.Resolution.Reason), + Message: summary, + } + } envelope := operationaltrust.EvidenceEnvelope{ ID: id, Source: source, @@ -506,9 +517,7 @@ func legacyAlertRecoveryEvidenceEnvelope( Completeness: operationaltrust.EvidencePartial, Confidence: operationaltrust.EvidenceUnknown, Permissions: operationaltrust.EvidencePermissionsUnknown, - Reason: &operationaltrust.EvidenceReason{ - Code: legacyAlertRecoveryEvidenceReason, - }, + Reason: reason, } return envelope, envelope.Validate() == nil } diff --git a/internal/alerts/operational_contract_test.go b/internal/alerts/operational_contract_test.go index 444cb7aba3..a4d0dc43bc 100644 --- a/internal/alerts/operational_contract_test.go +++ b/internal/alerts/operational_contract_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog" alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" @@ -744,3 +745,159 @@ func TestResolvedOccurrenceCallbackSurvivesLegacyPanic(t *testing.T) { t.Fatal("legacy panic prevented occurrence-qualified recovery") } } + +func TestReleasedMetricAlertCarriesResolutionInsteadOfRecovery(t *testing.T) { + manager := newUnifiedEvalParityManager(t) + store, err := eventlog.OpenInMemory() + if err != nil { + t.Fatalf("open in-memory event log: %v", err) + } + manager.SetEventLog(store) + manager.UpdateConfig(AlertConfig{ + Enabled: true, + NodeDefaults: ThresholdConfig{ + CPU: &HysteresisThreshold{Trigger: 80, Clear: 75}, + Memory: &HysteresisThreshold{Trigger: 85, Clear: 80}, + }, + }) + disableTestTimeThresholds(manager) + node := models.Node{ + ID: "pve-1", + Name: "pve-1", + Instance: "test", + Status: "online", + CPU: 0.95, + Memory: models.Memory{Total: 8 << 30, Used: 7 << 30, Usage: 95}, + } + manager.CheckNode(node) + if active := manager.GetActiveAlerts(); len(active) != 2 { + t.Fatalf("active alerts = %#v, want cpu and memory", active) + } + + resolvedByType := map[string]*ResolvedAlert{} + manager.SetResolvedAlertCallback(func(resolved *ResolvedAlert) { + resolvedByType[resolved.Alert.Type] = resolved + }) + release := func(metric string, resolution *AlertResolution) { + t.Helper() + spec, err := buildCanonicalMetricSpec(node.ID, node.Name, unifiedresources.ResourceType("node"), metric, nil) + if err != nil { + t.Fatalf("build %s spec: %v", metric, err) + } + manager.releaseCanonicalMetricAlert(spec, node.Name, node.Name, node.Instance, "node", 0, resolution) + } + + handover := &AlertResolution{ + Reason: AlertResolutionMovedToAgent, + SuccessorResourceID: "agent-pve-1", + SuccessorName: "pve-1 (Host Agent)", + } + release("memory", handover) + // The close is recorded as a copy: the caller's value cannot rewrite it. + handover.SuccessorName = "changed" + release("cpu", nil) + + if active := manager.GetActiveAlerts(); len(active) != 0 { + t.Fatalf("active alerts after release = %#v, want none: the close must still happen", active) + } + + moved := resolvedByType["memory"] + if moved == nil { + t.Fatal("memory release did not reach the resolved consumer") + } + resolution := moved.Alert.Resolution + if resolution == nil || + resolution.Reason != AlertResolutionMovedToAgent || + resolution.SuccessorResourceID != "agent-pve-1" || + resolution.SuccessorName != "pve-1 (Host Agent)" { + t.Fatalf("resolution = %+v, want moved to pve-1 (Host Agent)", resolution) + } + wantSummary := "Alert moved to pve-1 (Host Agent). This is not a recovery: check the agent for the current reading." + if got := resolution.Summary(); got != wantSummary { + t.Fatalf("Summary() = %q, want %q", got, wantSummary) + } + if got := resolution.Outcome(); got != "moved to pve-1 (Host Agent)" { + t.Fatalf("Outcome() = %q", got) + } + if got := resolution.Describe("Memory alert"); got != "Memory alert moved to pve-1 (Host Agent). This is not a recovery: check the agent for the current reading." { + t.Fatalf("Describe() = %q", got) + } + + record := moved.Alert.OperationalRecord + if record == nil || record.State != operationaltrust.OperationalResolved { + t.Fatalf("operational record = %+v, want resolved", record) + } + if err := record.Validate(); err != nil { + t.Fatalf("operational record Validate() error = %v", err) + } + transition := moved.Alert.LatestTransition + if transition == nil || transition.Reason != wantSummary { + t.Fatalf("resolve transition = %+v, want reason %q", transition, wantSummary) + } + if err := transition.Validate(); err != nil { + t.Fatalf("resolve transition Validate() error = %v", err) + } + closing := moved.Alert.Evidence[len(moved.Alert.Evidence)-1] + if closing.ID != transition.EvidenceIDs[0] || + closing.Reason == nil || + closing.Reason.Code != string(AlertResolutionMovedToAgent) || + closing.Reason.Message != wantSummary { + t.Fatalf("closing evidence = %+v, want the handover, not a recovery projection", closing) + } + + recovered := resolvedByType["cpu"] + if recovered == nil || recovered.Alert.Resolution != nil { + t.Fatalf("plain disabled release = %+v, want an ordinary resolve with no resolution", recovered) + } + if last := recovered.Alert.Evidence[len(recovered.Alert.Evidence)-1]; last.Reason == nil || + last.Reason.Code != legacyAlertRecoveryEvidenceReason { + t.Fatalf("plain release evidence = %+v, want legacy recovery projection", last.Reason) + } + + var historyResolution *AlertResolution + for _, entry := range manager.GetAlertHistory(10) { + if entry.Type == "memory" { + historyResolution = entry.Resolution + } + } + if historyResolution == nil || historyResolution.Reason != AlertResolutionMovedToAgent { + t.Fatalf("history resolution = %+v, want moved_to_agent", historyResolution) + } + + events, err := manager.AlertEvents(eventlog.Filter{AlertID: moved.Alert.ID, Types: []string{eventlog.TypeResolved}}) + if err != nil { + t.Fatalf("AlertEvents: %v", err) + } + if len(events) != 1 || + events[0].Reason != string(AlertResolutionMovedToAgent) || + events[0].Message != wantSummary { + t.Fatalf("resolved events = %+v, want one handover event", events) + } +} + +func TestResolvedHistoryRowKeepsResolutionWithoutEventLog(t *testing.T) { + firstObservedAt := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) + alert := &Alert{ + ID: "pve-1-memory", + Type: "memory", + Level: AlertLevelWarning, + ResourceID: "pve-1", + CanonicalSpecID: "metric-threshold:memory", + CanonicalState: "pve-1::metric-threshold:memory", + StartTime: firstObservedAt, + LastSeen: firstObservedAt, + } + ensureOperationalContract(alert, firstObservedAt) + history := newTestHistoryManager(t) + history.AddAlert(*alert) + manager := &Manager{historyManager: history} + + alert.Resolution = &AlertResolution{Reason: AlertResolutionMovedToAgent, SuccessorName: "pve-1 (Host Agent)"} + manager.newResolvedAlert(alert, firstObservedAt.Add(time.Minute), nil) + + entries := history.GetAllHistory(1) + if len(entries) != 1 || entries[0].Resolution == nil || + entries[0].Resolution.Reason != AlertResolutionMovedToAgent { + t.Fatalf("history entries = %+v, want the row to keep the moved_to_agent resolution", entries) + } +} diff --git a/internal/alerts/resolution.go b/internal/alerts/resolution.go new file mode 100644 index 0000000000..6ba3fbba54 --- /dev/null +++ b/internal/alerts/resolution.go @@ -0,0 +1,79 @@ +package alerts + +import ( + "fmt" + "strings" +) + +// AlertResolutionReason says why an alert closed when the close was not a +// recovery. The close still reaches every resolved consumer, so an incident +// opened from the alert does not stay open on the receiving side, but no +// surface may report the resource as healthy. +type AlertResolutionReason string + +const ( + // AlertResolutionMovedToAgent means a Pulse agent linked to the alert's + // resource now owns the same metric, so the agent's own alert settings + // decide whether the condition is still alerted on. + AlertResolutionMovedToAgent AlertResolutionReason = "moved_to_agent" +) + +// AlertResolution records why an alert closed without its condition being +// observed to clear. A nil resolution is an ordinary recovery. +type AlertResolution struct { + Reason AlertResolutionReason `json:"reason"` + // SuccessorResourceID and SuccessorName identify the resource that owns + // the alert from now on, named the way that resource's own alerts are. + SuccessorResourceID string `json:"successorResourceId,omitempty"` + SuccessorName string `json:"successorName,omitempty"` +} + +// Clone returns a copy of the resolution. +func (r *AlertResolution) Clone() *AlertResolution { + if r == nil { + return nil + } + clone := *r + return &clone +} + +// Outcome is the short form of a close that was not a recovery, for lists +// such as "moved to pve1 (Host Agent)". It is empty for an ordinary recovery. +func (r *AlertResolution) Outcome() string { + if r == nil { + return "" + } + switch r.Reason { + case AlertResolutionMovedToAgent: + successor := strings.TrimSpace(r.SuccessorName) + if successor == "" { + successor = "its Pulse agent" + } + return "moved to " + successor + default: + return "" + } +} + +// Summary is the one-line account of a close that was not a recovery. It is +// empty for an ordinary recovery. +func (r *AlertResolution) Summary() string { + return r.Describe("Alert") +} + +// Describe is Summary with the alert named by the caller, such as +// "Memory alert", so a message about one of several closes says which one. +func (r *AlertResolution) Describe(subject string) string { + if r == nil { + return "" + } + if subject = strings.TrimSpace(subject); subject == "" { + subject = "Alert" + } + switch r.Reason { + case AlertResolutionMovedToAgent: + return fmt.Sprintf("%s %s. This is not a recovery: check the agent for the current reading.", subject, r.Outcome()) + default: + return "" + } +} diff --git a/internal/notifications/notifications.go b/internal/notifications/notifications.go index adda188023..93ae16e0f3 100644 --- a/internal/notifications/notifications.go +++ b/internal/notifications/notifications.go @@ -432,6 +432,36 @@ func annotateResolvedMetadata(alert *alerts.Alert, resolvedAt time.Time) { alert.Metadata[metadataResolvedAt] = resolvedAt.Format(time.RFC3339) } +// resolvedAlertMessage is the one-line account of a resolved alert. A close +// that was not a recovery, such as a node alert that moved to the Pulse agent +// now monitoring the machine, says what happened instead of "now healthy". +func resolvedAlertMessage(alert *alerts.Alert) string { + if summary := resolvedAlertNotRecoveredSummary(alert); summary != "" { + return summary + } + return fmt.Sprintf("%s on %s is now healthy", alert.ResourceName, alert.Node) +} + +// resolvedAlertNotRecoveredSummary names the alert in its resolution summary, +// so one of several closes for a machine says which alert moved. It is empty +// for an ordinary recovery. +func resolvedAlertNotRecoveredSummary(alert *alerts.Alert) string { + label := alertTypeDisplay(alert.Type) + if label != "" { + label += " alert" + } + return alert.Resolution.Describe(label) +} + +// resolvedAlertListEntry names one alert in a grouped recovery list. +func resolvedAlertListEntry(alert *alerts.Alert) string { + entry := fmt.Sprintf("%s on %s", alert.ResourceName, alert.Node) + if outcome := alert.Resolution.Outcome(); outcome != "" { + entry += ", " + outcome + } + return entry +} + func quietHoursReplayAtForAlert(alert *alerts.Alert, now time.Time) *time.Time { if alert == nil || alert.Metadata == nil { return nil @@ -2267,6 +2297,9 @@ func buildResolvedNotificationContent(alertList []*alerts.Alert, resolvedAt time resolvedLabel := resolvedAt.Format(time.RFC3339) title := fmt.Sprintf("Pulse alert resolved: %s", primary.ResourceName) + if primary.Resolution.Outcome() != "" { + title = fmt.Sprintf("Pulse alert moved: %s", primary.ResourceName) + } if len(validAlerts) > 1 { title = fmt.Sprintf("Pulse alerts resolved (%d)", len(validAlerts)) } @@ -2282,6 +2315,10 @@ func buildResolvedNotificationContent(alertList []*alerts.Alert, resolvedAt time bodyBuilder.WriteString(alert.Message) bodyBuilder.WriteString("\n") } + if summary := resolvedAlertNotRecoveredSummary(alert); summary != "" { + bodyBuilder.WriteString(summary) + bodyBuilder.WriteString("\n") + } if !alert.StartTime.IsZero() { bodyBuilder.WriteString("Started: ") bodyBuilder.WriteString(alert.StartTime.Format(time.RFC3339)) @@ -2980,14 +3017,14 @@ func (n *NotificationManager) sendResolvedWebhook(webhook WebhookConfig, alertLi data.ResolvedAt = resolvedAt.Format(time.RFC3339) data.ResolvedAtISO = resolvedAt.Format(time.RFC3339) data.Duration = formatWebhookDuration(resolvedAt.Sub(alert.StartTime)) - data.Message = fmt.Sprintf("%s on %s is now healthy", alert.ResourceName, alert.Node) + data.Message = resolvedAlertMessage(alert) data.AlertCount = len(alertList) data.Alerts = alertList if len(alertList) > 1 { names := make([]string, 0, len(alertList)) for _, a := range alertList { if a != nil { - names = append(names, fmt.Sprintf("%s on %s", a.ResourceName, a.Node)) + names = append(names, resolvedAlertListEntry(a)) } } data.Message = fmt.Sprintf("%d alerts resolved: %s", len(alertList), strings.Join(names, "; ")) @@ -3036,28 +3073,31 @@ func (n *NotificationManager) sendResolvedWebhookNtfy(webhook WebhookConfig, ale return fmt.Errorf("rate limit exceeded for webhook %s", webhook.Name) } - // Build plain-text body + // Build plain-text body and title. A single alert that moved rather than + // recovered says so in both, and drops the green check. var body strings.Builder + title := "RESOLVED" + tags := "white_check_mark,pulse,resolved" if len(alertList) == 1 && alertList[0] != nil { a := alertList[0] - fmt.Fprintf(&body, "Resolved: %s on %s is now healthy", a.ResourceName, a.Node) + title = fmt.Sprintf("RESOLVED: %s", a.ResourceName) + if summary := resolvedAlertNotRecoveredSummary(a); summary != "" { + title = fmt.Sprintf("MOVED: %s", a.ResourceName) + tags = "arrow_right,pulse,resolved" + body.WriteString(summary) + } else { + fmt.Fprintf(&body, "Resolved: %s", resolvedAlertMessage(a)) + } } else { + title = fmt.Sprintf("RESOLVED: %d alerts", len(alertList)) fmt.Fprintf(&body, "%d alerts resolved at %s:\n", len(alertList), resolvedAt.Format(time.RFC822)) for _, a := range alertList { if a != nil { - fmt.Fprintf(&body, "- %s on %s\n", a.ResourceName, a.Node) + fmt.Fprintf(&body, "- %s\n", resolvedAlertListEntry(a)) } } } - // Build title - title := "RESOLVED" - if len(alertList) == 1 && alertList[0] != nil { - title = fmt.Sprintf("RESOLVED: %s", alertList[0].ResourceName) - } else { - title = fmt.Sprintf("RESOLVED: %d alerts", len(alertList)) - } - method := webhook.Method if method == "" { method = "POST" @@ -3071,7 +3111,7 @@ func (n *NotificationManager) sendResolvedWebhookNtfy(webhook WebhookConfig, ale req.Header.Set("Content-Type", "text/plain") req.Header.Set("Title", title) req.Header.Set("Priority", "default") - req.Header.Set("Tags", "white_check_mark,pulse,resolved") + req.Header.Set("Tags", tags) req.Header.Set("User-Agent", "Pulse-Monitoring/2.0") // Apply any custom headers from webhook config diff --git a/internal/notifications/resolved_handover_test.go b/internal/notifications/resolved_handover_test.go new file mode 100644 index 0000000000..b980c6fb56 --- /dev/null +++ b/internal/notifications/resolved_handover_test.go @@ -0,0 +1,183 @@ +package notifications + +import ( + "encoding/json" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" +) + +const movedSummary = "Memory alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading." + +// movedNodeMemoryAlert is a node memory alert that closed because the Pulse +// agent linked to the node took over its usage alerts, not because memory +// came back down. +func movedNodeMemoryAlert() *alerts.Alert { + return &alerts.Alert{ + ID: "pve1-memory", + Type: "memory", + Level: alerts.AlertLevelWarning, + ResourceName: "pve1", + ResourceID: "pve1", + Node: "pve1", + Instance: "https://pve.local:8006", + Message: "Node memory at 95.0%", + Value: 95, + Threshold: 90, + StartTime: time.Now().Add(-10 * time.Minute), + Resolution: &alerts.AlertResolution{ + Reason: alerts.AlertResolutionMovedToAgent, + SuccessorResourceID: "agent-pve1", + SuccessorName: "pve1 (Host Agent)", + }, + } +} + +// Exercise the normal entry points: the firing goes out, its receipt gates the +// close, and the close is persisted in the delivery queue and read back before +// it is rendered. +func TestMovedAlertCloseIsDeliveredAsMovedNotRecovered(t *testing.T) { + type message struct{ title, body, tags string } + received := make(chan message, 4) + server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("read notification: %v", err) + } + received <- message{r.Header.Get("Title"), string(body), r.Header.Get("Tags")} + w.WriteHeader(http.StatusAccepted) + })) + defer server.Close() + + webhook := WebhookConfig{ID: "ops", Name: "ops", URL: server.URL + "/topic", Enabled: true, Service: "ntfy"} + m := NewNotificationManagerWithDataDir("", t.TempDir()) + t.Cleanup(m.Stop) + m.webhookClient = server.Client() + if err := m.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil { + t.Fatal(err) + } + m.SetGroupingConfig(false, 0, false, false) + m.SetNotifyOnResolve(true) + m.AddWebhook(webhook) + + next := func() message { + t.Helper() + select { + case msg := <-received: + return msg + case <-time.After(10 * time.Second): + t.Fatal("notification not received") + return message{} + } + } + + alert := movedNodeMemoryAlert() + firing := alert.Clone() + firing.Resolution = nil + m.SendAlert(firing) + next() + recoveryJob := notificationDeliveryJob{Type: "webhook", Event: eventResolved, Alerts: []*alerts.Alert{alert}, WebhookConfig: &webhook} + deadline := time.Now().Add(10 * time.Second) + for len(m.filterResolvedJobsByDeliveryReceipt([]notificationDeliveryJob{recoveryJob})) != 1 { + if time.Now().After(deadline) { + t.Fatal("firing receipt never recorded") + } + time.Sleep(time.Millisecond) + } + + m.SendResolvedAlert(&alerts.ResolvedAlert{Alert: alert, ResolvedTime: time.Now()}) + closing := next() + if closing.title != "MOVED: pve1" || closing.body != movedSummary { + t.Fatalf("close = %+v, want MOVED title and the handover summary", closing) + } + if strings.Contains(closing.body, "healthy") || strings.Contains(closing.tags, "white_check_mark") { + t.Fatalf("close reads as a recovery: %+v", closing) + } +} + +func TestMovedAlertCloseRendersHonestlyOnEveryChannel(t *testing.T) { + capture := func(t *testing.T, service string) map[string]any { + t.Helper() + var gotBody []byte + server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotBody, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + nm := NewNotificationManager("https://pulse.local") + _ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1") + webhook := WebhookConfig{Name: service, URL: server.URL + "/hook", Enabled: true, Service: service, + CustomFields: map[string]string{"routing_key": "rk"}} + if err := nm.sendResolvedWebhook(webhook, []*alerts.Alert{movedNodeMemoryAlert()}, time.Now()); err != nil { + t.Fatalf("sendResolvedWebhook %s: %v", service, err) + } + var payload map[string]any + if err := json.Unmarshal(gotBody, &payload); err != nil { + t.Fatalf("unmarshal %s payload: %v", service, err) + } + return payload + } + + t.Run("discord says moved", func(t *testing.T) { + embed := capture(t, "discord")["embeds"].([]any)[0].(map[string]any) + description, _ := embed["description"].(string) + if !strings.Contains(description, movedSummary) || strings.Contains(description, "healthy") { + t.Fatalf("description = %q, want the handover summary", description) + } + }) + + t.Run("pagerduty still closes the incident", func(t *testing.T) { + payload := capture(t, "pagerduty") + if payload["event_action"] != "resolve" || payload["dedup_key"] != "pve1-memory" { + t.Fatalf("payload = %v, want a resolve for the node alert's incident", payload) + } + }) + + t.Run("grouped list names the move", func(t *testing.T) { + var gotBody []byte + server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotBody, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + nm := NewNotificationManager("https://pulse.local") + _ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1") + recovered := &alerts.Alert{ID: "vm-cpu", Type: "cpu", ResourceName: "web-vm-01", Node: "pve1", StartTime: time.Now().Add(-time.Minute)} + webhook := WebhookConfig{Name: "slack", URL: server.URL + "/hook", Enabled: true, Service: "slack"} + if err := nm.sendResolvedWebhook(webhook, []*alerts.Alert{movedNodeMemoryAlert(), recovered}, time.Now()); err != nil { + t.Fatalf("sendResolvedWebhook: %v", err) + } + body := string(gotBody) + if !strings.Contains(body, "pve1 on pve1, moved to pve1 (Host Agent)") || !strings.Contains(body, "web-vm-01 on pve1") { + t.Fatalf("grouped body = %s, want the moved alert marked and the recovery listed", body) + } + }) + + t.Run("email and apprise say moved", func(t *testing.T) { + title, htmlBody, textBody := buildResolvedNotificationContent([]*alerts.Alert{movedNodeMemoryAlert()}, time.Now(), "") + if title != "Pulse alert moved: pve1" { + t.Fatalf("title = %q", title) + } + if !strings.Contains(textBody, "Node memory at 95.0%\n"+movedSummary+"\n") { + t.Fatalf("text body = %q, want the breach followed by the handover summary", textBody) + } + if !strings.Contains(htmlBody, "This is not a recovery: check the agent for the current reading.") { + t.Fatalf("html body = %q", htmlBody) + } + }) + + t.Run("ordinary recovery is unchanged", func(t *testing.T) { + recovered := movedNodeMemoryAlert() + recovered.Resolution = nil + if got := resolvedAlertMessage(recovered); got != "pve1 on pve1 is now healthy" { + t.Fatalf("resolvedAlertMessage() = %q", got) + } + if title, _, _ := buildResolvedNotificationContent([]*alerts.Alert{recovered}, time.Now(), ""); title != "Pulse alert resolved: pve1" { + t.Fatalf("title = %q", title) + } + }) +} From 7f83d73cfd39b53f0d02360e947c993759c71d38 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:13:08 +0100 Subject: [PATCH 07/27] Tighten the drawer span tests The unit test only showed that a width write happened, so a doubled nudge or a missing forced layout read would still pass. It now spies on offsetWidth and requires exactly one forced read of the detail cell, at width 0px with the new colspan, when the requested span and the summary cells grow in the same update, and none when they shrink. The browser spec read every `thead th` under the table, which would also match headers of tables inside the drawer. It now reads only the outer table's own header row, and uses the frontend Prettier settings (single quotes, 100 columns) instead of Prettier's defaults. --- .../__tests__/InlineDetailTableRow.test.tsx | 55 +++---- .../98-inline-drawer-column-span.spec.ts | 136 ++++++------------ 2 files changed, 76 insertions(+), 115 deletions(-) diff --git a/frontend-modern/src/components/shared/__tests__/InlineDetailTableRow.test.tsx b/frontend-modern/src/components/shared/__tests__/InlineDetailTableRow.test.tsx index f8aa97bd76..989cb28536 100644 --- a/frontend-modern/src/components/shared/__tests__/InlineDetailTableRow.test.tsx +++ b/frontend-modern/src/components/shared/__tests__/InlineDetailTableRow.test.tsx @@ -115,7 +115,7 @@ describe('InlineDetailTableRow', () => { expect(detailCell()?.style.width).toBe(''); }); - it('nudges the detail cell width once when its span grows, not when it shrinks', async () => { + it('forces one layout of the widened detail cell when its span grows, none when it shrinks', async () => { const [showSystem, setShowSystem] = createSignal(false); render(() => ( @@ -127,39 +127,42 @@ describe('InlineDetailTableRow', () => { - +
Growing span detail
Seen
)); - const detailCell = () => screen.getByText('Growing span detail').closest('td')!; - await waitFor(() => expect(detailCell()).toHaveAttribute('colspan', '2')); + const detailCell = screen.getByText('Growing span detail').closest('td')!; + await waitFor(() => expect(detailCell).toHaveAttribute('colspan', '2')); - // jsdom has no layout, so record the style writes Chromium needs to + // jsdom has no layout, so record the forced layout reads Chromium needs to // recompute fixed-layout column widths for the widened cell. - const previousStyles: string[] = []; - const styleObserver = new MutationObserver((records) => { - for (const record of records) previousStyles.push(record.oldValue ?? ''); - }); - styleObserver.observe(detailCell(), { - attributes: true, - attributeFilter: ['style'], - attributeOldValue: true, - }); + const layoutReads: Array<{ width: string; colspan: string | null }> = []; + const offsetWidth = vi + .spyOn(HTMLElement.prototype, 'offsetWidth', 'get') + .mockImplementation(function (this: HTMLElement) { + if (this === detailCell) { + layoutReads.push({ width: this.style.width, colspan: this.getAttribute('colspan') }); + } + return 0; + }); + try { + // The requested span and the summary cells change in the same update. + setShowSystem(true); + await waitFor(() => expect(detailCell).toHaveAttribute('colspan', '3')); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(layoutReads).toEqual([{ width: '0px', colspan: '3' }]); + expect(detailCell.style.width).toBe(''); - setShowSystem(true); - await waitFor(() => expect(detailCell()).toHaveAttribute('colspan', '3')); - await new Promise((resolve) => setTimeout(resolve, 0)); - expect(previousStyles.some((style) => style.includes('width: 0px'))).toBe(true); - expect(detailCell().style.width).toBe(''); - - previousStyles.length = 0; - setShowSystem(false); - await waitFor(() => expect(detailCell()).toHaveAttribute('colspan', '2')); - await new Promise((resolve) => setTimeout(resolve, 0)); - expect(previousStyles).toEqual([]); - styleObserver.disconnect(); + layoutReads.length = 0; + setShowSystem(false); + await waitFor(() => expect(detailCell).toHaveAttribute('colspan', '2')); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(layoutReads).toEqual([]); + } finally { + offsetWidth.mockRestore(); + } }); it('re-measures for a summary cell span change but not for churn inside a cell', async () => { diff --git a/tests/integration/tests/98-inline-drawer-column-span.spec.ts b/tests/integration/tests/98-inline-drawer-column-span.spec.ts index 0aea3a4437..b639a2ac10 100644 --- a/tests/integration/tests/98-inline-drawer-column-span.spec.ts +++ b/tests/integration/tests/98-inline-drawer-column-span.spec.ts @@ -1,22 +1,17 @@ -import { - expect, - test as base, - type Locator, - type Page, -} from "@playwright/test"; +import { expect, test as base, type Locator, type Page } from '@playwright/test'; import { ensureAuthenticated, getMockMode, setMockMode, waitForDefaultMockRuntimeReady, -} from "./helpers"; +} from './helpers'; let mockModeWasEnabled: boolean | null = null; const HTTP_CREDENTIALS = { - username: "admin", - password: "adminadminadmin", + username: 'admin', + password: 'adminadminadmin', }; const test = base; @@ -38,13 +33,11 @@ async function ensureMockModeEnabled(page: Page): Promise { async function readLayout(drawer: Locator) { return drawer.evaluate((row: HTMLTableRowElement) => { - const table = row.closest("table")!; - const headers = Array.from( - table.querySelectorAll("thead th"), - ).filter( - (cell) => - getComputedStyle(cell).display !== "none" && - cell.getClientRects().length > 0, + const table = row.closest('table')!; + // Only the outer table's own header row: drawer content may hold tables. + const headerRows = table.tHead!.rows; + const headers = Array.from(headerRows[headerRows.length - 1].cells).filter( + (cell) => getComputedStyle(cell).display !== 'none' && cell.getClientRects().length > 0, ); return { headerSpan: headers.reduce((total, cell) => total + cell.colSpan, 0), @@ -56,16 +49,11 @@ async function readLayout(drawer: Locator) { }); } -async function expectAligned( - drawer: Locator, - headerSpan: number, -): Promise { +async function expectAligned(drawer: Locator, headerSpan: number): Promise { await expect(async () => { const layout = await readLayout(drawer); - expect(layout.headerSpan, "Visible header span").toBe(headerSpan); - expect(layout.drawerSpan, "Drawer colspan matches visible headers").toBe( - layout.headerSpan, - ); + expect(layout.headerSpan, 'Visible header span').toBe(headerSpan); + expect(layout.drawerSpan, 'Drawer colspan matches visible headers').toBe(layout.headerSpan); expect( Math.abs(layout.drawerRight - layout.tableRight), `Drawer reaches table right edge: ${JSON.stringify(layout)}`, @@ -76,12 +64,12 @@ async function expectAligned( ).toBeLessThanOrEqual(1); expect( Math.abs(layout.drawerRight - layout.headerRight), - "Drawer and header right edges", + 'Drawer and header right edges', ).toBeLessThanOrEqual(1); }).toPass({ timeout: 10_000 }); } -test.describe.serial("Inline machine drawer column span", () => { +test.describe.serial('Inline machine drawer column span', () => { test.setTimeout(240_000); test.afterAll(async ({ browser }) => { @@ -101,49 +89,38 @@ test.describe.serial("Inline machine drawer column span", () => { } }); - test("resynchronizes a mounted drawer after hiding and restoring System", async ({ + test('resynchronizes a mounted drawer after hiding and restoring System', async ({ page, }, testInfo) => { - test.skip( - testInfo.project.name !== "chromium", - "Chromium fixed-table layout regression", - ); + test.skip(testInfo.project.name !== 'chromium', 'Chromium fixed-table layout regression'); await page.setViewportSize({ width: 1440, height: 900 }); await ensureMockModeEnabled(page); - await page.goto("/standalone", { waitUntil: "domcontentloaded" }); + await page.goto('/standalone', { waitUntil: 'domcontentloaded' }); - const surface = page.getByTestId("standalone-page"); + const surface = page.getByTestId('standalone-page'); const summary = surface - .getByRole("row") + .getByRole('row') .filter({ has: page.locator('button[aria-controls^="agents-machine-drawer"]'), }) .first(); - const disclosure = summary.locator( - 'button[aria-controls^="agents-machine-drawer"]', - ); + const disclosure = summary.locator('button[aria-controls^="agents-machine-drawer"]'); await expect(disclosure).toBeVisible({ timeout: 60_000 }); - await summary.getByRole("cell").nth(1).click(); + await summary.getByRole('cell').nth(1).click(); - const drawer = surface.locator( - "tr[data-inline-platform-resource-detail-for]", - ); + const drawer = surface.locator('tr[data-inline-platform-resource-detail-for]'); await expect(drawer).toHaveCount(1); await expect(drawer).toBeVisible(); const originalRow = (await drawer.elementHandle())!; - await originalRow.evaluate((row) => - row.setAttribute("data-column-span-test", "original"), - ); + await originalRow.evaluate((row) => row.setAttribute('data-column-span-test', 'original')); const focusTarget = (await drawer - .getByRole("button", { name: /^Collapse .* details$/ }) + .getByRole('button', { name: /^Collapse .* details$/ }) .elementHandle())!; - await focusTarget.evaluate((button: HTMLElement) => - button.focus({ preventScroll: true }), - ); + await focusTarget.evaluate((button: HTMLElement) => button.focus({ preventScroll: true })); const scrollHandle = await drawer.evaluateHandle((row) => { return ( - Array.from(row.querySelectorAll("*")).find((element) => { + Array.from(row.querySelectorAll('*')).find((element) => { const style = getComputedStyle(element); return ( element.getClientRects().length > 0 && @@ -158,14 +135,8 @@ test.describe.serial("Inline machine drawer column span", () => { const scrollElement = scrollHandle.asElement(); const scrollOffset = scrollElement ? await scrollElement.evaluate((element) => { - element.scrollTop = Math.min( - 32, - element.scrollHeight - element.clientHeight, - ); - element.scrollLeft = Math.min( - 32, - element.scrollWidth - element.clientWidth, - ); + element.scrollTop = Math.min(32, element.scrollHeight - element.clientHeight); + element.scrollLeft = Math.min(32, element.scrollWidth - element.clientWidth); return { top: element.scrollTop, left: element.scrollLeft }; }) : null; @@ -174,19 +145,17 @@ test.describe.serial("Inline machine drawer column span", () => { expect( await originalRow.evaluate( (row) => - row.isConnected && - document.querySelector('[data-column-span-test="original"]') === - row, + row.isConnected && document.querySelector('[data-column-span-test="original"]') === row, ), - "Original drawer row stays mounted", + 'Original drawer row stays mounted', ).toBe(true); expect( await focusTarget.evaluate( (button) => document.activeElement === button && - button.closest("tr")?.hasAttribute("data-column-span-test"), + button.closest('tr')?.hasAttribute('data-column-span-test'), ), - "Original interactive element keeps focus inside the drawer", + 'Original interactive element keeps focus inside the drawer', ).toBe(true); if (scrollElement && scrollOffset) { const current = await scrollElement.evaluate((element) => ({ @@ -194,38 +163,31 @@ test.describe.serial("Inline machine drawer column span", () => { top: element.scrollTop, left: element.scrollLeft, })); - expect( - current.connected, - "Original inner scroll container stays mounted", - ).toBe(true); - expect( - current.top, - "Inner vertical scroll survives column changes", - ).toBe(scrollOffset.top); - expect( - current.left, - "Inner horizontal scroll survives column changes", - ).toBe(scrollOffset.left); + expect(current.connected, 'Original inner scroll container stays mounted').toBe(true); + expect(current.top, 'Inner vertical scroll survives column changes').toBe(scrollOffset.top); + expect(current.left, 'Inner horizontal scroll survives column changes').toBe( + scrollOffset.left, + ); } }; const initialSpan = (await readLayout(drawer)).headerSpan; - const view = surface.getByRole("button", { name: "View", exact: true }); - const preferences = surface.getByRole("region", { - name: "View preferences", + const view = surface.getByRole('button', { name: 'View', exact: true }); + const preferences = surface.getByRole('region', { + name: 'View preferences', exact: true, }); for (const visible of [false, true]) { await test.step( visible - ? "Show System after the reduced table has laid out" - : "Hide System with the drawer open", + ? 'Show System after the reduced table has laid out' + : 'Hide System with the drawer open', async () => { await view.click(); - await preferences.getByRole("button", { name: /^Columns\b/ }).click(); - const system = preferences.getByRole("checkbox", { - name: "System", + await preferences.getByRole('button', { name: /^Columns\b/ }).click(); + const system = preferences.getByRole('checkbox', { + name: 'System', exact: true, }); await expect(system).toBeChecked({ checked: !visible }); @@ -233,18 +195,14 @@ test.describe.serial("Inline machine drawer column span", () => { button.focus({ preventScroll: true }), ); // Native checkbox activation preserves drawer focus during the column mutation. - await system.evaluate((checkbox: HTMLInputElement) => - checkbox.click(), - ); + await system.evaluate((checkbox: HTMLInputElement) => checkbox.click()); await expect(system).toBeChecked({ checked: visible }); await drawer.evaluate( (row) => new Promise((resolve) => { // Force the reduced table to lay out before restoring the column. void row.getBoundingClientRect().width; - requestAnimationFrame(() => - requestAnimationFrame(() => resolve()), - ); + requestAnimationFrame(() => requestAnimationFrame(() => resolve())); }), ); await expectAligned(drawer, initialSpan - (visible ? 0 : 1)); From 0dd4d1690d0ce444387eab534f27a203a69ece3c Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:10:12 +0100 Subject: [PATCH 08/27] Stop presenting a silent agent's node temperature as live A host agent linked to a Proxmox node merges with it into one read-state row, and every PVE poll refreshes that row's LastSeen. The agent's last sensors stay in state after it stops reporting. The node temperature lookup checked neither: it returned the retained sensors stamped with the row's LastSeen, so each poll presented the silent agent's last reading as a fresh measurement. An open node temperature alert was re-observed every poll (LastSeen kept moving, so stale-alert cleanup never reached it), a stale hot reading could keep activating, node history recorded it as new samples, the UI showed it as current, and SSH collection was skipped as if a recent agent payload existed. Since #2569 evaluates node temperature for agent-backed nodes, this holds their temperature alerts open. The lookup now reads the agent source's own sighting through a new HostView.SourceStatus (matching the VM, container and disk views), uses the agent's sensors only inside the reporting lease that keeps the agent online, and stamps the reading with the agent's report time. Past the lease the node has no agent reading, which alert evaluation treats as missing rather than as evidence the node cooled down. The poller's carry-over of a previous reading when collection fails is also bounded to max(5m, 2 x PVE polling interval) and keeps its original timestamp. That branch does not engage today, because the previous-node projection carries the unified resource id and no temperature, so it never matches the polled node; the bound keeps it from presenting a frozen value as current once that projection is repaired. --- .../v6/internal/subsystems/monitoring.md | 17 ++++ .../internal/subsystems/unified-resources.md | 12 +++ internal/monitoring/host_agent_temps.go | 22 ++++- internal/monitoring/host_agent_temps_test.go | 98 +++++++++++++++++++ .../monitoring/monitor_alert_handling_test.go | 92 +++++++++++++++++ .../monitor_polling_node_helpers.go | 23 ++++- internal/unifiedresources/views.go | 12 +++ internal/unifiedresources/views_test.go | 40 ++++++++ 8 files changed, 313 insertions(+), 3 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 9ae9a5d83a..bad4963674 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -3695,6 +3695,23 @@ exists or the agent payload has no usable positive reading. Identity-only or zero-temperature SMART rows do not count as usable by themselves, but the runtime must not keep probing legacy SSH solely to augment an otherwise healthy agent temperature payload with SMART data. +A node temperature presented as available must describe the node now, because +node alert evaluation, node history writes, reporting, and the UI all read it as +a live measurement. A host agent linked to a Proxmox node merges into one host +row whose `LastSeen` every PVE poll keeps fresh, while the agent's last sensors +stay in state after it stops reporting, so the row's `LastSeen` cannot decide +whether those sensors are current. `internal/monitoring/host_agent_temps.go` +reads the agent's own sighting from the row's `SourceStatus(SourceAgent)`, feeds +the agent's sensors into the node only while that sighting is inside the +reporting lease that keeps the agent online (`hostAgentReportCurrent`, the same +`hostAgentHealthWindow` that `evaluateHostAgents` applies, so a slow-interval +agent's readings count exactly while the agent is shown online), and stamps the +reading with that report time. Past the lease the lookup falls back to the +cluster sensor cache, which keeps its own recency check. When every source +returns nothing, `internal/monitoring/monitor_polling_node_helpers.go` may carry +a previous reading, with its original `LastUpdate`, only inside the carry window +(twice the PVE polling interval, never under five minutes); an older reading is +dropped rather than re-presented as current. Legacy SSH temperature collection must also use the Pulse sensor-wrapper contract before falling back to raw lm-sensors output. `internal/monitoring/temperature.go` must request `/usr/local/sbin/pulse-sensors` when it exists, parse the wrapper diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index b91304e4d5..cc1ee9bd94 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -485,6 +485,18 @@ layer consumes it only as a fallback replacement for Proxmox LXC memory, so platform metric priority for a healthy agent is unchanged. `TestContainerViewLinkedAgentMemory` pins those boundaries. +**Host-row agent source freshness (6 October 2026)** + +`HostView.SourceStatus(source)` exposes the per-source delivery freshness the +registry already records, matching the VM, container, and physical-disk views. +A host agent linked to a Proxmox node merges into one row whose `LastSeen` +follows the freshest source, so consumers of agent-owned samples (monitoring's +linked-agent node temperature lookup) must read the agent source's own sighting +here instead of the row's `LastSeen`. The accessor returns a copy of the recorded +status and adds no freshness policy of its own. +`TestView_HostViewSourceStatusSeparatesAgentFromMergedRow` pins the merged-row +split. + ### Bounded incident-history selection Canonical history queries filter exact alert identifiers and observation windows diff --git a/internal/monitoring/host_agent_temps.go b/internal/monitoring/host_agent_temps.go index 5d10cabf3c..661caeaf6e 100644 --- a/internal/monitoring/host_agent_temps.go +++ b/internal/monitoring/host_agent_temps.go @@ -138,8 +138,19 @@ func (m *Monitor) getHostAgentTemperatureByID(nodeID, nodeName string) *models.T return m.getClusterSensorTemperature(nodeName) } - // Convert host agent sensor data to Temperature model - return convertUnifiedHostSensorsToTemperature(sensors, matchedHost.LastSeen()) + // An agent that stopped reporting keeps its last sensors in state. Those + // describe the machine as it was, so they must not keep feeding the node + // (and through it alerts and history) as a live reading. The row's own + // LastSeen cannot tell: a row merged with the Proxmox node stays fresh from + // every PVE poll, so read the agent source's sighting. + agentStatus, ok := matchedHost.SourceStatus(unifiedresources.SourceAgent) + if !ok || !hostAgentReportCurrent(agentStatus.LastSeen, matchedHost.IntervalSeconds(), time.Now()) { + return m.getClusterSensorTemperature(nodeName) + } + + // Convert host agent sensor data to Temperature model, stamped with the + // agent's report time rather than the merged row's. + return convertUnifiedHostSensorsToTemperature(sensors, agentStatus.LastSeen) } func convertUnifiedHostSensorsToTemperature(sensors *unifiedresources.HostSensorMeta, lastSeen time.Time) *models.Temperature { @@ -411,6 +422,13 @@ func canonicalSMARTDevicePath(device string) string { // isHostAgentTemperatureRecent checks if the host agent temperature data is recent enough to use. // We consider data stale if the host hasn't reported in more than 2 minutes. +// hostAgentReportCurrent reports whether an agent's last report is still inside +// the reporting lease that keeps the agent online (evaluateHostAgents uses the +// same window), so its sensor readings still describe the machine. +func hostAgentReportCurrent(lastSeen time.Time, intervalSeconds int, now time.Time) bool { + return !lastSeen.IsZero() && now.Sub(lastSeen) <= hostAgentHealthWindow(intervalSeconds) +} + func isHostAgentTemperatureRecent(lastSeen time.Time) bool { const staleDuration = 2 * time.Minute return time.Since(lastSeen) < staleDuration diff --git a/internal/monitoring/host_agent_temps_test.go b/internal/monitoring/host_agent_temps_test.go index b03dd61496..5885b47ea8 100644 --- a/internal/monitoring/host_agent_temps_test.go +++ b/internal/monitoring/host_agent_temps_test.go @@ -1,11 +1,14 @@ package monitoring import ( + "context" "testing" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/models" unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" "github.com/stretchr/testify/assert" ) @@ -459,6 +462,7 @@ func TestGetHostAgentTemperature(t *testing.T) { Sensors: models.HostSensorSummary{ TemperatureCelsius: map[string]float64{"cpu_package": 60.0}, }, + LastSeen: time.Now(), } m.state.UpsertHost(host) @@ -497,6 +501,7 @@ func TestGetHostAgentTemperature(t *testing.T) { Sensors: models.HostSensorSummary{ TemperatureCelsius: map[string]float64{"cpu_package": 65.0}, }, + LastSeen: time.Now(), } m.state.UpsertHost(host) @@ -663,6 +668,7 @@ func TestGetHostAgentTemperatureByID_LocalAgentTakesPriority(t *testing.T) { "cpu_package": 70.0, // local agent reports 70 }, }, + LastSeen: time.Now(), }) m.clusterSensorsCache["shared-node"] = clusterSensorsCacheEntry{ sensors: models.HostSensorSummary{ @@ -678,6 +684,98 @@ func TestGetHostAgentTemperatureByID_LocalAgentTakesPriority(t *testing.T) { assert.Equal(t, 70.0, result.CPUPackage, "local agent data should take priority over cluster cache") } +// An agent that stops reporting keeps its last sensors in state. Past the +// reporting lease that marks it offline, those sensors must not keep feeding the +// linked node as a live reading; the cluster cache (with its own recency) is +// still consulted. +func TestGetHostAgentTemperatureByID_IgnoresAgentPastReportingLease(t *testing.T) { + m := &Monitor{ + state: models.NewState(), + clusterSensorsCache: make(map[string]clusterSensorsCacheEntry), + } + m.state.UpsertHost(models.Host{ + ID: "host-silent", + Hostname: "silent-node", + LinkedNodeID: "node-silent", + IntervalSeconds: 30, + Sensors: models.HostSensorSummary{ + TemperatureCelsius: map[string]float64{"cpu_package": 95.0}, + }, + LastSeen: time.Now().Add(-hostAgentHealthWindow(30) - time.Minute), + }) + + assert.Nil(t, m.getHostAgentTemperatureByID("node-silent", "silent-node"), + "a silent agent's retained sensors must not be presented as a current reading") + + m.clusterSensorsCache["silent-node"] = clusterSensorsCacheEntry{ + sensors: models.HostSensorSummary{ + TemperatureCelsius: map[string]float64{"cpu_package": 58.0}, + }, + updatedAt: time.Now(), + } + if result := m.getHostAgentTemperatureByID("node-silent", "silent-node"); assert.NotNil(t, result) { + assert.Equal(t, 58.0, result.CPUPackage, "a recent cluster-cache reading still serves the node") + } + + m.state.UpsertHost(models.Host{ + ID: "host-silent", + Hostname: "silent-node", + LinkedNodeID: "node-silent", + IntervalSeconds: 30, + Sensors: models.HostSensorSummary{ + TemperatureCelsius: map[string]float64{"cpu_package": 71.0}, + }, + LastSeen: time.Now(), + }) + if result := m.getHostAgentTemperatureByID("node-silent", "silent-node"); assert.NotNil(t, result) { + assert.Equal(t, 71.0, result.CPUPackage, "a reporting agent's reading is used again") + } +} + +// When every temperature source returns nothing, a previous reading may be +// carried only inside the carry window and keeps its original timestamp; an +// older one is dropped rather than re-presented as current. +func TestCollectNodeTemperatureCarryIsBoundedByCarryWindow(t *testing.T) { + m := &Monitor{ + config: &config.Config{TemperatureMonitoringEnabled: true, PVEPollingInterval: 10 * time.Second}, + state: models.NewState(), + } + collectWithPrevious := func(prev *models.Temperature) *models.Temperature { + node := models.Node{ID: "pve1-node1", Name: "node1", Instance: "pve1"} + m.collectNodeTemperatureData( + context.Background(), "pve1", &config.PVEInstance{Name: "pve1"}, proxmox.Node{Node: "node1"}, + &node, []models.Node{{ID: node.ID, Temperature: prev}}, "online", + ) + return node.Temperature + } + + assert.Equal(t, 5*time.Minute, m.nodeTemperatureCarryWindow(), "the floor applies at the default polling interval") + m.config.PVEPollingInterval = 10 * time.Minute + assert.Equal(t, 20*time.Minute, m.nodeTemperatureCarryWindow(), "the window scales with a slow polling interval") + m.config.PVEPollingInterval = 10 * time.Second + + recent := time.Now().Add(-time.Minute) + if carried := collectWithPrevious(&models.Temperature{Available: true, CPUPackage: 92, LastUpdate: recent}); assert.NotNil(t, carried) { + assert.True(t, carried.Available) + assert.True(t, carried.LastUpdate.Equal(recent), "a carried reading keeps its original timestamp") + } + + stale := time.Now().Add(-m.nodeTemperatureCarryWindow() - time.Minute) + assert.Nil(t, collectWithPrevious(&models.Temperature{Available: true, CPUPackage: 92, LastUpdate: stale}), + "a reading older than the carry window must not be presented as current") + assert.Nil(t, collectWithPrevious(&models.Temperature{Available: true, CPUPackage: 92}), + "a reading with no timestamp cannot be shown to be recent") +} + +func TestHostAgentReportCurrent(t *testing.T) { + now := time.Now() + assert.False(t, hostAgentReportCurrent(time.Time{}, 30, now), "an agent that never reported has no current reading") + assert.True(t, hostAgentReportCurrent(now.Add(-hostAgentHealthWindow(30)), 30, now)) + assert.False(t, hostAgentReportCurrent(now.Add(-hostAgentHealthWindow(30)-time.Second), 30, now)) + // A slow agent's lease scales with its declared interval. + assert.True(t, hostAgentReportCurrent(now.Add(-8*time.Minute), 120, now)) +} + func TestGetHostAgentTemperatureByID_UsesUnifiedReadState(t *testing.T) { now := time.Now().UTC() registry := unifiedresources.NewRegistry(nil) diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go index 498bd35b8d..1bbef67f02 100644 --- a/internal/monitoring/monitor_alert_handling_test.go +++ b/internal/monitoring/monitor_alert_handling_test.go @@ -1,6 +1,7 @@ package monitoring import ( + "context" "encoding/json" "io" "net/http" @@ -13,10 +14,12 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/ai/memory" "github.com/rcourtman/pulse-go-rewrite/internal/alerts" "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog" + "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/notifications" unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" "github.com/rcourtman/pulse-go-rewrite/internal/websocket" + "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" "github.com/stretchr/testify/require" ) @@ -1466,3 +1469,92 @@ func TestMonitorDelayedPartialResolutionKeepsDestinationRecovery(t *testing.T) { t.Fatal("unannounced destination received a recovery") } } + +// A host agent linked to a Proxmox node merges into one read-state row that +// every PVE poll keeps fresh. When the agent stops reporting, its retained +// sensors must stop feeding the node's temperature: otherwise the frozen +// reading is re-observed every poll, an open temperature alert's LastSeen keeps +// moving, and stale-alert cleanup never reaches it. +func TestSilentLinkedAgentStopsRefreshingNodeTemperatureAlert(t *testing.T) { + manager := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore()) + t.Cleanup(manager.Stop) + alertConfig := manager.GetConfig() + alertConfig.Enabled = true + alertConfig.TimeThresholds = map[string]int{} + alertConfig.SuppressionWindow = 0 + alertConfig.NodeDefaults.Temperature = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75} + manager.UpdateConfig(alertConfig) + + registry := unifiedresources.NewRegistry(nil) + monitor := &Monitor{ + config: &config.Config{TemperatureMonitoringEnabled: true}, + state: models.NewState(), + resourceStore: unifiedresources.NewMonitorAdapter(registry), + } + node := models.Node{ + ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", + ConnectionHealth: "healthy", LinkedAgentID: "agent-1", + } + // ingest records a PVE poll that just saw the node, plus the agent as of its + // last report. + ingest := func(agentLastReport time.Time, agentStatus string) { + polledNode := node + polledNode.LastSeen = time.Now() + registry.IngestSnapshot(models.StateSnapshot{ + Nodes: []models.Node{polledNode}, + Hosts: []models.Host{{ + ID: "agent-1", Hostname: "node1", Status: agentStatus, LinkedNodeID: node.ID, + IntervalSeconds: 30, LastSeen: agentLastReport, + Sensors: models.HostSensorSummary{TemperatureCelsius: map[string]float64{"cpu_package": 92}}, + }}, + }) + } + // poll runs the node temperature step of a PVE poll (no SSH collector) and + // evaluates the node's alerts with the result. + poll := func() *models.Temperature { + _, prevNodes := monitor.snapshotPrevNodes("pve1") + polled := node + monitor.collectNodeTemperatureData( + context.Background(), "pve1", &config.PVEInstance{Name: "pve1"}, proxmox.Node{Node: node.Name}, + &polled, prevNodes, "online", + ) + manager.CheckNode(polled) + return polled.Temperature + } + temperatureAlert := func() alerts.Alert { + t.Helper() + for _, alert := range manager.GetActiveAlerts() { + if alert.Type == "temperature" && alert.ResourceID == node.ID { + return alert + } + } + t.Fatalf("expected an open temperature alert for %s", node.ID) + return alerts.Alert{} + } + + // While the agent reports, its hot reading feeds the node, stamped with the + // agent's report time, and opens the alert. + agentReport := time.Now().Add(-10 * time.Second) + ingest(agentReport, "online") + reading := poll() + require.NotNil(t, reading) + require.Equal(t, 92.0, reading.CPUPackage) + require.True(t, reading.LastUpdate.Equal(agentReport), "the reading carries the agent's own report time") + opened := temperatureAlert() + + // The agent goes silent while PVE polling continues: the merged row stays + // fresh, but the agent's retained sensors no longer count as a reading. + ingest(time.Now().Add(-hostAgentHealthWindow(30)-time.Minute), "offline") + time.Sleep(5 * time.Millisecond) + require.Nil(t, poll(), "a silent agent's retained sensors must not be presented as a current reading") + held := temperatureAlert() + require.True(t, held.LastSeen.Equal(opened.LastSeen), "a frozen agent reading must not keep the alert fresh") + require.Nil(t, poll()) + require.True(t, temperatureAlert().LastSeen.Equal(opened.LastSeen)) + + // The agent reports again and its reading is evaluated. + ingest(time.Now(), "online") + time.Sleep(5 * time.Millisecond) + require.NotNil(t, poll()) + require.True(t, temperatureAlert().LastSeen.After(held.LastSeen), "a fresh agent reading is evaluated") +} diff --git a/internal/monitoring/monitor_polling_node_helpers.go b/internal/monitoring/monitor_polling_node_helpers.go index 03156560d1..ced65961a2 100644 --- a/internal/monitoring/monitor_polling_node_helpers.go +++ b/internal/monitoring/monitor_polling_node_helpers.go @@ -129,6 +129,18 @@ func (m *Monitor) pveNodeUsesProviderScopedIdentity(instanceName string, instanc ) } +// nodeTemperatureCarryFloor is the shortest time a node may keep showing its +// last temperature after collection stops returning one, sized to ride out the +// SSH collector's early failure backoff steps (30s, 1m, 2m). +const nodeTemperatureCarryFloor = 5 * time.Minute + +// nodeTemperatureCarryWindow bounds how old a carried-over node temperature may +// be. Beyond it the reading no longer describes the node, so it stops being +// presented as available. +func (m *Monitor) nodeTemperatureCarryWindow() time.Duration { + return resourceStaleThresholdForPollInterval(effectivePVEPollingIntervalForConfig(m.config), nodeTemperatureCarryFloor) +} + func (m *Monitor) collectNodeTemperatureData( ctx context.Context, instanceName string, @@ -319,7 +331,16 @@ func (m *Monitor) collectNodeTemperatureData( } } - if prevTemp != nil { + if prevTemp != nil && time.Since(prevTemp.LastUpdate) > m.nodeTemperatureCarryWindow() { + // Collection has stayed broken past the carry window. Re-presenting + // the old reading would let alerts, history and the UI treat it as + // a live measurement forever, so the node has no reading now. + log.Debug(). + Str("node", node.Node). + Bool("isCluster", modelNode.IsClusterMember). + Time("lastUpdate", prevTemp.LastUpdate). + Msg("Dropped stale temperature data (collection has not returned a reading within the carry window)") + } else if prevTemp != nil { // Clone the previous temperature to avoid modifying historical data preserved := *prevTemp preserved.LastUpdate = prevTemp.LastUpdate // Keep original update time to indicate staleness diff --git a/internal/unifiedresources/views.go b/internal/unifiedresources/views.go index 4219d0469e..81667e27e1 100644 --- a/internal/unifiedresources/views.go +++ b/internal/unifiedresources/views.go @@ -1372,6 +1372,18 @@ func (v HostView) LastSeen() time.Time { return v.r.LastSeen } +// SourceStatus returns the canonical delivery freshness recorded for one +// source. On a merged row (a host agent linked to a Proxmox node) LastSeen +// follows the freshest source, so consumers of agent-owned samples such as +// sensors must read the agent source's own sighting here instead. +func (v HostView) SourceStatus(source DataSource) (SourceStatus, bool) { + if v.r == nil { + return SourceStatus{}, false + } + status, ok := v.r.SourceStatus[source] + return status, ok +} + func (v HostView) CPUPercent() float64 { if v.r == nil { return 0 diff --git a/internal/unifiedresources/views_test.go b/internal/unifiedresources/views_test.go index 4840036535..8c8bcc98cc 100644 --- a/internal/unifiedresources/views_test.go +++ b/internal/unifiedresources/views_test.go @@ -932,6 +932,46 @@ func TestView_HostViewIntegrationSource(t *testing.T) { } } +// A host agent linked to a Proxmox node merges into one row whose LastSeen +// follows the freshest source. Agent-owned samples (sensors) need the agent's +// own sighting, which SourceStatus exposes. +func TestView_HostViewSourceStatusSeparatesAgentFromMergedRow(t *testing.T) { + now := time.Now().UTC() + agentLastReport := now.Add(-30 * time.Minute) + registry := NewRegistry(nil) + registry.IngestSnapshot(models.StateSnapshot{ + Nodes: []models.Node{{ + ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", + LastSeen: now, LinkedAgentID: "agent-1", + }}, + Hosts: []models.Host{{ + ID: "agent-1", Hostname: "node1", Status: "offline", LinkedNodeID: "pve1-node1", + IntervalSeconds: 30, LastSeen: agentLastReport, + }}, + }) + + hosts := registry.Hosts() + if len(hosts) != 1 { + t.Fatalf("expected the agent and node to merge into one host row, got %d", len(hosts)) + } + host := hosts[0] + if !host.LastSeen().Equal(now) { + t.Fatalf("merged row LastSeen = %v, want the fresher Proxmox sighting %v", host.LastSeen(), now) + } + agent, ok := host.SourceStatus(SourceAgent) + if !ok || !agent.LastSeen.Equal(agentLastReport) { + t.Fatalf("agent source status = %+v (ok=%v), want the agent's own report time %v", agent, ok, agentLastReport) + } + if _, ok := host.SourceStatus(SourceTrueNAS); ok { + t.Fatal("a source that never reported must not have a status") + } + + var nilView HostView + if _, ok := nilView.SourceStatus(SourceAgent); ok { + t.Fatal("nil-backed view must report no source status") + } +} + func TestView_HostViewMetricsTargetAccessor(t *testing.T) { r := &Resource{ ID: "host-metrics", From 019963b60e33360567b7d9901f5fed6fdd675f3c Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:05:42 +0100 Subject: [PATCH 09/27] Give agent disk, array and sensor alerts their own resource type Host-agent alerts about something other than the machine itself (filesystems, disk temperatures, SMART disks, RAID and Unraid arrays, custom sensors) cloned the host's alert metadata and carried resourceType "agent". The alert card's Monitoring menu writes policy to their child resource id, yet it named the Pulse agent as the inventory owner, the copy meant for retiring the agent's own machine. CheckHost now stamps platformType "agent" on every host alert and gives child alerts agent-disk, agent-storage or agent-sensor, so the card still links to Machines and uses the neutral source-system copy. Webhook payloads carry the same resourceType values. Each child type keeps "agent" among its CanonicalResourceTypeKeys, and configuration re-evaluation now treats agent policy as final for agent-keyed alerts. Before, a SMART, RAID or sensor alert fell through to the guest switch, so disabling guest alerts resolved it on save and the next agent report raised and notified it again. A metric alert held between its clear and trigger thresholds returned before merging metadata, so one restored from a checkpoint written before platformType or these child types existed kept linking to the wrong page until it re-fired or cleared. The hold now refreshes resourceType and platformType only; value, message and last-seen time stay at the last breach. --- .../v6/internal/subsystems/alerts.md | 24 +- internal/alerts/canonical_metric.go | 37 +++ internal/alerts/config/identity.go | 7 + internal/alerts/config_runtime.go | 10 + internal/alerts/host.go | 92 ++++--- internal/alerts/host_unraid_lifecycle_test.go | 239 ++++++++++++++++++ .../alerts/migration_characterization_test.go | 90 +++++++ 7 files changed, 458 insertions(+), 41 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index a306e9a58a..76661a4486 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -2475,13 +2475,23 @@ reads that ahead of every fallback. The menu's owner copy comes from `frontend-modern/src/utils/resourceMonitoringPolicy.ts`, where an explicit platform outranks the resource-type guess and the agent-removal copy is kept to the agent's own machine: libvirt VMs and Unraid arrays in resource drawers -get the neutral source-system copy. Agent disk-usage, SMART disk and Unraid -array alerts still carry their host's `agent` metadata, so their menu names -the Pulse agent until those producers emit child-resource metadata. Alerts -restored from before `platformType` existed gain it at their next full -evaluation; a metric alert held between its clear and trigger thresholds keeps -the earlier fallback until then. Menu sentences open with the owner -label capitalised. +get the neutral source-system copy. Host-agent alerts +(`internal/alerts/host.go`) stamp `platformType` `agent`, and only the +machine's own alerts keep `resourceType` `agent`: filesystem, +disk-temperature and SMART alerts carry `agent-disk`, RAID and Unraid array +alerts `agent-storage`, and custom-sensor alerts `agent-sensor`, because the +menu writes policy to their child resource id, never to the machine. Each +child type keeps `agent` among its `CanonicalResourceTypeKeys`, and +configuration re-evaluation treats agent policy as final for host-agent +alerts (TrueNAS systems and vSphere hosts keep their platform policy), so the +storage and guest switches cannot resolve a SMART, RAID or sensor alert that +the next agent report would re-raise. A metric alert held between its clear +and trigger thresholds still refreshes `resourceType` and `platformType` from +its producer while its value, message and last-seen time stay at the last +breach, so a restored alert that is still firing or held picks up the current +classification at its next evaluation; one that resolves on that evaluation +keeps the stored classification in its resolved record. Menu sentences open +with the owner label capitalised. The retired dashboard recent-alert panel must not be reintroduced as a parallel alert surface. Alert summary/tone copy belongs to the alert overview presentation owner, and any future compact alert surface must compose the diff --git a/internal/alerts/canonical_metric.go b/internal/alerts/canonical_metric.go index fcfa75baa5..7e3a76580b 100644 --- a/internal/alerts/canonical_metric.go +++ b/internal/alerts/canonical_metric.go @@ -330,6 +330,15 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec return } + // A hold keeps the last breach's value and message, but what the + // alert is about is current: one restored from an older checkpoint, + // or raised before its producer named a child type or platform, + // must not link to the wrong page until it re-fires or clears. + if refreshMetricAlertClassification(existingAlert, alertMetadata) { + m.setActiveRecoveryAlert(existingAlert, storageKey) + m.saveActiveAlertsAsync("canonical metric classification") + } + if !triggered && primary == "" { // Hysteresis latch: below trigger but above recovery — hold // without refreshing, as the pre-cutover engine did. @@ -424,3 +433,31 @@ func alertspecsMetricTriggered(spec *alertspecs.MetricThresholdSpec, observed fl return false } } + +// metricAlertClassificationKeys name what a metric alert is about and which +// platform owns it. They describe the resource, not the reading. +var metricAlertClassificationKeys = []string{"resourceType", alertPlatformTypeKey} + +// refreshMetricAlertClassification copies the producer's current +// classification onto an open alert and reports whether anything changed. +func refreshMetricAlertClassification(alert *Alert, metadata map[string]interface{}) bool { + if alert == nil { + return false + } + changed := false + for _, key := range metricAlertClassificationKeys { + value, _ := metadata[key].(string) + if value == "" { + continue + } + if current, _ := alert.Metadata[key].(string); current == value { + continue + } + if alert.Metadata == nil { + alert.Metadata = make(map[string]interface{}, len(metricAlertClassificationKeys)) + } + alert.Metadata[key] = value + changed = true + } + return changed +} diff --git a/internal/alerts/config/identity.go b/internal/alerts/config/identity.go index fdaebaaa16..3908343ca8 100644 --- a/internal/alerts/config/identity.go +++ b/internal/alerts/config/identity.go @@ -57,6 +57,13 @@ func CanonicalResourceTypeKeys(resourceType string) []string { keys = addUnique(keys, "agent-disk") keys = addUnique(keys, "agent") keys = addUnique(keys, "storage") + case "agent-storage": + keys = addUnique(keys, "agent-storage") + keys = addUnique(keys, "agent") + keys = addUnique(keys, "storage") + case "agent-sensor": + keys = addUnique(keys, "agent-sensor") + keys = addUnique(keys, "agent") case "pbs": keys = addUnique(keys, "pbs") keys = addUnique(keys, "node") diff --git a/internal/alerts/config_runtime.go b/internal/alerts/config_runtime.go index 17d0ecf11e..6e803d5fa5 100644 --- a/internal/alerts/config_runtime.go +++ b/internal/alerts/config_runtime.go @@ -431,6 +431,16 @@ func (m *Manager) reevaluateActiveAlertsLocked() { continue } threshold = getThresholdForMetric(thresholds, metricType) + // Agent policy is the only policy for agent alerts. Falling + // through would let the storage or guest switches resolve a + // SMART, RAID or sensor alert that CheckHost re-raises on the + // next report. + if threshold == nil { + if isMetricThresholdAlertType(metricType) { + alertsToResolve = append(alertsToResolve, alertID) + } + continue + } } if alert.Type == "docker-host-offline" || diff --git a/internal/alerts/host.go b/internal/alerts/host.go index cc2969046f..ceb74961dd 100644 --- a/internal/alerts/host.go +++ b/internal/alerts/host.go @@ -28,6 +28,27 @@ func hostResourceID(hostID string) string { return fmt.Sprintf("agent:%s", trimmed) } +// Every alert CheckHost raises about something other than the machine itself +// (a filesystem, disk, array or custom sensor) targets a child resource ID, +// and the alert card's monitoring policy writes to that ID. These alerts +// name their own type so the card does not describe retiring the machine. +// Each type keeps "agent" among its alert policy keys +// (config.CanonicalResourceTypeKeys), so agent thresholds and switches still +// govern them. +const ( + hostDiskAlertResourceType = "agent-disk" // filesystems, disk temperatures, SMART disks + hostStorageAlertResourceType = "agent-storage" // RAID and Unraid arrays + hostSensorAlertResourceType = "agent-sensor" // custom sensors +) + +// hostChildAlertMetadata clones a host's base alert metadata for an alert +// about something the agent reports rather than the machine itself. +func hostChildAlertMetadata(base map[string]interface{}, resourceType string) map[string]interface{} { + metadata := cloneMetadata(base) + metadata["resourceType"] = resourceType + return metadata +} + func stripHostResourcePrefix(resourceID string) string { trimmed := strings.TrimSpace(resourceID) trimmed = strings.TrimPrefix(trimmed, "agent:") @@ -273,15 +294,16 @@ func (m *Manager) CheckHost(host models.Host) { instanceName := hostInstanceName(host) baseMetadata := map[string]interface{}{ - "resourceType": "agent", - "hostId": host.ID, - "hostname": host.Hostname, - "displayName": host.DisplayName, - "platform": host.Platform, - "osName": host.OSName, - "osVersion": host.OSVersion, - "agentVersion": host.AgentVersion, - "architecture": host.Architecture, + "resourceType": "agent", + alertPlatformTypeKey: string(unifiedresources.SourceAgent), + "hostId": host.ID, + "hostname": host.Hostname, + "displayName": host.DisplayName, + "platform": host.Platform, + "osName": host.OSName, + "osVersion": host.OSVersion, + "agentVersion": host.AgentVersion, + "architecture": host.Architecture, } if linkedNodeID := strings.TrimSpace(host.LinkedNodeID); linkedNodeID != "" { baseMetadata["linkedNodeId"] = linkedNodeID @@ -359,7 +381,7 @@ func (m *Manager) CheckHost(host models.Host) { tempResourceID := fmt.Sprintf("%s/disk_temp:%s", hostResourceID(host.ID), sanitizeHostComponent(disk.Device)) tempResourceName := fmt.Sprintf("%s (%s Temp)", hostDisplayName(host), disk.Device) - diskTempMetadata := cloneMetadata(baseMetadata) + diskTempMetadata := hostChildAlertMetadata(baseMetadata, hostDiskAlertResourceType) diskTempMetadata["metric"] = "diskTemperature" diskTempMetadata["device"] = disk.Device diskTempMetadata["temperature"] = disk.Temperature @@ -446,7 +468,7 @@ func (m *Manager) CheckHost(host models.Host) { continue } - diskMetadata := cloneMetadata(baseMetadata) + diskMetadata := hostChildAlertMetadata(baseMetadata, hostDiskAlertResourceType) diskMetadata["metric"] = "disk" diskMetadata["mountpoint"] = disk.Mountpoint diskMetadata["device"] = disk.Device @@ -517,7 +539,7 @@ func (m *Manager) CheckHost(host models.Host) { raidName := fmt.Sprintf("%s - %s (%s)", resourceName, array.Device, array.Level) raidSpecResourceID := fmt.Sprintf("%s/raid:%s", hostResourceID(host.ID), sanitizeRAIDDevice(array.Device)) - raidMetadata := cloneMetadata(baseMetadata) + raidMetadata := hostChildAlertMetadata(baseMetadata, hostStorageAlertResourceType) raidMetadata["metric"] = "raid" raidMetadata["raidDevice"] = array.Device raidMetadata["raidLevel"] = array.Level @@ -671,15 +693,16 @@ func (m *Manager) HandleHostTelemetryExpired(host models.Host) { m.clearHostUnraidAlerts(host.ID) } else { baseMetadata := map[string]interface{}{ - "resourceType": "agent", - "hostId": host.ID, - "hostname": host.Hostname, - "displayName": host.DisplayName, - "platform": host.Platform, - "osName": host.OSName, - "osVersion": host.OSVersion, - "agentVersion": host.AgentVersion, - "architecture": host.Architecture, + "resourceType": "agent", + alertPlatformTypeKey: string(unifiedresources.SourceAgent), + "hostId": host.ID, + "hostname": host.Hostname, + "displayName": host.DisplayName, + "platform": host.Platform, + "osName": host.OSName, + "osVersion": host.OSVersion, + "agentVersion": host.AgentVersion, + "architecture": host.Architecture, } if linkedNodeID := strings.TrimSpace(host.LinkedNodeID); linkedNodeID != "" { baseMetadata["linkedNodeId"] = linkedNodeID @@ -793,16 +816,17 @@ func (m *Manager) HandleHostOfflineWithCorrelation(host models.Host, correlation Message: fmt.Sprintf("Host '%s' is offline", resourceName), Correlation: correlation, Metadata: map[string]interface{}{ - "resourceType": "agent", - "hostId": host.ID, - "hostname": host.Hostname, - "displayName": host.DisplayName, - "platform": host.Platform, - "osName": host.OSName, - "osVersion": host.OSVersion, - "linkedNodeId": strings.TrimSpace(host.LinkedNodeID), - "linkedVmId": strings.TrimSpace(host.LinkedVMID), - "linkedContainerId": strings.TrimSpace(host.LinkedContainerID), + "resourceType": "agent", + alertPlatformTypeKey: string(unifiedresources.SourceAgent), + "hostId": host.ID, + "hostname": host.Hostname, + "displayName": host.DisplayName, + "platform": host.Platform, + "osName": host.OSName, + "osVersion": host.OSVersion, + "linkedNodeId": strings.TrimSpace(host.LinkedNodeID), + "linkedVmId": strings.TrimSpace(host.LinkedVMID), + "linkedContainerId": strings.TrimSpace(host.LinkedContainerID), }, AddToRecent: true, AddToHistory: true, @@ -965,7 +989,7 @@ func (m *Manager) syncHostCustomSensorAlerts(host models.Host, nodeName, instanc } } - metadata := cloneMetadata(baseMetadata) + metadata := hostChildAlertMetadata(baseMetadata, hostSensorAlertResourceType) metadata["metric"] = "customSensor" metadata["customSensorId"] = metric.ID metadata["customSensorName"] = metric.Name @@ -1393,7 +1417,7 @@ func (m *Manager) syncHostSMARTDiskAlert(host models.Host, disk models.HostDiskS reasonCodes := storageHealthReasonCodes(reasons) reasonSummaries := storageHealthReasonSummaries(reasons) - metadata := cloneMetadata(baseMetadata) + metadata := hostChildAlertMetadata(baseMetadata, hostDiskAlertResourceType) metadata["metric"] = alertType metadata["device"] = disk.Device metadata["model"] = disk.Model @@ -1474,7 +1498,7 @@ func (m *Manager) syncHostUnraidStorageAlert(host models.Host, nodeName, instanc reasonCodes := storageHealthReasonCodes(reasons) reasonSummaries := storageHealthReasonSummaries(reasons) - metadata := cloneMetadata(baseMetadata) + metadata := hostChildAlertMetadata(baseMetadata, hostStorageAlertResourceType) metadata["metric"] = "storageTopology" metadata["storagePlatform"] = "unraid" metadata["storageTopology"] = "array" diff --git a/internal/alerts/host_unraid_lifecycle_test.go b/internal/alerts/host_unraid_lifecycle_test.go index 3388516e35..8a9da9cbc6 100644 --- a/internal/alerts/host_unraid_lifecycle_test.go +++ b/internal/alerts/host_unraid_lifecycle_test.go @@ -305,3 +305,242 @@ func TestHostCustomSensorEscalationDelivery(t *testing.T) { }) } } + +// An agent reports things besides its own machine: filesystems, disks, +// arrays and custom sensors. Their alerts target a child resource, and the +// alert card's monitoring policy writes to that child, so they must not +// carry the machine's "agent" type: the card would offer the machine's +// retirement copy ("Agent removal remains available from Machines") for a +// policy that never touches the machine. platformType keeps every one of +// them linked to the agent's page. +func TestHostChildAlertsNameTheirOwnResourceType(t *testing.T) { + m := newTestManager(t) + m.ClearActiveAlerts() + m.mu.Lock() + m.config.TimeThresholds = map[string]int{} + m.config.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 70} + m.config.AgentDefaults.Disk = &HysteresisThreshold{Trigger: 80, Clear: 70} + m.config.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 50, Clear: 45} + m.mu.Unlock() + + pending := int64(2) + sensorValue := 23.0 + linuxHost := models.Host{ + ID: "child-linux", + Hostname: "storage-host", + DisplayName: "Storage Host", + Platform: "linux", + CPUUsage: 95, + Disks: []models.Disk{{ + Mountpoint: "/srv", + Device: "/dev/sdc1", + Usage: 92, + Total: 100, + Used: 92, + Free: 8, + }}, + Sensors: models.HostSensorSummary{ + SMART: []models.HostDiskSMART{{ + Device: "/dev/sda", + Model: "IronWolf", + Serial: "SERIAL-CHILD-1", + Health: "PASSED", + Temperature: 55, + Attributes: &models.SMARTAttributes{PendingSectors: &pending}, + }}, + Custom: []models.HostCustomSensorMetric{{ + ID: "queue_depth", + Name: "Queue depth", + Value: &sensorValue, + Status: "critical", + ObservedAt: time.Now().UTC(), + }}, + }, + RAID: []models.HostRAIDArray{{ + Device: "/dev/md0", + Level: "raid1", + State: "degraded", + TotalDevices: 2, + ActiveDevices: 1, + WorkingDevices: 1, + FailedDevices: 1, + }}, + } + unraidHost := models.Host{ + ID: "child-unraid", + Hostname: "tower", + DisplayName: "Tower", + Platform: "unraid", + Unraid: &models.HostUnraidStorage{ + ArrayStarted: true, + ArrayState: "STARTED", + Disks: []models.HostUnraidDisk{ + {Name: "disk1", Role: "data", Status: "online", Device: "/dev/sdb"}, + }, + }, + } + m.CheckHost(linuxHost) + m.CheckHost(unraidHost) + + want := map[string]string{ + "cpu": "agent", + "disk": "agent-disk", + "diskTemperature": "agent-disk", + "disk-health": "agent-disk", + "raid": "agent-storage", + "storage-topology": "agent-storage", + "custom-sensor": "agent-sensor", + } + seen := make(map[string]bool, len(want)) + for _, alert := range m.GetActiveAlerts() { + wantType, ok := want[alert.Type] + if !ok { + continue + } + seen[alert.Type] = true + if got := alert.Metadata["resourceType"]; got != wantType { + t.Errorf("%s alert resourceType = %v, want %s", alert.Type, got, wantType) + } + if got := alert.Metadata["platformType"]; got != "agent" { + t.Errorf("%s alert platformType = %v, want agent", alert.Type, got) + } + hostID, _ := alert.Metadata["hostId"].(string) + if wantType != "agent" && alert.ResourceID == hostResourceID(hostID) { + t.Errorf("%s alert targets the machine itself (%s), so it is not a child alert", alert.Type, alert.ResourceID) + } + } + for alertType := range want { + if !seen[alertType] { + t.Errorf("expected an active %s alert", alertType) + } + } +} + +// Child alerts keep the agent policy path. A configuration save that turns +// off storage or guest alerts must leave them alone: CheckHost only consults +// the agent switches, so resolving them here would re-raise and re-notify +// on the next report. +func TestHostChildAlertsIgnoreOtherPlatformSwitchesOnConfigSave(t *testing.T) { + m := newTestManager(t) + m.ClearActiveAlerts() + m.mu.Lock() + m.config.TimeThresholds = map[string]int{} + m.mu.Unlock() + + pending := int64(2) + sensorValue := 23.0 + host := models.Host{ + ID: "child-policy", + Hostname: "storage-host", + Platform: "linux", + Sensors: models.HostSensorSummary{ + SMART: []models.HostDiskSMART{{ + Device: "/dev/sda", + Serial: "SERIAL-CHILD-2", + Health: "PASSED", + Attributes: &models.SMARTAttributes{PendingSectors: &pending}, + }}, + Custom: []models.HostCustomSensorMetric{{ + ID: "queue_depth", + Name: "Queue depth", + Value: &sensorValue, + Status: "critical", + ObservedAt: time.Now().UTC(), + }}, + }, + RAID: []models.HostRAIDArray{{ + Device: "/dev/md0", + Level: "raid1", + State: "degraded", + TotalDevices: 2, + ActiveDevices: 1, + WorkingDevices: 1, + FailedDevices: 1, + }}, + } + m.CheckHost(host) + childTypes := []string{"disk-health", "raid", "custom-sensor"} + for _, alertType := range childTypes { + if !hasAlertType(m.GetActiveAlerts(), alertType) { + t.Fatalf("expected an active %s alert before the config save", alertType) + } + } + + m.mu.Lock() + m.config.DisableAllStorage = true + m.config.DisableAllGuests = true + m.reevaluateActiveAlertsLocked() + m.mu.Unlock() + + for _, alertType := range childTypes { + if !hasAlertType(m.GetActiveAlerts(), alertType) { + t.Errorf("%s alert was resolved by the storage or guest switch", alertType) + } + } + + m.mu.Lock() + m.config.DisableAllAgents = true + m.reevaluateActiveAlertsLocked() + m.mu.Unlock() + + for _, alertType := range childTypes { + if hasAlertType(m.GetActiveAlerts(), alertType) { + t.Errorf("%s alert survived disabling agent alerts", alertType) + } + } +} + +// The Proxmox node sweep removes alerts whose node is not a Proxmox node +// unless it recognises them as agent alerts, by an agent: resource id or the +// agent type. Child alerts no longer carry the agent type, so they must keep +// the agent: id their canonical spec gives them. +func TestHostChildAlertsSurviveProxmoxNodeCleanup(t *testing.T) { + m := newTestManager(t) + m.ClearActiveAlerts() + pending := int64(2) + sensorValue := 23.0 + host := models.Host{ + ID: "child-sweep", + Hostname: "storage-host", + Platform: "linux", + Sensors: models.HostSensorSummary{ + SMART: []models.HostDiskSMART{{ + Device: "/dev/sda", + Serial: "SERIAL-CHILD-3", + Health: "PASSED", + Attributes: &models.SMARTAttributes{PendingSectors: &pending}, + }}, + Custom: []models.HostCustomSensorMetric{{ + ID: "queue_depth", + Name: "Queue depth", + Value: &sensorValue, + Status: "critical", + ObservedAt: time.Now().UTC(), + }}, + }, + RAID: []models.HostRAIDArray{{ + Device: "/dev/md0", + Level: "raid1", + State: "degraded", + TotalDevices: 2, + ActiveDevices: 1, + WorkingDevices: 1, + FailedDevices: 1, + }}, + } + m.CheckHost(host) + childTypes := []string{"disk-health", "raid", "custom-sensor"} + for _, alertType := range childTypes { + if !hasAlertType(m.GetActiveAlerts(), alertType) { + t.Fatalf("expected an active %s alert", alertType) + } + } + + m.CleanupAlertsForNodes(map[string]bool{"pve1": true}) + + for _, alertType := range childTypes { + if !hasAlertType(m.GetActiveAlerts(), alertType) { + t.Errorf("Proxmox node cleanup removed an agent %s alert", alertType) + } + } +} diff --git a/internal/alerts/migration_characterization_test.go b/internal/alerts/migration_characterization_test.go index 17a2f075b9..d8814b3fa8 100644 --- a/internal/alerts/migration_characterization_test.go +++ b/internal/alerts/migration_characterization_test.go @@ -969,3 +969,93 @@ func TestAlertCharacterizationResolvedCallbackUsesCanonicalIDForCanonicalAliasCl t.Fatalf("expected resolved callback for %q", alertID) } } + +// A held threshold alert keeps the value and message of its last breach, but +// what the alert is about can be newer than the alert: one restored from a +// checkpoint written before its producer named a child type or stamped +// platformType would otherwise keep linking to the wrong page and naming the +// wrong policy owner until it re-fires or clears. +func TestAlertCharacterizationHeldMetricAlertRefreshesItsClassification(t *testing.T) { + type staleAlert struct { + resourceType string + platformType string + } + tests := []struct { + name string + alertID string + fire func(*Manager) + hold func(*Manager) + stale staleAlert + wantResourceType string + wantPlatform string + wantValue float64 + }{ + { + name: "agent filesystem", + alertID: canonicalMetricStateID("agent:held-host/disk:srv", "disk"), + fire: func(m *Manager) { + m.CheckHost(models.Host{ID: "held-host", Hostname: "held", Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sdc1", Usage: 92}}}) + }, + hold: func(m *Manager) { + m.CheckHost(models.Host{ID: "held-host", Hostname: "held", Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sdc1", Usage: 75}}}) + }, + stale: staleAlert{resourceType: "agent"}, + wantResourceType: "agent-disk", + wantPlatform: "agent", + wantValue: 92, + }, + { + name: "vSphere VM", + alertID: canonicalMetricStateID("vmware:vc-1:vm:vm-201", "disk"), + fire: func(m *Manager) { + m.CheckUnifiedResource(&UnifiedResourceInput{ID: "vmware:vc-1:vm:vm-201", Type: "vmware-vm", Name: "app-01", Node: "esxi-01", Instance: "Lab vCenter", Disk: &UnifiedResourceMetric{Percent: 94}}) + }, + hold: func(m *Manager) { + m.CheckUnifiedResource(&UnifiedResourceInput{ID: "vmware:vc-1:vm:vm-201", Type: "vmware-vm", Name: "app-01", Node: "esxi-01", Instance: "Lab vCenter", Disk: &UnifiedResourceMetric{Percent: 87}}) + }, + stale: staleAlert{resourceType: "vSphere VM"}, + wantResourceType: "vSphere VM", + wantPlatform: "vmware-vsphere", + wantValue: 94, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := newTestManager(t) + configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig()) + m.mu.Lock() + m.config.AgentDefaults.Disk = &HysteresisThreshold{Trigger: 80, Clear: 70} + m.mu.Unlock() + + tt.fire(m) + m.mu.Lock() + alert, ok := m.getActiveAlertNoLock(tt.alertID) + if !ok { + m.mu.Unlock() + t.Fatalf("expected active alert %q", tt.alertID) + } + alert.Metadata["resourceType"] = tt.stale.resourceType + delete(alert.Metadata, alertPlatformTypeKey) + m.mu.Unlock() + + tt.hold(m) + + m.mu.RLock() + defer m.mu.RUnlock() + held, ok := m.getActiveAlertNoLock(tt.alertID) + if !ok { + t.Fatal("held alert resolved; the reading should sit between clear and trigger") + } + if held.Value != tt.wantValue { + t.Fatalf("held alert value = %v, want the last breach %v", held.Value, tt.wantValue) + } + if got := held.Metadata["resourceType"]; got != tt.wantResourceType { + t.Fatalf("resourceType = %v, want %s", got, tt.wantResourceType) + } + if got := held.Metadata[alertPlatformTypeKey]; got != tt.wantPlatform { + t.Fatalf("platformType = %v, want %s", got, tt.wantPlatform) + } + }) + } +} From 56e88d5561e71da7a0ace363685808e8e4660645 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:18:35 +0100 Subject: [PATCH 10/27] Say what an agent update brings and what the CPU count is The outdated-agent notice told Machines users to update "for the latest agent command support and agent-managed platform detail". Proxmox, TrueNAS and vSphere used similar internal phrasing ("agent-contributed node detail", "in-guest telemetry"). It now reads "for the latest fixes and machine details", or node, system or VM details. It does not promise Patrol actions or command support: the notice selects hosts by version alone, and command execution is a separate setting that the update keeps as it is. A machine's CPU cell reads "22% (18)", and nothing explained the 18. The shared CPU bar's hover panel now names it "CPUs". That is logical CPUs on a host and configured vCPUs on a guest, so "Cores" would be wrong on hyperthreaded machines. --- .../subsystems/frontend-primitives.md | 7 +++ .../subsystems/performance-and-scalability.md | 5 ++ .../internal/subsystems/storage-recovery.md | 5 +- .../internal/subsystems/unified-resources.md | 3 ++ .../936d9cf319d0e486.json | 46 +++++++++++++++++++ .../components/Workloads/EnhancedCPUBar.tsx | 9 ++++ .../__tests__/EnhancedCPUBar.test.tsx | 14 ++++++ .../features/proxmox/ProxmoxPageSurface.tsx | 2 +- .../ProxmoxPageSurface.contract.test.tsx | 4 +- .../standalone/StandalonePageSurface.tsx | 2 +- .../__tests__/StandalonePageSurface.test.tsx | 4 +- .../features/truenas/TrueNASPageSurface.tsx | 2 +- .../TrueNASPageSurface.contract.test.tsx | 4 +- .../src/features/vmware/VmwarePageSurface.tsx | 2 +- .../VmwarePageSurface.contract.test.tsx | 2 +- 15 files changed, 95 insertions(+), 16 deletions(-) create mode 100644 frontend-modern/browser-verification/936d9cf319d0e486.json diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index aa974bf928..d6dcd4dd65 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -8366,6 +8366,13 @@ inventory and Storage search below the fold at desktop and phone widths, and from competing with real warnings for attention, while preserving every affected name on demand. The component test pins collapsed, expanded and collapsed-again states; 1440px, 768px and 390px browser checks verify placement and overflow. +Each platform page names what the update brings in the reader's words through +`missingLabel`: the latest fixes and machine, node, system or VM details. +The notice selects hosts by agent version alone, and command execution is a +separate setting the update preserves, so the notice must not promise Patrol +actions or command support. Internal phrasing such as "agent-contributed +detail", "agent-managed platform detail" or "in-guest telemetry" does not +belong in the notice. The shared `InlineNotice` action text uses opaque 800-level colors for its four tones. The current Tailwind configuration overrides several 900-level palette tokens with 25%-alpha colors for translucent backgrounds, so using those diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 50344121e9..7566602199 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -1122,6 +1122,11 @@ change may globally weaken the Task 03 lifecycle-state idempotency invariant. hover tooltips from synthesized mouse events; a row tap remains the primary drawer action. Tooltip suppression belongs to the shared hover-capability boundary rather than per-row viewport branches or listeners. + The shared `EnhancedCPUBar` hover panel names the CPU count that the bar + shows in parentheses beside the percentage ("22% (18)") as "CPUs": logical + CPUs on a host, configured vCPUs on a guest, so "Cores" would be wrong on + hyperthreaded hosts. It reads the `cores` prop the row already passes and + adds no per-row work. Ultra-narrow workload presentation belongs in the existing layout and column model: below 360 pixels it keeps five identity/health columns with a 40-percent identity track. It must not add per-row measurement, viewport diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index ebbc81311a..993581c65c 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -923,8 +923,9 @@ coverage, restore capability, recovery-job proof, storage-health verification, or evidence that a specific resource was protected or recovered. Proxmox page stale-agent notices are adjacent frontend and agent-lifecycle plumbing even though `ProxmoxPageSurface` is a storage/recovery canonical file. -Those notices may link an operator to scoped agent update commands for -agent-contributed node detail and command support, but they must not be +Those notices may link an operator to scoped agent update commands for the +latest fixes and node details (the notice's own wording; command execution is +a separate setting the update does not change), but they must not be interpreted as backup visibility, recovery readiness, restore capability, or a storage/recovery-owned command path. Suppressing those notices when the API has no deployable agent update target is likewise lifecycle/frontend behavior and diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index b91304e4d5..564eb08b9b 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -2370,6 +2370,9 @@ for the column when evidence later appears. The compact row action trigger chrome stays under the frontend-primitives `ActionIconButton` boundary rather than becoming a unified-resource-local button shell. +The Machines outdated-agent notice names what an update brings in plain +words, the latest fixes and machine details, and stays maintenance guidance +rather than a membership or health signal. Machines list search and online-state narrowing are frontend route state, not new unified-resource membership fields. `StandalonePageSurface.tsx` owns the `STANDALONE_QUERY_PARAMS` query/status projection and one composite diff --git a/frontend-modern/browser-verification/936d9cf319d0e486.json b/frontend-modern/browser-verification/936d9cf319d0e486.json new file mode 100644 index 0000000000..957411a378 --- /dev/null +++ b/frontend-modern/browser-verification/936d9cf319d0e486.json @@ -0,0 +1,46 @@ +{ + "version": 1, + "base_sha": "b19c6aaf709637adedd46de9f61307252ab1131d", + "verified_at": "2026-10-06T14:18:34Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Workloads/EnhancedCPUBar.tsx", + "frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx", + "frontend-modern/src/features/standalone/StandalonePageSurface.tsx", + "frontend-modern/src/features/truenas/TrueNASPageSurface.tsx", + "frontend-modern/src/features/vmware/VmwarePageSurface.tsx" + ], + "content_sha256": { + "frontend-modern/src/components/Workloads/EnhancedCPUBar.tsx": "7aad8eb3f5eb326aac32fb698e95b7fb76d3c2683a794dd104e0ed63e9080bf6", + "frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx": "3daf35d191a8df3d47c37d1813763730c1849a5097e3e707ecc6b3faf10143a6", + "frontend-modern/src/features/standalone/StandalonePageSurface.tsx": "620c1ebcc9fea62047bea73c4505a21c7ed3fc3f28791e50f4d07b3b5203d6d2", + "frontend-modern/src/features/truenas/TrueNASPageSurface.tsx": "52aa6b488694cbeb3103e83abe98cb739953afdc734d9f26b5b3d595f47fbbd0", + "frontend-modern/src/features/vmware/VmwarePageSurface.tsx": "cd6bd1afc48a59908b7b0940e9c8e33a084f3f55c513815218e53072db534ce1" + }, + "routes": [ + "/standalone/machines", + "/proxmox/overview", + "/truenas/overview", + "/vmware/overview" + ], + "viewports": [ + { + "width": 1440, + "height": 1000 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack on main b19c6aaf70 plus this change; Machines has three older agents and Proxmox five, TrueNAS and vSphere none (their notice copy is covered by the page contract tests)", + "Before: the Machines notice read 'Update them to v4.26.0 for the latest agent command support and agent-managed platform detail', Proxmox 'agent-contributed Proxmox node detail and command support'", + "After: 'Update them to v4.26.0 for the latest fixes and machine details' on Machines and '...fixes and node details' on Proxmox; the notice no longer promises Patrol actions or command support, which is a separate setting the update keeps", + "Before: the CPU cell's '22% (18)' had nothing explaining the 18; its hover panel listed Usage and Load (1m) only. After: the hover panel's first row reads 'CPUs 18'" + ], + "interactions": [ + "Hovered a machine's CPU bar and read the panel", + "Read the outdated-agent notice on each platform page" + ] +} diff --git a/frontend-modern/src/components/Workloads/EnhancedCPUBar.tsx b/frontend-modern/src/components/Workloads/EnhancedCPUBar.tsx index bca20abbd3..ff43f1b353 100644 --- a/frontend-modern/src/components/Workloads/EnhancedCPUBar.tsx +++ b/frontend-modern/src/components/Workloads/EnhancedCPUBar.tsx @@ -63,6 +63,15 @@ export function EnhancedCPUBar(props: EnhancedCPUBarProps) {
{props.model}
+ {/* The bar's "(18)" beside the percentage is this count: logical + CPUs on a host, configured vCPUs on a guest. */} + +
+ CPUs + {props.cores} +
+
+
Usage diff --git a/frontend-modern/src/components/Workloads/__tests__/EnhancedCPUBar.test.tsx b/frontend-modern/src/components/Workloads/__tests__/EnhancedCPUBar.test.tsx index 69401f4eae..8af6c7f263 100644 --- a/frontend-modern/src/components/Workloads/__tests__/EnhancedCPUBar.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/EnhancedCPUBar.test.tsx @@ -252,6 +252,20 @@ describe('EnhancedCPUBar', () => { expect(screen.getByText('CPU Details')).toBeInTheDocument(); }); + it('names the CPU count shown beside the percentage', async () => { + const { container } = render(() => ); + expect(container.textContent).toContain('(18)'); + await fireEvent.mouseEnter(getBarTrigger(container)); + const label = screen.getByText('CPUs'); + expect(label.closest('.flex.justify-between')?.textContent).toContain('18'); + }); + + it('omits the CPU row when the count is unknown', async () => { + const { container } = render(() => ); + await fireEvent.mouseEnter(getBarTrigger(container)); + expect(screen.queryByText('CPUs')).not.toBeInTheDocument(); + }); + it('shows usage value in tooltip', async () => { const { container } = render(() => ); await fireEvent.mouseEnter(getBarTrigger(container)); diff --git a/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx b/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx index d40ce5bf2c..c8db9e84d3 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx @@ -350,7 +350,7 @@ export function ProxmoxPageSurface() { { expect(screen.getByTestId('nodes-table')).toHaveAttribute('data-rows', '1'); const notice = screen.getByTestId('platform-outdated-agent-notice'); expect(notice).toHaveTextContent('delly runs an older Pulse agent (v5.1.34).'); - expect(notice).toHaveTextContent( - 'latest agent-contributed Proxmox node detail and command support', - ); + expect(notice).toHaveTextContent('latest fixes and node details'); expect(screen.getByRole('link', { name: 'Open agent upgrade commands' })).toHaveAttribute( 'href', '/settings/infrastructure/agent-doctor?agents=agent%3Aagent-delly', diff --git a/frontend-modern/src/features/standalone/StandalonePageSurface.tsx b/frontend-modern/src/features/standalone/StandalonePageSurface.tsx index 806407c6d2..908d94dad4 100644 --- a/frontend-modern/src/features/standalone/StandalonePageSurface.tsx +++ b/frontend-modern/src/features/standalone/StandalonePageSurface.tsx @@ -331,7 +331,7 @@ export function StandalonePageSurface() { { const notice = screen.getByTestId('platform-outdated-agent-notice'); expect(notice).toHaveTextContent('tower runs an older Pulse agent (v5.1.34).'); - expect(notice).toHaveTextContent( - 'latest agent command support and agent-managed platform detail', - ); + expect(notice).toHaveTextContent('latest fixes and machine details'); expect(screen.getByRole('link', { name: 'Open agent upgrade commands' })).toHaveAttribute( 'href', '/settings/infrastructure/agent-doctor?agents=agent%3Aagent-tower', diff --git a/frontend-modern/src/features/truenas/TrueNASPageSurface.tsx b/frontend-modern/src/features/truenas/TrueNASPageSurface.tsx index df6df6a535..60fd759874 100644 --- a/frontend-modern/src/features/truenas/TrueNASPageSurface.tsx +++ b/frontend-modern/src/features/truenas/TrueNASPageSurface.tsx @@ -189,7 +189,7 @@ export function TrueNASPageSurface() { { expect(screen.getByTestId('systems-table')).toHaveAttribute('data-rows', '1'); const notice = screen.getByTestId('platform-outdated-agent-notice'); expect(notice).toHaveTextContent('truenas-scale runs an older Pulse agent (v5.1.34).'); - expect(notice).toHaveTextContent( - 'latest agent-contributed TrueNAS system detail and command support', - ); + expect(notice).toHaveTextContent('latest fixes and system details'); expect(screen.getByRole('link', { name: 'Open agent upgrade commands' })).toHaveAttribute( 'href', '/settings/infrastructure/agent-doctor?agents=agent%3Aagent-truenas-scale', diff --git a/frontend-modern/src/features/vmware/VmwarePageSurface.tsx b/frontend-modern/src/features/vmware/VmwarePageSurface.tsx index e3aeeda59e..0e8d015897 100644 --- a/frontend-modern/src/features/vmware/VmwarePageSurface.tsx +++ b/frontend-modern/src/features/vmware/VmwarePageSurface.tsx @@ -352,7 +352,7 @@ export function VmwarePageSurface() { { expect(screen.getByTestId('hosts-table')).toHaveAttribute('data-rows', '1'); const notice = screen.getByTestId('platform-outdated-agent-notice'); expect(notice).toHaveTextContent('app-01 runs an older Pulse agent (v5.1.34).'); - expect(notice).toHaveTextContent('latest in-guest telemetry and command support.'); + expect(notice).toHaveTextContent('latest fixes and VM details.'); expect(screen.getByRole('link', { name: 'Open agent upgrade commands' })).toHaveAttribute( 'href', '/settings/infrastructure/agent-doctor?agents=agent%3Aagent-app-01', From e3a42d86c3c9be0dab2c8ba919b1e806fda1ef35 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:45:50 +0100 Subject: [PATCH 11/27] Evaluate host agents in mock mode like live agents checkMockAlerts evaluated mock nodes, guests, storage, PBS, PMG and Docker hosts but never host agents. Live agents are evaluated by CheckHost as each report lands; mock mode discards reports and the host health sweep never sees fixture agents, so no agent CPU, memory, disk, temperature or offline alert could open against mock data. A mock agent at 94% memory sat under an 85% trigger with no alert, and alert work on agents needed a private build to reproduce anything. Mock passes now evaluate every fixture agent with the live lifecycle boundaries: - offline fixtures go through HandleHostOfflineWithCorrelation, since CheckHost treats its input as a fresh report and marks it online; - agents run before nodes, so a node with an online linked agent hands its metric alerts to the agent on the first tick; - an agent that leaves the estate on a runtime mock config change goes through HandleHostRemoved, like a deleted live agent; - leaving mock mode routes the fixture agents through HandleHostRemoved, releasing hostname registrations that ClearActiveAlerts leaves behind and that would otherwise suppress a real node named pve1; a pass already in flight finds mock mode off and evaluates nothing. Evaluating agents exposed a mock-only reading: every Kubernetes pod took a single-pod share (10-80%) of its node's allocatable memory, so any node running four or more pods, and its linked agent, read a constant 100%. On the public demo's estate that became two permanent critical memory alerts no real cluster would raise. Pods now share 0.7 single-pod footprints per node (at most 58% of allocatable in steady state), and a 60% per-node ceiling clips the transient while rescheduled pods' readings settle, so those nodes stay under 82%. --- .../v6/internal/subsystems/agent-lifecycle.md | 15 ++ .../v6/internal/subsystems/monitoring.md | 52 +++++ .../subsystems/performance-and-scalability.md | 8 + internal/mock/generator.go | 91 ++++++++- internal/mock/generator_test.go | 138 +++++++++++++ internal/monitoring/monitor.go | 3 + internal/monitoring/monitor_alerts.go | 71 +++++++ ...nitor_host_agent_removal_lifecycle_test.go | 45 +++++ .../monitoring/monitor_mock_alerts_test.go | 189 ++++++++++++++++++ 9 files changed, 608 insertions(+), 4 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index b51b2d57dc..ea945c3572 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -8206,6 +8206,21 @@ or create agent continuity evidence. The removal lifecycle proof keeps a removed host blocked across a PBS storage sync so metric timestamp maintenance cannot become an accidental re-enrollment transition. +### Mock fixture agents reuse only the removal alert boundary + +`internal/monitoring/monitor.go` now holds the set of fixture agents the last +mock alert pass evaluated. When a runtime mock config change drops an agent +from the estate, `evaluateMockHostAgents` routes it through the alert +manager's `HandleHostRemoved`, the same alert boundary a deleted live agent +crosses. Leaving mock mode routes every fixture agent through the same call, +which also releases their hostname deduplication. Neither path writes a +removal tombstone, revokes a token, touches continuity evidence or admits a +report: fixture agents have no credentials or durable identity, and real +reports stay discarded while mock mode is on. +`TestMockHostAgentLeavingFixtureUsesRemovalLifecycle` in +`internal/monitoring/monitor_host_agent_removal_lifecycle_test.go` pins the +alert cleanup. + ### Deploy enrollment swaps credentials as one durable transition A deploy bootstrap token remains the live credential until Pulse can durably diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 9ae9a5d83a..a1c215ab3b 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -1596,6 +1596,58 @@ supporting inventory rather than a new batch of independent exits, so the confirmed host incident clears child alerts instead of producing one alert per container. +Mock alert evaluation covers host agents the same way. Live agents are +evaluated by `CheckHost` as each report lands. Mock mode discards reports, and +the `evaluateHostAgents` health sweep reads monitor state, which never holds +fixture agents, so `checkMockAlerts` evaluates every fixture agent itself +through `evaluateMockHostAgents`. Without that pass no agent CPU, memory, disk, +temperature or offline alert could open against mock data, while the same +reading on a real agent would alert. The pass keeps the live lifecycle +boundaries: + +- An offline fixture goes through `HandleHostOfflineWithCorrelation`, as a + lapsed live report does, because `CheckHost` treats its input as a fresh + report and would mark the host online. +- Agents are evaluated before nodes, so a node with an online linked agent + hands its CPU, memory and disk alerts to the agent on the first tick, as + host-agent hostname deduplication does in production. +- A runtime mock config change rebuilds the estate. An agent that leaves it + goes through `HandleHostRemoved`, as a deleted live agent does, so its alerts + and hostname deduplication do not outlive it. +- Leaving mock mode routes the fixture agents through `HandleHostRemoved`. + `ClearActiveAlerts` drops their alerts but not their hostname registrations, + and a real node named like a fixture agent (`pve1`) would otherwise keep its + metric alerts suppressed with no agent to own them. A pass whose snapshot + predates the switch finds mock mode off under the same lock and evaluates + nothing. Two overlapping passes can still evaluate an older snapshot after + a newer one; the next pass reconciles the set again. + +Fixture agents must carry readings a real agent could report. Mock Kubernetes +pods share 0.7 single-pod memory footprints per node, which keeps a node's pods +at or below 58% of allocatable memory in steady state, and a 60% per-node +ceiling clips the transient while rescheduled pods' readings settle. A dense +node and its linked agent therefore no longer read a constant 100% and raise a +critical alert no real cluster would. Whether a demo agent alerts at default thresholds depends on +the readings each boot draws. On 2026-10-06, private stacks with the public +demo's estate (8 nodes, 2 standalone agents and 3 Kubernetes node agents) +opened none on one boot and one (a standalone agent at 91% memory) on another. +With the agent memory trigger lowered to 60%, exactly the agents above it +alerted and their linked Proxmox nodes raised none, and shrinking the estate +through the mock settings API cleared the departed agents' alerts on the next +pass. Those are measurements, not test guarantees. The tests pin the +boundaries: +`TestCheckMockAlertsEvaluatesHostAgentsBeforeLinkedNodes`, +`TestCheckMockHostAlertsUsesHostLifecycleForOfflineFixtures`, +`TestLeavingMockModeReleasesFixtureAgentHostnames` and +`TestMockHostAgentPassAfterLeavingMockModeRegistersNothing` in +`internal/monitoring/monitor_mock_alerts_test.go`, +`TestMockHostAgentLeavingFixtureUsesRemovalLifecycle` in +`internal/monitoring/monitor_host_agent_removal_lifecycle_test.go`, and +`TestMockKubernetesDenseNodesStayBelowAllocatableMemory`, +`TestMockKubernetesRescheduledPodsDoNotSaturateReceivingNode` and +`TestMockKubernetesLoneHeavyPodStaysUnderNodeCeiling` in +`internal/mock/generator_test.go`. + Host and container-runtime disk collection supports an explicit include list for filesystems hidden by Pulse's automatic virtual/container filtering. The include list is bounded to that automatic filter; explicit disk exclusions diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 50344121e9..9bc2a19077 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -610,6 +610,14 @@ count times the number of shared storages. Demo Docker container names stay unique per host (numbered suffixes on reused profiles), so image "used by" lists and container rows never collapse two distinct containers into what reads as a duplicated row. +Mock Kubernetes pods share their node's allocatable memory: each active pod's +memory target is scaled by `mockKubernetesPodMemoryShare` over the active pods +on its node, which keeps a node's pods at or below 58% of allocatable memory in +steady state, and `capMockKubernetesNodePodMemory` clips them to 60% only while +rescheduled pods' smoothed readings settle, so a dense node does not pin itself +and its linked agent at 100%. The count and the clip are linear passes over the +cluster's pods in each usage update, keyed by node name, with no per-pod scan +of other pods or nodes. The browser applies resource deltas to its connection-scoped raw baseline, but canonicalizes and reconciles only changed resources plus the host-merge groups diff --git a/internal/mock/generator.go b/internal/mock/generator.go index 658d7c48a6..0e641a5cc1 100644 --- a/internal/mock/generator.go +++ b/internal/mock/generator.go @@ -1738,16 +1738,97 @@ func initializeMockKubernetesClusterUsage(cluster *models.KubernetesCluster, now } } + activePodsByNode := make(map[string]int, len(cluster.Nodes)) + for i := range cluster.Pods { + pod := &cluster.Pods[i] + nodeName := strings.TrimSpace(pod.NodeName) + if mockKubernetesPodIsActive(pod, nodeByName[nodeName]) { + activePodsByNode[nodeName]++ + } + } + clusterOffline := strings.EqualFold(strings.TrimSpace(cluster.Status), "offline") for i := range cluster.Pods { pod := &cluster.Pods[i] - node := nodeByName[strings.TrimSpace(pod.NodeName)] - updateMockKubernetesPodUsage(pod, node, now, randomize, clusterOffline) + nodeName := strings.TrimSpace(pod.NodeName) + updateMockKubernetesPodUsage(pod, nodeByName[nodeName], activePodsByNode[nodeName], now, randomize, clusterOffline) } + capMockKubernetesNodePodMemory(cluster, nodeByName) recomputeMockKubernetesNodeUsage(cluster, now) } +// mockKubernetesNodePodMemoryBudget is how many single-pod memory footprints +// the active pods on one node share. A pod's memory percent is measured +// against its node's allocatable memory, as the live agent reports it, so +// giving every pod a whole footprint (roughly 10-83%) committed any node +// running four or more pods past its allocatable memory. The node, and the +// host agent linked to it, then read a constant 100%. The largest footprint +// is about 83% (a Guaranteed StatefulSet pod at full burst), so a budget of +// 0.7 footprints keeps a node's pods at or below 58% in steady state, under +// mockKubernetesNodePodMemoryCeiling. +const mockKubernetesNodePodMemoryBudget = 0.7 + +// mockKubernetesPodMemoryShare scales one pod's memory target by the number +// of active pods on its node, so the node total stays within the budget plus +// system overhead however many pods are scheduled there. +func mockKubernetesPodMemoryShare(activePodsOnNode int) float64 { + if activePodsOnNode < 1 { + activePodsOnNode = 1 + } + return mockKubernetesNodePodMemoryBudget / float64(activePodsOnNode) +} + +// mockKubernetesNodePodMemoryCeiling bounds the fraction of a node's +// allocatable memory its active pods may hold together. Pod readings smooth +// toward their share of the budget, so they lag a change in the node's pod +// count: a pod rescheduled off a failed node arrives with a reading sized for +// its old node. Share-scaled targets stay under the ceiling, so it binds only +// on that transient, keeping the node, with its system overhead of up to 26%, +// below allocatable memory while readings settle. +const mockKubernetesNodePodMemoryCeiling = 0.6 + +// capMockKubernetesNodePodMemory applies the ceiling and reports how many +// nodes it clipped. +func capMockKubernetesNodePodMemory(cluster *models.KubernetesCluster, nodeByName map[string]*models.KubernetesNode) int { + usedByNode := make(map[string]int64, len(cluster.Nodes)) + for i := range cluster.Pods { + pod := &cluster.Pods[i] + nodeName := strings.TrimSpace(pod.NodeName) + if nodeByName[nodeName] != nil && mockKubernetesPodIsActive(pod, nodeByName[nodeName]) { + usedByNode[nodeName] += pod.UsageMemoryBytes + } + } + + scaleByNode := make(map[string]float64, len(usedByNode)) + for nodeName, used := range usedByNode { + node := nodeByName[nodeName] + allocMemory := node.AllocMemoryBytes + if allocMemory <= 0 { + allocMemory = node.CapacityMemoryBytes + } + ceiling := float64(allocMemory) * mockKubernetesNodePodMemoryCeiling + if allocMemory > 0 && float64(used) > ceiling { + scaleByNode[nodeName] = ceiling / float64(used) + } + } + if len(scaleByNode) == 0 { + return 0 + } + + for i := range cluster.Pods { + pod := &cluster.Pods[i] + nodeName := strings.TrimSpace(pod.NodeName) + scale, ok := scaleByNode[nodeName] + if !ok || !mockKubernetesPodIsActive(pod, nodeByName[nodeName]) { + continue + } + pod.UsageMemoryPercent *= scale + pod.UsageMemoryBytes = int64(float64(pod.UsageMemoryBytes) * scale) + } + return len(scaleByNode) +} + // reconcileMockKubernetesPodScheduling keeps pod placement coherent with // node readiness. It runs on every mock update tick, both mid-tick (with // randomize) and again after the demo scenario re-pins pod profiles, so it @@ -1916,6 +1997,7 @@ func normalizeMockKubernetesNodeCapacity(node *models.KubernetesNode) { func updateMockKubernetesPodUsage( pod *models.KubernetesPod, node *models.KubernetesNode, + activePodsOnNode int, now time.Time, randomize bool, clusterOffline bool, @@ -2002,8 +2084,9 @@ func updateMockKubernetesPodUsage( baseMemory := 9.0 + float64(mockStableHash64(seedID, "mem-base")%20) burstMemory := 14.0 + float64(mockStableHash64(seedID, "mem-burst")%36) - targetMemory := (baseMemory + activity*burstMemory) * (0.92 + ownerScale*0.12) - pod.UsageMemoryPercent = clampFloat(smoothMetricToward(pod.UsageMemoryPercent, targetMemory, 0.08), 3, 97) + memoryShare := mockKubernetesPodMemoryShare(activePodsOnNode) + targetMemory := (baseMemory + activity*burstMemory) * (0.92 + ownerScale*0.12) * memoryShare + pod.UsageMemoryPercent = clampFloat(smoothMetricToward(pod.UsageMemoryPercent, targetMemory, 0.08), 3*memoryShare, 97) pod.UsageMemoryBytes = int64(float64(allocMemory) * (pod.UsageMemoryPercent / 100.0)) baseNetIn := float64((24 + int(mockStableHash64(seedID, "netin-base")%180)) * 1024) diff --git a/internal/mock/generator_test.go b/internal/mock/generator_test.go index 98772cd2ff..0c76b8346d 100644 --- a/internal/mock/generator_test.go +++ b/internal/mock/generator_test.go @@ -478,6 +478,144 @@ func TestBuildFixtureStateCreatesHostEntriesForKubernetesNodes(t *testing.T) { } } +// newMockKubernetesTestCluster builds ready, schedulable nodes with a fixed +// number of running pods each, so placement does not depend on the random +// readiness the fixture generator rolls. +func newMockKubernetesTestCluster(nodeCount, podsPerNode int) models.KubernetesCluster { + cluster := models.KubernetesCluster{ID: "usage-test", Name: "usage-test", Status: "online"} + for n := 0; n < nodeCount; n++ { + nodeName := fmt.Sprintf("usage-test-%d", n) + cluster.Nodes = append(cluster.Nodes, models.KubernetesNode{ + UID: nodeName + "-uid", + Name: nodeName, + Ready: true, + CapacityCPU: 8, + AllocCPU: 8, + CapacityMemoryBytes: 32 << 30, + AllocMemoryBytes: 32 << 30, + }) + for p := 0; p < podsPerNode; p++ { + cluster.Pods = append(cluster.Pods, models.KubernetesPod{ + UID: fmt.Sprintf("%s-pod-%d-uid", nodeName, p), + Name: fmt.Sprintf("%s-pod-%d", nodeName, p), + Namespace: "apps", + NodeName: nodeName, + Phase: "Running", + OwnerKind: "Deployment", + }) + } + } + return cluster +} + +func TestMockKubernetesDenseNodesStayBelowAllocatableMemory(t *testing.T) { + cluster := newMockKubernetesTestCluster(3, 13) + data := models.StateSnapshot{KubernetesClusters: []models.KubernetesCluster{cluster}} + for _, node := range cluster.Nodes { + data.Hosts = append(data.Hosts, models.Host{ID: "host-" + node.Name, Hostname: node.Name, Status: "online"}) + } + + start := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) + for tick := 0; tick < 40; tick++ { + initializeMockKubernetesClusterUsage(&data.KubernetesClusters[0], start.Add(time.Duration(tick)*15*time.Second), false) + // The mock tick re-derives each linked host's memory from its node. + syncMockKubernetesNodeHosts(&data) + + podMemory := make(map[string]float64, len(cluster.Nodes)) + for _, pod := range data.KubernetesClusters[0].Pods { + podMemory[pod.NodeName] += pod.UsageMemoryPercent + } + for _, node := range data.KubernetesClusters[0].Nodes { + if node.UsageMemoryBytes >= node.AllocMemoryBytes || node.UsageMemoryPercent >= 100 { + t.Fatalf("tick %d: node %s with 13 running pods is pinned at allocatable memory (%.1f%%)", tick, node.Name, node.UsageMemoryPercent) + } + // Steady-state shares stay under the ceiling, which clips to it + // exactly; a total at the ceiling means it bound without a transient. + if podMemory[node.Name] >= mockKubernetesNodePodMemoryCeiling*100-0.01 { + t.Fatalf("tick %d: node %s pods hold %.2f%%, the ceiling bound in steady state", tick, node.Name, podMemory[node.Name]) + } + } + for _, host := range data.Hosts { + if host.Memory.Usage >= 100 { + t.Fatalf("tick %d: linked host %s memory = %.1f%%, want the node's unsaturated reading", tick, host.Hostname, host.Memory.Usage) + } + } + } +} + +func TestMockKubernetesRescheduledPodsDoNotSaturateReceivingNode(t *testing.T) { + cluster := newMockKubernetesTestCluster(3, 8) + start := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) + for tick := 0; tick < 40; tick++ { + initializeMockKubernetesClusterUsage(&cluster, start.Add(time.Duration(tick)*15*time.Second), false) + } + + // Two nodes fail: their sixteen pods move to the survivor carrying + // readings sized for nodes that each ran a third as many pods. + cluster.Nodes[0].Ready = false + cluster.Nodes[1].Ready = false + initializeMockKubernetesClusterUsage(&cluster, start.Add(40*15*time.Second), false) + + survivor := cluster.Nodes[2] + running := 0 + for _, pod := range cluster.Pods { + if pod.NodeName == survivor.Name { + running++ + } + } + if running != 24 { + t.Fatalf("survivor runs %d pods after the failures, want all 24", running) + } + // Pods hold at most the ceiling and system overhead tops out at 26%. + if limit := (mockKubernetesNodePodMemoryCeiling + 0.26) * 100; survivor.UsageMemoryPercent > limit+1e-9 { + t.Fatalf("survivor memory = %.1f%% right after rescheduling, want at most %.1f%%", survivor.UsageMemoryPercent, limit) + } +} + +func TestMockKubernetesPodMemoryShareKeepsNodeWithinBudget(t *testing.T) { + for pods := 0; pods <= 64; pods++ { + share := mockKubernetesPodMemoryShare(pods) + if share <= 0 || share > 1 { + t.Fatalf("share(%d) = %f, want (0, 1]", pods, share) + } + if committed := float64(pods) * share; committed > mockKubernetesNodePodMemoryBudget+1e-9 { + t.Fatalf("%d pods commit %.2f single-pod footprints, want at most %.2f", pods, committed, mockKubernetesNodePodMemoryBudget) + } + } +} + +func TestMockKubernetesLoneHeavyPodStaysUnderNodeCeiling(t *testing.T) { + // A five-minute cadence lets smoothing track each target closely, so the + // pod reaches its real peaks. + setMockUpdateInterval(5 * time.Minute) + t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) }) + + // The heaviest footprint: the highest base and burst seeds on a + // Guaranteed StatefulSet pod. + uid, best := "", uint64(0) + for i := 0; i < 2000; i++ { + candidate := fmt.Sprintf("lone-pod-%d", i) + weight := mockStableHash64(candidate, "mem-base")%20 + mockStableHash64(candidate, "mem-burst")%36 + if uid == "" || weight > best { + uid, best = candidate, weight + } + } + cluster := newMockKubernetesTestCluster(1, 1) + cluster.Pods[0].UID = uid + cluster.Pods[0].OwnerKind = "StatefulSet" + cluster.Pods[0].QoSClass = "Guaranteed" + + start := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC) + peak := 0.0 + for tick := 0; tick < 288; tick++ { + initializeMockKubernetesClusterUsage(&cluster, start.Add(time.Duration(tick)*5*time.Minute), false) + peak = math.Max(peak, cluster.Pods[0].UsageMemoryPercent) + } + if peak >= mockKubernetesNodePodMemoryCeiling*100-0.01 { + t.Fatalf("lone pod %s peaked at %.2f%%: the ceiling clipped a steady-state reading", uid, peak) + } +} + func TestGenerateNodesBoundsDirectConfiguration(t *testing.T) { cfg := DefaultConfig cfg.NodeCount = 1 << 30 diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 53182cf222..baab181eb9 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -1263,6 +1263,8 @@ type Monitor struct { recoveryIngestPending []recoveryIngestBatch mockMetricsCancel context.CancelFunc mockMetricsWg sync.WaitGroup + mockHostAgentsMu sync.Mutex + mockHostAgents map[string]models.Host // Fixture agents evaluated by the last mock alert pass dockerChecker DockerChecker // Optional Docker checker for LXC containers dockerCheckerConfiguredAt time.Time // Last time the Docker checker was configured dockerCheckAllowedVMIDs map[int]struct{} // Optional VMID allowlist gating the LXC Docker socket probe; empty means all guests @@ -4567,6 +4569,7 @@ func (m *Monitor) SetMockMode(enable bool) error { return err } m.alertManager.ClearActiveAlerts() + m.forgetMockHostAgents() m.mu.Lock() m.resetStateLocked() m.metricsHistory.Reset() diff --git a/internal/monitoring/monitor_alerts.go b/internal/monitoring/monitor_alerts.go index 46445c3ec7..96fecbde9e 100644 --- a/internal/monitoring/monitor_alerts.go +++ b/internal/monitoring/monitor_alerts.go @@ -686,6 +686,16 @@ func (m *Monitor) checkMockAlerts() { guestsChecked++ } + // Check alerts for host agents. Live agents are evaluated as each report + // lands, and mock mode discards reports and keeps fixture agents out of + // the monitor state the host health sweep reads, so without this pass no + // agent CPU, memory, disk, temperature or offline alert can open against + // mock data. Agents go before nodes so a linked node hands its CPU, memory + // and disk alerts to its agent on the first tick, as the host-agent + // hostname deduplication does in production. + log.Debug().Int("hostCount", len(state.Hosts)).Msg("checking host agent alerts") + m.evaluateMockHostAgents(state.Hosts, state.Nodes) + // Check alerts for each node for _, node := range state.Nodes { m.alertManager.CheckNode(node) @@ -727,6 +737,67 @@ func (m *Monitor) checkMockAlerts() { mock.UpdateAlertSnapshots(m.alertManager.GetActiveAlerts(), m.alertManager.GetRecentlyResolved()) } +// evaluateMockHostAgents evaluates every fixture agent and remembers the set. +// A runtime mock config change rebuilds the estate, so an agent can leave it +// between passes; it then goes through HandleHostRemoved, as a deleted live +// agent does, or its alerts and hostname deduplication would outlive it. +func (m *Monitor) evaluateMockHostAgents(hosts []models.Host, nodes []models.Node) { + m.mockHostAgentsMu.Lock() + defer m.mockHostAgentsMu.Unlock() + + // A pass that took its snapshot before the monitor left mock mode must + // not re-register fixture agents after forgetMockHostAgents ran. + if !mock.IsMockEnabled() { + return + } + + current := make(map[string]models.Host, len(hosts)) + for _, host := range hosts { + if host.ID != "" { + current[host.ID] = host + } + } + // Remove first so a departed agent cannot unregister a hostname that a + // remaining agent registers on this same pass. + for id, host := range m.mockHostAgents { + if _, ok := current[id]; !ok { + m.alertManager.HandleHostRemoved(host) + } + } + for _, host := range hosts { + m.checkMockHostAlerts(host, nodes) + } + m.mockHostAgents = current +} + +// forgetMockHostAgents removes the fixture agents when the monitor leaves mock +// mode. ClearActiveAlerts drops their alerts but not their hostname +// deduplication, so a real node named like a fixture agent (pve1) would keep +// its CPU, memory and disk alerts suppressed with no agent to own them. The +// removal also clears anything a pass already in flight recreated; mock mode +// is off before this runs, so no later pass can register them again. +func (m *Monitor) forgetMockHostAgents() { + m.mockHostAgentsMu.Lock() + defer m.mockHostAgentsMu.Unlock() + + for _, host := range m.mockHostAgents { + m.alertManager.HandleHostRemoved(host) + } + m.mockHostAgents = nil +} + +// checkMockHostAlerts applies the live host-agent boundary to a fixture. An +// offline fixture has no fresh telemetry, so it goes through the host +// connectivity lifecycle that evaluateHostAgents uses for a lapsed report. +// CheckHost treats its input as a fresh report and would mark the host online. +func (m *Monitor) checkMockHostAlerts(host models.Host, nodes []models.Node) { + if strings.EqualFold(strings.TrimSpace(host.Status), "offline") { + m.alertManager.HandleHostOfflineWithCorrelation(host, sharedSystemAlertCorrelationForHost(host, nodes)) + return + } + m.alertManager.CheckHost(host) +} + // checkMockDockerHostAlerts preserves the same evidence boundary as live // agent monitoring. An explicitly offline fixture is missing fresh container // telemetry; its last container states must not be reinterpreted as a fresh diff --git a/internal/monitoring/monitor_host_agent_removal_lifecycle_test.go b/internal/monitoring/monitor_host_agent_removal_lifecycle_test.go index 3b9eac2272..7459a3c610 100644 --- a/internal/monitoring/monitor_host_agent_removal_lifecycle_test.go +++ b/internal/monitoring/monitor_host_agent_removal_lifecycle_test.go @@ -812,3 +812,48 @@ func TestUnifiedStorageMetricSyncPreservesHostRemovalBlock(t *testing.T) { t.Fatal("storage metric sync allowed a removed host to report with its denied token") } } + +func TestMockHostAgentLeavingFixtureUsesRemovalLifecycle(t *testing.T) { + mustSetMockEnabled(t, true) + t.Cleanup(func() { mustSetMockEnabled(t, false) }) + manager := newMockHostAlertTestManager(t) + monitor := &Monitor{alertManager: manager} + + memory := models.Memory{Total: 32 << 30, Used: 30 << 30, Free: 2 << 30, Usage: 93.75} + kept := models.Host{ID: "host-linux-1", Hostname: "apollo-114", Status: "online", Memory: memory} + departed := models.Host{ID: "host-node-pve9", Hostname: "pve9", Status: "online", Memory: memory} + + monitor.evaluateMockHostAgents([]models.Host{kept, departed}, nil) + // A runtime mock config change rebuilt the estate without pve9. + monitor.evaluateMockHostAgents([]models.Host{kept}, nil) + + keptAlert := false + for _, alert := range manager.GetActiveAlerts() { + switch alert.ResourceID { + case "agent:" + departed.ID: + t.Fatalf("agent that left the mock estate kept alert %s", alert.ID) + case "agent:" + kept.ID: + keptAlert = keptAlert || alert.Type == "memory" + } + } + if !keptAlert { + t.Fatal("agent still in the mock estate lost its memory alert") + } + + // The departed agent's hostname deduplication is released too, so the + // node it was linked to owns its metric alerts again. + node := models.Node{ + ID: "mock-cluster-1-pve9", + Name: "pve9", + Instance: "mock-cluster-1", + Status: "online", + Memory: memory, + } + manager.CheckNode(node) + for _, alert := range manager.GetActiveAlerts() { + if alert.ResourceID == node.ID && alert.Type == "memory" { + return + } + } + t.Fatal("node pve9 raised no memory alert after its agent left the mock estate") +} diff --git a/internal/monitoring/monitor_mock_alerts_test.go b/internal/monitoring/monitor_mock_alerts_test.go index bd0bb9b10a..66e69d616a 100644 --- a/internal/monitoring/monitor_mock_alerts_test.go +++ b/internal/monitoring/monitor_mock_alerts_test.go @@ -1,9 +1,12 @@ package monitoring import ( + "strings" "testing" + "time" "github.com/rcourtman/pulse-go-rewrite/internal/alerts" + "github.com/rcourtman/pulse-go-rewrite/internal/mock" "github.com/rcourtman/pulse-go-rewrite/internal/models" ) @@ -50,3 +53,189 @@ func TestCheckMockDockerHostAlertsUsesHostLifecycleForOfflineFixtures(t *testing t.Fatalf("offline mock host alert = %+v, want canonical host connectivity incident", active[0]) } } + +func newMockHostAlertTestManager(t *testing.T) *alerts.Manager { + t.Helper() + manager := alerts.NewManagerWithDataDir(t.TempDir()) + t.Cleanup(manager.Stop) + config := manager.GetConfig() + config.Enabled = true + config.ActivationState = alerts.ActivationPending + config.TimeThresholds = map[string]int{} + config.MetricTimeThresholds = nil + config.BackupDefaults.Enabled = false + config.AgentDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 1, Clear: 0.5} + config.NodeDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 1, Clear: 0.5} + manager.UpdateConfig(config) + return manager +} + +func TestCheckMockAlertsEvaluatesHostAgentsBeforeLinkedNodes(t *testing.T) { + mustSetMockEnabled(t, true) + t.Cleanup(func() { mustSetMockEnabled(t, false) }) + manager := newMockHostAlertTestManager(t) + + monitor := &Monitor{alertManager: manager} + monitor.checkMockAlerts() + + // Agents reading well above the 1% trigger must alert. A NotReady + // Kubernetes node's agent validly reads 0%, and a later fixture read may + // drift slightly from the pass's own snapshot, so the bound leaves margin. + state := mock.CurrentFixtureGraph().State + onlineHosts := make(map[string]models.Host, len(state.Hosts)) + for _, host := range state.Hosts { + if !strings.EqualFold(strings.TrimSpace(host.Status), "offline") && host.Memory.HasKnownUsage() && host.Memory.Usage >= 5 { + onlineHosts[host.ID] = host + } + } + if len(onlineHosts) == 0 { + t.Fatal("mock fixture has no online host agents reading above 5% memory") + } + + active := manager.GetActiveAlerts() + agentMemoryAlerts := 0 + alertByResourceMetric := make(map[string]bool, len(active)) + for _, alert := range active { + alertByResourceMetric[alert.ResourceID+"/"+alert.Type] = true + if alert.Type != "memory" || !strings.HasPrefix(alert.ResourceID, "agent:") { + continue + } + if _, ok := onlineHosts[strings.TrimPrefix(alert.ResourceID, "agent:")]; ok { + agentMemoryAlerts++ + } + } + if agentMemoryAlerts != len(onlineHosts) { + t.Fatalf("mock pass opened %d agent memory alerts at a 1%% trigger, want one per online agent above 5%% (%d)", agentMemoryAlerts, len(onlineHosts)) + } + + // A node whose linked agent reported on the same pass must hand its + // memory alert to the agent on the first tick, as hostname + // deduplication does for live agents. + linkedNodes := 0 + for _, node := range state.Nodes { + host, ok := onlineHosts[strings.TrimSpace(node.LinkedAgentID)] + if !ok || !strings.EqualFold(host.Hostname, node.Name) { + continue + } + linkedNodes++ + if alertByResourceMetric[node.ID+"/memory"] { + t.Fatalf("node %s raised its own memory alert although linked agent %s reported on the same pass", node.Name, host.ID) + } + } + if linkedNodes == 0 { + t.Fatal("mock fixture has no online node with a linked agent") + } +} + +func TestCheckMockHostAlertsUsesHostLifecycleForOfflineFixtures(t *testing.T) { + manager := newMockHostAlertTestManager(t) + monitor := &Monitor{alertManager: manager} + + host := models.Host{ + ID: "host-linux-9", + Hostname: "branch-gateway", + DisplayName: "Branch Gateway", + Platform: "linux", + Status: "offline", + Memory: models.Memory{Total: 16 << 30, Used: 15 << 30, Usage: 93.75}, + } + + // Three offline observations confirm the connectivity incident. CheckHost + // would treat the stale fixture as a fresh report: it marks the host + // online and raises the stale 94% memory reading instead. + monitor.checkMockHostAlerts(host, nil) + monitor.checkMockHostAlerts(host, nil) + monitor.checkMockHostAlerts(host, nil) + + active := manager.GetActiveAlerts() + if len(active) != 1 || active[0].Type != alerts.HostOfflineAlertType || active[0].ResourceID != "agent:host-linux-9" { + got := make([]string, 0, len(active)) + for _, alert := range active { + got = append(got, alert.ResourceID+" "+alert.Type) + } + t.Fatalf("offline mock agent alerts = %v, want only the host-offline incident for agent:host-linux-9", got) + } + + host.Status = "online" + monitor.checkMockHostAlerts(host, nil) + for _, alert := range manager.GetActiveAlerts() { + if alert.Type == alerts.HostOfflineAlertType { + t.Fatalf("recovered mock agent kept its offline incident %s", alert.ID) + } + } +} + +func TestLeavingMockModeReleasesFixtureAgentHostnames(t *testing.T) { + setMockSamplerTestEnv(t, time.Hour, 5*time.Minute) + manager := newMockHostAlertTestManager(t) + monitor := &Monitor{ + state: models.NewState(), + alertManager: manager, + metricsHistory: NewMetricsHistory(10, time.Hour), + } + mustSetMonitorMockMode(t, monitor, true) + t.Cleanup(func() { mustSetMockEnabled(t, false) }) + monitor.checkMockAlerts() + + registeredPVE1 := false + for _, host := range mock.CurrentFixtureGraph().State.Hosts { + if host.Hostname == "pve1" && !strings.EqualFold(host.Status, "offline") { + registeredPVE1 = true + } + } + if !registeredPVE1 { + t.Fatal("mock fixture has no online agent named pve1") + } + mustSetMonitorMockMode(t, monitor, false) + + // A real node named like a fixture agent, with no agent of its own, + // must own its metric alerts once the monitor shows live data again. + node := models.Node{ + ID: "real-pve1", + Name: "pve1", + Instance: "real", + Status: "online", + Memory: models.Memory{Total: 64 << 30, Used: 60 << 30, Free: 4 << 30, Usage: 93.75}, + } + manager.CheckNode(node) + for _, alert := range manager.GetActiveAlerts() { + if alert.ResourceID == node.ID && alert.Type == "memory" { + return + } + } + t.Fatal("real node pve1 raised no memory alert: the fixture agent's hostname deduplication outlived mock mode") +} + +func TestMockHostAgentPassAfterLeavingMockModeRegistersNothing(t *testing.T) { + manager := newMockHostAlertTestManager(t) + monitor := &Monitor{alertManager: manager} + mustSetMockEnabled(t, false) + + // A pass whose snapshot predates leaving mock mode reaches the agent step + // after forgetMockHostAgents ran. + stale := models.Host{ + ID: "host-node-mock-cluster-1-pve1", + Hostname: "pve1", + Status: "online", + Memory: models.Memory{Total: 32 << 30, Used: 30 << 30, Free: 2 << 30, Usage: 93.75}, + } + monitor.evaluateMockHostAgents([]models.Host{stale}, nil) + + if active := manager.GetActiveAlerts(); len(active) != 0 { + t.Fatalf("stale mock pass opened %d alerts in live mode", len(active)) + } + node := models.Node{ + ID: "real-pve1", + Name: "pve1", + Instance: "real", + Status: "online", + Memory: models.Memory{Total: 64 << 30, Used: 60 << 30, Free: 4 << 30, Usage: 93.75}, + } + manager.CheckNode(node) + for _, alert := range manager.GetActiveAlerts() { + if alert.ResourceID == node.ID && alert.Type == "memory" { + return + } + } + t.Fatal("stale mock pass registered fixture hostname pve1 and suppressed the real node's memory alert") +} From 602e2d04cd51a316f1fd14ca5ad3bee7f611c5ea Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:19:17 +0100 Subject: [PATCH 12/27] Move the handover tests off the end of the contract test file main appended platform-stamping tests to the end of operational_contract_test.go after this branch appended the handover tests there, so the two insertions conflicted. Placing the handover tests beside the resolved-history test lets main merge in cleanly. No test changes. --- internal/alerts/operational_contract_test.go | 312 +++++++++---------- 1 file changed, 156 insertions(+), 156 deletions(-) diff --git a/internal/alerts/operational_contract_test.go b/internal/alerts/operational_contract_test.go index a4d0dc43bc..51922249de 100644 --- a/internal/alerts/operational_contract_test.go +++ b/internal/alerts/operational_contract_test.go @@ -413,6 +413,162 @@ func TestResolvedOperationalContractUpdatesHistorySnapshot(t *testing.T) { } } +func TestReleasedMetricAlertCarriesResolutionInsteadOfRecovery(t *testing.T) { + manager := newUnifiedEvalParityManager(t) + store, err := eventlog.OpenInMemory() + if err != nil { + t.Fatalf("open in-memory event log: %v", err) + } + manager.SetEventLog(store) + manager.UpdateConfig(AlertConfig{ + Enabled: true, + NodeDefaults: ThresholdConfig{ + CPU: &HysteresisThreshold{Trigger: 80, Clear: 75}, + Memory: &HysteresisThreshold{Trigger: 85, Clear: 80}, + }, + }) + disableTestTimeThresholds(manager) + node := models.Node{ + ID: "pve-1", + Name: "pve-1", + Instance: "test", + Status: "online", + CPU: 0.95, + Memory: models.Memory{Total: 8 << 30, Used: 7 << 30, Usage: 95}, + } + manager.CheckNode(node) + if active := manager.GetActiveAlerts(); len(active) != 2 { + t.Fatalf("active alerts = %#v, want cpu and memory", active) + } + + resolvedByType := map[string]*ResolvedAlert{} + manager.SetResolvedAlertCallback(func(resolved *ResolvedAlert) { + resolvedByType[resolved.Alert.Type] = resolved + }) + release := func(metric string, resolution *AlertResolution) { + t.Helper() + spec, err := buildCanonicalMetricSpec(node.ID, node.Name, unifiedresources.ResourceType("node"), metric, nil) + if err != nil { + t.Fatalf("build %s spec: %v", metric, err) + } + manager.releaseCanonicalMetricAlert(spec, node.Name, node.Name, node.Instance, "node", 0, resolution) + } + + handover := &AlertResolution{ + Reason: AlertResolutionMovedToAgent, + SuccessorResourceID: "agent-pve-1", + SuccessorName: "pve-1 (Host Agent)", + } + release("memory", handover) + // The close is recorded as a copy: the caller's value cannot rewrite it. + handover.SuccessorName = "changed" + release("cpu", nil) + + if active := manager.GetActiveAlerts(); len(active) != 0 { + t.Fatalf("active alerts after release = %#v, want none: the close must still happen", active) + } + + moved := resolvedByType["memory"] + if moved == nil { + t.Fatal("memory release did not reach the resolved consumer") + } + resolution := moved.Alert.Resolution + if resolution == nil || + resolution.Reason != AlertResolutionMovedToAgent || + resolution.SuccessorResourceID != "agent-pve-1" || + resolution.SuccessorName != "pve-1 (Host Agent)" { + t.Fatalf("resolution = %+v, want moved to pve-1 (Host Agent)", resolution) + } + wantSummary := "Alert moved to pve-1 (Host Agent). This is not a recovery: check the agent for the current reading." + if got := resolution.Summary(); got != wantSummary { + t.Fatalf("Summary() = %q, want %q", got, wantSummary) + } + if got := resolution.Outcome(); got != "moved to pve-1 (Host Agent)" { + t.Fatalf("Outcome() = %q", got) + } + if got := resolution.Describe("Memory alert"); got != "Memory alert moved to pve-1 (Host Agent). This is not a recovery: check the agent for the current reading." { + t.Fatalf("Describe() = %q", got) + } + + record := moved.Alert.OperationalRecord + if record == nil || record.State != operationaltrust.OperationalResolved { + t.Fatalf("operational record = %+v, want resolved", record) + } + if err := record.Validate(); err != nil { + t.Fatalf("operational record Validate() error = %v", err) + } + transition := moved.Alert.LatestTransition + if transition == nil || transition.Reason != wantSummary { + t.Fatalf("resolve transition = %+v, want reason %q", transition, wantSummary) + } + if err := transition.Validate(); err != nil { + t.Fatalf("resolve transition Validate() error = %v", err) + } + closing := moved.Alert.Evidence[len(moved.Alert.Evidence)-1] + if closing.ID != transition.EvidenceIDs[0] || + closing.Reason == nil || + closing.Reason.Code != string(AlertResolutionMovedToAgent) || + closing.Reason.Message != wantSummary { + t.Fatalf("closing evidence = %+v, want the handover, not a recovery projection", closing) + } + + recovered := resolvedByType["cpu"] + if recovered == nil || recovered.Alert.Resolution != nil { + t.Fatalf("plain disabled release = %+v, want an ordinary resolve with no resolution", recovered) + } + if last := recovered.Alert.Evidence[len(recovered.Alert.Evidence)-1]; last.Reason == nil || + last.Reason.Code != legacyAlertRecoveryEvidenceReason { + t.Fatalf("plain release evidence = %+v, want legacy recovery projection", last.Reason) + } + + var historyResolution *AlertResolution + for _, entry := range manager.GetAlertHistory(10) { + if entry.Type == "memory" { + historyResolution = entry.Resolution + } + } + if historyResolution == nil || historyResolution.Reason != AlertResolutionMovedToAgent { + t.Fatalf("history resolution = %+v, want moved_to_agent", historyResolution) + } + + events, err := manager.AlertEvents(eventlog.Filter{AlertID: moved.Alert.ID, Types: []string{eventlog.TypeResolved}}) + if err != nil { + t.Fatalf("AlertEvents: %v", err) + } + if len(events) != 1 || + events[0].Reason != string(AlertResolutionMovedToAgent) || + events[0].Message != wantSummary { + t.Fatalf("resolved events = %+v, want one handover event", events) + } +} + +func TestResolvedHistoryRowKeepsResolutionWithoutEventLog(t *testing.T) { + firstObservedAt := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) + alert := &Alert{ + ID: "pve-1-memory", + Type: "memory", + Level: AlertLevelWarning, + ResourceID: "pve-1", + CanonicalSpecID: "metric-threshold:memory", + CanonicalState: "pve-1::metric-threshold:memory", + StartTime: firstObservedAt, + LastSeen: firstObservedAt, + } + ensureOperationalContract(alert, firstObservedAt) + history := newTestHistoryManager(t) + history.AddAlert(*alert) + manager := &Manager{historyManager: history} + + alert.Resolution = &AlertResolution{Reason: AlertResolutionMovedToAgent, SuccessorName: "pve-1 (Host Agent)"} + manager.newResolvedAlert(alert, firstObservedAt.Add(time.Minute), nil) + + entries := history.GetAllHistory(1) + if len(entries) != 1 || entries[0].Resolution == nil || + entries[0].Resolution.Reason != AlertResolutionMovedToAgent { + t.Fatalf("history entries = %+v, want the row to keep the moved_to_agent resolution", entries) + } +} + func TestMergeOperationalRecurrenceReopensStableCause(t *testing.T) { firstObservedAt := time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC) resolvedAt := firstObservedAt.Add(5 * time.Minute) @@ -745,159 +901,3 @@ func TestResolvedOccurrenceCallbackSurvivesLegacyPanic(t *testing.T) { t.Fatal("legacy panic prevented occurrence-qualified recovery") } } - -func TestReleasedMetricAlertCarriesResolutionInsteadOfRecovery(t *testing.T) { - manager := newUnifiedEvalParityManager(t) - store, err := eventlog.OpenInMemory() - if err != nil { - t.Fatalf("open in-memory event log: %v", err) - } - manager.SetEventLog(store) - manager.UpdateConfig(AlertConfig{ - Enabled: true, - NodeDefaults: ThresholdConfig{ - CPU: &HysteresisThreshold{Trigger: 80, Clear: 75}, - Memory: &HysteresisThreshold{Trigger: 85, Clear: 80}, - }, - }) - disableTestTimeThresholds(manager) - node := models.Node{ - ID: "pve-1", - Name: "pve-1", - Instance: "test", - Status: "online", - CPU: 0.95, - Memory: models.Memory{Total: 8 << 30, Used: 7 << 30, Usage: 95}, - } - manager.CheckNode(node) - if active := manager.GetActiveAlerts(); len(active) != 2 { - t.Fatalf("active alerts = %#v, want cpu and memory", active) - } - - resolvedByType := map[string]*ResolvedAlert{} - manager.SetResolvedAlertCallback(func(resolved *ResolvedAlert) { - resolvedByType[resolved.Alert.Type] = resolved - }) - release := func(metric string, resolution *AlertResolution) { - t.Helper() - spec, err := buildCanonicalMetricSpec(node.ID, node.Name, unifiedresources.ResourceType("node"), metric, nil) - if err != nil { - t.Fatalf("build %s spec: %v", metric, err) - } - manager.releaseCanonicalMetricAlert(spec, node.Name, node.Name, node.Instance, "node", 0, resolution) - } - - handover := &AlertResolution{ - Reason: AlertResolutionMovedToAgent, - SuccessorResourceID: "agent-pve-1", - SuccessorName: "pve-1 (Host Agent)", - } - release("memory", handover) - // The close is recorded as a copy: the caller's value cannot rewrite it. - handover.SuccessorName = "changed" - release("cpu", nil) - - if active := manager.GetActiveAlerts(); len(active) != 0 { - t.Fatalf("active alerts after release = %#v, want none: the close must still happen", active) - } - - moved := resolvedByType["memory"] - if moved == nil { - t.Fatal("memory release did not reach the resolved consumer") - } - resolution := moved.Alert.Resolution - if resolution == nil || - resolution.Reason != AlertResolutionMovedToAgent || - resolution.SuccessorResourceID != "agent-pve-1" || - resolution.SuccessorName != "pve-1 (Host Agent)" { - t.Fatalf("resolution = %+v, want moved to pve-1 (Host Agent)", resolution) - } - wantSummary := "Alert moved to pve-1 (Host Agent). This is not a recovery: check the agent for the current reading." - if got := resolution.Summary(); got != wantSummary { - t.Fatalf("Summary() = %q, want %q", got, wantSummary) - } - if got := resolution.Outcome(); got != "moved to pve-1 (Host Agent)" { - t.Fatalf("Outcome() = %q", got) - } - if got := resolution.Describe("Memory alert"); got != "Memory alert moved to pve-1 (Host Agent). This is not a recovery: check the agent for the current reading." { - t.Fatalf("Describe() = %q", got) - } - - record := moved.Alert.OperationalRecord - if record == nil || record.State != operationaltrust.OperationalResolved { - t.Fatalf("operational record = %+v, want resolved", record) - } - if err := record.Validate(); err != nil { - t.Fatalf("operational record Validate() error = %v", err) - } - transition := moved.Alert.LatestTransition - if transition == nil || transition.Reason != wantSummary { - t.Fatalf("resolve transition = %+v, want reason %q", transition, wantSummary) - } - if err := transition.Validate(); err != nil { - t.Fatalf("resolve transition Validate() error = %v", err) - } - closing := moved.Alert.Evidence[len(moved.Alert.Evidence)-1] - if closing.ID != transition.EvidenceIDs[0] || - closing.Reason == nil || - closing.Reason.Code != string(AlertResolutionMovedToAgent) || - closing.Reason.Message != wantSummary { - t.Fatalf("closing evidence = %+v, want the handover, not a recovery projection", closing) - } - - recovered := resolvedByType["cpu"] - if recovered == nil || recovered.Alert.Resolution != nil { - t.Fatalf("plain disabled release = %+v, want an ordinary resolve with no resolution", recovered) - } - if last := recovered.Alert.Evidence[len(recovered.Alert.Evidence)-1]; last.Reason == nil || - last.Reason.Code != legacyAlertRecoveryEvidenceReason { - t.Fatalf("plain release evidence = %+v, want legacy recovery projection", last.Reason) - } - - var historyResolution *AlertResolution - for _, entry := range manager.GetAlertHistory(10) { - if entry.Type == "memory" { - historyResolution = entry.Resolution - } - } - if historyResolution == nil || historyResolution.Reason != AlertResolutionMovedToAgent { - t.Fatalf("history resolution = %+v, want moved_to_agent", historyResolution) - } - - events, err := manager.AlertEvents(eventlog.Filter{AlertID: moved.Alert.ID, Types: []string{eventlog.TypeResolved}}) - if err != nil { - t.Fatalf("AlertEvents: %v", err) - } - if len(events) != 1 || - events[0].Reason != string(AlertResolutionMovedToAgent) || - events[0].Message != wantSummary { - t.Fatalf("resolved events = %+v, want one handover event", events) - } -} - -func TestResolvedHistoryRowKeepsResolutionWithoutEventLog(t *testing.T) { - firstObservedAt := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) - alert := &Alert{ - ID: "pve-1-memory", - Type: "memory", - Level: AlertLevelWarning, - ResourceID: "pve-1", - CanonicalSpecID: "metric-threshold:memory", - CanonicalState: "pve-1::metric-threshold:memory", - StartTime: firstObservedAt, - LastSeen: firstObservedAt, - } - ensureOperationalContract(alert, firstObservedAt) - history := newTestHistoryManager(t) - history.AddAlert(*alert) - manager := &Manager{historyManager: history} - - alert.Resolution = &AlertResolution{Reason: AlertResolutionMovedToAgent, SuccessorName: "pve-1 (Host Agent)"} - manager.newResolvedAlert(alert, firstObservedAt.Add(time.Minute), nil) - - entries := history.GetAllHistory(1) - if len(entries) != 1 || entries[0].Resolution == nil || - entries[0].Resolution.Reason != AlertResolutionMovedToAgent { - t.Fatalf("history entries = %+v, want the row to keep the moved_to_agent resolution", entries) - } -} From d4e94a7e349ab2bc870a46866c9b37b3f0f42ba9 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:36:09 +0100 Subject: [PATCH 13/27] Keep the availability result readable and give names room MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A check observed from several locations showed "failed…" in its Result cell, with a "2 locations · 2/2 reporting" chip spilling past the column. The chip was given flex next to the badge's own inline-flex, which left it inline beside the result, so the result clipped at every width up to 1440px. The result and the chip now share one line. The result keeps its full width at the cell's right edge, and the chip, now "2/2 locations reporting" as Settings words it, truncates to its left with the full text on hover. Rows stay one line because the table windows them at a single measured height. On phones, Method and Target took the room and cut check names to "Workshop do…". They now hide there, as the row drawer still shows them, and the Method cell truncates with an ellipsis instead of clipping. --- .../subsystems/frontend-primitives.md | 6 ++ .../internal/subsystems/unified-resources.md | 8 +++ .../46c7544c2f493da0.json | 51 ++++++++++++++++ .../standalone/AvailabilityChecksTable.tsx | 58 ++++++++++++------- .../AvailabilityChecksTable.test.tsx | 44 ++++++++++++++ 5 files changed, 145 insertions(+), 22 deletions(-) create mode 100644 frontend-modern/browser-verification/46c7544c2f493da0.json diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index aa974bf928..be7ee95ae8 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -7558,6 +7558,12 @@ inside the fleet surface with explicit copy while current status and resource navigation remain usable. Desktop and phone layouts must keep every tile keyboard-operable, preserve the textual legend, and avoid horizontal clipping at fleet scale. +In the table, a secondary `MetadataBadge` beside a cell's primary value shares +one flex line with it: the value is `shrink-0` and the badge `min-w-0` with a +truncating label, so the value never clips and rows keep the single height +`PlatformWindowedRows` measures. A second line under the value would break +that windowing. Passing `flex` to the badge alongside its own `inline-flex` +left the display to stylesheet order, which clipped the availability result. The Machines page must not pretend its machine list is a generic overview: the default tab is `Machines`, the Machines table is only for Pulse Agent-backed resources with host telemetry, and the full availability-check row list belongs diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index b91304e4d5..d739d3722b 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -3460,6 +3460,14 @@ with the offline threshold on hover, not an "N/M" fraction. Recent check timing and fuller failure context may stay in tooltip or drawer detail, but the table row must not duplicate the same probe protocol and result text across both identity and metric cells. +The probe source chip ("via Edge 01", or "2/2 locations reporting" for a +multi-location check, the same wording Settings uses) shares the result's +single line: the result keeps its full width at the cell's right edge and the +chip takes the room left, truncating with its full text on hover. Before, the +chip pushed "failed" out of its column at every width up to 1440px. Rows stay +one line because the table windows them at a single measured height. On +phones the method and target columns hide so the check name reads, and both +stay in the row drawer. That same frontend-owned compatibility boundary must remain intentionally narrow. Shared resource adapters may admit explicit aliases such as `host`, `truenas`, and `ceph`, and VMware detail mappers may project typed metadata diff --git a/frontend-modern/browser-verification/46c7544c2f493da0.json b/frontend-modern/browser-verification/46c7544c2f493da0.json new file mode 100644 index 0000000000..eb1a480d20 --- /dev/null +++ b/frontend-modern/browser-verification/46c7544c2f493da0.json @@ -0,0 +1,51 @@ +{ + "version": 1, + "base_sha": "c4db7a3fba35a14ee41675483726213644b31edc", + "verified_at": "2026-10-06T14:33:38Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/features/standalone/AvailabilityChecksTable.tsx" + ], + "content_sha256": { + "frontend-modern/src/features/standalone/AvailabilityChecksTable.tsx": "d8bfb5052c0c150ebf1c11e1672f8776e1caa159805a66774ebaaf57bd66a6a2" + }, + "routes": [ + "/standalone/availability" + ], + "viewports": [ + { + "width": 390, + "height": 844 + }, + { + "width": 600, + "height": 844 + }, + { + "width": 768, + "height": 1000 + }, + { + "width": 1024, + "height": 1000 + }, + { + "width": 1440, + "height": 1000 + }, + { + "width": 1920, + "height": 1000 + } + ], + "states": [ + "Private mock stack on main b19c6aaf70 plus this change (rebased onto c4db7a3fba, whose backup-alert revalidation touches no frontend file); seven availability checks, one of them (Frontend Swarm service) observed from two locations and failing", + "Before: that row's Result cell read 'failed\u2026' with a '2 locations \u00b7 2/2 reporting' chip spilling past the column (the chip's flex and the badge's inline-flex left it inline); the result clipped at every width up to 1440. On phones the check names were cut to 'Workshop do\u2026', 'Frontend Sw\u2026' (7 rows clipped by up to 77px) while Method and Target took the room", + "After: one line per row, 32-33px at 390, 768 and 1440; the result ('failed') is whole at the cell's right edge and the source chip ('2/2 locations reporting', Settings' wording) truncates to its left with the full text on hover. On phones Method and Target hide (still in the drawer) and check names read in full except one by 3px; the Method cell now truncates with an ellipsis where it clipped" + ], + "interactions": [ + "Measured every visible header and cell for clipping at six widths", + "Measured row heights at three widths", + "Hovered the source chip for its full text" + ] +} diff --git a/frontend-modern/src/features/standalone/AvailabilityChecksTable.tsx b/frontend-modern/src/features/standalone/AvailabilityChecksTable.tsx index 1d7debdaee..18506c5dc2 100644 --- a/frontend-modern/src/features/standalone/AvailabilityChecksTable.tsx +++ b/frontend-modern/src/features/standalone/AvailabilityChecksTable.tsx @@ -275,12 +275,12 @@ export const AvailabilityChecksTable: Component<{ Check Method Target @@ -329,7 +329,7 @@ export const AvailabilityChecksTable: Component<{ const target = () => formatTarget(check); const probeSource = () => (availability()?.locations?.length ?? 0) > 1 - ? `${availability()?.locations?.length} locations · ${availability()?.reportingLocations ?? 0}/${availability()?.expectedLocations ?? availability()?.locations?.length ?? 0} reporting` + ? `${availability()?.reportingLocations ?? 0}/${availability()?.expectedLocations ?? availability()?.locations?.length ?? 0} locations reporting` : getProbeSourceChipLabel( props.probeAgentOptions ?? [], availability()?.probeAgentId, @@ -366,13 +366,17 @@ export const AvailabilityChecksTable: Component<{
+ {/* On a phone the check name needs the room; the method + and target stay in the row drawer. */} - {method()} + + {method()} + {target()} @@ -381,22 +385,32 @@ export const AvailabilityChecksTable: Component<{ - - {result()} - - - {(sourceLabel) => ( - - {sourceLabel()} - - )} - + {/* One line, as the table windows rows at a single + height: the result stays whole at the right edge + and the probe source takes what room is left, + truncating with its full text on hover. */} +
+ + {(sourceLabel) => ( + + {sourceLabel()} + + )} + + + {result()} + +

{alert.description}

+ + {(detail) =>

{detail()}

} +
diff --git a/frontend-modern/src/features/alerts/AlertHistoryTableAlertRow.tsx b/frontend-modern/src/features/alerts/AlertHistoryTableAlertRow.tsx index 9187526f50..13d1b3b797 100644 --- a/frontend-modern/src/features/alerts/AlertHistoryTableAlertRow.tsx +++ b/frontend-modern/src/features/alerts/AlertHistoryTableAlertRow.tsx @@ -74,7 +74,7 @@ export function AlertHistoryTableAlertRow(props: AlertHistoryTableAlertRowProps) - + {historyStatusPresentation().label} diff --git a/frontend-modern/src/features/alerts/alertHistoryModel.ts b/frontend-modern/src/features/alerts/alertHistoryModel.ts index bd2d89af25..097a560da5 100644 --- a/frontend-modern/src/features/alerts/alertHistoryModel.ts +++ b/frontend-modern/src/features/alerts/alertHistoryModel.ts @@ -1,5 +1,9 @@ import type { Alert } from '@/types/api'; import type { Resource } from '@/types/resource'; +import { + ALERT_HISTORY_MOVED_STATUS, + getAlertResolutionDetail, +} from '@/utils/alertIncidentPresentation'; import { isPulseSystemAlert } from '@/utils/alertScope'; import { alertTypeDisplayLabel, unifiedTypeToAlertDisplayType } from './helpers'; @@ -25,6 +29,8 @@ export interface HistoryItem { title: string; rawAlertType?: string; description?: string; + // Set when the alert closed without recovering: what happened instead. + closeDetail?: string; acknowledged?: boolean; systemAlert?: boolean; } @@ -206,11 +212,17 @@ export function buildAlertHistoryItems({ alertHistory.forEach((alert) => { if (activeAlertIds.has(alert.id)) return; const systemScoped = isPulseSystemAlert(alert); + // A close that was not a recovery must not read as "resolved". + const closeDetail = getAlertResolutionDetail(alert.resolution); items.push({ id: alert.id, source: 'alert', - status: alert.acknowledged ? 'acknowledged' : 'resolved', + status: closeDetail + ? ALERT_HISTORY_MOVED_STATUS + : alert.acknowledged + ? 'acknowledged' + : 'resolved', startTime: alert.startTime, endTime: alert.lastSeen, duration: formatAlertHistoryDuration(alert.startTime, alert.lastSeen, now), @@ -231,6 +243,7 @@ export function buildAlertHistoryItems({ title: alertTypeDisplayLabel(alert.type), rawAlertType: alert.type, description: alert.message, + ...(closeDetail ? { closeDetail } : {}), acknowledged: alert.acknowledged, }); }); diff --git a/frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx b/frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx index 76bbcbe03d..72ac52cd43 100644 --- a/frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx +++ b/frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx @@ -68,6 +68,7 @@ import { copyToClipboard } from '@/utils/clipboard'; import type { PatrolAutonomyLevel } from '@/api/patrol'; import { getAttentionFlappingPresentation, + getAttentionHandoverClose, getLinkedPatrolFindings, getPatrolRememberedDecisionPresentation, partitionPatrolAttention, @@ -971,7 +972,10 @@ function AttentionDetail(props: {
- + @@ -984,6 +988,9 @@ function AttentionDetail(props: {

{loaded().item.plainLanguageSummary}

+ + {(handover) =>

{handover()}

} +
{(flapping) => (

@@ -1793,10 +1800,17 @@ function severityTone(item: AttentionItem): MetadataBadgeTone { } } -function StateBadge(props: { item: AttentionItem }) { +// movedClose marks a resolved item whose condition moved to another owner: it +// did not recover, so it must not wear the green "Resolved" badge. +function StateBadge(props: { item: AttentionItem; movedClose?: boolean }) { return ( - - {formatLabel(props.item.state)} + + {props.movedClose ? 'Moved' : formatLabel(props.item.state)} ); } diff --git a/frontend-modern/src/features/patrol/__tests__/PatrolAttentionWorkbench.test.tsx b/frontend-modern/src/features/patrol/__tests__/PatrolAttentionWorkbench.test.tsx index 3c5ff2b423..e3a01ebc10 100644 --- a/frontend-modern/src/features/patrol/__tests__/PatrolAttentionWorkbench.test.tsx +++ b/frontend-modern/src/features/patrol/__tests__/PatrolAttentionWorkbench.test.tsx @@ -988,6 +988,56 @@ describe('PatrolAttentionWorkbench', () => { expect(screen.getByText('Show all 11 transitions')).toBeInTheDocument(); }); + it('shows an alert handed to its Pulse agent as moved, not recovered', async () => { + const reason = + 'Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading.'; + const moved = item({ + id: 'record-moved', + operationalRecordId: 'record-moved', + subjectResourceId: 'pve1', + subjectResourceName: 'pve1', + subjectResourceType: 'node', + kind: 'memory', + title: 'Memory on pve1', + plainLanguageSummary: 'Memory usage at 95%', + severity: 'warning', + state: 'resolved', + protectionPosture: undefined, + }); + const movedDetail = detail(moved); + movedDetail.timeline = [ + ...movedDetail.timeline, + { + id: 'transition-moved', + operationalRecordId: 'record-moved', + from: 'open', + to: 'resolved', + at: '2026-07-19T07:45:00Z', + cause: 'ownership_transferred', + causeKey: 'memory:pve1', + evidenceIds: ['evidence-handover'], + reason, + }, + ]; + window.history.replaceState({}, '', '/patrol?attention=record-moved'); + apiMocks.getList.mockResolvedValue(listResponse([], summary())); + apiMocks.getDetail.mockResolvedValue(movedDetail); + renderWorkbench(); + + const panel = await screen.findByRole('complementary', { name: 'pve1 · Memory' }); + expect(await within(panel).findByText('Moved')).toBeInTheDocument(); + expect(within(panel).queryByText('Resolved')).not.toBeInTheDocument(); + // The account sits under the summary, outside the collapsed history. + const summaryLine = within(panel).getByText('Memory usage at 95%'); + expect(summaryLine.nextElementSibling).toHaveTextContent(reason); + expect(summaryLine.nextElementSibling?.closest('details')).toBeNull(); + + fireEvent.click(within(panel).getByText('Evidence and history')); + expect(within(panel).getByText('Open to Resolved')).toBeInTheDocument(); + expect(within(panel).getByText(/Ownership transferred/)).toBeInTheDocument(); + expect(within(panel).queryByText(/Recovery evidence/)).not.toBeInTheDocument(); + }); + it('opens the canonical governed action review from an eligible attention item', async () => { const actionOffer = { targetResourceId: 'docker:host-1/container-1', diff --git a/frontend-modern/src/features/patrol/__tests__/patrolHomePresentation.test.ts b/frontend-modern/src/features/patrol/__tests__/patrolHomePresentation.test.ts index 95d4ccdd28..9b07da74a7 100644 --- a/frontend-modern/src/features/patrol/__tests__/patrolHomePresentation.test.ts +++ b/frontend-modern/src/features/patrol/__tests__/patrolHomePresentation.test.ts @@ -1,7 +1,8 @@ import { describe, expect, it } from 'vitest'; import type { PatrolObjective } from '@/api/patrol'; -import type { AttentionItem } from '@/api/patrolAttention'; +import type { AttentionItem, AttentionItemDetail } from '@/api/patrolAttention'; import { + getAttentionHandoverClose, getPatrolAttentionDecisionReason, getPatrolObjectiveProtectionSummary, partitionPatrolAttention, @@ -145,3 +146,62 @@ describe('patrol home presentation', () => { }); }); }); + +describe('getAttentionHandoverClose', () => { + const transition = ( + overrides: Partial, + ): AttentionItemDetail['timeline'][number] => ({ + id: 'transition-1', + operationalRecordId: 'record-1', + from: 'open', + to: 'resolved', + at: '2026-08-14T08:00:00Z', + cause: 'recovery_evidence', + causeKey: 'memory:pve1', + evidenceIds: ['evidence-1'], + ...overrides, + }); + const reason = + 'Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading.'; + + it('returns the handover account only for a record closed by an ownership transfer', () => { + const resolved = attentionItem({ state: 'resolved' }); + expect( + getAttentionHandoverClose({ + item: resolved, + timeline: [transition({ cause: 'ownership_transferred', reason })], + }), + ).toBe(reason); + expect( + getAttentionHandoverClose({ + item: resolved, + timeline: [transition({ cause: 'ownership_transferred' })], + }), + ).toBe('This alert moved to another owner when it closed. It did not recover.'); + expect(getAttentionHandoverClose({ item: resolved, timeline: [transition({})] })).toBeNull(); + }); + + it('follows the latest close, so a later recovery or reopen wins', () => { + const handover = transition({ cause: 'ownership_transferred', reason }); + const reopened = transition({ + id: 'transition-2', + from: 'resolved', + to: 'open', + at: '2026-08-14T09:00:00Z', + cause: 'detector_decision', + }); + const recovered = transition({ id: 'transition-3', at: '2026-08-14T10:00:00Z' }); + expect( + getAttentionHandoverClose({ + item: attentionItem({ state: 'resolved' }), + timeline: [handover, reopened, recovered], + }), + ).toBeNull(); + expect( + getAttentionHandoverClose({ + item: attentionItem({ state: 'open' }), + timeline: [handover, reopened], + }), + ).toBeNull(); + }); +}); diff --git a/frontend-modern/src/features/patrol/patrolHomePresentation.ts b/frontend-modern/src/features/patrol/patrolHomePresentation.ts index 78d15e94ce..29ae650713 100644 --- a/frontend-modern/src/features/patrol/patrolHomePresentation.ts +++ b/frontend-modern/src/features/patrol/patrolHomePresentation.ts @@ -1,5 +1,5 @@ import type { PatrolAutonomyLevel, PatrolObjective } from '@/api/patrol'; -import type { AttentionFlapping, AttentionItem } from '@/api/patrolAttention'; +import type { AttentionFlapping, AttentionItem, AttentionItemDetail } from '@/api/patrolAttention'; import type { UnifiedFinding } from '@/stores/aiIntelligence'; export interface PatrolAutonomyExperience { @@ -207,6 +207,26 @@ export function getAttentionFlappingPresentation( return getFlappingPresentation(flapping.transitionCount, flapping.windowHours); } +/** + * A resolved item whose close handed the condition to another record, such as + * a node metric its Pulse agent took over, did not recover. Returns the + * account to show instead of a healthy "Resolved", or null for a recovery. + */ +export function getAttentionHandoverClose( + detail: Pick | null | undefined, +): string | null { + if (!detail || detail.item.state !== 'resolved') return null; + let close: AttentionItemDetail['timeline'][number] | undefined; + for (const transition of detail.timeline ?? []) { + if (transition.to !== 'resolved') continue; + if (!close || Date.parse(transition.at) >= Date.parse(close.at)) close = transition; + } + if (close?.cause !== 'ownership_transferred') return null; + return ( + close.reason?.trim() || 'This alert moved to another owner when it closed. It did not recover.' + ); +} + const normalizeResourceKey = (value: string | undefined): string => (value ?? '').trim().toLowerCase(); diff --git a/frontend-modern/src/pages/__tests__/Alerts.helpers.test.ts b/frontend-modern/src/pages/__tests__/Alerts.helpers.test.ts index 75e996aa76..31016b5670 100644 --- a/frontend-modern/src/pages/__tests__/Alerts.helpers.test.ts +++ b/frontend-modern/src/pages/__tests__/Alerts.helpers.test.ts @@ -100,6 +100,7 @@ import { buildProjectedOverrides, normalizeRawOverridesConfig, } from '@/features/alerts/alertOverridesModel'; +import { buildAlertHistoryItems } from '@/features/alerts/alertHistoryModel'; import { getAlertIncidentAcknowledgedBadgeClass, getAlertIncidentEventFilterActionButtonClass, @@ -1320,3 +1321,59 @@ describe('Unified selector parity', () => { ); }); }); + +describe('alert history close outcome', () => { + it('lists a node alert handed to its Pulse agent as moved, not resolved', () => { + const base = { + type: 'memory', + level: 'warning' as const, + resourceName: 'pve1', + node: 'pve1', + instance: 'pve1', + value: 95, + threshold: 85, + startTime: '2026-10-06T09:00:00Z', + lastSeen: '2026-10-06T11:00:00Z', + acknowledged: true, + }; + const items = buildAlertHistoryItems({ + activeAlerts: {}, + alertHistory: [ + { + ...base, + id: 'pve1-memory', + resourceId: 'pve1', + message: 'Memory usage at 95%', + resolution: { + reason: 'moved_to_agent', + successorResourceId: 'agent-pve1', + successorName: 'pve1 (Host Agent)', + }, + }, + { + ...base, + id: 'pve1-cpu', + resourceId: 'pve1', + message: 'CPU usage at 90%', + acknowledged: false, + }, + ], + getResource: () => undefined, + allResources: [], + now: Date.parse('2026-10-06T12:00:00Z'), + }); + + const moved = items.find((item) => item.id === 'pve1-memory'); + // Acknowledgement does not hide the outcome: the condition went to the agent. + expect(moved?.status).toBe('moved'); + expect(moved?.description).toBe('Memory usage at 95%'); + expect(moved?.closeDetail).toBe( + 'Moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading.', + ); + const recovered = items.find((item) => item.id === 'pve1-cpu'); + expect(recovered?.status).toBe('resolved'); + expect(recovered).not.toHaveProperty('closeDetail'); + + expect(alertHistoryTableAlertRowSource).toContain('title={props.alert.closeDetail}'); + }); +}); diff --git a/frontend-modern/src/types/api.ts b/frontend-modern/src/types/api.ts index 5e7da6d6de..f1a16f2483 100644 --- a/frontend-modern/src/types/api.ts +++ b/frontend-modern/src/types/api.ts @@ -1380,6 +1380,18 @@ export interface Alert { latestTransition?: LifecycleTransition; transitions?: LifecycleTransition[]; evidence?: EvidenceEnvelope[]; + // Set only on a closed alert whose condition did not clear, such as a node + // metric handed to its Pulse agent. Absent for an ordinary recovery. + resolution?: AlertResolution; +} + +// Why an alert closed without recovering (alerts.AlertResolution). +export interface AlertResolution { + reason: 'moved_to_agent' | (string & {}); + successorResourceId?: string; + successorName?: string; + // The alert engine's one-line account; carried on state snapshots only. + summary?: string; } export interface ResolvedAlert extends Alert { diff --git a/frontend-modern/src/types/operationalTrust.ts b/frontend-modern/src/types/operationalTrust.ts index feeb98a9d2..67ebc0bb7c 100644 --- a/frontend-modern/src/types/operationalTrust.ts +++ b/frontend-modern/src/types/operationalTrust.ts @@ -97,6 +97,7 @@ export type TransitionCause = | 'suppression' | 'suppression_expired' | 'recovery_evidence' + | 'ownership_transferred' | 'collection_stale' | 'collection_unknown'; diff --git a/frontend-modern/src/utils/__tests__/alertIncidentPresentation.test.ts b/frontend-modern/src/utils/__tests__/alertIncidentPresentation.test.ts index 2e4ad59696..deebbc497c 100644 --- a/frontend-modern/src/utils/__tests__/alertIncidentPresentation.test.ts +++ b/frontend-modern/src/utils/__tests__/alertIncidentPresentation.test.ts @@ -10,8 +10,10 @@ import { ALERT_RESOURCE_INCIDENT_REFRESH_LABEL, ALERT_RESOURCE_INCIDENT_SAVE_NOTE_LABEL, ALERT_RESOURCE_INCIDENT_SAVING_NOTE_LABEL, + ALERT_HISTORY_MOVED_STATUS, getAlertHistoryStatusPresentation, getAlertIncidentLevelBadgeClass, + getAlertResolutionDetail, getAlertIncidentTimelineDialogTitle, getAlertHistoryInvestigationDialogTitle, getAlertResourceIncidentAcknowledgedByLabel, @@ -103,6 +105,29 @@ describe('alertIncidentPresentation', () => { className: 'text-xs px-2 py-0.5 rounded-sm bg-surface-hover text-base-content', rowClassName: '', }); + + expect(getAlertHistoryStatusPresentation(ALERT_HISTORY_MOVED_STATUS)).toEqual({ + label: 'moved to agent', + className: + 'text-xs px-2 py-0.5 rounded-sm bg-blue-100 dark:bg-blue-900/25 text-blue-700 dark:text-blue-300', + rowClassName: '', + }); + }); + + it('says a handed-over alert did not recover, and leaves recoveries alone', () => { + expect(getAlertResolutionDetail(undefined)).toBeUndefined(); + expect( + getAlertResolutionDetail({ reason: 'moved_to_agent', successorName: 'pve1 (Host Agent)' }), + ).toBe( + 'Moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading.', + ); + expect(getAlertResolutionDetail({ reason: 'moved_to_agent' })).toBe( + 'Moved to its Pulse agent. This is not a recovery: check the agent for the current reading.', + ); + // A future reason the UI has no copy for still shows the engine's account. + expect( + getAlertResolutionDetail({ reason: 'replaced', summary: 'Alert replaced by a new check.' }), + ).toBe('Alert replaced by a new check.'); }); it('returns canonical resource incident loading and empty-state copy', () => { diff --git a/frontend-modern/src/utils/alertIncidentPresentation.ts b/frontend-modern/src/utils/alertIncidentPresentation.ts index 5649918d23..af6b6d0721 100644 --- a/frontend-modern/src/utils/alertIncidentPresentation.ts +++ b/frontend-modern/src/utils/alertIncidentPresentation.ts @@ -1,3 +1,4 @@ +import type { AlertResolution } from '@/types/api'; import { getAlertSeverityBadgeClass } from '@/utils/alertSeverityPresentation'; export type AlertIncidentLevel = 'warning' | 'critical' | string | null | undefined; @@ -76,6 +77,24 @@ export function getAlertIncidentLevelBadgeClass(level: AlertIncidentLevel): stri return getAlertSeverityBadgeClass(String(level ?? 'warning')); } +// History status for an alert that closed because its Pulse agent took the +// metric over. The condition did not clear; the agent's own alert owns it. +export const ALERT_HISTORY_MOVED_STATUS = 'moved'; + +/** + * The account of a close that was not a recovery, for history rows and their + * tooltips. Returns undefined for an ordinary recovery so callers keep their + * "resolved" wording. + */ +export function getAlertResolutionDetail(resolution?: AlertResolution | null): string | undefined { + if (!resolution) return undefined; + if (resolution.reason === 'moved_to_agent') { + const successor = resolution.successorName?.trim() || 'its Pulse agent'; + return `Moved to ${successor}. This is not a recovery: check the agent for the current reading.`; + } + return resolution.summary?.trim() || undefined; +} + export function getAlertHistoryStatusPresentation( status?: string | null, ): AlertHistoryStatusPresentation { @@ -99,6 +118,17 @@ export function getAlertHistoryStatusPresentation( }; } + // Closed without recovering: neither a live problem on this row nor a + // healthy outcome, so it gets its own neutral-blue badge. + if (normalized === ALERT_HISTORY_MOVED_STATUS) { + return { + label: 'moved to agent', + className: + 'text-xs px-2 py-0.5 rounded-sm bg-blue-100 dark:bg-blue-900/25 text-blue-700 dark:text-blue-300', + rowClassName: '', + }; + } + return { label: normalized || 'resolved', className: 'text-xs px-2 py-0.5 rounded-sm bg-surface-hover text-base-content', diff --git a/internal/ai/alert_adapter.go b/internal/ai/alert_adapter.go index a8a5cd4d35..f34652fd83 100644 --- a/internal/ai/alert_adapter.go +++ b/internal/ai/alert_adapter.go @@ -2,6 +2,7 @@ package ai import ( "fmt" + "strings" "time" "github.com/rcourtman/pulse-go-rewrite/internal/alerts" @@ -56,6 +57,7 @@ func (a *AlertManagerAdapter) GetRecentlyResolved(minutes int) []ResolvedAlertIn AlertInfo: convertAlertFromModels(&resolved.Alert), ResolvedTime: resolved.ResolvedTime, Duration: formatDuration(resolved.ResolvedTime.Sub(resolved.StartTime)), + Resolution: resolvedAlertResolution(resolved.Resolution), } result = append(result, info) } @@ -98,6 +100,7 @@ func (a *AlertManagerAdapter) GetAlertHistory(resourceID string, limit int) []Re AlertInfo: convertAlertFromModels(&resolved.Alert), ResolvedTime: resolved.ResolvedTime, Duration: formatDuration(resolved.ResolvedTime.Sub(resolved.StartTime)), + Resolution: resolvedAlertResolution(resolved.Resolution), } result = append(result, info) if len(result) >= limit { @@ -245,3 +248,12 @@ func formatDuration(d time.Duration) string { } return fmt.Sprintf("%d days", days) } + +// resolvedAlertResolution is the one-line account of a close that was not a +// recovery, empty for an ordinary resolve. +func resolvedAlertResolution(resolution *models.AlertResolution) string { + if resolution == nil { + return "" + } + return strings.TrimSpace(resolution.Summary) +} diff --git a/internal/ai/alert_provider.go b/internal/ai/alert_provider.go index 6c36e68ece..3b7a3b8852 100644 --- a/internal/ai/alert_provider.go +++ b/internal/ai/alert_provider.go @@ -31,6 +31,9 @@ type ResolvedAlertInfo struct { AlertInfo ResolvedTime time.Time `json:"resolved_time"` Duration string `json:"total_duration"` // How long the alert lasted + // Resolution is set only when the alert closed without recovering, such + // as a node metric handed to its Pulse agent. + Resolution string `json:"resolution,omitempty"` } // AlertProvider provides access to the current alert state @@ -131,6 +134,12 @@ func (s *Service) buildAlertContext() string { } for i := 0; i < limit; i++ { a := recentlyResolved[i] + if resolution := strings.TrimSpace(a.Resolution); resolution != "" { + sections = append(sections, fmt.Sprintf("- **%s** on %s: %s (lasted %s, closed %s ago) %s", + a.Type, a.ResourceName, a.Message, a.Duration, + formatTimeAgo(a.ResolvedTime), resolution)) + continue + } sections = append(sections, fmt.Sprintf("- **%s** on %s: %s (lasted %s, resolved %s ago)", a.Type, a.ResourceName, a.Message, a.Duration, formatTimeAgo(a.ResolvedTime))) diff --git a/internal/ai/alert_provider_test.go b/internal/ai/alert_provider_test.go index c223df3060..26680472d2 100644 --- a/internal/ai/alert_provider_test.go +++ b/internal/ai/alert_provider_test.go @@ -5,6 +5,8 @@ import ( "strings" "testing" "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/models" ) type stubAlertProvider struct { @@ -151,6 +153,61 @@ func TestService_buildAlertContext_NoActiveWithResolved(t *testing.T) { } } +// The assistant reads recently resolved alerts through the adapter, so a node +// alert handed to its Pulse agent must reach the prompt as a handover and not +// as "resolved", which the model would read as the node being healthy. +func TestService_buildAlertContext_HandoverIsNotARecovery(t *testing.T) { + now := time.Now() + summary := "Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading." + adapter := NewAlertManagerAdapter(&stubAlertManager{resolved: []models.ResolvedAlert{ + { + Alert: models.Alert{ + ID: "pve1-memory", + Type: "memory", + Level: "warning", + ResourceID: "pve1", + ResourceName: "pve1", + Message: "Memory usage at 95%", + StartTime: now.Add(-time.Hour), + Resolution: &models.AlertResolution{Reason: "moved_to_agent", Summary: summary}, + }, + ResolvedTime: now.Add(-time.Minute), + }, + { + Alert: models.Alert{ + ID: "pve2-cpu", + Type: "cpu", + Level: "warning", + ResourceID: "pve2", + ResourceName: "pve2", + Message: "CPU usage at 90%", + StartTime: now.Add(-time.Hour), + }, + ResolvedTime: now.Add(-time.Minute), + }, + }}) + + resolved := adapter.GetRecentlyResolved(30) + if len(resolved) != 2 || resolved[0].Resolution != summary || resolved[1].Resolution != "" { + t.Fatalf("GetRecentlyResolved = %+v, want only the handover to carry a resolution", resolved) + } + if history := adapter.GetAlertHistory("pve1", 5); len(history) != 1 || history[0].Resolution != summary { + t.Fatalf("GetAlertHistory = %+v, want the handover resolution", history) + } + + s := &Service{} + s.SetAlertProvider(adapter) + ctx := s.buildAlertContext() + if !strings.Contains(ctx, "- **memory** on pve1: Memory usage at 95% (lasted ") || + !strings.Contains(ctx, ", closed 1 minute ago) "+summary) { + t.Fatalf("expected the handover line, got: %s", ctx) + } + if !strings.Contains(ctx, "- **cpu** on pve2: CPU usage at 90% (lasted ") || + !strings.Contains(ctx, ", resolved 1 minute ago)") { + t.Fatalf("expected the ordinary recovery line, got: %s", ctx) + } +} + func TestService_buildTargetAlertContext(t *testing.T) { s := &Service{} s.SetAlertProvider(&stubAlertProvider{ diff --git a/internal/ai/memory/incidents.go b/internal/ai/memory/incidents.go index eaab0dd572..8729c83cef 100644 --- a/internal/ai/memory/incidents.go +++ b/internal/ai/memory/incidents.go @@ -474,6 +474,17 @@ func (s *IncidentStore) RecordAlertUnacknowledged(alert *alerts.Alert, user stri s.saveAsync() } +// alertResolvedEventSummary is the timeline line for an alert's close: the +// handover account when it closed without recovering, else "Alert resolved". +func alertResolvedEventSummary(alert *alerts.Alert) string { + if alert != nil { + if summary := alert.Resolution.Summary(); summary != "" { + return summary + } + } + return "Alert resolved" +} + // RecordAlertResolved records a resolved event and closes the incident. func (s *IncidentStore) RecordAlertResolved(alert *alerts.Alert, resolvedAt time.Time) { if alert == nil { @@ -507,7 +518,7 @@ func (s *IncidentStore) RecordAlertResolved(alert *alerts.Alert, resolvedAt time shell.OccurrenceClosedAt = cloneTime(resolvedAt) if !s.projectsFromCanonicalLocked() { - s.addEventAtLocked(shell, IncidentEventAlertResolved, resolvedAt, "Alert resolved", map[string]interface{}{ + s.addEventAtLocked(shell, IncidentEventAlertResolved, resolvedAt, alertResolvedEventSummary(alert), map[string]interface{}{ "resolved_at": resolvedAt.Format(time.RFC3339), }) } @@ -563,7 +574,7 @@ func (s *IncidentStore) EnsureAlertOccurrence(alert *alerts.Alert, resolvedAt *t if resolvedAt != nil && !resolvedAt.IsZero() && (shell.OccurrenceRefiredAt == nil || !resolvedAt.Before(*shell.OccurrenceRefiredAt)) { shell.OccurrenceClosedAt = cloneTime(*resolvedAt) if !hasIncidentEventType(shell.Events, IncidentEventAlertResolved) { - s.addEventAtLocked(shell, IncidentEventAlertResolved, *resolvedAt, "Alert resolved", map[string]interface{}{ + s.addEventAtLocked(shell, IncidentEventAlertResolved, *resolvedAt, alertResolvedEventSummary(alert), map[string]interface{}{ "resolved_at": resolvedAt.Format(time.RFC3339), "projection_source": incidentSnapshotSource, }) @@ -1113,6 +1124,13 @@ func incidentEventSummaryFromChange(change unifiedresources.ResourceChange, even case IncidentEventAlertUnsnoozed: return "Alert notifications resumed" case IncidentEventAlertResolved: + // A handover closes the alert without recovering; its change reason + // says so and where the condition went. + if _, ok := stringMetadata(change.Metadata, unifiedresources.MetadataAlertResolution); ok { + if reason := strings.TrimSpace(change.Reason); reason != "" { + return reason + } + } return "Alert resolved" default: if summary := strings.TrimSpace(change.Reason); summary != "" { diff --git a/internal/ai/memory/incidents_test.go b/internal/ai/memory/incidents_test.go index 0d218a9a2a..94d8930593 100644 --- a/internal/ai/memory/incidents_test.go +++ b/internal/ai/memory/incidents_test.go @@ -239,6 +239,39 @@ func TestIncidentStore_EqualTimestampLifecycleKeepsResolvedState(t *testing.T) { } } +// Without a canonical resource timeline the store writes its own close event, +// which must say the alert moved rather than resolved. +func TestIncidentStore_HandoverCloseIsNotARecovery(t *testing.T) { + store := NewIncidentStore(IncidentStoreConfig{MaxIncidents: 10, MaxEventsPerIncident: 10, MaxAgeDays: 30}) + startedAt := time.Now().UTC().Truncate(time.Second).Add(-time.Hour) + movedAt := startedAt.Add(30 * time.Minute) + alert := &alerts.Alert{ + ID: "pve1-memory", + Type: "memory", + ResourceID: "pve1", + ResourceName: "pve1", + Message: "Memory usage at 95%", + StartTime: startedAt, + LastSeen: startedAt, + Resolution: &alerts.AlertResolution{Reason: alerts.AlertResolutionMovedToAgent, SuccessorName: "pve1 (Host Agent)"}, + } + summary := "Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading." + + store.RecordAlertFired(alert) + store.RecordAlertResolved(alert, movedAt) + timeline := store.GetTimelineByAlertAt(alert.ID, startedAt) + if timeline == nil || len(timeline.Events) != 2 || timeline.Events[1].Summary != summary { + t.Fatalf("recorded timeline = %#v, want the handover close", timeline) + } + + snapshot := alert.Clone() + snapshot.ID = "pve1-memory-imported" + repaired := store.EnsureAlertOccurrence(snapshot, &movedAt) + if repaired == nil || len(repaired.Events) != 2 || repaired.Events[1].Summary != summary { + t.Fatalf("repaired timeline = %#v, want the handover close", repaired) + } +} + func TestIncidentStore_ProjectsCanonicalTimelineWhenAttached(t *testing.T) { store := NewIncidentStore(IncidentStoreConfig{ MaxIncidents: 10, diff --git a/internal/ai/patrol_ai.go b/internal/ai/patrol_ai.go index 41e3feacfa..ac6c9072fc 100644 --- a/internal/ai/patrol_ai.go +++ b/internal/ai/patrol_ai.go @@ -3499,6 +3499,12 @@ func (p *PatrolService) seedHealthAndAlertsState(snap patrolRuntimeState, scoped sb.WriteString("# Recently Resolved Alerts\n") for _, r := range alerts { ago := seedFormatTimeAgo(now, r.ResolvedTime) + // A handover closes the alert without the condition clearing, so + // it must not read as the resource having recovered. + if r.Resolution != nil && strings.TrimSpace(r.Resolution.Summary) != "" { + sb.WriteString(fmt.Sprintf("- %s — closed %s. %s\n", seedAlertSubject(r.Alert), ago, strings.TrimSpace(r.Resolution.Summary))) + continue + } sb.WriteString(fmt.Sprintf("- %s — resolved %s\n", seedAlertSubject(r.Alert), ago)) } sb.WriteString("\n") diff --git a/internal/ai/patrol_ai_more_test.go b/internal/ai/patrol_ai_more_test.go index b7f168e86d..7613f10a47 100644 --- a/internal/ai/patrol_ai_more_test.go +++ b/internal/ai/patrol_ai_more_test.go @@ -922,6 +922,42 @@ func TestSeedHealthAndAlerts_NoIssues(t *testing.T) { } } +// Patrol's seed must not tell the model a node recovered when its alert only +// moved to the node's Pulse agent. +func TestSeedHealthAndAlerts_HandoverIsNotARecovery(t *testing.T) { + ps := NewPatrolService(nil, nil) + cfg := DefaultPatrolConfig() + now := time.Now() + summary := "Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading." + state := models.StateSnapshot{ + RecentlyResolved: []models.ResolvedAlert{ + { + Alert: models.Alert{ + ResourceName: "pve1", + Message: "Memory usage at 95%", + Resolution: &models.AlertResolution{Reason: "moved_to_agent", Summary: summary}, + }, + ResolvedTime: now.Add(-3 * time.Minute), + }, + { + Alert: models.Alert{ResourceName: "pve2", Message: "CPU usage at 90%"}, + ResolvedTime: now.Add(-5 * time.Minute), + }, + }, + } + + out := ps.seedHealthAndAlertsState(patrolRuntimeStateForTest(ps, state), nil, cfg, now) + if !strings.Contains(out, "- pve1: Memory usage at 95% — closed 3m ago. "+summary+"\n") { + t.Fatalf("expected the handover line, got: %s", out) + } + if strings.Contains(out, "pve1: Memory usage at 95% — resolved") { + t.Fatalf("handover reported as a recovery: %s", out) + } + if !strings.Contains(out, "- pve2: CPU usage at 90% — resolved 5m ago\n") { + t.Fatalf("expected the ordinary recovery line, got: %s", out) + } +} + func TestSeedHealthAndAlerts_UnknownDiskHealthDoesNotClaimSMARTPassed(t *testing.T) { ps := NewPatrolService(nil, nil) cfg := DefaultPatrolConfig() diff --git a/internal/ai/tools/data_types.go b/internal/ai/tools/data_types.go index db3360534a..39c0043c25 100644 --- a/internal/ai/tools/data_types.go +++ b/internal/ai/tools/data_types.go @@ -2364,4 +2364,9 @@ type ResolvedAlertSummary struct { Threshold float64 `json:"threshold,omitempty"` StartTime time.Time `json:"start_time"` ResolvedTime time.Time `json:"resolved_time"` + // Resolution is set only when the alert closed without recovering, such + // as a node metric handed to its Pulse agent; the successor resource + // owns the condition from then on. + Resolution string `json:"resolution,omitempty"` + SuccessorResourceID string `json:"successor_resource_id,omitempty"` } diff --git a/internal/ai/tools/tools_alerts.go b/internal/ai/tools/tools_alerts.go index 252d0dc7f1..1d94770676 100644 --- a/internal/ai/tools/tools_alerts.go +++ b/internal/ai/tools/tools_alerts.go @@ -19,7 +19,7 @@ func (e *PulseToolExecutor) registerAlertsTools() { Actions: - list: List active threshold alerts (CPU > 80%, disk full, etc.) - findings: List AI patrol findings (detected issues) -- resolved: List recently resolved alerts +- resolved: List recently resolved alerts; one with a resolution closed without recovering - resolve: Mark a finding as resolved - dismiss: Dismiss a finding as not an issue @@ -417,7 +417,7 @@ func (e *PulseToolExecutor) executeListResolvedAlerts(_ context.Context, args ma break } - alerts = append(alerts, ResolvedAlertSummary{ + summary := ResolvedAlertSummary{ ID: alert.ID, Type: alert.Type, Level: alert.Level, @@ -430,7 +430,12 @@ func (e *PulseToolExecutor) executeListResolvedAlerts(_ context.Context, args ma Threshold: alert.Threshold, StartTime: alert.StartTime, ResolvedTime: alert.ResolvedTime, - }) + } + if resolution := alert.Resolution; resolution != nil { + summary.Resolution = resolution.Summary + summary.SuccessorResourceID = resolution.SuccessorResourceID + } + alerts = append(alerts, summary) } if alerts == nil { diff --git a/internal/ai/tools/tools_metrics_alerts_test.go b/internal/ai/tools/tools_metrics_alerts_test.go index 5f5e6aa1bb..ada13ba204 100644 --- a/internal/ai/tools/tools_metrics_alerts_test.go +++ b/internal/ai/tools/tools_metrics_alerts_test.go @@ -215,6 +215,49 @@ func TestExecuteListResolvedAlerts(t *testing.T) { if len(resp.Alerts) != 1 || resp.Alerts[0].ID != "a1" { t.Fatalf("unexpected resolved alerts: %+v", resp) } + if resp.Alerts[0].Resolution != "" || resp.Alerts[0].SuccessorResourceID != "" { + t.Fatalf("a recovery must not carry a resolution: %+v", resp.Alerts[0]) + } +} + +// A node alert handed to its Pulse agent closes without recovering, so the +// assistant must see that, and where the condition went, instead of a plain +// resolve it would read as the node being healthy again. +func TestExecuteListResolvedAlertsReportsHandover(t *testing.T) { + alertProv := &mockAlertProvider{} + executor := NewPulseToolExecutor(ExecutorConfig{}) + executor.alertProvider = alertProv + now := time.Now() + summary := "Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading." + alertProv.On("GetRecentlyResolved", 24*60).Return([]models.ResolvedAlert{{ + Alert: models.Alert{ + ID: "pve1-memory", + Type: "memory", + Level: "warning", + ResourceID: "pve1", + ResourceName: "pve1", + Message: "Memory usage at 95%", + StartTime: now.Add(-time.Hour), + Resolution: &models.AlertResolution{ + Reason: "moved_to_agent", + SuccessorResourceID: "agent-pve1", + SuccessorName: "pve1 (Host Agent)", + Summary: summary, + }, + }, + ResolvedTime: now, + }}) + + result, _ := executor.executeListResolvedAlerts(context.Background(), map[string]interface{}{}) + var resp ResolvedAlertsResponse + if err := json.Unmarshal([]byte(result.Content[0].Text), &resp); err != nil { + t.Fatalf("decode resolved alerts: %v", err) + } + if len(resp.Alerts) != 1 || + resp.Alerts[0].Resolution != summary || + resp.Alerts[0].SuccessorResourceID != "agent-pve1" { + t.Fatalf("resolved alerts = %+v, want the handover and its successor", resp.Alerts) + } } func TestExecuteListAlertsAndFindings(t *testing.T) { diff --git a/internal/alerts/history_test.go b/internal/alerts/history_test.go index 37c066d232..89430dc711 100644 --- a/internal/alerts/history_test.go +++ b/internal/alerts/history_test.go @@ -7,7 +7,9 @@ import ( "testing" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" ) // newTestHistoryManager creates a HistoryManager using a temp directory @@ -1058,3 +1060,62 @@ func TestSaveHistoryWithRetry_RestoresBackupOnFailure(t *testing.T) { _ = hm } + +// The state snapshot, the websocket and the assistant read recently resolved +// alerts through GetRecentlyResolved, so a handover must reach them as one +// and not as a recovery. +func TestRecentlyResolvedCarriesHandoverResolution(t *testing.T) { + manager := newUnifiedEvalParityManager(t) + manager.UpdateConfig(AlertConfig{ + Enabled: true, + NodeDefaults: ThresholdConfig{ + CPU: &HysteresisThreshold{Trigger: 80, Clear: 75}, + Memory: &HysteresisThreshold{Trigger: 85, Clear: 80}, + }, + }) + disableTestTimeThresholds(manager) + node := models.Node{ + ID: "pve-1", + Name: "pve-1", + Instance: "test", + Status: "online", + CPU: 0.95, + Memory: models.Memory{Total: 8 << 30, Used: 7 << 30, Usage: 95}, + } + manager.CheckNode(node) + release := func(metric string, resolution *AlertResolution) { + t.Helper() + spec, err := buildCanonicalMetricSpec(node.ID, node.Name, unifiedresources.ResourceType("node"), metric, nil) + if err != nil { + t.Fatalf("build %s spec: %v", metric, err) + } + manager.releaseCanonicalMetricAlert(spec, node.Name, node.Name, node.Instance, "node", 0, resolution) + } + release("memory", &AlertResolution{ + Reason: AlertResolutionMovedToAgent, + SuccessorResourceID: "agent-pve-1", + SuccessorName: "pve-1 (Host Agent)", + }) + release("cpu", nil) + + byType := map[string]models.ResolvedAlert{} + for _, resolved := range manager.GetRecentlyResolved() { + byType[resolved.Type] = resolved + } + moved, ok := byType["memory"] + if !ok { + t.Fatalf("recently resolved = %+v, want the moved memory alert", byType) + } + want := models.AlertResolution{ + Reason: "moved_to_agent", + SuccessorResourceID: "agent-pve-1", + SuccessorName: "pve-1 (Host Agent)", + Summary: "Alert moved to pve-1 (Host Agent). This is not a recovery: check the agent for the current reading.", + } + if moved.Resolution == nil || *moved.Resolution != want { + t.Fatalf("moved resolution = %+v, want %+v", moved.Resolution, want) + } + if recovered, ok := byType["cpu"]; !ok || recovered.Resolution != nil { + t.Fatalf("recovered cpu alert = %+v, want an ordinary resolve with no resolution", recovered) + } +} diff --git a/internal/alerts/operational_contract.go b/internal/alerts/operational_contract.go index f59fe89a1b..5da5ff5e6e 100644 --- a/internal/alerts/operational_contract.go +++ b/internal/alerts/operational_contract.go @@ -285,6 +285,7 @@ func markOperationalResolved( } alert.Evidence = appendOperationalEvidence(alert.Evidence, recoveryEvidence.Clone()) recoveryEvidenceIDs := []string{recoveryEvidence.ID} + cause := resolvedTransitionCause(alert.Resolution) from := record.State record.State = operationaltrust.OperationalResolved record.StateChangedAt = resolvedAt @@ -295,7 +296,7 @@ func markOperationalResolved( from, operationaltrust.OperationalResolved, resolvedAt, - operationaltrust.TransitionRecoveryEvidence, + cause, record.CauseKey, recoveryEvidenceIDs, ) @@ -308,7 +309,7 @@ func markOperationalResolved( From: from, To: operationaltrust.OperationalResolved, At: resolvedAt, - Cause: operationaltrust.TransitionRecoveryEvidence, + Cause: cause, CauseKey: record.CauseKey, EvidenceIDs: recoveryEvidenceIDs, Reason: alert.Resolution.Summary(), @@ -319,6 +320,16 @@ func markOperationalResolved( ) } +// resolvedTransitionCause names why a record entered resolved. Only a close +// with no typed resolution is a recovery; a handover keeps its own cause so +// timelines and alert-quality counts never report it as one. +func resolvedTransitionCause(resolution *AlertResolution) operationaltrust.TransitionCause { + if resolution != nil && resolution.Reason == AlertResolutionMovedToAgent { + return operationaltrust.TransitionOwnershipTransferred + } + return operationaltrust.TransitionRecoveryEvidence +} + func (m *Manager) newResolvedAlert( alert *Alert, resolvedAt time.Time, diff --git a/internal/alerts/operational_contract_test.go b/internal/alerts/operational_contract_test.go index f483dd11b0..c15a8b48f5 100644 --- a/internal/alerts/operational_contract_test.go +++ b/internal/alerts/operational_contract_test.go @@ -504,6 +504,15 @@ func TestReleasedMetricAlertCarriesResolutionInsteadOfRecovery(t *testing.T) { if err := transition.Validate(); err != nil { t.Fatalf("resolve transition Validate() error = %v", err) } + // Patrol's timeline prints the cause above the reason, so a handover + // must not be labelled as recovery evidence. + if transition.Cause != operationaltrust.TransitionOwnershipTransferred { + t.Fatalf("resolve transition cause = %q, want %q", transition.Cause, operationaltrust.TransitionOwnershipTransferred) + } + if recorded := moved.Alert.Transitions[len(moved.Alert.Transitions)-1]; recorded.ID != transition.ID || + recorded.Cause != operationaltrust.TransitionOwnershipTransferred { + t.Fatalf("recorded timeline transition = %+v, want the ownership transfer", recorded) + } closing := moved.Alert.Evidence[len(moved.Alert.Evidence)-1] if closing.ID != transition.EvidenceIDs[0] || closing.Reason == nil || @@ -520,6 +529,10 @@ func TestReleasedMetricAlertCarriesResolutionInsteadOfRecovery(t *testing.T) { last.Reason.Code != legacyAlertRecoveryEvidenceReason { t.Fatalf("plain release evidence = %+v, want legacy recovery projection", last.Reason) } + if recovered.Alert.LatestTransition == nil || + recovered.Alert.LatestTransition.Cause != operationaltrust.TransitionRecoveryEvidence { + t.Fatalf("plain release transition = %+v, want recovery_evidence", recovered.Alert.LatestTransition) + } var historyResolution *AlertResolution for _, entry := range manager.GetAlertHistory(10) { diff --git a/internal/alerts/read_model.go b/internal/alerts/read_model.go index a30630c5ba..68382bc6ea 100644 --- a/internal/alerts/read_model.go +++ b/internal/alerts/read_model.go @@ -301,7 +301,8 @@ func (m *Manager) GetRecentlyResolved() []models.ResolvedAlert { StartTime: exported.StartTime, Acknowledged: exported.Acknowledged, // exported is a deep clone, so the map is already private. - Metadata: exported.Metadata, + Metadata: exported.Metadata, + Resolution: modelAlertResolution(exported.Resolution), }, ResolvedTime: alert.ResolvedTime, }) @@ -309,6 +310,20 @@ func (m *Manager) GetRecentlyResolved() []models.ResolvedAlert { return resolved } +// modelAlertResolution projects a non-recovery close into the state model, so +// snapshot and assistant readers can tell a handover from a recovery. +func modelAlertResolution(resolution *AlertResolution) *models.AlertResolution { + if resolution == nil { + return nil + } + return &models.AlertResolution{ + Reason: string(resolution.Reason), + SuccessorResourceID: resolution.SuccessorResourceID, + SuccessorName: resolution.SuccessorName, + Summary: resolution.Summary(), + } +} + // GetResolvedAlert returns a copy of a recently resolved alert by ID. func (m *Manager) GetResolvedAlert(alertID string) *ResolvedAlert { // Write lock: getResolvedAlertNoLock backfills the resolvedAlias map on a diff --git a/internal/models/deepcopy.go b/internal/models/deepcopy.go index 15d487f630..184633c596 100644 --- a/internal/models/deepcopy.go +++ b/internal/models/deepcopy.go @@ -1337,6 +1337,10 @@ func cloneAlert(src Alert) Alert { dest.Metadata[k] = v } } + if src.Resolution != nil { + resolution := *src.Resolution + dest.Resolution = &resolution + } return dest } diff --git a/internal/models/metrics_types_test.go b/internal/models/metrics_types_test.go index 86d4df09d0..8c29f1aeb9 100644 --- a/internal/models/metrics_types_test.go +++ b/internal/models/metrics_types_test.go @@ -96,6 +96,43 @@ func TestAlertLastSeenWireContract(t *testing.T) { } } +// A resolved alert that closed without recovering says so on the state and +// websocket payloads; an ordinary recovery carries no resolution at all. +func TestAlertResolutionWireContract(t *testing.T) { + recovered, err := json.Marshal(ResolvedAlert{Alert: Alert{ID: "alert-1"}}) + if err != nil { + t.Fatalf("marshal recovered alert: %v", err) + } + if strings.Contains(string(recovered), "resolution") { + t.Fatalf("payload = %s, a recovery must not carry a resolution", recovered) + } + + moved := ResolvedAlert{Alert: Alert{ID: "alert-1", Resolution: &AlertResolution{ + Reason: "moved_to_agent", + SuccessorResourceID: "agent-pve-1", + SuccessorName: "pve-1 (Host Agent)", + Summary: "Alert moved to pve-1 (Host Agent). This is not a recovery: check the agent for the current reading.", + }}} + payload, err := json.Marshal(moved) + if err != nil { + t.Fatalf("marshal moved alert: %v", err) + } + want := `"resolution":{"reason":"moved_to_agent","successorResourceId":"agent-pve-1","successorName":"pve-1 (Host Agent)","summary":"Alert moved to pve-1 (Host Agent). This is not a recovery: check the agent for the current reading."}` + if !strings.Contains(string(payload), want) { + t.Fatalf("payload = %s, want %s", payload, want) + } + + // The state store keeps its own copy of each resolution. + state := NewState() + state.UpdateRecentlyResolved([]ResolvedAlert{moved}) + moved.Resolution.SuccessorName = "changed" + snapshot := state.GetSnapshot().RecentlyResolved + if len(snapshot) != 1 || snapshot[0].Resolution == nil || + snapshot[0].Resolution.SuccessorName != "pve-1 (Host Agent)" { + t.Fatalf("snapshot = %+v, want the resolution as it was recorded", snapshot) + } +} + func TestDockerHostCollectionModeWireContract(t *testing.T) { payload, err := json.Marshal(DockerHost{ ID: "docker-host-1", diff --git a/internal/models/models.go b/internal/models/models.go index 5f1f2af5b8..71cc2c57fa 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -110,6 +110,20 @@ type Alert struct { // Metadata carries alert-engine annotations (notably resourceType) so the // frontend can classify an alert without re-deriving resource identity. Metadata map[string]interface{} `json:"metadata,omitempty"` + // Resolution is set only on a resolved alert whose close was not a + // recovery, so readers never report its resource as healthy. + Resolution *AlertResolution `json:"resolution,omitempty"` +} + +// AlertResolution is the state projection of the alert engine's +// AlertResolution: why an alert closed without its condition clearing. +type AlertResolution struct { + Reason string `json:"reason"` + SuccessorResourceID string `json:"successorResourceId,omitempty"` + SuccessorName string `json:"successorName,omitempty"` + // Summary is the alert engine's one-line account, such as "Alert moved + // to pve1 (Host Agent). This is not a recovery: ...". + Summary string `json:"summary,omitempty"` } // ResolvedAlert represents a recently resolved alert diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go index 498bd35b8d..5159501c59 100644 --- a/internal/monitoring/monitor_alert_handling_test.go +++ b/internal/monitoring/monitor_alert_handling_test.go @@ -238,6 +238,68 @@ func TestMonitor_HandleAlertLifecycle_WritesCanonicalChanges(t *testing.T) { } } +// A node alert handed to its Pulse agent closes without recovering. The +// resource history and the alert's incident timeline (the Alerts history row +// expansion) must say where it went, not "Alert resolved: Memory usage at 95%". +func TestMonitor_HandleAlertLifecycle_HandoverCloseIsNotARecovery(t *testing.T) { + resourceStore := unifiedresources.NewMemoryStore() + incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{}) + m := &Monitor{ + incidentStore: incidentStore, + resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(resourceStore)), + } + incidentStore.SetResourceTimelineStore(m.resourceStore.(memory.IncidentTimelineStore)) + + startedAt := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) + movedAt := startedAt.Add(2 * time.Hour) + alert := &alerts.Alert{ + ID: "pve1-memory", + Type: "memory", + Level: alerts.AlertLevelWarning, + ResourceID: "pve1", + Message: "Memory usage at 95%", + Value: 95, + Threshold: 85, + StartTime: startedAt, + } + m.handleAlertLifecycleEvent(alerts.LifecycleEvent{Type: eventlog.TypeFired, OccurredAt: startedAt, Alert: alert}) + closed := alert.Clone() + closed.Resolution = &alerts.AlertResolution{ + Reason: alerts.AlertResolutionMovedToAgent, + SuccessorResourceID: "agent-pve1", + SuccessorName: "pve1 (Host Agent)", + } + m.handleAlertLifecycleEvent(alerts.LifecycleEvent{Type: eventlog.TypeResolved, OccurredAt: movedAt, Alert: closed}) + + summary := "Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading." + changes, err := resourceStore.GetRecentChanges("pve1", time.Time{}, 10) + require.NoError(t, err) + require.Len(t, changes, 2) + require.Equal(t, unifiedresources.ChangeAlertResolved, changes[0].Kind) + require.Equal(t, summary, changes[0].Reason) + require.Equal(t, "moved_to_agent", changes[0].Metadata[unifiedresources.MetadataAlertResolution]) + require.Equal(t, unifiedresources.ChangeAlertFired, changes[1].Kind) + require.NotContains(t, changes[1].Metadata, unifiedresources.MetadataAlertResolution) + + timeline := incidentStore.GetTimelineByAlertAt(alert.ID, startedAt) + require.NotNil(t, timeline) + require.Len(t, timeline.Events, 2) + require.Equal(t, memory.IncidentEventAlertResolved, timeline.Events[1].Type) + require.Equal(t, summary, timeline.Events[1].Summary) + + // An ordinary recovery keeps its existing wording. + recovered := alert.Clone() + recovered.ID = "pve1-cpu" + recovered.Type = "cpu" + recovered.Message = "CPU usage at 90%" + m.handleAlertLifecycleEvent(alerts.LifecycleEvent{Type: eventlog.TypeFired, OccurredAt: startedAt, Alert: recovered}) + m.handleAlertLifecycleEvent(alerts.LifecycleEvent{Type: eventlog.TypeResolved, OccurredAt: movedAt.Add(time.Minute), Alert: recovered}) + recoveredTimeline := incidentStore.GetTimelineByAlertAt(recovered.ID, startedAt) + require.NotNil(t, recoveredTimeline) + require.Len(t, recoveredTimeline.Events, 2) + require.Equal(t, "Alert resolved", recoveredTimeline.Events[1].Summary) +} + func TestPausedDeliveryStillBuildsTimelineThroughRealAlertLifecycle(t *testing.T) { manager := alerts.NewManagerWithDataDir(t.TempDir()) t.Cleanup(manager.Stop) diff --git a/internal/monitoring/monitor_alerts.go b/internal/monitoring/monitor_alerts.go index 46445c3ec7..f7aeb761d8 100644 --- a/internal/monitoring/monitor_alerts.go +++ b/internal/monitoring/monitor_alerts.go @@ -576,7 +576,7 @@ func (m *Monitor) recordAlertTimelineChange(alert *alerts.Alert, kind unifiedres return } - change := unifiedresources.BuildAlertTimelineChange(alert.ResourceID, kind, occurredAt, actor, unifiedresources.AlertTimelineChange{ + timelineChange := unifiedresources.AlertTimelineChange{ AlertIdentifier: alert.ID, AlertStartedAt: alert.StartTime, AlertType: alert.Type, @@ -585,7 +585,12 @@ func (m *Monitor) recordAlertTimelineChange(alert *alerts.Alert, kind unifiedres AlertValue: alert.Value, AlertThreshold: alert.Threshold, AlertMetadata: alert.Metadata, - }) + } + if summary := alert.Resolution.Summary(); summary != "" { + timelineChange.ResolutionReason = string(alert.Resolution.Reason) + timelineChange.ResolutionSummary = summary + } + change := unifiedresources.BuildAlertTimelineChange(alert.ResourceID, kind, occurredAt, actor, timelineChange) if change == nil { return } diff --git a/internal/operationaltrust/contracts.go b/internal/operationaltrust/contracts.go index 19b1a1b20e..303ebf857b 100644 --- a/internal/operationaltrust/contracts.go +++ b/internal/operationaltrust/contracts.go @@ -493,8 +493,12 @@ const ( TransitionSuppression TransitionCause = "suppression" TransitionSuppressionExpired TransitionCause = "suppression_expired" TransitionRecoveryEvidence TransitionCause = "recovery_evidence" - TransitionCollectionStale TransitionCause = "collection_stale" - TransitionCollectionUnknown TransitionCause = "collection_unknown" + // TransitionOwnershipTransferred closes a record whose condition another + // record now owns, such as a node metric a linked Pulse agent took over. + // The condition was not observed to clear, so it is not a recovery. + TransitionOwnershipTransferred TransitionCause = "ownership_transferred" + TransitionCollectionStale TransitionCause = "collection_stale" + TransitionCollectionUnknown TransitionCause = "collection_unknown" ) func (cause TransitionCause) valid() bool { @@ -505,6 +509,7 @@ func (cause TransitionCause) valid() bool { TransitionSuppression, TransitionSuppressionExpired, TransitionRecoveryEvidence, + TransitionOwnershipTransferred, TransitionCollectionStale, TransitionCollectionUnknown: return true @@ -617,6 +622,13 @@ func (transition LifecycleTransition) validateWithoutID() error { if len(canonicalIDs(transition.EvidenceIDs)) == 0 { return errors.New("recovery transition requires evidence") } + case TransitionOwnershipTransferred: + if transition.To != OperationalResolved { + return errors.New("ownership transfer must enter resolved state") + } + if len(canonicalIDs(transition.EvidenceIDs)) == 0 { + return errors.New("ownership transfer requires evidence") + } case TransitionCollectionStale: if transition.To != OperationalStale { return errors.New("collection-stale transition must enter stale state") diff --git a/internal/operationaltrust/contracts_test.go b/internal/operationaltrust/contracts_test.go index 13039d5932..80381c4a26 100644 --- a/internal/operationaltrust/contracts_test.go +++ b/internal/operationaltrust/contracts_test.go @@ -235,6 +235,48 @@ func TestLifecycleTransitionEnforcesOperationalSemantics(t *testing.T) { } } +// A close that hands the condition to another record is not a recovery, so it +// has its own cause. It may only close the record, and only with the evidence +// that recorded the handover. +func TestOwnershipTransferOnlyClosesWithEvidence(t *testing.T) { + at := time.Date(2026, 10, 6, 15, 0, 0, 0, time.UTC) + if _, err := NewTransitionID( + "record-1", + OperationalOpen, + OperationalResolved, + at, + TransitionOwnershipTransferred, + "memory-high", + []string{"evidence-handover"}, + ); err != nil { + t.Fatalf("open-to-resolved ownership transfer error = %v", err) + } + if _, err := NewTransitionID( + "record-1", + OperationalAcknowledged, + OperationalResolved, + at, + TransitionOwnershipTransferred, + "memory-high", + nil, + ); err == nil || !strings.Contains(err.Error(), "requires evidence") { + t.Fatalf("evidenceless ownership transfer error = %v", err) + } + for _, to := range []OperationalState{OperationalResolving, OperationalOpen, OperationalStale} { + if _, err := NewTransitionID( + "record-1", + OperationalAcknowledged, + to, + at, + TransitionOwnershipTransferred, + "memory-high", + []string{"evidence-handover"}, + ); err == nil || !strings.Contains(err.Error(), "must enter resolved") { + t.Fatalf("ownership transfer into %q error = %v", to, err) + } + } +} + func TestOperationalRecordRequiresResolutionEvidence(t *testing.T) { firstObserved := time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC) resolvedAt := firstObserved.Add(5 * time.Minute) diff --git a/internal/unifiedresources/incident_changes.go b/internal/unifiedresources/incident_changes.go index abde5af5c3..8d39473401 100644 --- a/internal/unifiedresources/incident_changes.go +++ b/internal/unifiedresources/incident_changes.go @@ -16,6 +16,9 @@ const ( MetadataAlertMessage = "alert_message" MetadataAlertValue = "alert_value" MetadataAlertThreshold = "alert_threshold" + // MetadataAlertResolution is the reason code of a resolve that was not a + // recovery, such as "moved_to_agent". Ordinary recoveries omit it. + MetadataAlertResolution = "alert_resolution" MetadataCommand = "command" MetadataSuccess = "success" MetadataOutputExcerpt = "output_excerpt" @@ -38,6 +41,11 @@ type AlertTimelineChange struct { AlertValue float64 AlertThreshold float64 AlertMetadata map[string]any + // ResolutionReason and ResolutionSummary describe a resolve that was not + // a recovery. They are empty for an ordinary recovery and ignored on + // every other lifecycle kind. + ResolutionReason string + ResolutionSummary string } // BuildAlertTimelineChange constructs a canonical resource change for alert @@ -74,6 +82,9 @@ func BuildAlertTimelineChange(resourceID string, kind ChangeKind, occurredAt tim if !alert.AlertStartedAt.IsZero() { change.Metadata[MetadataAlertStartedAt] = alert.AlertStartedAt.UTC().Format(time.RFC3339Nano) } + if reason := strings.TrimSpace(alert.ResolutionReason); kind == ChangeAlertResolved && reason != "" { + change.Metadata[MetadataAlertResolution] = reason + } for key, value := range cloneChangeMetadata(alert.AlertMetadata) { if _, exists := change.Metadata[key]; exists { continue @@ -178,6 +189,11 @@ func alertChangeReason(kind ChangeKind, alert AlertTimelineChange) string { case ChangeAlertUnsnoozed: return "Alert notifications resumed" case ChangeAlertResolved: + // A close that was not a recovery must not read as one: the alert's + // own message describes the condition that is still unresolved. + if summary := strings.TrimSpace(alert.ResolutionSummary); summary != "" { + return summary + } if message != "" { return fmt.Sprintf("Alert resolved: %s", message) } From a89c08a59e408dc42c2b95aa870a56e2087041e9 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:17:57 +0100 Subject: [PATCH 18/27] Drop recovery colours and check marks from moved alert closes A second review found the moved-alert text was honest but the visuals still said recovered: Discord kept its green embed, both Teams cards kept recovery green, Mattermost led with a check emoji, and a grouped ntfy batch holding a moved alert kept the green check tag. A check mark on a phone is read before the sentence under it. Resolved webhook data now carries NotRecovered when any close in the delivery was not a recovery. The built-in templates switch to neutral colours and an arrow when it is set, and ntfy uses the arrow tag for a single alert or a batch. Titles still say Resolved, which stays true for the closed alert, and ordinary recoveries keep their green. The render tests now use an isolated data directory, stop their managers, and hand the captured request over a channel, so they no longer touch a shared queue or race on the captured body. --- .../v6/internal/subsystems/notifications.md | 26 ++-- internal/notifications/notifications.go | 19 ++- .../notifications/resolved_handover_test.go | 144 ++++++++++++------ internal/notifications/webhook_enhanced.go | 4 + internal/notifications/webhook_templates.go | 8 +- 5 files changed, 141 insertions(+), 60 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/notifications.md b/docs/release-control/v6/internal/subsystems/notifications.md index bb8bba2518..0afc28d582 100644 --- a/docs/release-control/v6/internal/subsystems/notifications.md +++ b/docs/release-control/v6/internal/subsystems/notifications.md @@ -1229,16 +1229,22 @@ receives `resolve` for the alert's dedup key. Only the wording changes. agent for the current reading.") in place of " on is now healthy" for every templated webhook `.Message`. Grouped lists append the short outcome ("moved to pve1 (Host Agent)") to the moved entry. ntfy titles a -single moved alert `MOVED: ` and swaps the green check tag for -`arrow_right`. Email and Apprise title it `Pulse alert moved: ` and -add the summary after the breach message. The resolution rides the Alert JSON, -so it survives the delivery queue round trip and appears in the default JSON -payload's `alerts` array. Built-in template titles still say Resolved, which is -accurate for the closed alert, and a custom template that hard-codes recovery -text keeps it. Issue #1682 is the precedent: a recipient called "is now -healthy" a false factual claim when an alert closed without recovering. +single moved alert `MOVED: `. Email and Apprise title it `Pulse alert +moved: ` and add the summary after the breach message. Recovery +visuals follow the same rule: when any close in a resolved delivery was not a +recovery, `WebhookPayloadData.NotRecovered` is set and the built-in templates +drop them (Discord's green embed becomes the neutral blue, both Teams cards +lose recovery green, Mattermost's check emoji becomes `:arrow_right:`), and +ntfy swaps its green check tag for `arrow_right`, for a single alert or a batch. +Built-in titles still say Resolved, which is accurate for the closed alert; +custom templates can read `.NotRecovered`, and one that hard-codes recovery +text keeps it. The resolution rides the Alert JSON, so it survives the delivery +queue round trip and appears in the default JSON payload's `alerts` array. +Issue #1682 is the precedent: a recipient called "is now healthy" a false +factual claim when an alert closed without recovering. Verification: `resolved_handover_test.go` drives the firing, its receipt and the queued close through the normal entry points for ntfy, and renders Discord, -PagerDuty, a grouped Slack list and email/Apprise content, plus the unchanged -ordinary recovery wording. +both Teams cards, Mattermost, PagerDuty, a grouped Slack list, a grouped ntfy +batch and email/Apprise content, plus the unchanged ordinary recovery wording +and visuals. diff --git a/internal/notifications/notifications.go b/internal/notifications/notifications.go index 93ae16e0f3..71f467b967 100644 --- a/internal/notifications/notifications.go +++ b/internal/notifications/notifications.go @@ -453,6 +453,17 @@ func resolvedAlertNotRecoveredSummary(alert *alerts.Alert) string { return alert.Resolution.Describe(label) } +// resolvedAlertsIncludeNotRecovered reports whether any close in a resolved +// notification was not a recovery, so recovery visuals must not be used. +func resolvedAlertsIncludeNotRecovered(alertList []*alerts.Alert) bool { + for _, alert := range alertList { + if alert != nil && alert.Resolution.Outcome() != "" { + return true + } + } + return false +} + // resolvedAlertListEntry names one alert in a grouped recovery list. func resolvedAlertListEntry(alert *alerts.Alert) string { entry := fmt.Sprintf("%s on %s", alert.ResourceName, alert.Node) @@ -3018,6 +3029,7 @@ func (n *NotificationManager) sendResolvedWebhook(webhook WebhookConfig, alertLi data.ResolvedAtISO = resolvedAt.Format(time.RFC3339) data.Duration = formatWebhookDuration(resolvedAt.Sub(alert.StartTime)) data.Message = resolvedAlertMessage(alert) + data.NotRecovered = resolvedAlertsIncludeNotRecovered(alertList) data.AlertCount = len(alertList) data.Alerts = alertList if len(alertList) > 1 { @@ -3074,16 +3086,19 @@ func (n *NotificationManager) sendResolvedWebhookNtfy(webhook WebhookConfig, ale } // Build plain-text body and title. A single alert that moved rather than - // recovered says so in both, and drops the green check. + // recovered says so in both, and no batch holding such a close carries + // the green check. var body strings.Builder title := "RESOLVED" tags := "white_check_mark,pulse,resolved" + if resolvedAlertsIncludeNotRecovered(alertList) { + tags = "arrow_right,pulse,resolved" + } if len(alertList) == 1 && alertList[0] != nil { a := alertList[0] title = fmt.Sprintf("RESOLVED: %s", a.ResourceName) if summary := resolvedAlertNotRecoveredSummary(a); summary != "" { title = fmt.Sprintf("MOVED: %s", a.ResourceName) - tags = "arrow_right,pulse,resolved" body.WriteString(summary) } else { fmt.Fprintf(&body, "Resolved: %s", resolvedAlertMessage(a)) diff --git a/internal/notifications/resolved_handover_test.go b/internal/notifications/resolved_handover_test.go index b980c6fb56..f144a178be 100644 --- a/internal/notifications/resolved_handover_test.go +++ b/internal/notifications/resolved_handover_test.go @@ -99,64 +99,108 @@ func TestMovedAlertCloseIsDeliveredAsMovedNotRecovered(t *testing.T) { } } -func TestMovedAlertCloseRendersHonestlyOnEveryChannel(t *testing.T) { - capture := func(t *testing.T, service string) map[string]any { - t.Helper() - var gotBody []byte - server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotBody, _ = io.ReadAll(r.Body) - w.WriteHeader(http.StatusOK) - })) - defer server.Close() - nm := NewNotificationManager("https://pulse.local") - _ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1") - webhook := WebhookConfig{Name: service, URL: server.URL + "/hook", Enabled: true, Service: service, - CustomFields: map[string]string{"routing_key": "rk"}} - if err := nm.sendResolvedWebhook(webhook, []*alerts.Alert{movedNodeMemoryAlert()}, time.Now()); err != nil { - t.Fatalf("sendResolvedWebhook %s: %v", service, err) - } - var payload map[string]any - if err := json.Unmarshal(gotBody, &payload); err != nil { - t.Fatalf("unmarshal %s payload: %v", service, err) - } - return payload +// captureResolvedWebhook renders one resolved delivery through an isolated +// manager and returns what the destination received. +func captureResolvedWebhook(t *testing.T, service string, alertList []*alerts.Alert) ([]byte, http.Header) { + t.Helper() + type request struct { + body []byte + header http.Header } + received := make(chan request, 1) + server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + received <- request{body: body, header: r.Header.Clone()} + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + nm := NewNotificationManagerWithDataDir("https://pulse.local", t.TempDir()) + t.Cleanup(nm.Stop) + if err := nm.UpdateAllowedPrivateCIDRs("127.0.0.1"); err != nil { + t.Fatal(err) + } + webhook := WebhookConfig{Name: service, URL: server.URL + "/hook", Enabled: true, Service: service, + CustomFields: map[string]string{"routing_key": "rk"}} + if err := nm.sendResolvedWebhook(webhook, alertList, time.Now()); err != nil { + t.Fatalf("sendResolvedWebhook %s: %v", service, err) + } + select { + case got := <-received: + return got.body, got.header + case <-time.After(5 * time.Second): + t.Fatalf("%s destination received nothing", service) + return nil, nil + } +} - t.Run("discord says moved", func(t *testing.T) { - embed := capture(t, "discord")["embeds"].([]any)[0].(map[string]any) +func decodeResolvedPayload(t *testing.T, service string, body []byte) map[string]any { + t.Helper() + var payload map[string]any + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("unmarshal %s payload: %v\n%s", service, err, body) + } + return payload +} + +func TestMovedAlertCloseRendersHonestlyOnEveryChannel(t *testing.T) { + recovered := &alerts.Alert{ID: "vm-cpu", Type: "cpu", ResourceName: "web-vm-01", Node: "pve1", StartTime: time.Now().Add(-time.Minute)} + + t.Run("discord says moved without the recovery colour", func(t *testing.T) { + body, _ := captureResolvedWebhook(t, "discord", []*alerts.Alert{movedNodeMemoryAlert()}) + embed := decodeResolvedPayload(t, "discord", body)["embeds"].([]any)[0].(map[string]any) description, _ := embed["description"].(string) if !strings.Contains(description, movedSummary) || strings.Contains(description, "healthy") { t.Fatalf("description = %q, want the handover summary", description) } + if embed["color"] != float64(3447003) { + t.Fatalf("embed color = %v, want the neutral 3447003, not recovery green", embed["color"]) + } + }) + + t.Run("teams cards drop recovery green", func(t *testing.T) { + body, _ := captureResolvedWebhook(t, "teams", []*alerts.Alert{movedNodeMemoryAlert()}) + if got := decodeResolvedPayload(t, "teams", body)["themeColor"]; got != "0076D7" { + t.Fatalf("teams themeColor = %v, want 0076D7", got) + } + body, _ = captureResolvedWebhook(t, "teams-adaptive", []*alerts.Alert{movedNodeMemoryAlert()}) + if !strings.Contains(string(body), `"color": "Accent"`) || strings.Contains(string(body), `"color": "Good"`) { + t.Fatalf("teams adaptive card = %s, want the Accent title colour", body) + } + }) + + t.Run("mattermost drops the check mark", func(t *testing.T) { + body, _ := captureResolvedWebhook(t, "mattermost", []*alerts.Alert{movedNodeMemoryAlert()}) + text, _ := decodeResolvedPayload(t, "mattermost", body)["text"].(string) + if strings.Contains(text, ":white_check_mark:") || !strings.Contains(text, ":arrow_right: **RESOLVED**") || !strings.Contains(text, movedSummary) { + t.Fatalf("mattermost text = %q", text) + } }) t.Run("pagerduty still closes the incident", func(t *testing.T) { - payload := capture(t, "pagerduty") + body, _ := captureResolvedWebhook(t, "pagerduty", []*alerts.Alert{movedNodeMemoryAlert()}) + payload := decodeResolvedPayload(t, "pagerduty", body) if payload["event_action"] != "resolve" || payload["dedup_key"] != "pve1-memory" { t.Fatalf("payload = %v, want a resolve for the node alert's incident", payload) } }) t.Run("grouped list names the move", func(t *testing.T) { - var gotBody []byte - server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotBody, _ = io.ReadAll(r.Body) - w.WriteHeader(http.StatusOK) - })) - defer server.Close() - nm := NewNotificationManager("https://pulse.local") - _ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1") - recovered := &alerts.Alert{ID: "vm-cpu", Type: "cpu", ResourceName: "web-vm-01", Node: "pve1", StartTime: time.Now().Add(-time.Minute)} - webhook := WebhookConfig{Name: "slack", URL: server.URL + "/hook", Enabled: true, Service: "slack"} - if err := nm.sendResolvedWebhook(webhook, []*alerts.Alert{movedNodeMemoryAlert(), recovered}, time.Now()); err != nil { - t.Fatalf("sendResolvedWebhook: %v", err) - } - body := string(gotBody) - if !strings.Contains(body, "pve1 on pve1, moved to pve1 (Host Agent)") || !strings.Contains(body, "web-vm-01 on pve1") { + body, _ := captureResolvedWebhook(t, "slack", []*alerts.Alert{movedNodeMemoryAlert(), recovered}) + if !strings.Contains(string(body), "pve1 on pve1, moved to pve1 (Host Agent)") || !strings.Contains(string(body), "web-vm-01 on pve1") { t.Fatalf("grouped body = %s, want the moved alert marked and the recovery listed", body) } }) + t.Run("grouped ntfy batch drops the check mark", func(t *testing.T) { + body, header := captureResolvedWebhook(t, "ntfy", []*alerts.Alert{movedNodeMemoryAlert(), recovered}) + if tags := header.Get("Tags"); strings.Contains(tags, "white_check_mark") || !strings.Contains(tags, "arrow_right") { + t.Fatalf("ntfy tags = %q, want no check mark on a batch holding a move", tags) + } + if !strings.Contains(string(body), "- pve1 on pve1, moved to pve1 (Host Agent)\n") { + t.Fatalf("ntfy body = %q", body) + } + }) + t.Run("email and apprise say moved", func(t *testing.T) { title, htmlBody, textBody := buildResolvedNotificationContent([]*alerts.Alert{movedNodeMemoryAlert()}, time.Now(), "") if title != "Pulse alert moved: pve1" { @@ -170,14 +214,26 @@ func TestMovedAlertCloseRendersHonestlyOnEveryChannel(t *testing.T) { } }) - t.Run("ordinary recovery is unchanged", func(t *testing.T) { - recovered := movedNodeMemoryAlert() - recovered.Resolution = nil - if got := resolvedAlertMessage(recovered); got != "pve1 on pve1 is now healthy" { + t.Run("ordinary recovery keeps its wording and visuals", func(t *testing.T) { + plain := movedNodeMemoryAlert() + plain.Resolution = nil + if got := resolvedAlertMessage(plain); got != "pve1 on pve1 is now healthy" { t.Fatalf("resolvedAlertMessage() = %q", got) } - if title, _, _ := buildResolvedNotificationContent([]*alerts.Alert{recovered}, time.Now(), ""); title != "Pulse alert resolved: pve1" { + if title, _, _ := buildResolvedNotificationContent([]*alerts.Alert{plain}, time.Now(), ""); title != "Pulse alert resolved: pve1" { t.Fatalf("title = %q", title) } + body, _ := captureResolvedWebhook(t, "discord", []*alerts.Alert{plain}) + if embed := decodeResolvedPayload(t, "discord", body)["embeds"].([]any)[0].(map[string]any); embed["color"] != float64(3066993) { + t.Fatalf("ordinary recovery embed color = %v, want green 3066993", embed["color"]) + } + body, _ = captureResolvedWebhook(t, "mattermost", []*alerts.Alert{plain}) + if text, _ := decodeResolvedPayload(t, "mattermost", body)["text"].(string); !strings.Contains(text, ":white_check_mark: **RESOLVED**") { + t.Fatalf("ordinary mattermost text = %q", text) + } + _, header := captureResolvedWebhook(t, "ntfy", []*alerts.Alert{plain}) + if tags := header.Get("Tags"); tags != "white_check_mark,pulse,resolved" { + t.Fatalf("ordinary ntfy tags = %q", tags) + } }) } diff --git a/internal/notifications/webhook_enhanced.go b/internal/notifications/webhook_enhanced.go index 5c390fd1c6..c586c2e9fb 100644 --- a/internal/notifications/webhook_enhanced.go +++ b/internal/notifications/webhook_enhanced.go @@ -65,6 +65,10 @@ type WebhookPayloadData struct { Event string // "alert" or "resolved" ResolvedAt string // RFC3339 timestamp when alert was resolved ResolvedAtISO string // Same as ResolvedAt (alias for template consistency) + // NotRecovered marks a resolved notification in which at least one close + // was not a recovery (an alert that moved to a linked Pulse agent), so + // templates can drop recovery colours and check marks. + NotRecovered bool // Tenant identity (multi-tenant orgs and MSP client runtimes). // Empty on plain single-tenant installs. diff --git a/internal/notifications/webhook_templates.go b/internal/notifications/webhook_templates.go index 8c9f8db4f0..5e4b2bf152 100644 --- a/internal/notifications/webhook_templates.go +++ b/internal/notifications/webhook_templates.go @@ -56,7 +56,7 @@ func GetWebhookTemplates() []WebhookTemplate { "embeds": [{ "title": "Resolved: {{.ResourceName | jsonString}}", "description": "{{.Message | jsonString}}", - "color": 3066993, + "color": {{if .NotRecovered}}3447003{{else}}3066993{{end}}, "fields": [ {"name": "Resource", "value": "{{.ResourceName | jsonString}}", "inline": true}, {"name": "Node", "value": "{{.Node | jsonString}}", "inline": true}, @@ -241,7 +241,7 @@ func GetWebhookTemplates() []WebhookTemplate { ResolvedPayloadTemplate: `{ "@type": "MessageCard", "@context": "http://schema.org/extensions", - "themeColor": "2DC72D", + "themeColor": "{{if .NotRecovered}}0076D7{{else}}2DC72D{{end}}", "summary": "Resolved: {{.ResourceName | jsonString}}", {{if .Mention}}"text": "{{.Mention | jsonString}}",{{end}} "sections": [{ @@ -372,7 +372,7 @@ func GetWebhookTemplates() []WebhookTemplate { "text": "Resolved: {{.ResourceName | jsonString}}", "weight": "Bolder", "size": "Large", - "color": "Good" + "color": "{{if .NotRecovered}}Accent{{else}}Good{{end}}" }, { "type": "TextBlock", @@ -521,7 +521,7 @@ View in Pulse: {{.Instance}}`, ResolvedPayloadTemplate: `{ "username": "Pulse Monitoring", "icon_url": "https://raw.githubusercontent.com/rcourtman/Pulse/main/frontend-modern/public/android-chrome-192x192.png", - "text": "{{if .Mention}}{{.Mention | jsonString}}\n\n{{end}}:white_check_mark: **RESOLVED**\n\n**{{.ResourceName | jsonString}}** on **{{.Node | jsonString}}**\n\n{{.Message | jsonString}}\n\n| Detail | Value |\n|:-------|:------|\n| Resource | {{.ResourceName | jsonString}} |\n| Node | {{.Node | jsonString}} |\n| Type | {{.Type | title | jsonString}} |\n| Duration | {{.Duration | jsonString}} |\n| Resolved At | {{.ResolvedAt | jsonString}} |\n| Alert Identifier | {{.ID | jsonString}} |" + "text": "{{if .Mention}}{{.Mention | jsonString}}\n\n{{end}}{{if .NotRecovered}}:arrow_right:{{else}}:white_check_mark:{{end}} **RESOLVED**\n\n**{{.ResourceName | jsonString}}** on **{{.Node | jsonString}}**\n\n{{.Message | jsonString}}\n\n| Detail | Value |\n|:-------|:------|\n| Resource | {{.ResourceName | jsonString}} |\n| Node | {{.Node | jsonString}} |\n| Type | {{.Type | title | jsonString}} |\n| Duration | {{.Duration | jsonString}} |\n| Resolved At | {{.ResolvedAt | jsonString}} |\n| Alert Identifier | {{.ID | jsonString}} |" }`, Instructions: "1. In Mattermost, go to Integrations > Incoming Webhooks\n2. Create a new webhook and select the channel\n3. Copy the webhook URL and paste it here\n\nNote: This template uses Markdown formatting which is fully supported by Mattermost.", }, From 549161bf6fa0b04bfc02f896566eb32939b29800 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:26:59 +0100 Subject: [PATCH 19/27] Stop presenting a silent agent's disk readings as collected A host agent that stops reporting keeps its last SMART rows in state. The Proxmox physical-disk merge copied that agent's temperature and I/O counters into the disk together with its "available" collection state, the registry merged the agent's retained rows into the canonical disk the same way, and every full disk poll wrote the frozen temperature to SMART history as a new sample. A dead agent's disk temperatures therefore stayed "collected" in the API and kept extending the history chart indefinitely. Expiring the agent's reporting lease is now the point where those readings stop being current. ExpireHostTelemetry keeps the SMART temperature and I/O values as last-known evidence but marks them unavailable ("host agent stopped reporting"), as the physical-disk contract asks for retained evidence. Because MergeStatus lets any available state win, a Proxmox disk row still holding the earlier copy would mask that, both on skipped disk polls and when the PVE host is down and never disk-polled again. diskinventory.MergeReportedStatus lets a source's later word on its own field supersede the availability it supplied earlier; the monitoring merge and both registry ingest orders use it. Every SMART temperature history writer now records a temperature only when its current state is collected, so a retained reading (an agent past its lease, or a disk in standby) is not written as a new sample. Agents before 6.2 send no collection provenance, so their merged temperature is recorded as collected from "host_agent" instead of inheriting Proxmox's "unsupported", keeping their history intact. Expiry is now compare-and-set on the report time the evaluation judged stale, so a report accepted in between is not expired. SMART attributes, wearout and health stay last-known on purpose: they are durable health evidence, and dropping them would resolve disk-health and low-life alerts when an agent goes quiet. Disk temperature alerts are evaluated from the agent's own report, so they never read this merged value. The node-temperature path (agent sensors feeding node.Temperature.SMART, which the disk merge labels proxmox_node_smart) is gated by the agent's sighting in #2582; a carried node reading can still stamp a disk as collected and belongs to that path. --- .../v6/internal/subsystems/agent-lifecycle.md | 6 + .../v6/internal/subsystems/monitoring.md | 16 +- .../internal/subsystems/unified-resources.md | 7 + internal/models/host_transient_expiry_test.go | 98 +++++++- internal/models/models.go | 50 +++- .../issue1595_collection_trust_test.go | 44 ++++ internal/monitoring/monitor.go | 50 +++- internal/monitoring/monitor_agents.go | 4 +- .../physical_disk_roundtrip_test.go | 228 ++++++++++++++++++ internal/unifiedresources/registry.go | 10 +- .../registry_merge_policy_test.go | 77 ++++++ pkg/diskinventory/status.go | 29 +++ pkg/diskinventory/status_test.go | 41 ++++ 13 files changed, 648 insertions(+), 12 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index b51b2d57dc..7c84b24cf5 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -728,6 +728,12 @@ into monitoring's models. Absent evidence has to carry its declared sentinel rather than a zero value that reads as a real measurement: an absent physical-disk view projects `Wearout` as `unifiedresources.WearoutUnreported`, never `0`, which would announce a spent disk the agent never reported. +When the host heartbeat lease expires, `State.ExpireHostTelemetry` keeps that +agent's SMART temperature and I/O counters as last-known values but marks them +`unavailable` ("host agent stopped reporting"); the agent's next accepted +report replaces them. Expiry is compare-and-set on the report time the offline +sweep judged stale, so a report admitted between that judgement and the expiry +is never expired. An enabled availability target assigned to a host agent creates an agent-lifecycle lease for that exact target/agent pairing. First assignment diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 9ae9a5d83a..10dbab5822 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -3684,7 +3684,21 @@ transiently `unavailable`, provider/controller `unsupported`, or unexpectedly `missing`. Normalization may retain the last known value when the current observation is not available, but it must preserve the current state and reason so API and UI consumers do not present retained evidence as freshly -collected. Unified-resource physical-disk round trips must retain named +collected. A host agent's SMART temperature and I/O counters follow its +reporting lease: once `State.ExpireHostTelemetry` expires the agent, they stay +as last-known values marked `unavailable` ("host agent stopped reporting"), +and the linked Proxmox disk merge and the registry let the agent's own later +state supersede the availability it supplied earlier +(`diskinventory.MergeReportedStatus`), so neither a skipped disk poll nor a +host that is never disk-polled again can carry it forward. Expiry is +compare-and-set on the report time the evaluation judged stale, so a report +accepted in between is never expired. Every SMART temperature history writer +records a temperature only when its current collection state is available (or +predates collection state), so a retained reading has to keep its non-available +state to stay out of history; a path that relabels a carried reading as +available, such as a node-temperature carry stamped `proxmox_node_smart`, is +not covered by this rule. +Unified-resource physical-disk round trips must retain named `StorageGroup` membership rather than degrading it to the generic `Used` filesystem label. That same host-agent temperature boundary must prefer a recent linked host-agent diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index b91304e4d5..452757ae82 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -3516,6 +3516,13 @@ the five field statuses for serial, temperature, I/O, controller, and pool. `unavailable`, `unsupported`, or `missing` state may retain a prior value for continuity, but the state and reason must survive so the consumer cannot claim fresh evidence or synthesize controller-level activity for one member. +The one exception is a source withdrawing its own evidence: when the agent +row reports a field as no longer available, that supersedes an `available` +state the Proxmox row still carries from the same source +(`diskinventory.MergeReportedStatus`), in either ingest order. A host agent +past its reporting lease must not stay "collected" through the Proxmox row's +copy of the agent's own state, including when its host is down and no disk +poll refreshes that copy. Cross-source correlation compares normalized serial and WWN values across fields without truncation, allowing a PVE bare-hex array-volume serial to join diff --git a/internal/models/host_transient_expiry_test.go b/internal/models/host_transient_expiry_test.go index 4aab4e2d9e..d91c8bfd28 100644 --- a/internal/models/host_transient_expiry_test.go +++ b/internal/models/host_transient_expiry_test.go @@ -1,6 +1,11 @@ package models -import "testing" +import ( + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" +) func TestExpireHostTelemetryClearsTransientStorageOperationsOnly(t *testing.T) { state := NewState() @@ -24,7 +29,7 @@ func TestExpireHostTelemetryClearsTransientStorageOperationsOnly(t *testing.T) { }, }) - host, changed := state.ExpireHostTelemetry("storage-host") + host, changed := state.ExpireHostTelemetry("storage-host", time.Time{}) if !changed { t.Fatal("expected host telemetry to change") } @@ -47,3 +52,92 @@ func TestExpireHostTelemetryClearsTransientStorageOperationsOnly(t *testing.T) { t.Fatalf("RAID last-known health was lost: %+v", host.RAID[0]) } } + +func TestExpireHostTelemetryMarksSMARTReadingsNoLongerCollected(t *testing.T) { + state := NewState() + judged := time.Now().Add(-time.Hour) + state.UpsertHost(Host{ + ID: "agent", + Status: "online", + LastSeen: judged, + Sensors: HostSensorSummary{SMART: []HostDiskSMART{ + { + Device: "sda", Serial: "CURRENT", Temperature: 41, Health: "PASSED", + IO: &DiskIO{Device: "sda", ReadBytes: 1000}, + Collection: &diskinventory.CollectionStatus{ + Serial: diskinventory.Available("smartctl"), + Temperature: diskinventory.Available("smartctl"), + IO: diskinventory.Available("kernel_diskstats"), + }, + }, + // Agents before collection provenance report only what they collected. + {Device: "sdb", Serial: "LEGACY", Temperature: 38, IO: &DiskIO{Device: "sdb"}}, + { + Device: "sdc", Serial: "ASLEEP", Standby: true, + Collection: &diskinventory.CollectionStatus{ + Temperature: diskinventory.Unavailable("smartctl", "disk is in standby"), + IO: diskinventory.Unsupported("controller", "per-member counters unavailable"), + }, + }, + }}, + }) + + host, changed := state.ExpireHostTelemetry("agent", judged) + if !changed { + t.Fatal("expected host telemetry to change") + } + stopped := diskinventory.Unavailable("smartctl", "host agent stopped reporting") + current := host.Sensors.SMART[0] + if current.Temperature != 41 || current.IO == nil || current.IO.ReadBytes != 1000 { + t.Fatalf("last-known SMART readings were dropped: %+v", current) + } + if current.Collection.Temperature != stopped || + current.Collection.IO != diskinventory.Unavailable("kernel_diskstats", "host agent stopped reporting") { + t.Fatalf("expired readings still claim to be collected: %+v", current.Collection) + } + if current.Collection.Serial != diskinventory.Available("smartctl") { + t.Fatalf("disk identity evidence must stay last-known: %+v", current.Collection.Serial) + } + legacy := host.Sensors.SMART[1] + if legacy.Temperature != 38 || legacy.Collection == nil || + legacy.Collection.Temperature.State != diskinventory.FieldUnavailable || + legacy.Collection.IO.State != diskinventory.FieldUnavailable { + t.Fatalf("provenance-less readings not expired: %+v", legacy) + } + asleep := host.Sensors.SMART[2].Collection + if asleep.Temperature != diskinventory.Unavailable("smartctl", "disk is in standby") || + asleep.IO.State != diskinventory.FieldUnsupported { + t.Fatalf("readings that were already not collected must keep their state: %+v", asleep) + } + + if _, changed := state.ExpireHostTelemetry("agent", judged); changed { + t.Fatal("re-evaluating an expired host must not report a change") + } + stored, _ := state.GetHost("agent") + if stored.Sensors.SMART[0].Collection.Temperature != stopped { + t.Fatalf("expiry was not kept in state: %+v", stored.Sensors.SMART[0].Collection) + } +} + +// Expiry is judged on a host snapshot. A report accepted before the expiry +// takes the state lock must not be expired on the strength of that judgement. +func TestExpireHostTelemetryLeavesHostThatReportedSinceJudgement(t *testing.T) { + state := NewState() + judged := time.Now().Add(-time.Hour) + fresh := Host{ + ID: "agent", Status: "online", LastSeen: time.Now(), + Sensors: HostSensorSummary{SMART: []HostDiskSMART{{ + Device: "sda", Temperature: 41, + Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}, + }}}, + } + state.UpsertHost(fresh) + + if host, changed := state.ExpireHostTelemetry("agent", judged); changed || host.ID != "" { + t.Fatalf("a host that reported after the stale judgement was expired: %+v changed=%v", host, changed) + } + stored, _ := state.GetHost("agent") + if stored.Status != "online" || stored.Sensors.SMART[0].Collection.Temperature.State != diskinventory.FieldAvailable { + t.Fatalf("fresh report was altered: %+v", stored) + } +} diff --git a/internal/models/models.go b/internal/models/models.go index 5f1f2af5b8..d20dd7c66f 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -5634,7 +5634,13 @@ func (s *State) SetHostStatus(hostID, status string) bool { // ExpireHostTelemetry marks a host offline and clears transient operation // claims that cannot remain authoritative after the reporting lease expires. // Static topology and health counters remain available as last-known context. -func (s *State) ExpireHostTelemetry(hostID string) (Host, bool) { +// SMART temperature and I/O readings stay as last-known values, but their +// collection state says they are no longer collected. +// +// lastSeen is the report time the caller judged stale. A host that has +// reported since then is left untouched and nothing is returned, so a report +// accepted between that judgement and this call is never expired. +func (s *State) ExpireHostTelemetry(hostID string, lastSeen time.Time) (Host, bool) { s.mu.Lock() defer s.mu.Unlock() @@ -5642,6 +5648,9 @@ func (s *State) ExpireHostTelemetry(hostID string) (Host, bool) { if host.ID != hostID { continue } + if !host.LastSeen.Equal(lastSeen) { + return Host{}, false + } changed := host.Status != "offline" host.Status = "offline" for idx := range host.RAID { @@ -5658,6 +5667,11 @@ func (s *State) ExpireHostTelemetry(hostID string) (Host, bool) { host.Unraid.SyncProgress = 0 changed = true } + for idx := range host.Sensors.SMART { + if expireHostSMARTReadings(&host.Sensors.SMART[idx]) { + changed = true + } + } s.Hosts[i] = host if changed { s.LastUpdate = time.Now() @@ -5667,6 +5681,40 @@ func (s *State) ExpireHostTelemetry(hostID string) (Host, bool) { return Host{}, false } +const hostAgentStoppedReportingReason = "host agent stopped reporting" + +// expireHostSMARTReadings marks the temperature and I/O counters of one SMART +// row as no longer collected, keeping their values as last-known evidence. +// Reports from agents that predate collection provenance carry no state, so a +// present reading is treated as collected. It reports whether anything changed. +func expireHostSMARTReadings(disk *HostDiskSMART) bool { + collection := diskinventory.CloneStatus(disk.Collection) + if collection == nil { + collection = &diskinventory.CollectionStatus{} + } + temperature := expireCollectedReading(&collection.Temperature, disk.Temperature > 0) + io := expireCollectedReading(&collection.IO, disk.IO != nil) + if !temperature && !io { + return false + } + disk.Collection = collection + return true +} + +func expireCollectedReading(status *diskinventory.FieldStatus, hasValue bool) bool { + switch status.State { + case diskinventory.FieldAvailable: + case "": + if !hasValue { + return false + } + default: + return false + } + *status = diskinventory.Unavailable(status.Source, hostAgentStoppedReportingReason) + return true +} + // TouchHost updates the last seen timestamp for a host. func (s *State) TouchHost(hostID string, ts time.Time) bool { s.mu.Lock() diff --git a/internal/monitoring/issue1595_collection_trust_test.go b/internal/monitoring/issue1595_collection_trust_test.go index 83587605fd..2c4835d3c9 100644 --- a/internal/monitoring/issue1595_collection_trust_test.go +++ b/internal/monitoring/issue1595_collection_trust_test.go @@ -10,6 +10,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" + "github.com/rcourtman/pulse-go-rewrite/pkg/metrics" ) type issue1595TopologyFixture struct { @@ -316,3 +317,46 @@ func hasIssue1595Source(sources []unifiedresources.DataSource, want unifiedresou } return false } + +// An agent report's SMART row may carry a temperature its collector did not +// collect this time (retained across a standby or failed probe). Only a +// collected temperature becomes a history sample. +func TestWriteHostSMARTMetricsRecordsOnlyCollectedTemperatures(t *testing.T) { + storeConfig := metrics.DefaultConfig(t.TempDir()) + storeConfig.FlushInterval = time.Hour + store, err := metrics.NewStore(storeConfig) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + m := &Monitor{metricsStore: store} + + host := models.Host{ID: "agent-1", Hostname: "node1", Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{ + {Device: "sda", Serial: "COLLECTED1", Temperature: 41, + Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}}, + {Device: "sdb", Serial: "LEGACY1", Temperature: 38}, + {Device: "sdc", Serial: "RETAINED1", Temperature: 36, + Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}}, + }}} + now := time.Now() + m.writeHostSMARTMetrics(host, now) + store.Flush() + + for _, tc := range []struct { + disk models.HostDiskSMART + want int + }{ + {host.Sensors.SMART[0], 1}, + {host.Sensors.SMART[1], 1}, + {host.Sensors.SMART[2], 0}, + } { + id := unifiedresources.HostSMARTDiskSourceID(host, tc.disk) + points, err := store.Query("disk", id, "smart_temp", now.Add(-time.Minute), now.Add(time.Minute), 0) + if err != nil { + t.Fatal(err) + } + if len(points) != tc.want { + t.Fatalf("%s: temperature samples = %d, want %d", tc.disk.Serial, len(points), tc.want) + } + } +} diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 53182cf222..e5c5c89632 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -648,7 +648,10 @@ func mergeHostAgentSMARTIntoDisks(disks []models.PhysicalDisk, nodes []models.No ioCopy := *matched.IO updated[i].IO = &ioCopy } - updated[i].Collection = diskinventory.MergeStatus(updated[i].Collection, matched.Collection) + // The agent's later word on a field it supplied (its lease expired, or + // the disk went into standby) must win over the copy a skipped disk + // poll carries forward from the previous merge. + updated[i].Collection = diskinventory.MergeReportedStatus(updated[i].Collection, hostAgentSMARTCollection(*matched)) if shouldUseHostAgentPhysicalDiskHealth(updated[i].Health, matched.Health) { updated[i].Health = matched.Health @@ -658,6 +661,28 @@ func mergeHostAgentSMARTIntoDisks(disks []models.PhysicalDisk, nodes []models.No return updated } +// hostAgentLegacySource is the provenance recorded for a temperature from an +// agent that predates collection provenance (before 6.2). Those agents only +// send readings they collected. +const hostAgentLegacySource = "host_agent" + +// hostAgentSMARTCollection returns the agent's collection state for one SMART +// row, giving a provenance-less temperature the legacy agent source so that +// history and later lease expiry treat it like any other agent reading. +func hostAgentSMARTCollection(smart models.HostDiskSMART) *diskinventory.CollectionStatus { + collection := diskinventory.CloneStatus(smart.Collection) + if collection == nil { + collection = &diskinventory.CollectionStatus{} + } + switch { + case collection.Temperature.State == "" && smart.Temperature > 0 && !smart.Standby: + collection.Temperature = diskinventory.Available(hostAgentLegacySource) + case collection.Temperature.State != "" && strings.TrimSpace(collection.Temperature.Source) == "": + collection.Temperature.Source = hostAgentLegacySource + } + return collection +} + func shouldUseHostAgentPhysicalDiskHealth(existing, incoming string) bool { incoming = strings.ToUpper(strings.TrimSpace(incoming)) if incoming == "" || incoming == "UNKNOWN" { @@ -914,8 +939,7 @@ func (m *Monitor) writeSMARTMetrics(disk models.PhysicalDisk, now time.Time) { return } - // Temperature (always write if > 0) - if disk.Temperature > 0 && m.metricsHistory != nil { + if diskTemperatureCollected(disk.Temperature, disk.Collection) && m.metricsHistory != nil { m.metricsHistory.AddDiskMetric(resourceID, "smart_temp", float64(disk.Temperature), now) } @@ -929,6 +953,22 @@ func (m *Monitor) writeSMARTMetrics(disk models.PhysicalDisk, now time.Time) { } } +// diskTemperatureCollected reports whether a disk temperature was collected by +// its current observation. Normalization may carry a last-known temperature +// when it was not (standby, an agent past its reporting lease); that value must +// not be recorded as a new history sample. A temperature without collection +// state predates the contract and keeps the old behavior. +func diskTemperatureCollected(temperature int, collection *diskinventory.CollectionStatus) bool { + if temperature <= 0 { + return false + } + if collection == nil { + return true + } + state := collection.Temperature.State + return state == "" || state == diskinventory.FieldAvailable +} + // smartMetricStoreWrites builds the persisted SMART writes for one physical // disk observation. The caller owns batching so the unified sync can anchor the // observation time and drop exact replays (#1966). @@ -949,8 +989,7 @@ func (m *Monitor) smartMetricStoreWrites(disk models.PhysicalDisk, resourceID st }) } - // Temperature (always write if > 0) - if disk.Temperature > 0 { + if diskTemperatureCollected(disk.Temperature, disk.Collection) { appendWrite("smart_temp", float64(disk.Temperature)) } @@ -5984,6 +6023,7 @@ func (m *Monitor) syncUnifiedPhysicalDiskMetrics(store ResourceStoreInterface) { RPM: resource.PhysicalDisk.RPM, Used: resource.PhysicalDisk.Used, SmartAttributes: smartAttributesFromUnifiedMeta(resource.PhysicalDisk.SMART), + Collection: diskinventory.CloneStatus(resource.PhysicalDisk.Collection), LastChecked: resource.LastSeen, } if disk.Serial == "" { diff --git a/internal/monitoring/monitor_agents.go b/internal/monitoring/monitor_agents.go index 4ab39a48ea..5fcf15d549 100644 --- a/internal/monitoring/monitor_agents.go +++ b/internal/monitoring/monitor_agents.go @@ -4086,7 +4086,7 @@ func (m *Monitor) writeHostSMARTMetrics(host models.Host, now time.Time) { continue } - if disk.Temperature > 0 { + if diskTemperatureCollected(disk.Temperature, disk.Collection) { m.metricsStore.Write("disk", resourceID, "smart_temp", float64(disk.Temperature), now) } @@ -5090,7 +5090,7 @@ func (m *Monitor) evaluateHostAgents(now time.Time) { Bool("lastSeenZero", host.LastSeen.IsZero()). Msg("Host agent appears offline") } - if expiredHost, changed := m.state.ExpireHostTelemetry(host.ID); expiredHost.ID != "" { + if expiredHost, changed := m.state.ExpireHostTelemetry(host.ID, host.LastSeen); expiredHost.ID != "" { hostCopy = expiredHost resourceRefreshNeeded = resourceRefreshNeeded || changed } else { diff --git a/internal/monitoring/physical_disk_roundtrip_test.go b/internal/monitoring/physical_disk_roundtrip_test.go index 64060eaa9a..19c9f8a187 100644 --- a/internal/monitoring/physical_disk_roundtrip_test.go +++ b/internal/monitoring/physical_disk_roundtrip_test.go @@ -11,6 +11,9 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/notifications" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" + "github.com/rcourtman/pulse-go-rewrite/pkg/metrics" + "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" ) // Monitor construction restores notification destinations before admitting @@ -326,3 +329,228 @@ func TestMergeHostAgentSMARTIntoDisks_AgentWearoutFillsUnreportedPVELife(t *test t.Fatalf("agent endurance did not fill unreported PVE life: wearout=%d", result[0].Wearout) } } + +type silentAgentDiskPVEClient struct { + fakeStorageClient +} + +// The Proxmox inventory carries no temperature or I/O counters; for this disk +// both come only from the node's linked host agent. +func (*silentAgentDiskPVEClient) GetDisks(ctx context.Context, node string) ([]proxmox.Disk, error) { + return []proxmox.Disk{{ + DevPath: "/dev/sdb", + Model: "WDC WD40EFRX", + Serial: "WD-SILENT1", + Type: "hdd", + Health: "PASSED", + Wearout: 100, + Size: 4000787030016, + }}, nil +} + +// A host agent that stops reporting keeps its last SMART rows in state. Once +// its reporting lease expires, the linked Proxmox disk may retain the agent's +// last temperature and I/O counters, but the disk state, the canonical +// resource and temperature history must stop treating them as collected. +func TestSilentLinkedAgentSMARTIsRetainedButNotPresentedAsCollected(t *testing.T) { + t.Setenv("PULSE_DATA_DIR", t.TempDir()) + now := time.Now() + state := models.NewState() + state.UpdateNodesForInstance("pve1", []models.Node{{ + ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", + LastSeen: now, LinkedAgentID: "agent-1", + }}) + report := func(seen time.Time, temperature int, readBytes uint64) { + state.UpsertHost(models.Host{ + ID: "agent-1", Hostname: "node1", LinkedNodeID: "pve1-node1", + Status: "online", IntervalSeconds: 30, LastSeen: seen, + Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{ + Device: "/dev/sdb", Serial: "WD-SILENT1", Type: "sata", Health: "PASSED", + Temperature: temperature, + IO: &models.DiskIO{Device: "sdb", ReadBytes: readBytes}, + Collection: &diskinventory.CollectionStatus{ + Serial: diskinventory.Available("smartctl"), + Temperature: diskinventory.Available("smartctl"), + IO: diskinventory.Available("kernel_diskstats"), + }, + }}}, + }) + } + + alertManager := alerts.NewManager() + t.Cleanup(alertManager.Stop) + storeConfig := metrics.DefaultConfig(t.TempDir()) + storeConfig.FlushInterval = time.Hour + store, err := metrics.NewStore(storeConfig) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + history := NewMetricsHistory(100, time.Hour) + m := &Monitor{ + state: state, resourceStore: adapter, metricsHistory: history, metricsStore: store, + alertManager: alertManager, startTime: now.Add(-time.Hour), + lastPhysicalDiskPoll: make(map[string]time.Time), + } + pveNodes := []proxmox.Node{{Node: "node1", Status: "online"}} + nodeStatus := map[string]string{"node1": "online"} + + disk := func() models.PhysicalDisk { + t.Helper() + disks := state.GetSnapshot().PhysicalDisks + if len(disks) != 1 { + t.Fatalf("physical disks = %+v, want one", disks) + } + return disks[0] + } + fullPoll := func() models.PhysicalDisk { + t.Helper() + adapter.PopulateFromSnapshot(state.GetSnapshot()) + started := time.Now() + delete(m.lastPhysicalDiskPoll, "pve1") + m.maybePollPhysicalDisksAsync(context.Background(), "pve1", &config.PVEInstance{}, &silentAgentDiskPVEClient{}, pveNodes, nodeStatus, nil) + deadline := time.Now().Add(3 * time.Second) + for { + if disks := state.GetSnapshot().PhysicalDisks; len(disks) == 1 && !disks[0].LastChecked.Before(started) { + return disks[0] + } + if time.Now().After(deadline) { + t.Fatal("full physical disk poll did not land in state") + } + time.Sleep(10 * time.Millisecond) + } + } + skippedPoll := func() models.PhysicalDisk { + t.Helper() + adapter.PopulateFromSnapshot(state.GetSnapshot()) + m.lastPhysicalDiskPoll["pve1"] = time.Now() + m.maybePollPhysicalDisksAsync(context.Background(), "pve1", &config.PVEInstance{}, nil, nil, nil, nil) + return disk() + } + // Chart samples, and the persisted values. The store keys samples by + // second, so polls run back to back in this test can share one row. + temperatureSamples := func(d models.PhysicalDisk) (int, []float64) { + t.Helper() + id := unifiedresources.PhysicalDiskMetricID(d) + chart := len(history.GetDiskMetrics(id, "smart_temp", time.Hour)) + store.Flush() + points, err := store.Query("disk", id, "smart_temp", now.Add(-time.Hour), time.Now().Add(time.Hour), 0) + if err != nil { + t.Fatal(err) + } + stored := make([]float64, 0, len(points)) + for _, point := range points { + stored = append(stored, point.Value) + } + return chart, stored + } + canonicalCollection := func() *diskinventory.CollectionStatus { + t.Helper() + adapter.PopulateFromSnapshot(state.GetSnapshot()) + views := adapter.PhysicalDisks() + if len(views) != 1 { + t.Fatalf("canonical physical disks = %d, want one", len(views)) + } + if views[0].Temperature() != 41 { + t.Fatalf("canonical disk temperature = %d, want the retained 41", views[0].Temperature()) + } + return views[0].Collection() + } + assertRetainedNotCollected := func(step string, got models.PhysicalDisk) { + t.Helper() + if got.Temperature != 41 || got.IO == nil || got.IO.ReadBytes != 1000 { + t.Fatalf("%s: last-known temperature and I/O were not retained: temp=%d io=%+v", step, got.Temperature, got.IO) + } + if got.Collection == nil || + got.Collection.Temperature.State != diskinventory.FieldUnavailable || + got.Collection.IO.State != diskinventory.FieldUnavailable { + t.Fatalf("%s: silent agent's readings presented as collected: %+v", step, got.Collection) + } + } + + report(now, 41, 1000) + got := fullPoll() + if got.Temperature != 41 || got.Collection == nil || got.Collection.Temperature.State != diskinventory.FieldAvailable { + t.Fatalf("reporting agent temperature not merged: temp=%d collection=%+v", got.Temperature, got.Collection) + } + if chart, stored := temperatureSamples(got); chart != 1 || len(stored) != 1 || stored[0] != 41 { + t.Fatalf("reporting agent temperature history: chart=%d stored=%v, want one 41 sample", chart, stored) + } + + // The agent stops reporting; its lease expires on the next evaluation. + state.TouchHost("agent-1", now.Add(-hostAgentHealthWindow(30)-time.Minute)) + m.evaluateHostAgents(now) + + // No disk poll runs when the whole host is down (the PVE poll fails before + // reaching it), so the canonical disk must reflect the expiry on its own + // even though the Proxmox disk row still carries the earlier merge. + if collection := canonicalCollection(); collection == nil || + collection.Temperature.State != diskinventory.FieldUnavailable || + collection.IO.State != diskinventory.FieldUnavailable { + t.Fatalf("canonical disk presents the silent agent's readings as collected before any disk poll: %+v", collection) + } + assertRetainedNotCollected("skipped poll", skippedPoll()) + got = fullPoll() + assertRetainedNotCollected("full poll", got) + if chart, stored := temperatureSamples(got); chart != 1 || len(stored) != 1 { + t.Fatalf("silent agent's retained temperature was recorded as new history: chart=%d stored=%v", chart, stored) + } + if collection := canonicalCollection(); collection == nil || + collection.Temperature.State != diskinventory.FieldUnavailable || + collection.IO.State != diskinventory.FieldUnavailable { + t.Fatalf("canonical disk presents the silent agent's readings as collected: %+v", collection) + } + + // A resumed agent is collected again. + report(time.Now(), 39, 2000) + m.evaluateHostAgents(time.Now()) + got = fullPoll() + if got.Temperature != 39 || got.Collection.Temperature.State != diskinventory.FieldAvailable || + got.IO == nil || got.IO.ReadBytes != 2000 || got.Collection.IO.State != diskinventory.FieldAvailable { + t.Fatalf("resumed agent readings not collected: temp=%d io=%+v collection=%+v", got.Temperature, got.IO, got.Collection) + } + if chart, stored := temperatureSamples(got); chart != 2 || len(stored) == 0 || stored[len(stored)-1] != 39 { + t.Fatalf("resumed agent temperature history: chart=%d stored=%v, want a new 39 sample", chart, stored) + } +} + +// Agents before 6.2 send SMART rows without collection provenance. Their +// temperature is still a collected reading (history records it), and once the +// lease expires the disk must stop presenting it as collected even when a +// skipped disk poll carries the previous merge forward. +func TestMergeHostAgentSMARTIntoDisks_LegacyAgentTemperatureFollowsLease(t *testing.T) { + proxmoxDisk := models.PhysicalDisk{ + ID: "pve1-node1-sdb", Node: "node1", Instance: "pve1", DevPath: "/dev/sdb", Serial: "LEGACY1", + Collection: &diskinventory.CollectionStatus{ + Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature"), + }, + } + nodes := []models.Node{{Name: "node1", LinkedAgentID: "agent-1"}} + state := models.NewState() + state.UpsertHost(models.Host{ID: "agent-1", Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{ + Device: "/dev/sdb", Serial: "LEGACY1", Temperature: 38, + }}}}) + + reporting := mergeHostAgentSMARTIntoDisks([]models.PhysicalDisk{proxmoxDisk}, nodes, state.GetHosts())[0] + if reporting.Temperature != 38 || reporting.Collection.Temperature != diskinventory.Available(hostAgentLegacySource) { + t.Fatalf("legacy agent temperature not recorded as collected: temp=%d collection=%+v", reporting.Temperature, reporting.Collection) + } + if !diskTemperatureCollected(reporting.Temperature, reporting.Collection) { + t.Fatal("a reporting legacy agent's temperature must reach history") + } + + state.ExpireHostTelemetry("agent-1", time.Time{}) + for name, disk := range map[string]models.PhysicalDisk{ + "full poll": proxmoxDisk, + "skipped poll": reporting, + } { + got := mergeHostAgentSMARTIntoDisks([]models.PhysicalDisk{disk}, nodes, state.GetHosts())[0] + if got.Temperature != 38 || got.Collection.Temperature.State != diskinventory.FieldUnavailable { + t.Fatalf("%s: silent legacy agent temperature presented as collected: temp=%d collection=%+v", name, got.Temperature, got.Collection) + } + if diskTemperatureCollected(got.Temperature, got.Collection) { + t.Fatalf("%s: a silent legacy agent's retained temperature must not reach history", name) + } + } +} diff --git a/internal/unifiedresources/registry.go b/internal/unifiedresources/registry.go index 34992538c1..cb2f10a48a 100644 --- a/internal/unifiedresources/registry.go +++ b/internal/unifiedresources/registry.go @@ -3719,11 +3719,19 @@ func (rr *ResourceRegistry) mergeInto(existing *Resource, incoming Resource, sou if previous.IO != nil { existing.PhysicalDisk.IO = clonePhysicalDiskIOMeta(previous.IO) } - existing.PhysicalDisk.Collection = diskinventory.MergeStatus( + // The agent's own report is the later word on fields it supplied; + // the Proxmox row may still carry an earlier copy of them. + existing.PhysicalDisk.Collection = diskinventory.MergeReportedStatus( incoming.PhysicalDisk.Collection, previous.Collection, ) } + if source == SourceAgent && previous != nil { + existing.PhysicalDisk.Collection = diskinventory.MergeReportedStatus( + previous.Collection, + incoming.PhysicalDisk.Collection, + ) + } } if existing.PhysicalDisk != nil && (mergedPhysicalDisk || len(incoming.Incidents) > 0) { existing.PhysicalDisk.Risk = physicalDiskRiskFromMeta(existing.PhysicalDisk, existing.Incidents) diff --git a/internal/unifiedresources/registry_merge_policy_test.go b/internal/unifiedresources/registry_merge_policy_test.go index 51b27e416b..83c8fda61a 100644 --- a/internal/unifiedresources/registry_merge_policy_test.go +++ b/internal/unifiedresources/registry_merge_policy_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) func TestLinkedMergeAllowsOneSidedNodeHostLinkWhenHostnameCorroborates(t *testing.T) { @@ -557,3 +558,79 @@ func TestProxmoxOneWayLinkRejectsHostLocalEndpointCorroboration(t *testing.T) { t.Fatal("explicit private report-IP corroboration lost") } } + +// The Proxmox disk row carries a copy of the agent's collection state from the +// last disk poll. When the agent's own row later withdraws a reading (its +// reporting lease expired), the canonical disk must follow the agent in either +// ingest order, even if no disk poll has refreshed the Proxmox copy (a host that +// is down entirely is never disk-polled). +func TestPhysicalDiskMergeFollowsAgentWithdrawalOfItsOwnReadings(t *testing.T) { + proxmoxDisk := func(agent diskinventory.FieldStatus) Resource { + return Resource{ + Type: ResourceTypePhysicalDisk, Name: "WDC", Status: StatusOnline, + PhysicalDisk: &PhysicalDiskMeta{ + DevPath: "/dev/sdb", Serial: "WD-SILENT1", Temperature: 41, + Collection: &diskinventory.CollectionStatus{ + Serial: diskinventory.Available("proxmox_disks"), + Temperature: agent, + Pool: diskinventory.Available("proxmox_zfs"), + }, + }, + } + } + agentDisk := func(temperature diskinventory.FieldStatus) Resource { + return Resource{ + Type: ResourceTypePhysicalDisk, Name: "WDC", Status: StatusOnline, + PhysicalDisk: &PhysicalDiskMeta{ + DevPath: "/dev/sdb", Serial: "WD-SILENT1", Temperature: 41, + Collection: &diskinventory.CollectionStatus{ + Serial: diskinventory.Available("smartctl"), + Temperature: temperature, + }, + }, + } + } + identity := ResourceIdentity{MachineID: "WD-SILENT1", Hostnames: []string{"node1"}} + collected := diskinventory.Available("smartctl") + expired := diskinventory.Unavailable("smartctl", "host agent stopped reporting") + + for _, tc := range []struct { + name string + proxmoxFirst bool + agent diskinventory.FieldStatus + want diskinventory.FieldStatus + }{ + {"proxmox then expired agent", true, expired, expired}, + {"expired agent then proxmox", false, expired, expired}, + {"proxmox then reporting agent", true, collected, collected}, + {"reporting agent then proxmox", false, collected, collected}, + } { + t.Run(tc.name, func(t *testing.T) { + registry := NewRegistry(nil) + // The Proxmox row still holds the earlier "collected" copy. + ingestProxmox := func() { registry.ingest(SourceProxmox, "pve1-node1-sdb", proxmoxDisk(collected), identity) } + ingestAgent := func() { registry.ingest(SourceAgent, "agent-1-sdb", agentDisk(tc.agent), identity) } + if tc.proxmoxFirst { + ingestProxmox() + ingestAgent() + } else { + ingestAgent() + ingestProxmox() + } + disks := registry.ListByType(ResourceTypePhysicalDisk) + if len(disks) != 1 || disks[0].PhysicalDisk == nil || disks[0].PhysicalDisk.Collection == nil { + t.Fatalf("expected one merged disk, got %+v", disks) + } + collection := disks[0].PhysicalDisk.Collection + if collection.Temperature != tc.want { + t.Fatalf("temperature collection = %+v, want %+v", collection.Temperature, tc.want) + } + if disks[0].PhysicalDisk.Temperature != 41 { + t.Fatalf("last-known temperature was not retained: %d", disks[0].PhysicalDisk.Temperature) + } + if collection.Pool != diskinventory.Available("proxmox_zfs") { + t.Fatalf("Proxmox-owned evidence changed: %+v", collection.Pool) + } + }) + } +} diff --git a/pkg/diskinventory/status.go b/pkg/diskinventory/status.go index 68e98b4839..7041ab9883 100644 --- a/pkg/diskinventory/status.go +++ b/pkg/diskinventory/status.go @@ -89,6 +89,35 @@ func MergeStatus(existing, incoming *CollectionStatus) *CollectionStatus { return &merged } +// MergeReportedStatus merges a source's own report into existing state that +// may still hold an earlier copy of that report. It behaves like MergeStatus, +// except that a field the report no longer marks available supersedes an +// available state that existing carries from the same source: the source's +// later word on its own field wins, so a stale copy cannot keep claiming the +// field was collected. +func MergeReportedStatus(existing, reported *CollectionStatus) *CollectionStatus { + merged := MergeStatus(existing, reported) + if existing == nil || reported == nil { + return merged + } + merged.Serial = supersedeFieldStatus(merged.Serial, existing.Serial, reported.Serial) + merged.Temperature = supersedeFieldStatus(merged.Temperature, existing.Temperature, reported.Temperature) + merged.IO = supersedeFieldStatus(merged.IO, existing.IO, reported.IO) + merged.Controller = supersedeFieldStatus(merged.Controller, existing.Controller, reported.Controller) + merged.Pool = supersedeFieldStatus(merged.Pool, existing.Pool, reported.Pool) + return merged +} + +func supersedeFieldStatus(merged, existing, reported FieldStatus) FieldStatus { + source := strings.TrimSpace(existing.Source) + if existing.State == FieldAvailable && + reported.State != "" && reported.State != FieldAvailable && + source != "" && strings.EqualFold(source, strings.TrimSpace(reported.Source)) { + return reported + } + return merged +} + func mergeFieldStatus(existing, incoming FieldStatus) FieldStatus { if incoming.State == "" { return existing diff --git a/pkg/diskinventory/status_test.go b/pkg/diskinventory/status_test.go index 4c0bba2883..238e4b7097 100644 --- a/pkg/diskinventory/status_test.go +++ b/pkg/diskinventory/status_test.go @@ -24,3 +24,44 @@ func TestMergeStatusPrefersAvailableEvidencePerField(t *testing.T) { t.Fatalf("missing I/O state was not retained: %+v", got.IO) } } + +func TestMergeReportedStatusLetsTheSourceWithdrawItsOwnAvailability(t *testing.T) { + existing := &CollectionStatus{ + Serial: Available("smartctl"), + Temperature: Available("smartctl"), + IO: Available("kernel_diskstats"), + Pool: Available("proxmox_zfs"), + } + reported := &CollectionStatus{ + Serial: Available("smartctl"), + Temperature: Unavailable("smartctl", "host agent stopped reporting"), + IO: Unavailable("kernel_diskstats", "host agent stopped reporting"), + Pool: Unavailable("zpool", "command failed"), + } + + got := MergeReportedStatus(existing, reported) + if got.Temperature != reported.Temperature || got.IO != reported.IO { + t.Fatalf("the source's later word on its own fields was ignored: %+v", got) + } + if got.Serial != Available("smartctl") { + t.Fatalf("still-available field changed: %+v", got.Serial) + } + if got.Pool != Available("proxmox_zfs") { + t.Fatalf("another source's available evidence was withdrawn: %+v", got.Pool) + } + + // A copy that is already weaker never overrides a current report, and the + // rule is plain MergeStatus when either side is absent. + if got := MergeReportedStatus(reported, existing); got.Temperature != Available("smartctl") { + t.Fatalf("current report lost to a weaker copy: %+v", got.Temperature) + } + if got := MergeReportedStatus(nil, reported); got.Temperature != reported.Temperature { + t.Fatalf("nil existing: %+v", got) + } + if got := MergeReportedStatus(existing, nil); got.Temperature != existing.Temperature { + t.Fatalf("nil report: %+v", got) + } + if existing.Temperature != Available("smartctl") { + t.Fatal("MergeReportedStatus mutated its input") + } +} From 6413cda1712a701e82c3504a9923a42377bf54bf Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:28:07 +0100 Subject: [PATCH 20/27] Judge legacy disk temperature alerts by the lowest trigger on save A disk temperature alert persisted before CheckHost recorded its disk type has no diskType metadata until its next firing evaluation. A config save in that window judged it against the agent threshold. With a 75C agent default and a 70C nvme entry, a 72C alert was resolved on save and raised again by the next report. Without a known disk type, a save now judges the alert against the lowest enabled trigger the host's disks can use: the agent threshold or any DiskTempByType entry. A save therefore cannot resolve an alert its disk type would still fire, and turning the threshold off still resolves it. --- .../v6/internal/subsystems/alerts.md | 27 ++++++++------ internal/alerts/host.go | 26 +++++++++++++- internal/alerts/host_unraid_lifecycle_test.go | 35 +++++++++++++++++++ 3 files changed, 76 insertions(+), 12 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index bbfbba386d..1c5fa9f1ff 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -1598,8 +1598,10 @@ Host-agent SMART disk temperature alerts (`diskTemperature`, resource `agent:/disk_temp:`) follow one effective threshold: the resolved agent `DiskTemperature`, refined by the disk type's `DiskTempByType` entry unless an explicit host or linked-resource override set it -(`hostDiskTemperatureThresholdNoLock` in `internal/alerts/host.go`). When that -threshold is off, `CheckHost` clears the host's disk temperature alerts. The +(`hostDiskTemperatureThresholdNoLock` in `internal/alerts/host.go`). When the +host's resolved `DiskTemperature` is off, `CheckHost` clears every disk +temperature alert on the host. A disabled `DiskTempByType` entry clears only +the alerts of disks of that type, through the disabled metric spec. The all-agents switch, a disabled host override, agent removal and confirmed agent offline clear them with the host's other disk alerts. Before this, nothing cleared them once the agent-level threshold was off. @@ -1607,15 +1609,17 @@ nothing cleared them once the agent-level threshold was off. A config save judges an open disk temperature alert against the same threshold (`resolveHostAlertThresholdsNoLock` refines it by the alert's `diskType` metadata, and `metric_runtime.go` classifies `diskTemperature` as a -threshold metric), so turning the threshold off resolves the alert -immediately. Every firing evaluation refreshes `diskType`, so an alert at or -above its trigger always carries it. Only an alert held in its recovery band -since before that metadata existed (May 2026) falls back to the agent -threshold. Its reading is already below its trigger, so the fallback can -resolve it early or leave it to the evaluator, but it cannot resolve an alert -the next report raises again. +threshold metric), so turning the host's `DiskTemperature` off resolves the +alert immediately. An alert persisted before `CheckHost` recorded `diskType` (May +2026) carries no disk type until its next firing evaluation, so a save judges +it against the lowest enabled trigger any of the host's disks can use: the +agent threshold or a `DiskTempByType` entry +(`lowestHostDiskTemperatureThresholdNoLock`). A save therefore cannot resolve +an alert its disk type would still fire. -For every threshold metric, a config save resolves an alert whose reading is +For a metric alert that reaches the threshold comparison in +`reevaluateActiveAlertsLocked` (docker-host alerts return before it), a config +save resolves an alert whose reading is below the new trigger. That includes readings inside a valid recovery band, which the evaluator would hold. A clear level at or above the trigger is no recovery band (`buildCanonicalMetricSpec` drops it), and a reading at the @@ -1623,7 +1627,8 @@ trigger stays firing, as it does in the reducer, so a save no longer resolves an alert the next evaluation raises again. `TestCheckHostClearsDiskTemperatureAlertsWhenThresholdTurnsOff`, `TestHostDiskTemperatureAlertsClearWhenAgentLeaves`, -`TestConfigSaveResolvesDiskTemperatureAlertAgainstItsDiskTypeThreshold` and +`TestConfigSaveResolvesDiskTemperatureAlertAgainstItsDiskTypeThreshold`, +`TestConfigSaveKeepsDiskTemperatureAlertWithoutDiskType` and `TestConfigSaveKeepsAlertTheEvaluatorKeepsWithoutRecoveryBand` in `internal/alerts/host_unraid_lifecycle_test.go` pin these paths. diff --git a/internal/alerts/host.go b/internal/alerts/host.go index 6084b5c158..5fb2692605 100644 --- a/internal/alerts/host.go +++ b/internal/alerts/host.go @@ -143,7 +143,13 @@ func (m *Manager) resolveHostAlertThresholdsNoLock(alert *Alert, resourceID stri if alert != nil && alert.Type == "diskTemperature" { override, exists := m.hostThresholdOverrideNoLock(hostID, linkedNodeID, linkedVMID, linkedContainerID) overridden := exists && override.DiskTemperature != nil - thresholds.DiskTemperature = m.hostDiskTemperatureThresholdNoLock(thresholds.DiskTemperature, overridden, metadataStringValue(alert.Metadata, "diskType")) + if diskType, known := alert.Metadata["diskType"].(string); known || overridden { + thresholds.DiskTemperature = m.hostDiskTemperatureThresholdNoLock(thresholds.DiskTemperature, overridden, diskType) + } else { + // Alerts persisted before CheckHost recorded diskType carry no + // disk type until their next firing evaluation. + thresholds.DiskTemperature = m.lowestHostDiskTemperatureThresholdNoLock(thresholds.DiskTemperature) + } } return thresholds } @@ -934,6 +940,24 @@ func (m *Manager) hostDiskTemperatureThresholdNoLock(hostThreshold *HysteresisTh return hostThreshold } +// lowestHostDiskTemperatureThresholdNoLock returns the lowest enabled +// threshold any SMART disk of the host can be evaluated against. An alert +// whose disk type is unknown is judged against it, so a config save never +// resolves an alert its disk type would still fire. Callers must hold m.mu. +func (m *Manager) lowestHostDiskTemperatureThresholdNoLock(hostThreshold *HysteresisThreshold) *HysteresisThreshold { + if hostThreshold == nil || hostThreshold.Trigger <= 0 { + return hostThreshold + } + lowest := hostThreshold + for _, th := range m.config.DiskTempByType { + if th.Trigger > 0 && th.Trigger < lowest.Trigger { + t := th + lowest = &t + } + } + return lowest +} + // clearHostDiskTemperatureAlerts resolves every SMART disk temperature alert // for a host. func (m *Manager) clearHostDiskTemperatureAlerts(hostID string) { diff --git a/internal/alerts/host_unraid_lifecycle_test.go b/internal/alerts/host_unraid_lifecycle_test.go index 300f35b26f..afb88c6c8c 100644 --- a/internal/alerts/host_unraid_lifecycle_test.go +++ b/internal/alerts/host_unraid_lifecycle_test.go @@ -326,6 +326,41 @@ func TestConfigSaveResolvesDiskTemperatureAlertAgainstItsDiskTypeThreshold(t *te } } +func TestConfigSaveKeepsDiskTemperatureAlertWithoutDiskType(t *testing.T) { + m := configureDiskTempTypeHostManager(t) + + m.mu.Lock() + m.config.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 75, Clear: 70} + m.mu.Unlock() + + host := hostWithSMARTDiskTemp("host-temp-legacy", "nvme", 72) + alertID := hostDiskTempAlertID(host) + m.CheckHost(host) + alert, exists := testLookupActiveAlert(t, m, alertID) + if !exists { + t.Fatalf("expected nvme disk temperature alert at 72C, active: %v", alertKeys(m)) + } + // Restored from a release that did not record the disk type. + m.mu.Lock() + delete(alert.Metadata, "diskType") + m.mu.Unlock() + + // 72C is above the nvme trigger (70) but below the agent default (75). + cfg := m.GetConfig() + cfg.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 85, Clear: 80} + m.UpdateConfig(cfg) + if _, exists := testLookupActiveAlert(t, m, alertID); !exists { + t.Fatalf("config save resolved a disk temperature alert of unknown disk type that its nvme trigger still fires, active: %v", alertKeys(m)) + } + + cfg = m.GetConfig() + cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0, Clear: 0} + m.UpdateConfig(cfg) + if _, exists := testLookupActiveAlert(t, m, alertID); exists { + t.Fatalf("disk temperature alert of unknown disk type stayed active after the threshold was turned off, active: %v", alertKeys(m)) + } +} + func TestConfigSaveKeepsAlertTheEvaluatorKeepsWithoutRecoveryBand(t *testing.T) { // Normalization keeps a 70/75 entry, and the evaluator treats a clear // level at or above the trigger as no recovery band, firing from 70C. From aa252362a002945d5995deca4b81b49ef17f40a6 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:37:59 +0100 Subject: [PATCH 21/27] Prove a standby disk's retained temperature stays out of history A disk that goes into standby keeps its pre-sleep temperature through preserveUnavailablePhysicalDiskEvidence, and before the collection-state gate every full disk poll wrote that value to SMART history as a new sample. The silent-agent test covers lease expiry only; this pins the standby path the same gate fixes, starting from a disk that already has a recorded temperature. --- .../physical_disk_roundtrip_test.go | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/internal/monitoring/physical_disk_roundtrip_test.go b/internal/monitoring/physical_disk_roundtrip_test.go index 19c9f8a187..74bca727c3 100644 --- a/internal/monitoring/physical_disk_roundtrip_test.go +++ b/internal/monitoring/physical_disk_roundtrip_test.go @@ -554,3 +554,82 @@ func TestMergeHostAgentSMARTIntoDisks_LegacyAgentTemperatureFollowsLease(t *test } } } + +// A disk that goes into standby keeps its pre-sleep temperature as retained +// evidence (preserveUnavailablePhysicalDiskEvidence), with the agent's standby +// state. Full disk polls during standby must not record that value again. +func TestStandbyDiskRetainedTemperatureIsNotRecordedAsHistory(t *testing.T) { + t.Setenv("PULSE_DATA_DIR", t.TempDir()) + state := models.NewState() + state.UpdateNodesForInstance("pve1", []models.Node{{ + ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", + LastSeen: time.Now(), LinkedAgentID: "agent-1", + }}) + report := func(row models.HostDiskSMART) { + row.Device, row.Serial, row.Type, row.Health = "/dev/sdb", "WD-SILENT1", "sata", "PASSED" + state.UpsertHost(models.Host{ + ID: "agent-1", Hostname: "node1", LinkedNodeID: "pve1-node1", Status: "online", + IntervalSeconds: 30, LastSeen: time.Now(), + Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{row}}, + }) + } + alertManager := alerts.NewManager() + t.Cleanup(alertManager.Stop) + storeConfig := metrics.DefaultConfig(t.TempDir()) + storeConfig.FlushInterval = time.Hour + store, err := metrics.NewStore(storeConfig) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + history := NewMetricsHistory(100, time.Hour) + m := &Monitor{ + state: state, resourceStore: adapter, metricsHistory: history, metricsStore: store, + alertManager: alertManager, startTime: time.Now().Add(-time.Hour), + lastPhysicalDiskPoll: make(map[string]time.Time), + } + fullPoll := func() models.PhysicalDisk { + t.Helper() + adapter.PopulateFromSnapshot(state.GetSnapshot()) + started := time.Now() + delete(m.lastPhysicalDiskPoll, "pve1") + m.maybePollPhysicalDisksAsync(context.Background(), "pve1", &config.PVEInstance{}, &silentAgentDiskPVEClient{}, + []proxmox.Node{{Node: "node1", Status: "online"}}, map[string]string{"node1": "online"}, nil) + deadline := time.Now().Add(3 * time.Second) + for { + if disks := state.GetSnapshot().PhysicalDisks; len(disks) == 1 && !disks[0].LastChecked.Before(started) { + return disks[0] + } + if time.Now().After(deadline) { + t.Fatal("full physical disk poll did not land in state") + } + time.Sleep(10 * time.Millisecond) + } + } + samples := func(d models.PhysicalDisk) int { + return len(history.GetDiskMetrics(unifiedresources.PhysicalDiskMetricID(d), "smart_temp", time.Hour)) + } + + report(models.HostDiskSMART{Temperature: 38, Collection: &diskinventory.CollectionStatus{ + Temperature: diskinventory.Available("smartctl"), + }}) + if got := fullPoll(); got.Temperature != 38 || samples(got) != 1 { + t.Fatalf("awake disk temperature not recorded: temp=%d samples=%d", got.Temperature, samples(got)) + } + + // The agent keeps reporting, but the disk is now asleep. + report(models.HostDiskSMART{Standby: true, Collection: &diskinventory.CollectionStatus{ + Temperature: diskinventory.Unavailable("smartctl", "disk is in standby"), + }}) + for poll := 0; poll < 2; poll++ { + got := fullPoll() + if got.Temperature != 38 || got.Collection == nil || got.Collection.Temperature.State != diskinventory.FieldUnavailable { + t.Fatalf("poll %d: standby disk lost its retained temperature or claims it was collected: temp=%d collection=%+v", + poll, got.Temperature, got.Collection) + } + if n := samples(got); n != 1 { + t.Fatalf("poll %d: standby disk's pre-sleep temperature recorded as new history: samples=%d", poll, n) + } + } +} From 7c8e9099a1c08d925142613807cbbd358536afb3 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:39:29 +0100 Subject: [PATCH 22/27] Clear agent disk temperature alerts when the disk leaves A host-agent SMART disk temperature alert stayed active for a day once its disk left the agent's report (removed, replaced, or a renamed device path). CheckHost only evaluated disks still listed and node cleanup exempts agent: alerts, so only the 24-hour stale sweep resolved the alert or dropped a pending threshold run. CheckHost now tracks which disks a non-empty SMART list reports. Once a disk with an alert or a pending threshold run has been missing from three consecutive non-empty reports, the alert clears and the pending run is dropped. The Windows, FreeBSD and controller-multiplexed Linux collectors drop a disk whose probe fails, so a single omission does not count as the disk leaving, and the count restarts when the disk is reported again. An empty SMART list holds every alert, because the agent omits the list when collection fails or is unsupported. A listed disk in standby or without a temperature also holds its alert. Neither shows the disk cooled, and SMART health alerts already hold through standby. Turning the threshold off or removing the agent now drops pending runs and absence counts as well as alerts. Cleanup matches the disk_temp resource prefix alone, which also catches alerts raised under the original disk_temperature type. --- .../v6/internal/subsystems/alerts.md | 29 ++++ internal/alerts/host.go | 85 +++++++++- internal/alerts/host_unraid_lifecycle_test.go | 153 ++++++++++++++++++ internal/alerts/manager.go | 2 + 4 files changed, 265 insertions(+), 4 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 1c5fa9f1ff..c0b0274c87 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -1632,6 +1632,35 @@ an alert the next evaluation raises again. `TestConfigSaveKeepsAlertTheEvaluatorKeepsWithoutRecoveryBand` in `internal/alerts/host_unraid_lifecycle_test.go` pin these paths. +### Agent disk temperature alerts clear when their disk leaves the report + +`CheckHost` also clears a disk temperature alert, and drops its pending +threshold run, once its device is missing from three consecutive non-empty +SMART lists (`cleanupHostDiskTemperatureAlerts`, a seen map keyed by resource +ID like `cleanupHostCustomSensorAlerts`). A removed, replaced or renamed disk +never sends the reading that would resolve its alert, and +`shouldPreserveAlertOutsideNodeCleanup` exempts `agent:` alerts from node +cleanup, so before this only the 24-hour stale-alert sweep in +`cleanupStaleMaps` resolved it. One omission is not proof the disk left: the +Windows, FreeBSD and controller-multiplexed Linux collectors drop a disk whose +probe fails, so the count restarts whenever the disk is reported again. Three +reports in a row of failed probes on such a disk still clear its alert. + +An empty SMART list skips this cleanup and does not count toward the three, +because the agent omits the list when SMART collection fails or is +unsupported. A listed disk in standby or without a temperature also holds its +alert, since neither shows the disk cooled: standby reports avoid waking the +disk, and most failed Linux probes still list the disk with identity only. +SMART health alerts hold through standby the same way. A held alert resolves +through the normal recovery path, including any configured recovery delay, +once the disk reports cool readings again; with no evaluation for 24 hours, +the stale-alert sweep resolves it. Turning the threshold off, or removing the +agent, drops the host's pending disk temperature runs and absence counts as +well as its alerts. +`TestCheckHostClearsDiskTemperatureAlertWhenDiskLeavesSMARTReport` in +`internal/alerts/host_unraid_lifecycle_test.go` pins the clear, the holds, the +restarted count and the pending-run and count cleanup. + ### Configured flapping thresholds remain reachable Every accepted positive `FlappingThreshold`, including values above ten, must diff --git a/internal/alerts/host.go b/internal/alerts/host.go index 5fb2692605..04ead920c3 100644 --- a/internal/alerts/host.go +++ b/internal/alerts/host.go @@ -355,15 +355,21 @@ func (m *Manager) CheckHost(host models.Host) { } if thresholds.DiskTemperature != nil && thresholds.DiskTemperature.Trigger > 0 { + // An empty SMART list means collection failed or is unsupported for + // this report, not that every disk left, so existing alerts are held. if len(host.Sensors.SMART) > 0 { + seenDiskTemps := make(map[string]struct{}, len(host.Sensors.SMART)) for _, disk := range host.Sensors.SMART { + // A listed disk in standby, or without a temperature after a + // failed probe, is still present. Its alert holds until a fresh + // reading resolves it instead of clearing and re-raising. + tempResourceID := hostDiskTemperatureResourceID(host.ID, disk.Device) + seenDiskTemps[tempResourceID] = struct{}{} if disk.Temperature > 0 && !disk.Standby { m.mu.RLock() effectiveTempThreshold := m.hostDiskTemperatureThresholdNoLock(thresholds.DiskTemperature, diskTempOverridden, disk.Type) m.mu.RUnlock() - // Use specific resource ID for the disk: hostID/disk-temp:device - tempResourceID := hostDiskTemperatureResourcePrefix(host.ID) + sanitizeHostComponent(disk.Device) tempResourceName := fmt.Sprintf("%s (%s Temp)", hostDisplayName(host), disk.Device) diskTempMetadata := cloneMetadata(baseMetadata) @@ -386,6 +392,9 @@ func (m *Manager) CheckHost(host models.Host) { m.checkMetricWithCanonicalSpec(spec, tempResourceName, nodeName, disk.Device, "agent", float64(disk.Temperature), effectiveTempThreshold, &metricOptions{Metadata: diskTempMetadata}) } } + // A disk missing from consecutive non-empty reports was removed, + // replaced or renamed, so no later reading will resolve its alert. + m.cleanupHostDiskTemperatureAlerts(host.ID, seenDiskTemps) } } else { // Disk temperature alerting is off for this host, so no later reading @@ -634,6 +643,9 @@ func (m *Manager) HandleHostRemoved(host models.Host) { m.clearHostRAIDAlerts(host.ID) m.clearHostUnraidAlerts(host.ID) m.clearHostCustomSensorAlerts(host.ID) + // No later report will close the host's pending disk temperature runs or + // reset its disk absence counts. + m.clearHostDiskTemperatureAlerts(host.ID) } // HandleHostTelemetryExpired re-evaluates transient storage-operation evidence @@ -924,6 +936,12 @@ func hostDiskTemperatureResourcePrefix(hostID string) string { return hostResourceID(hostID) + "/disk_temp:" } +// hostDiskTemperatureResourceID is the resource ID of the SMART disk +// temperature alert for one device on a host. +func hostDiskTemperatureResourceID(hostID, device string) string { + return hostDiskTemperatureResourcePrefix(hostID) + sanitizeHostComponent(device) +} + // hostDiskTemperatureThresholdNoLock returns the threshold for one SMART disk // given the host's resolved disk temperature threshold. An enabled threshold // gives way to the disk type's DiskTempByType entry unless an explicit host or @@ -961,6 +979,20 @@ func (m *Manager) lowestHostDiskTemperatureThresholdNoLock(hostThreshold *Hyster // clearHostDiskTemperatureAlerts resolves every SMART disk temperature alert // for a host. func (m *Manager) clearHostDiskTemperatureAlerts(hostID string) { + m.cleanupHostDiskTemperatureAlerts(hostID, nil) +} + +// hostDiskTemperatureAbsenceConfirmations is how many consecutive non-empty +// SMART reports must omit a disk before its temperature alert clears. The +// Windows, FreeBSD and controller-multiplexed Linux collectors drop a disk +// whose probe fails, so one omission is not evidence the disk left. +const hostDiskTemperatureAbsenceConfirmations = 3 + +// cleanupHostDiskTemperatureAlerts resolves a host's SMART disk temperature +// alerts, and drops their pending threshold runs, once their resource ID has +// been missing from seen for hostDiskTemperatureAbsenceConfirmations calls in +// a row. A nil seen resolves all of them at once. +func (m *Manager) cleanupHostDiskTemperatureAlerts(hostID string, seen map[string]struct{}) { if hostID == "" { return } @@ -970,11 +1002,56 @@ func (m *Manager) clearHostDiskTemperatureAlerts(hostID string) { m.mu.Lock() defer m.mu.Unlock() + // Absent resources map to the storage keys of their active alerts; a + // resource with only a pending run maps to none. + absent := make(map[string][]string) for storageKey, alert := range m.activeAlerts { - if alert == nil || alert.Type != "diskTemperature" { + // Only CheckHost raises alerts under this prefix. Matching on it + // alone also catches alerts raised before the type became + // diskTemperature ("disk_temperature", January 2026). + if alert == nil || !strings.HasPrefix(alert.ResourceID, resourcePrefix) { continue } - if strings.HasPrefix(alert.ResourceID, resourcePrefix) { + if _, exists := seen[alert.ResourceID]; exists { + continue + } + absent[alert.ResourceID] = append(absent[alert.ResourceID], storageKey) + } + for _, resourceID := range m.core.PendingResourceIDs() { + if !strings.HasPrefix(resourceID, resourcePrefix) { + continue + } + if _, exists := seen[resourceID]; exists { + continue + } + if _, exists := absent[resourceID]; !exists { + absent[resourceID] = nil + } + } + + if m.hostDiskTempAbsences == nil { + m.hostDiskTempAbsences = make(map[string]int) + } + // A disk seen again, or whose state is gone, restarts its count. + for resourceID := range m.hostDiskTempAbsences { + if !strings.HasPrefix(resourceID, resourcePrefix) { + continue + } + if _, exists := absent[resourceID]; !exists { + delete(m.hostDiskTempAbsences, resourceID) + } + } + for resourceID, storageKeys := range absent { + if seen != nil { + m.hostDiskTempAbsences[resourceID]++ + if m.hostDiskTempAbsences[resourceID] < hostDiskTemperatureAbsenceConfirmations { + continue + } + } + delete(m.hostDiskTempAbsences, resourceID) + // A departed disk never sends the reading that would close its run. + m.core.DropPendingForResource(resourceID) + for _, storageKey := range storageKeys { m.clearAlertNoLock(storageKey) } } diff --git a/internal/alerts/host_unraid_lifecycle_test.go b/internal/alerts/host_unraid_lifecycle_test.go index afb88c6c8c..ae5d09a3f7 100644 --- a/internal/alerts/host_unraid_lifecycle_test.go +++ b/internal/alerts/host_unraid_lifecycle_test.go @@ -294,6 +294,159 @@ func TestHostDiskTemperatureAlertsClearWhenAgentLeaves(t *testing.T) { }) } +func TestCheckHostClearsDiskTemperatureAlertWhenDiskLeavesSMARTReport(t *testing.T) { + const hostID = "host-temp-absent" + smartDisk := func(device string, temperature int) models.HostDiskSMART { + return models.HostDiskSMART{Device: device, Model: "test-disk", Type: "sata", Temperature: temperature} + } + alertIDFor := func(device string) string { + return canonicalMetricStateID(hostDiskTemperatureResourceID(hostID, device), "diskTemperature") + } + sdaActive := func(m *Manager) bool { + _, exists := testLookupActiveAlert(t, m, alertIDFor("/dev/sda")) + return exists + } + // sda runs hot (sata trigger 55, clear 50); sdb stays cool. + hotHost := func(t *testing.T) (*Manager, models.Host) { + m := configureDiskTempTypeHostManager(t) + host := hostWithSMARTDiskTemp(hostID, "sata", 60) + host.Sensors.SMART = []models.HostDiskSMART{smartDisk("/dev/sda", 60), smartDisk("/dev/sdb", 40)} + m.CheckHost(host) + if !sdaActive(m) { + t.Fatalf("expected sda disk temperature alert at 60C, active: %v", alertKeys(m)) + } + return m, host + } + + cases := []struct { + name string + next []models.HostDiskSMART + wantHeld bool + wantRaised string + }{ + {name: "disk removed", next: []models.HostDiskSMART{smartDisk("/dev/sdb", 40)}}, + {name: "device renamed", next: []models.HostDiskSMART{smartDisk("/dev/sdc", 60), smartDisk("/dev/sdb", 40)}, wantRaised: "/dev/sdc"}, + // No fresh reading says the disk cooled or left, so the alert holds. + {name: "SMART collection empty", next: nil, wantHeld: true}, + {name: "disk in standby", next: []models.HostDiskSMART{{Device: "/dev/sda", Type: "sata", Temperature: 45, Standby: true}, smartDisk("/dev/sdb", 40)}, wantHeld: true}, + {name: "probe returned no temperature", next: []models.HostDiskSMART{{Device: "/dev/sda", Type: "sata"}, smartDisk("/dev/sdb", 40)}, wantHeld: true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + m, host := hotHost(t) + + host.Sensors.SMART = tc.next + for report := 1; report <= hostDiskTemperatureAbsenceConfirmations; report++ { + m.CheckHost(host) + wantActive := tc.wantHeld || report < hostDiskTemperatureAbsenceConfirmations + if active := sdaActive(m); active != wantActive { + t.Fatalf("report %d: sda disk temperature alert active = %v, want %v; active: %v", report, active, wantActive, alertKeys(m)) + } + } + if tc.wantRaised != "" { + if _, exists := testLookupActiveAlert(t, m, alertIDFor(tc.wantRaised)); !exists { + t.Fatalf("expected %s disk temperature alert, active: %v", tc.wantRaised, alertKeys(m)) + } + } + if !tc.wantHeld { + return + } + + // A held alert still resolves on the disk's next cool reading. + host.Sensors.SMART = []models.HostDiskSMART{smartDisk("/dev/sda", 45), smartDisk("/dev/sdb", 40)} + m.CheckHost(host) + if sdaActive(m) { + t.Fatalf("held sda disk temperature alert outlived a 45C reading, active: %v", alertKeys(m)) + } + }) + } + + t.Run("intermittently dropped disk keeps its alert", func(t *testing.T) { + m, host := hotHost(t) + present := host.Sensors.SMART + dropped := []models.HostDiskSMART{smartDisk("/dev/sdb", 40)} + + // The run of omissions restarts whenever the disk is reported again. + for _, smart := range [][]models.HostDiskSMART{dropped, dropped, present, dropped, dropped} { + host.Sensors.SMART = smart + m.CheckHost(host) + if !sdaActive(m) { + t.Fatalf("sda disk temperature alert cleared by an omission run shorter than %d reports, active: %v", hostDiskTemperatureAbsenceConfirmations, alertKeys(m)) + } + } + }) + + // pendingHost leaves a hot sda inside a 300s alert delay, so sda holds a + // pending run and no alert. + pendingHost := func(t *testing.T) (*Manager, models.Host, func() bool) { + m := configureDiskTempTypeHostManager(t) + m.mu.Lock() + m.config.TimeThresholds = map[string]int{"agent": 300} + m.mu.Unlock() + sdaResourceID := hostDiskTemperatureResourceID(hostID, "/dev/sda") + sdaPending := func() bool { + m.mu.RLock() + defer m.mu.RUnlock() + for _, resourceID := range m.core.PendingResourceIDs() { + if resourceID == sdaResourceID { + return true + } + } + return false + } + + host := hostWithSMARTDiskTemp(hostID, "sata", 60) + host.Sensors.SMART = []models.HostDiskSMART{smartDisk("/dev/sda", 60), smartDisk("/dev/sdb", 40)} + m.CheckHost(host) + if sdaActive(m) || !sdaPending() { + t.Fatalf("expected a pending sda run inside the 300s delay, active: %v", alertKeys(m)) + } + return m, host, sdaPending + } + + t.Run("departed disk drops its pending run", func(t *testing.T) { + m, host, sdaPending := pendingHost(t) + + host.Sensors.SMART = []models.HostDiskSMART{smartDisk("/dev/sdb", 40)} + for report := 1; report <= hostDiskTemperatureAbsenceConfirmations; report++ { + m.CheckHost(host) + wantPending := report < hostDiskTemperatureAbsenceConfirmations + if pending := sdaPending(); pending != wantPending { + t.Fatalf("report %d: pending sda run = %v, want %v", report, pending, wantPending) + } + } + }) + + t.Run("threshold off drops pending runs", func(t *testing.T) { + m, host, sdaPending := pendingHost(t) + + m.mu.Lock() + m.config.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0, Clear: 0} + m.mu.Unlock() + m.CheckHost(host) + if sdaPending() { + t.Fatal("pending sda run survived its threshold being turned off; re-enabling would inherit its old start time") + } + }) + + t.Run("removed host leaves no pending run or absence count", func(t *testing.T) { + m, host, sdaPending := pendingHost(t) + + host.Sensors.SMART = []models.HostDiskSMART{smartDisk("/dev/sdb", 40)} + m.CheckHost(host) + m.HandleHostRemoved(host) + if sdaPending() { + t.Fatal("pending sda run outlived its removed host") + } + m.mu.RLock() + defer m.mu.RUnlock() + if len(m.hostDiskTempAbsences) != 0 { + t.Fatalf("absence counts outlived their removed host: %v", m.hostDiskTempAbsences) + } + }) +} + func TestConfigSaveResolvesDiskTemperatureAlertAgainstItsDiskTypeThreshold(t *testing.T) { m := configureDiskTempTypeHostManager(t) diff --git a/internal/alerts/manager.go b/internal/alerts/manager.go index 53aa04e84f..64b7093c4d 100644 --- a/internal/alerts/manager.go +++ b/internal/alerts/manager.go @@ -78,6 +78,7 @@ type Manager struct { unifiedIncidentConfirmations map[string]int // Track consecutive provider-incident observations before activation unifiedIncidentFirstSeen map[string]time.Time // Preserve the first confirmed observation as lifecycle start unifiedIncidentRecoveries map[string]int // Track consecutive healthy observations before provider-incident recovery + hostDiskTempAbsences map[string]int // Track consecutive SMART reports missing a disk that holds temperature alert state dockerRestartTracking map[string]*dockerRestartRecord // Track restart counts and times for restart loop detection dockerUpdateFirstSeen map[string]time.Time // Track when image updates were first detected for alert delay // Stable identity tracking prevents update-delay resets when host IDs churn. @@ -233,6 +234,7 @@ func NewManagerWithDataDir(dataDir string, options ...ManagerOption) *Manager { unifiedIncidentConfirmations: make(map[string]int), unifiedIncidentFirstSeen: make(map[string]time.Time), unifiedIncidentRecoveries: make(map[string]int), + hostDiskTempAbsences: make(map[string]int), dockerRestartTracking: make(map[string]*dockerRestartRecord), dockerUpdateFirstSeen: make(map[string]time.Time), dockerUpdateFirstSeenByIdentity: make(map[string]time.Time), From a649dd9b45ee66dc767351eddf55783cbcd094ed Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:43:14 +0100 Subject: [PATCH 23/27] Tint Kubernetes node rows for their drawer's alerts A Kubernetes node that runs a Pulse agent is an agent row, and the agent's alerts are keyed "agent:" and labelled with the hostname. The nodes table matched only the row id and display name, so a node at 100% memory with a critical alert stayed untinted while its drawer listed the alert. The rows now use getUnifiedResourceAlertStyles, the set the drawer reads, and drop the tint while expanded like the Docker host and Machines rows. The overview's "Nodes needing attention" looked only at readiness and cordons, so that node was missing from the list too. It now ranks each node by the stronger of its state and its unacknowledged open alerts, NotReady and critical alerts first, and the nodes table's default order uses the same rank so a critical node no longer sorts below a degraded one. An acknowledged alert drops out of both, as it does from the tint. vSphere host and TrueNAS system rows keep the id-plus-host-name match. Their VMs', pools' and apps' alerts carry the host name as their node, the TrueNAS drawer states the same pool problems, and no producer keys alerts on those rows' "agent:" aliases (a Pulse agent on a TrueNAS box stays a separate row). A test now pins the TrueNAS pool rollup. --- .../subsystems/frontend-primitives.md | 3 +- .../internal/subsystems/unified-resources.md | 12 ++- .../83848f2d5ef2a326.json | 51 +++++++++++ .../kubernetes/KubernetesNodesTable.tsx | 35 +++++--- .../kubernetes/KubernetesPageSurface.tsx | 18 +++- .../__tests__/KubernetesNodesTable.test.tsx | 86 +++++++++++++++++- .../KubernetesPageSurface.contract.test.tsx | 68 ++++++++++++++ .../__tests__/kubernetesPageModel.test.ts | 89 +++++++++++++++++++ .../kubernetes/kubernetesPageModel.ts | 49 +++++++--- .../__tests__/TrueNASSystemsTable.test.tsx | 43 ++++++++- 10 files changed, 426 insertions(+), 28 deletions(-) create mode 100644 frontend-modern/browser-verification/83848f2d5ef2a326.json diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index db7c72aded..a98a3b15eb 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -4995,7 +4995,8 @@ name, so it keeps the shared single-line rhythm. Drawers for unified resources websocket `activeAlerts` map through `getAlertsForUnifiedResource` (`frontend-modern/src/utils/alerts.ts`), most severe first, never from a resource-embedded list, and show each alert's resource name and metric beside -its text like the other drawers. Their table rows take the platform host rows' +its text like the other drawers. Their table rows (Docker hosts, Machines, +Kubernetes nodes) take the platform host rows' red (critical) or yellow tint from the same set through `getUnifiedResourceAlertStyles`, only for unacknowledged alerts and never on an expanded row, so a row and its drawer agree. The remaining Overview rows are additive diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 9a954da020..b459d393da 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -2230,7 +2230,17 @@ may repeat beside it. The Docker hosts (`DockerHostsTable.tsx`) and Machines `getUnifiedResourceAlertStyles`, so a collapsed row never looks healthy while its drawer lists an unacknowledged alert (an expanded row drops the tint while its drawer shows the alerts); acknowledged alerts stay in the drawer without -tinting the row. +tinting the row. Kubernetes node rows (`KubernetesNodesTable.tsx`) tint the +same way: a node that runs a Pulse agent is an agent row whose alerts are keyed +`agent:` and carry the hostname, so the earlier id-and-display-name match +left it untinted while its drawer listed a critical alert. The Kubernetes +overview's "Nodes needing attention" and the nodes table's default order rank +a node by the stronger of its readiness state and those unacknowledged alerts +(`getKubernetesNodeAttentionRank`), so a Ready node at 100% memory is named +beside the NotReady ones. vSphere host and TrueNAS system rows keep +`getAlertStyles` with the host name: their VMs', pools' and apps' alerts carry +that name as their node, and no producer keys alerts on those rows' `agent:` +aliases (a Pulse agent on a TrueNAS box stays a separate resource). Machine and host overview cards that render compact system, hardware, disk, and temperature facts must also compose the frontend-primitives `InfoCardKeyValueRow`. Mobile rows retain their condensed endpoint layout; diff --git a/frontend-modern/browser-verification/83848f2d5ef2a326.json b/frontend-modern/browser-verification/83848f2d5ef2a326.json new file mode 100644 index 0000000000..edffd9048a --- /dev/null +++ b/frontend-modern/browser-verification/83848f2d5ef2a326.json @@ -0,0 +1,51 @@ +{ + "version": 1, + "base_sha": "3b290640a51ef0f488f9cd06b4aa644adbdcc6b3", + "verified_at": "2026-10-06T15:43:08Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/features/kubernetes/KubernetesNodesTable.tsx", + "frontend-modern/src/features/kubernetes/KubernetesPageSurface.tsx", + "frontend-modern/src/features/kubernetes/kubernetesPageModel.ts" + ], + "content_sha256": { + "frontend-modern/src/features/kubernetes/KubernetesNodesTable.tsx": "8861f6f170dd9ebab4eebe949e3a311af46916aa10d8d2d2768a33e3d64b2d1a", + "frontend-modern/src/features/kubernetes/KubernetesPageSurface.tsx": "9a2096f3ff967cec93024466d15c1b5117f08d89985c1955965d336a38d03bc0", + "frontend-modern/src/features/kubernetes/kubernetesPageModel.ts": "0c2f0be19aa05f8e07f1167273b26b17b0643c73855df8246fcf5f591d3ae5b8" + }, + "routes": [ + "/kubernetes/overview", + "/kubernetes/nodes" + ], + "viewports": [ + { + "width": 1440, + "height": 1000 + }, + { + "width": 1280, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Private mock stack (backend built from base 3b290640a5 on its own port, Vite serving this tree); mock mode skips CheckHost, so the private binary added that call through a go build overlay of internal/monitoring/monitor_alerts.go that is not part of this commit", + "Kubernetes nodes: Prod Euw1 K8s 02 (agent row agent-c8db98b4db98ed5c) has a backend-raised critical alert keyed agent:host-k8s-k8s-production-1-k8s-production-1-node-2-955a86fe (node prod-euw1-k8s-02, Agent memory at 100.0%); before the change its row was untinted while its drawer listed the alert, now the row is red-50 and its drawer lists the same alert", + "Kubernetes nodes default order: K8s 02 (Ready, critical alert) first, then K8s 03 (Degraded), then K8s 01 (Ready); K8s 01 and K8s 03 untinted with no drawer alerts", + "Kubernetes overview: Nodes needing attention lists K8s 02 (red) then K8s 03 (Degraded); before the change it listed only K8s 03", + "After acknowledging the K8s 02 alert through the private stack API: its row is untinted, it drops out of Nodes needing attention, the nodes table falls back to K8s 03, K8s 01, K8s 02, and its drawer still lists the alert as Acknowledged; unacknowledged again afterwards", + "Expanded K8s 02 row drops the tint (surface hover background) while its drawer shows the alert", + "Light theme at 1440px and 390px and dark theme at 1280px: red-50 / red-950/25 on K8s 02 only, no horizontal overflow at 390px", + "TrueNAS and vSphere tables unchanged: truenas-main stays amber for its pool alerts (node truenas-main) and its drawer lists the pool problems; esxi-07 amber for its own alert", + "No page errors or console errors" + ], + "interactions": [ + "Open Kubernetes overview and Nodes, read each node row's tint and order", + "Expand and collapse the K8s 02 row drawer and read its Needs attention list", + "Acknowledge then unacknowledge the K8s 02 memory alert and re-read the overview list, row tint, order and drawer", + "Repeat the overview and nodes checks at the 390px phone viewport (row tap to expand) and in dark mode" + ] +} diff --git a/frontend-modern/src/features/kubernetes/KubernetesNodesTable.tsx b/frontend-modern/src/features/kubernetes/KubernetesNodesTable.tsx index 0d939ae8c9..dae497764b 100644 --- a/frontend-modern/src/features/kubernetes/KubernetesNodesTable.tsx +++ b/frontend-modern/src/features/kubernetes/KubernetesNodesTable.tsx @@ -6,7 +6,7 @@ import { StackedMemoryBar } from '@/components/Workloads/StackedMemoryBar'; import { TableCell, TableRow } from '@/components/shared/Table'; import { ResourceNameWithWebInterfaceLink } from '@/components/shared/WebInterfaceLink'; import { asTrimmedString } from '@/utils/stringUtils'; -import { getAlertStyles } from '@/utils/alerts'; +import { getUnifiedResourceAlertStyles } from '@/utils/alerts'; import { useWebSocket } from '@/contexts/appRuntime'; import { useAlertsActivation } from '@/stores/alertsActivation'; import { unifiedPlatformOverrideIdCandidates } from '@/features/alerts/alertOverridesModel'; @@ -41,6 +41,7 @@ import type { Resource } from '@/types/resource'; import { compareKubernetesNodes, filterKubernetesResources, + getKubernetesNodeAttentionRank, kubernetesClusterLabel, mapKubernetesNodeStatus, type KubernetesResourceStatusFilter, @@ -145,19 +146,27 @@ export const KubernetesNodesTable: Component<{ const drawer = createPlatformResourceDetailState({ idPrefix: 'kubernetes-node-drawer' }); const resolveResourceLabel = createPlatformResourceLabelResolver(() => props.resources); // User-controlled sorting layered over the attention-first default: rows - // are pre-sorted by the status compare, so a user sort keeps that order - // for ties and the table falls straight back to it when the sort clears. + // are pre-sorted by attention (status and open alerts), then the status + // compare, so a user sort keeps that order for ties and the table falls + // straight back to it when the sort clears. const sort = createPlatformTableSortState({ storageKey: props.sortStorageKey ?? 'kubernetesNodes', sortKeys: KUBERNETES_NODE_SORT_KEYS, descendingFirst: ['cpu', 'memory', 'uptime', 'capacity'], }); - const sortedRows = createMemo(() => - sort.sortRows( - [...tableState.filtered()].sort(compareKubernetesNodes), + const sortedRows = createMemo(() => { + const rows = tableState.filtered(); + const enabled = alertsEnabled(); + const ranks = new Map( + rows.map((node) => [node.id, getKubernetesNodeAttentionRank(node, activeAlerts, enabled)]), + ); + return sort.sortRows( + [...rows].sort( + (a, b) => (ranks.get(a.id) ?? 2) - (ranks.get(b.id) ?? 2) || compareKubernetesNodes(a, b), + ), getKubernetesNodeSortValue, - ), - ); + ); + }); return ( indicator().variant !== 'muted'; const detailRowId = () => drawer.detailRowId(node); const isExpanded = () => drawer.isExpanded(node); + // The same open alerts the node's drawer lists: a node that + // runs a Pulse agent has its alerts keyed on the agent and + // labelled with its hostname, so matching the row's id or + // display name finds none of them. const nodeAlertStyles = createMemo(() => - getAlertStyles(node.id, activeAlerts, alertsEnabled(), name()), + getUnifiedResourceAlertStyles(node, activeAlerts, alertsEnabled()), ); const nodeAlertBg = () => { const s = nodeAlertStyles(); @@ -380,7 +393,9 @@ export const KubernetesNodesTable: Component<{ return ( <> drawer.toggle(node)} > diff --git a/frontend-modern/src/features/kubernetes/KubernetesPageSurface.tsx b/frontend-modern/src/features/kubernetes/KubernetesPageSurface.tsx index 311aa3f67b..be03791131 100644 --- a/frontend-modern/src/features/kubernetes/KubernetesPageSurface.tsx +++ b/frontend-modern/src/features/kubernetes/KubernetesPageSurface.tsx @@ -9,8 +9,10 @@ import { collectOutdatedAgentHosts, formatAgentVersionDisplay, } from '@/features/platformPage/agentVersion'; +import { useWebSocket } from '@/contexts/appRuntime'; import { useUnifiedResources, type UnifiedResourceFacets } from '@/hooks/useUnifiedResources'; import { buildKubernetesPath, KUBERNETES_QUERY_PARAMS } from '@/routing/resourceLinks'; +import { useAlertsActivation } from '@/stores/alertsActivation'; import { updateStore } from '@/stores/updates'; import { PLATFORM_HEALTH_FILTER_OPTIONS, @@ -710,14 +712,22 @@ function KubernetesOverview(props: KubernetesOverviewProps) { return posture.podAttention + posture.deploymentAttention; }); - const nodesNeedingAttention = createMemo(() => getKubernetesNodesNeedingAttention(props.model())); + const { activeAlerts } = useWebSocket(); + const alertsActivation = useAlertsActivation(); + const nodesNeedingAttention = createMemo(() => + getKubernetesNodesNeedingAttention( + props.model(), + activeAlerts, + alertsActivation.detectionEnabled(), + ), + ); return (

{/* Problems lead, inventory follows: active health signals, then the - nodes that are not Ready, before the cluster summary and workloads. - Filtering stays on the Nodes and Events tabs. + nodes that are not Ready or have an open alert, before the cluster + summary and workloads. Filtering stays on the Nodes and Events tabs. */} 0}>
@@ -738,7 +748,7 @@ function KubernetesOverview(props: KubernetesOverviewProps) { sortStorageKey="kubernetesNodesNeedingAttention" emptyIcon={k8sIcon()} emptyTitle="All nodes are Ready" - emptyDescription="Nodes appear here when they are not Ready, cordoned or degraded." + emptyDescription="Nodes appear here when they are not Ready, cordoned, degraded or alerting." showToolbar={false} />
diff --git a/frontend-modern/src/features/kubernetes/__tests__/KubernetesNodesTable.test.tsx b/frontend-modern/src/features/kubernetes/__tests__/KubernetesNodesTable.test.tsx index bdc81eba19..59791206c1 100644 --- a/frontend-modern/src/features/kubernetes/__tests__/KubernetesNodesTable.test.tsx +++ b/frontend-modern/src/features/kubernetes/__tests__/KubernetesNodesTable.test.tsx @@ -1,6 +1,8 @@ import { cleanup, fireEvent, render, screen } from '@solidjs/testing-library'; +import { createStore, reconcile } from 'solid-js/store'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { Alert } from '@/types/api'; import type { Resource } from '@/types/resource'; import { KubernetesNodesTable } from '../KubernetesNodesTable'; @@ -12,8 +14,11 @@ vi.mock('@/components/Workloads/StackedMemoryBar', () => ({ StackedMemoryBar: () =>
, })); +// The websocket's activeAlerts is a Solid store, so rows must follow it live. +const activeAlertsRef = vi.hoisted(() => ({ current: {} as Record })); + vi.mock('@/contexts/appRuntime', () => ({ - useWebSocket: () => ({ activeAlerts: [] }), + useWebSocket: () => ({ activeAlerts: activeAlertsRef.current }), })); vi.mock('@/stores/alertsActivation', () => ({ @@ -51,8 +56,12 @@ const makeNodeResource = (overrides: Partial = {}): Resource => ({ ...overrides, }); +const [activeAlerts, setActiveAlerts] = createStore>({}); +activeAlertsRef.current = activeAlerts; + afterEach(() => { cleanup(); + setActiveAlerts(reconcile({})); vi.clearAllMocks(); }); @@ -146,4 +155,79 @@ describe('KubernetesNodesTable', () => { 'false', ); }); + it('tints a node row for the open alerts its drawer lists and leads with it', () => { + const agentAlert = (id: string, resourceId: string, overrides: Partial = {}): Alert => ({ + id, + type: 'memory', + level: 'critical', + resourceId, + resourceName: 'Prod Euw1 K8s 02', + node: 'prod-euw1-k8s-02', + instance: '', + message: 'Agent memory at 100.0%', + value: 100, + threshold: 90, + startTime: '2026-10-06T10:00:00Z', + acknowledged: false, + ...overrides, + }); + // A node that runs a Pulse agent is an agent row; its alerts are keyed + // "agent:" and carry the hostname, never the row id or name. + const agentNode = (id: string, name: string, agentId: string, ready = true) => + makeNodeResource({ + id, + name, + displayName: name, + type: 'agent', + agent: { agentId }, + kubernetes: { ...makeNodeResource().kubernetes, nodeName: id, ready }, + } as Partial); + setActiveAlerts({ + memory: agentAlert('memory', 'agent:host-k8s-node-2'), + }); + + const { container } = render(() => ( + } + emptyTitle="No nodes" + emptyDescription="No nodes" + showToolbar={false} + /> + )); + + const row = (id: string) => + container.querySelector(`[data-kubernetes-node-row="${id}"]`) as HTMLElement; + const order = () => + [...container.querySelectorAll('[data-kubernetes-node-row]')].map((element) => + element.getAttribute('data-kubernetes-node-row'), + ); + expect(row('node-2')).toHaveClass('bg-red-50'); + expect(row('node-1')).not.toHaveClass('bg-red-50'); + // A Ready node with a critical alert leads, ahead of a cordoned node. + expect(order()).toEqual(['node-2', 'node-3', 'node-1']); + + // The tint and the order follow the live alert store: acknowledging + // clears both, and a new warning tints the row amber. + setActiveAlerts('memory', 'acknowledged', true); + expect(row('node-2')).not.toHaveClass('bg-red-50'); + expect(order()).toEqual(['node-3', 'node-1', 'node-2']); + setActiveAlerts('memory', { acknowledged: false, level: 'warning' }); + expect(row('node-2')).toHaveClass('bg-yellow-50'); + // An expanded row drops the tint while its drawer lists the alert. + fireEvent.click(row('node-2')); + expect(row('node-2')).not.toHaveClass('bg-yellow-50'); + fireEvent.click(row('node-2')); + expect(row('node-2')).toHaveClass('bg-yellow-50'); + setActiveAlerts(reconcile({})); + expect(row('node-2')).not.toHaveClass('bg-yellow-50'); + }); }); diff --git a/frontend-modern/src/features/kubernetes/__tests__/KubernetesPageSurface.contract.test.tsx b/frontend-modern/src/features/kubernetes/__tests__/KubernetesPageSurface.contract.test.tsx index 44bb874f08..8badbd48f1 100644 --- a/frontend-modern/src/features/kubernetes/__tests__/KubernetesPageSurface.contract.test.tsx +++ b/frontend-modern/src/features/kubernetes/__tests__/KubernetesPageSurface.contract.test.tsx @@ -1,12 +1,15 @@ import { Route, Router } from '@solidjs/router'; import { cleanup, render, screen } from '@solidjs/testing-library'; +import { createStore, reconcile } from 'solid-js/store'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Alert } from '@/types/api'; import type { Resource } from '@/types/resource'; import { KubernetesPageSurface } from '../KubernetesPageSurface'; const mockUseUnifiedResources = vi.fn(); const mockPathname = vi.hoisted(() => vi.fn(() => '/')); const mockVersionInfo = vi.hoisted(() => vi.fn()); +const activeAlertsRef = vi.hoisted(() => ({ current: {} as Record })); const makeResource = (resource: Partial & Pick): Resource => ({ name: resource.id, @@ -40,6 +43,14 @@ vi.mock('@/hooks/useUnifiedResources', () => ({ useUnifiedResources: (...args: unknown[]) => mockUseUnifiedResources(...args), })); +vi.mock('@/contexts/appRuntime', () => ({ + useWebSocket: () => ({ activeAlerts: activeAlertsRef.current }), +})); + +vi.mock('@/stores/alertsActivation', () => ({ + useAlertsActivation: () => ({ detectionEnabled: () => true }), +})); + vi.mock('@/stores/updates', () => ({ updateStore: { versionInfo: mockVersionInfo, @@ -129,6 +140,7 @@ vi.mock('../KubernetesNodesTable', () => ({
resource.id).join(',')} data-title={props.title} data-sort-key={props.sortStorageKey} /> @@ -159,6 +171,9 @@ vi.mock('../KubernetesStorageTable', () => ({ ), })); +const [activeAlerts, setActiveAlerts] = createStore>({}); +activeAlertsRef.current = activeAlerts; + describe('KubernetesPageSurface contract', () => { beforeEach(() => { mockPathname.mockReturnValue('/'); @@ -167,6 +182,7 @@ describe('KubernetesPageSurface contract', () => { afterEach(() => { cleanup(); + setActiveAlerts(reconcile({})); vi.clearAllMocks(); }); @@ -207,6 +223,58 @@ describe('KubernetesPageSurface contract', () => { ).toBeTruthy(); }); + it('names a Ready node with an open alert among the nodes needing attention', () => { + setResources([ + makeResource({ id: 'cluster-1', type: 'k8s-cluster' }), + makeResource({ + id: 'node-memory', + type: 'agent', + sources: ['agent', 'kubernetes'], + agent: { agentId: 'host-k8s-02' }, + kubernetes: { ready: true } as Resource['kubernetes'], + } as Partial & Pick), + makeResource({ + id: 'node-ok', + type: 'k8s-node', + kubernetes: { ready: true } as Resource['kubernetes'], + }), + ]); + renderSurface(); + // Every node is Ready, so nothing needs attention until an alert opens. + expect(screen.queryByTestId('nodes-table')).toBeNull(); + + // The node's agent raises it: keyed on the agent, labelled with the hostname. + setActiveAlerts({ + memory: { + id: 'memory', + type: 'memory', + level: 'critical', + resourceId: 'agent:host-k8s-02', + resourceName: 'Prod Euw1 K8s 02', + node: 'prod-euw1-k8s-02', + instance: '', + message: 'Agent memory at 100.0%', + value: 100, + threshold: 90, + startTime: '2026-10-06T10:00:00Z', + acknowledged: false, + }, + }); + + const attention = screen.getByTestId('nodes-table'); + expect(attention).toHaveAttribute('data-title', 'Nodes needing attention'); + expect(attention).toHaveAttribute('data-ids', 'node-memory'); + + // Acknowledged: someone owns it, and the row is no longer tinted either. + setActiveAlerts('memory', 'acknowledged', true); + expect(screen.queryByTestId('nodes-table')).toBeNull(); + setActiveAlerts('memory', 'acknowledged', false); + expect(screen.getByTestId('nodes-table')).toHaveAttribute('data-ids', 'node-memory'); + // Resolved: the alert leaves the store and the node leaves the list. + setActiveAlerts(reconcile({})); + expect(screen.queryByTestId('nodes-table')).toBeNull(); + }); + it('shows no attention table when every node is Ready', () => { setResources([ makeResource({ id: 'cluster-1', type: 'k8s-cluster' }), diff --git a/frontend-modern/src/features/kubernetes/__tests__/kubernetesPageModel.test.ts b/frontend-modern/src/features/kubernetes/__tests__/kubernetesPageModel.test.ts index 8511c16e62..0079bef0d3 100644 --- a/frontend-modern/src/features/kubernetes/__tests__/kubernetesPageModel.test.ts +++ b/frontend-modern/src/features/kubernetes/__tests__/kubernetesPageModel.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from 'vitest'; +import type { Alert } from '@/types/api'; import type { Resource } from '@/types/resource'; import { + getKubernetesNodeAttentionRank, getKubernetesNodesNeedingAttention, splitKubernetesNameTail, isSingleKubernetesCluster, @@ -1178,4 +1180,91 @@ describe('getKubernetesNodesNeedingAttention', () => { expect(result.map((entry: { id: string }) => entry.id)).toEqual(['a-down', 'c-cordoned']); }); + + // A node running a Pulse agent is an agent row whose alerts are keyed + // "agent:" and labelled with its hostname. + const agentNode = (id: string, agentId: string, kubernetes: Record) => + ({ + id, + name: id, + type: 'agent', + status: 'online', + agent: { agentId }, + kubernetes, + }) as never; + const alert = (id: string, resourceId: string, overrides: Partial = {}): Alert => ({ + id, + type: 'memory', + level: 'critical', + resourceId, + resourceName: id, + node: id, + instance: '', + message: id, + value: 100, + threshold: 90, + startTime: '2026-10-06T10:00:00Z', + acknowledged: false, + ...overrides, + }); + + it('names Ready nodes with an unacknowledged open alert, critical ones with NotReady', () => { + const nodes = [ + agentNode('a-ready', 'host-a', { ready: true }), + agentNode('b-memory', 'host-b', { ready: true }), + agentNode('c-cordoned', 'host-c', { ready: true, unschedulable: true }), + agentNode('d-down', 'host-d', { ready: false }), + agentNode('e-disk', 'host-e', { ready: true }), + agentNode('f-acked', 'host-f', { ready: true }), + ]; + const activeAlerts = { + memory: alert('memory', 'agent:host-b'), + // A component alert the drawer lists counts for its node too. + disk: alert('disk', 'agent:host-e/disk:root', { type: 'disk', level: 'warning' }), + acked: alert('acked', 'agent:host-f', { acknowledged: true }), + }; + + const result = getKubernetesNodesNeedingAttention({ nodes } as never, activeAlerts, true); + + // Within a rank, node state orders: NotReady ahead of a Ready node. + expect(result.map((entry: { id: string }) => entry.id)).toEqual([ + 'd-down', + 'b-memory', + 'c-cordoned', + 'e-disk', + ]); + // With alert detection off the list falls back to node state alone. + expect( + getKubernetesNodesNeedingAttention({ nodes } as never, activeAlerts, false).map( + (entry: { id: string }) => entry.id, + ), + ).toEqual(['d-down', 'c-cordoned']); + }); + + it('counts alerts keyed on a node without an agent by its own id', () => { + // Unified Kubernetes metric and incident alerts carry the resource id and + // the cluster as their node. + const nodes = [node('k8s-node-a', { ready: true }), node('k8s-node-b', { ready: true })]; + const activeAlerts = { + cpu: alert('cpu', 'k8s-node-b', { type: 'cpu', level: 'warning', node: 'prod-west' }), + }; + + expect( + getKubernetesNodesNeedingAttention({ nodes } as never, activeAlerts, true).map( + (entry: { id: string }) => entry.id, + ), + ).toEqual(['k8s-node-b']); + }); + + it('ranks a node by the stronger of its state and its open alerts', () => { + const ready = agentNode('ready', 'host-ready', { ready: true }); + const cordoned = agentNode('cordoned', 'host-cordoned', { ready: true, unschedulable: true }); + const warning = { w: alert('w', 'agent:host-ready', { level: 'warning' }) }; + const critical = { c: alert('c', 'agent:host-cordoned') }; + + expect(getKubernetesNodeAttentionRank(ready, {}, true)).toBe(2); + expect(getKubernetesNodeAttentionRank(ready, warning, true)).toBe(1); + expect(getKubernetesNodeAttentionRank(cordoned, {}, true)).toBe(1); + expect(getKubernetesNodeAttentionRank(cordoned, critical, true)).toBe(0); + }); }); diff --git a/frontend-modern/src/features/kubernetes/kubernetesPageModel.ts b/frontend-modern/src/features/kubernetes/kubernetesPageModel.ts index 14cb794a50..d205da31b7 100644 --- a/frontend-modern/src/features/kubernetes/kubernetesPageModel.ts +++ b/frontend-modern/src/features/kubernetes/kubernetesPageModel.ts @@ -4,7 +4,9 @@ import type { ResourceKubernetesPodContainerStatus, ResourceType, } from '@/types/resource'; +import type { Alert } from '@/types/api'; import type { StatusIndicator, StatusIndicatorVariant } from '@/utils/status'; +import { getUnifiedResourceAlertStyles } from '@/utils/alerts'; import { resolveResourcePlatformType } from '@/utils/sourcePlatforms'; import { matchesSearchTermSplit, splitSearchExclusions } from '@/utils/searchQuery'; @@ -801,17 +803,44 @@ const countKubernetesDanger = ( mapper: (resource: Resource) => StatusIndicator, ): number => resources.filter((resource) => mapper(resource).variant === 'danger').length; -// Nodes the overview should name before any inventory: NotReady first, then -// cordoned or degraded ones. "2/3 nodes" on the cluster row says something is -// wrong but not which node; this list answers that. -export function getKubernetesNodesNeedingAttention(model: KubernetesPageModel): Resource[] { - const rank = (node: Resource) => { - const variant = mapKubernetesNodeStatus(node).variant; - return variant === 'danger' ? 0 : variant === 'warning' ? 1 : 2; - }; +// Attention order shared by the overview's "Nodes needing attention" list and +// the nodes table's default sort: 0 for a NotReady node or one with an +// unacknowledged critical alert, 1 for a cordoned or degraded node or one with +// another unacknowledged alert, 2 otherwise. The alerts are the ones the +// node's row tint and drawer use, so a Ready node at 100% memory ranks with +// the NotReady ones instead of below every degraded node. +export function getKubernetesNodeAttentionRank( + node: Resource, + activeAlerts: Record, + alertsEnabled: boolean, +): 0 | 1 | 2 { + const variant = mapKubernetesNodeStatus(node).variant; + const statusRank = variant === 'danger' ? 0 : variant === 'warning' ? 1 : 2; + const alertStyles = getUnifiedResourceAlertStyles(node, activeAlerts, alertsEnabled); + const alertRank = !alertStyles.hasUnacknowledgedAlert + ? 2 + : alertStyles.severity === 'critical' + ? 0 + : 1; + return Math.min(statusRank, alertRank) as 0 | 1 | 2; +} + +// Nodes the overview should name before any inventory, in attention order. +// "2/3 nodes" on the cluster row says something is wrong but not which node; +// this list answers that. +export function getKubernetesNodesNeedingAttention( + model: KubernetesPageModel, + activeAlerts: Record = {}, + alertsEnabled = false, +): Resource[] { return model.nodes - .filter((node) => rank(node) < 2) - .sort((a, b) => rank(a) - rank(b) || a.name.localeCompare(b.name)); + .map((node) => ({ + node, + rank: getKubernetesNodeAttentionRank(node, activeAlerts, alertsEnabled), + })) + .filter((entry) => entry.rank < 2) + .sort((a, b) => a.rank - b.rank || compareKubernetesNodes(a.node, b.node)) + .map((entry) => entry.node); } export function buildKubernetesOverviewPosture( diff --git a/frontend-modern/src/features/truenas/__tests__/TrueNASSystemsTable.test.tsx b/frontend-modern/src/features/truenas/__tests__/TrueNASSystemsTable.test.tsx index 401b95b28e..26249d44c4 100644 --- a/frontend-modern/src/features/truenas/__tests__/TrueNASSystemsTable.test.tsx +++ b/frontend-modern/src/features/truenas/__tests__/TrueNASSystemsTable.test.tsx @@ -9,8 +9,10 @@ vi.mock('@/components/shared/responsive', () => ({ ResponsiveMetricCell: () =>
, })); +const activeAlertsRef = vi.hoisted(() => ({ current: {} as Record })); + vi.mock('@/contexts/appRuntime', () => ({ - useWebSocket: () => ({ activeAlerts: {} as Record }), + useWebSocket: () => ({ activeAlerts: activeAlertsRef.current }), })); vi.mock('@/stores/alertsActivation', () => ({ useAlertsActivation: () => ({ @@ -43,6 +45,7 @@ const makeSystem = (overrides: Partial & Pick): Resour afterEach(() => { cleanup(); + activeAlertsRef.current = {}; }); describe('TrueNASSystemsTable', () => { @@ -107,4 +110,42 @@ describe('TrueNASSystemsTable', () => { expect(container.querySelector('[data-truenas-system-row="truenas-healthy"]')).toBeNull(); expect(screen.getByText('1 of 2 systems')).toBeInTheDocument(); }); + it("tints a system row for its pools' alerts, which carry the system hostname", () => { + // TrueNAS pool, dataset, disk and app alerts are keyed on the child + // resource and name the system in "node". The system's drawer states the + // same pool problem through its health issue, so the row matches by + // hostname as well as id; no producer keys a TrueNAS system's alerts on + // its "agent:" alias. + activeAlertsRef.current = { + pool: { + id: 'pool', + type: 'truenas-pool-health', + level: 'warning', + resourceId: 'storage-archive', + resourceName: 'archive', + node: 'truenas-main', + instance: 'TrueNAS', + message: 'Pool archive is DEGRADED', + value: 0, + threshold: 0, + startTime: '2026-10-06T10:00:00Z', + acknowledged: false, + }, + }; + + const { container } = render(() => ( + } + emptyTitle="No systems" + emptyDescription="No systems" + showToolbar={false} + /> + )); + + const row = (id: string) => container.querySelector(`[data-truenas-system-row="${id}"]`); + expect(row('truenas-main')).toHaveClass('bg-yellow-50'); + expect(row('truenas-backup')).not.toHaveClass('bg-yellow-50'); + }); }); From 4cc76209b282447db185a9703c7fb6b737505258 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:48:29 +0100 Subject: [PATCH 24/27] Record the browser pass for the combined Machines table #2579 (row alert tint) and #2578 (shared relative-time clock) each carried a receipt for their own AgentsMachinesTable.tsx, and both merged without conflict, so main's combined file is covered by neither. The combined table was checked on a private mock stack: an alerted machine keeps its red row tint at 1600px light and 390px dark, its drawer lists the alert, and its seen age moves from "just now" to "3h ago" when the clock jumps three hours with no data change. --- .../0434175b9a7f1326.json | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 frontend-modern/browser-verification/0434175b9a7f1326.json diff --git a/frontend-modern/browser-verification/0434175b9a7f1326.json b/frontend-modern/browser-verification/0434175b9a7f1326.json new file mode 100644 index 0000000000..2530f2aecb --- /dev/null +++ b/frontend-modern/browser-verification/0434175b9a7f1326.json @@ -0,0 +1,38 @@ +{ + "version": 1, + "base_sha": "bffd21f47026ca2408ee1d0b5bb071c02e60c623", + "verified_at": "2026-10-06T15:48:24Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx" + ], + "content_sha256": { + "frontend-modern/src/features/standalone/AgentsMachinesTable.tsx": "db24691402c341a2ceec3135d6425c03c9706b6cb57630f9c0efc2e956179264" + }, + "routes": [ + "/standalone/machines" + ], + "viewports": [ + { + "width": 1600, + "height": 1000 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "Combined AgentsMachinesTable.tsx on main after #2579 (row alert tint) and #2578 (shared relative-time clock) merged; private mock stack built from bffd21f470", + "A critical memory alert for Apollo-114 (resourceId agent-acdfdee2953587fb) injected into the websocket activeAlerts and /api/alerts/active; Spectrum-709 left without alerts", + "1600px light: Apollo-114 row bg-red-50 (rgb 254,242,242), Spectrum-709 untinted; with Last seen hidden the identity title read \"| seen just now\"", + "Playwright page.clock fast-forwarded 3h with /ws held and REST replayed: Apollo-114 title \"| seen 3h ago\" and the red tint unchanged, Spectrum-709 still untinted", + "390px dark: Apollo-114 row dark:bg-red-950/25, Seen cell \"just now\" -> \"3h ago\" after the same 3h jump with the tint unchanged", + "Apollo-114 drawer lists the injected \"Memory usage is 96%\" alert at both widths; no page errors" + ], + "interactions": [ + "Opened View > Choose which columns to display and unchecked Last seen on Machines (1600px)", + "page.clock.fastForward 3h with data held", + "Clicked the Apollo-114 row to open its drawer" + ] +} From f4c304037351fa10dd4e32598ec77fc1bde95216 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:33:08 +0100 Subject: [PATCH 25/27] Read a linked agent's own heartbeat in host snapshots A host agent linked to a Proxmox node merges into one unified host row. When the agent stops reporting but PVE polling continues, that row stays online and its LastSeen follows the PVE sighting. hostFromReadStateView copied both into models.Host, so a silent agent looked current to every consumer of the projection: - linkedHostForNode accepted it, and resolveNodeDisk kept using the agent's last filesystem summary as the node's disk, feeding disk alerts and history with a frozen reading. Its retained ZFS datasets kept merging into the node's pools too. - Agent connections built from HostsSnapshot (the Connections list and the update-readiness agent check) aged the agent from the PVE poll, so a dead agent stayed Active. The projection now takes LastSeen from the agent source's own sighting and reports the agent offline once that sighting is past its reporting lease, the same hostAgentHealthWindow evaluateHostAgents applies. Rows without an agent sighting, such as vSphere and TrueNAS hosts, keep the row's values. The node falls back to the Proxmox rootfs reading while the agent is silent and returns to the agent summary when it reports. --- .../v6/internal/subsystems/agent-lifecycle.md | 12 ++++ .../v6/internal/subsystems/monitoring.md | 19 +++++ internal/monitoring/monitor.go | 24 ++++++- .../monitoring/monitor_host_agents_test.go | 69 +++++++++++++++++++ 4 files changed, 122 insertions(+), 2 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index ea945c3572..93d863fc62 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -8797,3 +8797,15 @@ no update status, instead of reporting "no update available" (#2353). Agent registration, enrolment, install, update, removal and report identity are unchanged; the per-image memo lives in the registry checker and is pruned each collection cycle to the images in use. + +### Host snapshots report a linked agent's own heartbeat + +`internal/monitoring/monitor.go` changed only so a host produced from the read +state carries the agent source's own last report and reads offline past the +agent's reporting lease, even when the agent's row is merged with a Proxmox +node that PVE polling keeps fresh. The agent connection on the Connections +list and the update-readiness agent-continuity check therefore age a silent +agent from its last report instead of from the PVE poll, and mark it stale at +their own heartbeat cutoff (`fleethealth.AgentStaleThreshold`, which is +separate from the monitoring reporting lease). Agent registration, enrolment, +install, update, removal and report identity are unchanged. diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 17b73c02fd..0353b909c1 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -2638,6 +2638,25 @@ online/running/healthy vocabulary. The diagnostics subject set is unchanged: snapshot (real agents), which now matches what agent-only surfaces show once integration-backed ledger rows are excluded. +A host produced by `hostFromReadStateView` describes the agent, so its +`Status` and `LastSeen` are the agent's own. A host agent linked to a Proxmox +node merges into one row whose `LastSeen` follows its freshest source and +whose status stays online while PVE polling continues, so the row's values +would present a silent agent as online and current. The projection reads the +agent source's sighting from `HostView.SourceStatus(SourceAgent)`: the host +carries that report time and reads `offline` once it is past the reporting +lease (`hostAgentReportCurrent`, the `hostAgentHealthWindow` that +`evaluateHostAgents` applies to the host in state). Rows without an agent +sighting, such as vSphere and TrueNAS integration hosts, keep the row's +values. `linkedHostForNode` therefore stops handing a silent agent's retained +filesystem summary and ZFS datasets to its node, whose disk falls back to the +Proxmox rootfs or `/nodes` reading before disk alerts and history read it, and +agent connections built from `HostsSnapshot()` (the Connections list and the +update-readiness agent-continuity check) age the agent from its last report +rather than the PVE poll time, applying their own heartbeat cutoff. +`TestSilentLinkedAgentStopsFeedingNodeDisk` drives the disk and projection +path through `evaluateHostAgents` and the registry-backed store refresh. + Unified Agent host reports now make module readiness and updater/config lifecycle evidence monitoring-owned observed state. Monitoring preserves the last successful one-shot update transition across subsequent reports, forwards diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index baab181eb9..54b6e634c9 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -3577,6 +3577,7 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host { if trimmed := strings.TrimSpace(view.Name()); trimmed != "" && trimmed != view.Hostname() { displayName = trimmed } + status, lastSeen := hostAgentLivenessFromReadStateView(view, time.Now()) return models.Host{ ID: firstNonEmptyString(view.AgentID(), view.ID()), @@ -3598,10 +3599,10 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host { RAID: hostRAIDFromReadStateView(view.RAID()), Unraid: hostUnraidFromReadStateView(view.Unraid()), Ceph: hostCephFromReadStateView(view.Ceph()), - Status: string(view.Status()), + Status: status, UptimeSeconds: view.UptimeSeconds(), IntervalSeconds: view.IntervalSeconds(), - LastSeen: view.LastSeen(), + LastSeen: lastSeen, AgentVersion: view.AgentVersion(), IntegrationSource: view.IntegrationSource(), MachineID: view.MachineID(), @@ -3624,6 +3625,25 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host { } } +// hostAgentLivenessFromReadStateView returns the status and last report of the +// agent behind a read-state host row. A row merged with a Proxmox node takes +// LastSeen from its freshest source and stays online while PVE polling +// continues, so a silent agent would look current: its retained disk summary +// and datasets kept feeding the node, and its connection read as active. A +// models.Host describes the agent, so read the agent source's own sighting +// and lease, as evaluateHostAgents does for the host in state. +func hostAgentLivenessFromReadStateView(view *unifiedresources.HostView, now time.Time) (string, time.Time) { + status, lastSeen := string(view.Status()), view.LastSeen() + agentSighting, ok := view.SourceStatus(unifiedresources.SourceAgent) + if !ok || agentSighting.LastSeen.IsZero() { + return status, lastSeen + } + if !hostAgentReportCurrent(agentSighting.LastSeen, view.IntervalSeconds(), now) { + status = "offline" + } + return status, agentSighting.LastSeen +} + func vmFromReadStateView(view *unifiedresources.VMView) models.VM { if view == nil { return models.VM{} diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go index f14c2d1d55..3ef3414d7d 100644 --- a/internal/monitoring/monitor_host_agents_test.go +++ b/internal/monitoring/monitor_host_agents_test.go @@ -24,6 +24,7 @@ import ( agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker" agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" "github.com/rcourtman/pulse-go-rewrite/pkg/metrics" + "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" ) func TestNewMonitorRoutesStartupCustomSensorWarningBeforeStart(t *testing.T) { @@ -6843,3 +6844,71 @@ func TestBroadcastProjectionListsRegistryOnceAndKeepsLiveChanges(t *testing.T) { t.Fatal("removed host lifecycle surface disappeared") } } + +// A host agent linked to a Proxmox node merges into one read-state row that +// every PVE poll keeps online and fresh. Once the agent stops reporting, its +// retained filesystem summary must stop standing in for the node's disk (and +// through it disk alerts and history), and the projected agent must read as +// offline with its own last report, so its connection shows the missed +// heartbeat instead of the PVE poll time. +func TestSilentLinkedAgentStopsFeedingNodeDisk(t *testing.T) { + state := models.NewState() + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m := &Monitor{config: &config.Config{}, state: state, resourceStore: adapter} + + node := models.Node{ + ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", + ConnectionHealth: "healthy", LinkedAgentID: "agent-1", + } + rootFS := &proxmox.NodeStatus{RootFS: &proxmox.RootFS{Total: 200, Used: 50, Free: 150}} + // poll records a PVE poll that just saw the node, with the agent as of its + // last report, and resolves the node's disk the way pollPVENode does. + poll := func(agentLastReport time.Time) (models.Disk, string, models.Host) { + t.Helper() + polledNode := node + polledNode.LastSeen = time.Now() + state.UpdateNodes([]models.Node{polledNode}) + state.UpsertHost(models.Host{ + ID: "agent-1", Hostname: "node1", Status: "online", LinkedNodeID: node.ID, + IntervalSeconds: 30, LastSeen: agentLastReport, + Disks: []models.Disk{{Mountpoint: "/", Type: "ext4", Total: 1000, Used: 900, Free: 100, Usage: 90}}, + }) + m.evaluateHostAgents(time.Now()) + m.refreshUnifiedResourceStoreAfterAgentStateChange() + + disk, source := m.resolveNodeDisk("pve1", node.ID, node.Name, proxmox.Node{Node: node.Name}, rootFS) + for _, host := range m.HostsSnapshot() { + if host.ID == "agent-1" { + return disk, source, host + } + } + t.Fatal("HostsSnapshot does not list agent-1") + return disk, source, models.Host{} + } + + agentReport := time.Now().Add(-10 * time.Second) + disk, source, host := poll(agentReport) + if source != "agent" || disk.Used != 900 { + t.Fatalf("reporting agent: node disk = %+v from %q, want the agent's summary", disk, source) + } + if host.Status != "online" || !host.LastSeen.Equal(agentReport) { + t.Fatalf("reporting agent projected as %q last seen %s, want online at its report %s", host.Status, host.LastSeen, agentReport) + } + + silentSince := time.Now().Add(-hostAgentHealthWindow(30) - time.Minute) + disk, source, host = poll(silentSince) + if source != "node-status-rootfs" || disk.Used != 50 { + t.Fatalf("silent agent: node disk = %+v from %q, want Proxmox rootfs", disk, source) + } + if linked := m.linkedHostForNode("pve1", node.ID, node.Name); linked != nil { + t.Fatalf("silent agent still linked to the node for disk and dataset data: %+v", linked) + } + if host.Status != "offline" || !host.LastSeen.Equal(silentSince) { + t.Fatalf("silent agent projected as %q last seen %s, want offline at its last report %s", host.Status, host.LastSeen, silentSince) + } + + disk, source, host = poll(time.Now()) + if source != "agent" || disk.Used != 900 || host.Status != "online" { + t.Fatalf("agent reporting again: node disk = %+v from %q, agent %q; want the agent's summary and online", disk, source, host.Status) + } +} From 0c130a4855e4467c5056292abab1d9103f0b7c01 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:46:16 +0100 Subject: [PATCH 26/27] Key previous Proxmox nodes by source ID so poll carry-overs match The poller reads the previous poll's nodes back from the unified read state, but previousNodeFromView returned the registry's unified resource ID and no temperature. Every lookup keyed on the Proxmox source ID missed: nodeLastOnline in the offline grace policy, the temperature carry, the CPU low/record tracking (reset to the current reading every poll) and the node memory carry. When a whole instance poll failed, markPVEInstanceNodesUnreachable wrote the projected nodes back, so each failed poll re-ingested the node under a new unified ID. Cluster members also lost the cluster identity their canonical ID derives from, so they changed ID for the length of an outage. The projection now returns the source ID, the stored temperature details, the cluster name and the linked agent ID, and previousNodesForInstance restores the config-derived identity stamps (provider-scoped flag, native aliases, TLS fingerprint) the way pollPVENode writes them. Reviving those paths needed four bounds: - The node memory carry uses the poller's own validated snapshot rather than the read state, which holds the agent's reading on a node merged with a host agent, and keeps used + cache + free equal to the total. - A carried temperature keeps a lapsed host agent's lease. A reading stamped no later than the agent's last report is not carried, so the five-minute carry window cannot hold a silent agent's reading in alerts and history past its lease, which #2582 guarantees. - A carried reading keeps the node's CPU and GPU temperatures only. Its SMART and NVMe rows also stamp the node's physical disks, so a disk that went to standby would be stamped and recorded with its pre-sleep value. - Outage preservation and authoritative membership reconciliation keep a node's temperature only under those rules and drop it when the node goes offline. TestPollPVEInstanceMarksStaleNodesOfflineWhenGetNodesReturnsEmpty relied on the dead nodeLastOnline lookup and now ages that sighting too. The silent-agent test from #2582 now rebuilds the store as production does and feeds the polled node back, which is what exercises the carry. --- .../v6/internal/subsystems/monitoring.md | 34 +- internal/monitoring/host_agent_temps.go | 101 +++-- .../memory_trust_characterization_test.go | 6 +- .../monitoring/monitor_additional_test.go | 427 ++++++++++++++++++ .../monitoring/monitor_alert_handling_test.go | 28 +- .../monitoring/monitor_extra_coverage_test.go | 22 +- internal/monitoring/monitor_memory_test.go | 5 + internal/monitoring/monitor_polling_node.go | 32 +- .../monitor_polling_node_helpers.go | 42 +- internal/monitoring/monitor_previous_state.go | 33 +- internal/monitoring/monitor_pve.go | 31 +- internal/monitoring/monitor_pve_cluster.go | 2 + .../monitoring/node_memory_sources_test.go | 1 - 13 files changed, 671 insertions(+), 93 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 40873c0536..ba87140d89 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -3772,7 +3772,39 @@ cluster sensor cache, which keeps its own recency check. When every source returns nothing, `internal/monitoring/monitor_polling_node_helpers.go` may carry a previous reading, with its original `LastUpdate`, only inside the carry window (twice the PVE polling interval, never under five minutes); an older reading is -dropped rather than re-presented as current. +dropped rather than re-presented as current. A carried reading also keeps a +lapsed host agent's lease. Agent readings are stamped with the agent's report +time, so `carriedTemperatureOutlivesAgentLease` drops a carried reading stamped +no later than the last report of the node's matched agent (by link, then +hostname, whether or not that report had sensors) once that agent has lapsed. +Readings record no source, so the bound is conservative: an SSH, cluster-cache +or agent-plus-SSH merged reading (stamped with the agent's time) of that age is +dropped too, being already older than the lease, while a reading stamped after +that report keeps the carry window. A carried reading (`carriedNodeTemperature`) +keeps only the node's CPU and GPU temperatures: its SMART and NVMe rows would +also stamp the node's physical disks, whose freshness and standby state a +carried copy cannot follow (a disk that spun down would be stamped and recorded +with its pre-sleep value), so they are left out, and nothing is carried when +only they remain. +The poller reads its previous poll's nodes back from the unified read state +through `internal/monitoring/monitor_previous_state.go`. `nodeLastOnline` and +the temperature carry with its CPU low/record tracking match them by node ID, +which is the Proxmox source ID, so the projection must return +`NodeView.SourceID()`, never the unified resource ID, and carry the stored +`TemperatureDetails`. Nodes preserved through an instance outage +(`preserveOrExpireNodes`) or authoritative membership reconciliation are written +back into state from that projection, so it also restores the identity inputs +the registry derives a node's canonical ID and cross-view merges from: cluster +name, provider-scoped flag, native aliases, TLS fingerprint and linked agent ID, +the config-derived ones stamped the way `pollPVENode` stamps them. With the +unified ID a node re-keys on every failed poll, and without its cluster +identity a cluster member changes canonical ID for the length of an outage. +A preserved node keeps its +temperature only under the same carry rules and loses it when it goes +offline. Node memory carry-over after a failed status read uses the poller's own +validated `NodeMemorySnapshot`, not the read state, which holds the agent's +reading on a node merged with a host agent, and keeps used, cache and free +summing to the total. Legacy SSH temperature collection must also use the Pulse sensor-wrapper contract before falling back to raw lm-sensors output. `internal/monitoring/temperature.go` must request `/usr/local/sbin/pulse-sensors` when it exists, parse the wrapper diff --git a/internal/monitoring/host_agent_temps.go b/internal/monitoring/host_agent_temps.go index 661caeaf6e..8b3fc4d3e0 100644 --- a/internal/monitoring/host_agent_temps.go +++ b/internal/monitoring/host_agent_temps.go @@ -86,44 +86,7 @@ func (m *Monitor) getHostAgentTemperatureByID(nodeID, nodeName string) *models.T return nil } - hosts := readState.Hosts() - if len(hosts) == 0 { - // No host agents at all — check cluster sensor cache as fallback - return m.getClusterSensorTemperature(nodeName) - } - - var matchedHost *unifiedresources.HostView - - // First, try to find a host agent that is explicitly linked to this node - // via LinkedNodeID. This is the most reliable method and handles duplicate - // hostnames correctly. - if nodeID != "" { - for i := range hosts { - if hosts[i].LinkedNodeID() == nodeID { - matchedHost = hosts[i] - log.Debug(). - Str("nodeID", nodeID). - Str("hostAgentID", hosts[i].ID()). - Str("hostname", hosts[i].Hostname()). - Msg("Matched host agent to node via LinkedNodeID") - break - } - } - } - - // Fallback: match by hostname if no linked host was found - // This maintains backwards compatibility for setups where linking hasn't occurred yet - if matchedHost == nil { - nodeLower := strings.ToLower(strings.TrimSpace(nodeName)) - for i := range hosts { - hostnameLower := strings.ToLower(strings.TrimSpace(hosts[i].Hostname())) - if hostnameLower == nodeLower { - matchedHost = hosts[i] - break - } - } - } - + matchedHost := hostAgentForNode(readState.Hosts(), nodeID, nodeName) if matchedHost == nil { // No directly-linked host agent found — check cluster sensor cache return m.getClusterSensorTemperature(nodeName) @@ -153,6 +116,68 @@ func (m *Monitor) getHostAgentTemperatureByID(nodeID, nodeName string) *models.T return convertUnifiedHostSensorsToTemperature(sensors, agentStatus.LastSeen) } +// carriedTemperatureOutlivesAgentLease reports whether a temperature a node +// carries over from an earlier poll may have come from the node's host agent +// after that agent stopped reporting. Agent readings are stamped with the +// agent's report time and only count inside its reporting lease, so a reading +// stamped no later than a lapsed agent's last report is treated as the agent's +// and is not carried. Readings carry no source, so an SSH or cluster-cache +// reading of that age is dropped too; it is already older than the lease, and +// the bound errs toward a gap rather than a stale value. A reading stamped +// after that report keeps the ordinary carry window. +func (m *Monitor) carriedTemperatureOutlivesAgentLease(nodeID, nodeName string, temp *models.Temperature, now time.Time) bool { + if temp == nil { + return false + } + readState := m.GetUnifiedReadStateOrSnapshot() + if readState == nil { + return false + } + matchedHost := hostAgentForNode(readState.Hosts(), nodeID, nodeName) + if matchedHost == nil { + return false + } + agentStatus, ok := matchedHost.SourceStatus(unifiedresources.SourceAgent) + if !ok || agentStatus.LastSeen.IsZero() || + hostAgentReportCurrent(agentStatus.LastSeen, matchedHost.IntervalSeconds(), now) { + return false + } + return !temp.LastUpdate.After(agentStatus.LastSeen) +} + +// hostAgentForNode returns the host agent for a Proxmox node: the agent linked +// to the node ID, else one whose hostname matches. This correctly handles +// clusters where multiple nodes may have the same hostname (e.g., "px1" on +// different IPs). +func hostAgentForNode(hosts []*unifiedresources.HostView, nodeID, nodeName string) *unifiedresources.HostView { + // First, try to find a host agent that is explicitly linked to this node + // via LinkedNodeID. This is the most reliable method and handles duplicate + // hostnames correctly. + if nodeID != "" { + for i := range hosts { + if hosts[i].LinkedNodeID() == nodeID { + log.Debug(). + Str("nodeID", nodeID). + Str("hostAgentID", hosts[i].ID()). + Str("hostname", hosts[i].Hostname()). + Msg("Matched host agent to node via LinkedNodeID") + return hosts[i] + } + } + } + + // Fallback: match by hostname if no linked host was found + // This maintains backwards compatibility for setups where linking hasn't occurred yet + nodeLower := strings.ToLower(strings.TrimSpace(nodeName)) + for i := range hosts { + hostnameLower := strings.ToLower(strings.TrimSpace(hosts[i].Hostname())) + if hostnameLower == nodeLower { + return hosts[i] + } + } + return nil +} + func convertUnifiedHostSensorsToTemperature(sensors *unifiedresources.HostSensorMeta, lastSeen time.Time) *models.Temperature { if sensors == nil { return nil diff --git a/internal/monitoring/memory_trust_characterization_test.go b/internal/monitoring/memory_trust_characterization_test.go index 60c583c3a1..366164e81d 100644 --- a/internal/monitoring/memory_trust_characterization_test.go +++ b/internal/monitoring/memory_trust_characterization_test.go @@ -203,7 +203,7 @@ func TestPollPVENodeMemoryTrustCharacterization(t *testing.T) { MaxCPU: 8, } - modelNode, _, _, err := mon.pollPVENode(context.Background(), "test", &mon.config.PVEInstances[0], client, node, "healthy", nil, nil) + modelNode, _, _, err := mon.pollPVENode(context.Background(), "test", &mon.config.PVEInstances[0], client, node, "healthy", nil) if err != nil { t.Fatalf("pollPVENode() error = %v", err) } @@ -258,7 +258,7 @@ func TestPollPVENodePreservesPreviousSnapshotDuringTransientFallback(t *testing. MaxCPU: 8, } - first, _, _, err := mon.pollPVENode(context.Background(), "test", &mon.config.PVEInstances[0], client, node, "healthy", nil, nil) + first, _, _, err := mon.pollPVENode(context.Background(), "test", &mon.config.PVEInstances[0], client, node, "healthy", nil) if err != nil { t.Fatalf("first pollPVENode() error = %v", err) } @@ -274,7 +274,6 @@ func TestPollPVENodePreservesPreviousSnapshotDuringTransientFallback(t *testing. client, node, "healthy", - map[string]models.Memory{first.ID: first.Memory}, nil, ) if err != nil { @@ -302,7 +301,6 @@ func TestPollPVENodePreservesPreviousSnapshotDuringTransientFallback(t *testing. client, node, "healthy", - map[string]models.Memory{second.ID: second.Memory}, nil, ) if err != nil { diff --git a/internal/monitoring/monitor_additional_test.go b/internal/monitoring/monitor_additional_test.go index bc771e5d42..6a9b566dae 100644 --- a/internal/monitoring/monitor_additional_test.go +++ b/internal/monitoring/monitor_additional_test.go @@ -1158,3 +1158,430 @@ func TestTenantDeletionWaitsForSlowMonitoringLoop(t *testing.T) { } mtm.FinishTenantDeletion("deletion") } + +// The poller reads the previous poll's nodes back through the unified read +// state. Those lookups key on the Proxmox source ID, so the projection must +// return it (not the registry's unified resource ID) and carry the stored +// temperature; otherwise the node's CPU low/record values reset to the +// current reading every poll and a failed collection never carries over. +func TestPreviousNodeProjectionFeedsPollerTemperatureTracking(t *testing.T) { + instanceCfg := config.PVEInstance{Name: "pve1", Host: "https://10.0.0.1:8006"} + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m := &Monitor{ + config: &config.Config{TemperatureMonitoringEnabled: true, PVEInstances: []config.PVEInstance{instanceCfg}}, + state: models.NewState(), + resourceStore: adapter, + } + base := models.Node{ID: "pve1-node1", NodeIdentity: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", Type: "node"} + + // publish rebuilds the unified store from state the way the monitor does, + // alongside the host agents' latest reports. + var hosts []models.Host + publish := func() { + snapshot := m.state.GetSnapshot() + snapshot.Hosts = hosts + adapter.PopulateFromSnapshot(snapshot) + } + // poll delivers an agent report (none when agentReading is 0), runs one PVE + // poll's temperature step against the previous poll's nodes, and publishes + // the polled node. + poll := func(agentReading float64) *models.Temperature { + t.Helper() + hosts = nil + if agentReading > 0 { + hosts = []models.Host{{ + ID: "agent-1", Hostname: "node1", Status: "online", LinkedNodeID: base.ID, IntervalSeconds: 30, LastSeen: time.Now(), + Sensors: models.HostSensorSummary{TemperatureCelsius: map[string]float64{"cpu_package": agentReading}}, + }} + } + publish() + prevNodes := m.snapshotPrevNodes("pve1") + polled := base + polled.LastSeen = time.Now() + m.collectNodeTemperatureData(context.Background(), "pve1", &instanceCfg, proxmox.Node{Node: "node1"}, &polled, prevNodes, "online") + m.state.UpdateNodesForInstance("pve1", []models.Node{polled}) + publish() + return polled.Temperature + } + + if temp := poll(50); temp == nil || temp.CPUMin != 50 || temp.CPUMaxRecord != 50 { + t.Fatalf("first reading = %#v, want min and record initialised to 50", temp) + } + prevNodes := m.snapshotPrevNodes("pve1") + if len(prevNodes) != 1 || prevNodes[0].ID != base.ID || prevNodes[0].Temperature == nil { + t.Fatalf("previous nodes = %#v, want source ID %q with its temperature", prevNodes, base.ID) + } + + if temp := poll(62); temp == nil || temp.CPUPackage != 62 || temp.CPUMin != 50 || temp.CPUMaxRecord != 62 { + t.Fatalf("second reading = %#v, want current 62, low 50, record 62", temp) + } + third := poll(45) + if third == nil || third.CPUPackage != 45 || third.CPUMin != 45 || third.CPUMaxRecord != 62 { + t.Fatalf("third reading = %#v, want current 45, low 45, record 62", third) + } + + // No agent and no SSH collector: the last reading carries over with its + // original timestamp, inside the carry window. + carried := poll(0) + if carried == nil || carried.CPUPackage != 45 || !carried.LastUpdate.Equal(third.LastUpdate) { + t.Fatalf("carried reading = %#v, want the previous 45 with its original timestamp", carried) + } +} + +// When a whole Proxmox instance stops answering, its last-known nodes are +// written back into state through the offline grace policy. That writeback +// must keep each node the same resource: the poller's source ID and the +// config-derived identity the registry derives a cluster member's canonical ID +// from. Past grace the node goes offline and stops presenting a temperature. +func TestPVEOutageWritebackKeepsNodeIdentity(t *testing.T) { + for _, tc := range []struct { + name string + instances []config.PVEInstance + wantID string + providerScoped bool + }{ + { + name: "cluster member", + instances: []config.PVEInstance{{Name: "pve-a", Host: "https://10.0.0.1:8006", IsCluster: true, ClusterName: "lab"}}, + wantID: "lab-node1", + }, + { + name: "cluster name shared by two connections", + instances: []config.PVEInstance{ + {Name: "pve-a", Host: "https://10.0.0.1:8006", IsCluster: true, ClusterName: "lab"}, + {Name: "pve-b", Host: "https://10.0.1.1:8006", IsCluster: true, ClusterName: "lab"}, + }, + wantID: "pve-a-node1", + providerScoped: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m := &Monitor{ + config: &config.Config{PVEInstances: tc.instances}, + state: models.NewState(), + resourceStore: adapter, + nodeLastOnline: make(map[string]time.Time), + } + instanceCfg := m.getInstanceConfig("pve-a") + if got := m.pveNodeID("pve-a", instanceCfg, "node1"); got != tc.wantID { + t.Fatalf("pveNodeID = %q, want %q", got, tc.wantID) + } + now := time.Now() + polled := m.placeholderNodeForInstance("pve-a", instanceCfg, "node1") + polled.Status = "online" + polled.ConnectionHealth = "healthy" + polled.LastSeen = now + polled.Temperature = &models.Temperature{CPUPackage: 55, Available: true, HasCPU: true, LastUpdate: now} + if polled.ProviderScopedIdentity != tc.providerScoped { + t.Fatalf("ProviderScopedIdentity = %v, want %v", polled.ProviderScopedIdentity, tc.providerScoped) + } + m.nodeLastOnline[polled.ID] = now + m.state.UpdateNodesForInstance("pve-a", []models.Node{polled}) + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + unifiedID := adapter.Nodes()[0].ID() + + for cycle := 0; cycle < 3; cycle++ { + m.markPVEInstanceNodesUnreachable("pve-a") + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + stateNodes := m.state.GetSnapshot().Nodes + if len(stateNodes) != 1 || stateNodes[0].ID != tc.wantID || stateNodes[0].ClusterName != "lab" || + stateNodes[0].ProviderScopedIdentity != tc.providerScoped { + t.Fatalf("cycle %d: state nodes = %#v, want %q kept with its cluster identity", cycle, stateNodes, tc.wantID) + } + if stateNodes[0].Status != "online" || stateNodes[0].Temperature == nil { + t.Fatalf("cycle %d: node within grace = %#v, want online with its recent temperature", cycle, stateNodes[0]) + } + if nodes := adapter.Nodes(); len(nodes) != 1 || nodes[0].ID() != unifiedID { + t.Fatalf("cycle %d: unified nodes = %v, want the single resource %q", cycle, nodes, unifiedID) + } + } + + // The process last saw the node online beyond the grace period. + m.nodeLastOnline[polled.ID] = now.Add(-2 * nodeOfflineGracePeriod) + polled.LastSeen = now.Add(-2 * nodeOfflineGracePeriod) + m.state.UpdateNodesForInstance("pve-a", []models.Node{polled}) + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + m.markPVEInstanceNodesUnreachable("pve-a") + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + expired := m.state.GetSnapshot().Nodes + if len(expired) != 1 || expired[0].ID != tc.wantID || expired[0].Status != "offline" || expired[0].Temperature != nil { + t.Fatalf("expired nodes = %#v, want %q offline without a temperature", expired, tc.wantID) + } + if nodes := adapter.Nodes(); len(nodes) != 1 || nodes[0].ID() != unifiedID { + t.Fatalf("unified nodes after expiry = %v, want the single resource %q", nodes, unifiedID) + } + }) + } +} + +// Within grace a preserved node keeps its temperature only while the reading +// is inside the same carry window a failed collection gets. +func TestPreserveOrExpireNodesBoundsGraceTemperature(t *testing.T) { + m := &Monitor{config: &config.Config{}, state: models.NewState(), nodeLastOnline: make(map[string]time.Time)} + now := time.Now() + m.nodeLastOnline["pve-node1"] = now + stale := now.Add(-m.nodeTemperatureCarryWindow() - time.Minute) + preserved := m.preserveOrExpireNodes([]models.Node{{ + ID: "pve-node1", Name: "node1", Status: "online", LastSeen: now, + Temperature: &models.Temperature{CPUPackage: 70, Available: true, LastUpdate: stale}, + }}) + if len(preserved) != 1 || preserved[0].Status != "online" || preserved[0].Temperature != nil { + t.Fatalf("preserved = %#v, want online within grace without the stale temperature", preserved) + } +} + +// When detailed node status fails, the poller may carry its own last trusted +// memory reading for one cycle. On a node merged with a host agent, the read +// state holds the agent's memory, which is not the Proxmox value the snapshot +// vouches for, so the carry must use the snapshot and keep its used, cache +// and free split summing to the total. +func TestPollPVENodeCarriesOwnMemorySnapshot(t *testing.T) { + t.Setenv("PULSE_DATA_DIR", t.TempDir()) + const gib = uint64(1024 * 1024 * 1024) + + mon := newTestPVEMonitor("test") + defer mon.alertManager.Stop() + defer mon.notificationMgr.Stop() + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + mon.resourceStore = adapter + + client := &stubPVEClient{nodeStatus: &proxmox.NodeStatus{ + Memory: &proxmox.MemoryStatus{Total: 16 * gib, Used: 12 * gib, Free: 1 * gib, Available: 8 * gib}, + }} + node := proxmox.Node{Node: "node1", Status: "online", MaxMem: 16 * gib, Mem: 15 * gib, MaxCPU: 8} + + first, _, _, err := mon.pollPVENode(context.Background(), "test", &mon.config.PVEInstances[0], client, node, "healthy", nil) + if err != nil { + t.Fatalf("first pollPVENode() error = %v", err) + } + if !first.Memory.HasKnownUsage() || first.Memory.Used != int64(8*gib) || first.Memory.Cache == 0 { + t.Fatalf("first.Memory = %+v, want 8 GiB used with reclaimable cache split out", first.Memory) + } + mon.state.UpdateNodesForInstance("test", []models.Node{first}) + snapshot := mon.state.GetSnapshot() + snapshot.Hosts = []models.Host{{ + ID: "agent-1", Hostname: "node1", Status: "online", LinkedNodeID: first.ID, IntervalSeconds: 30, LastSeen: time.Now(), + Memory: models.Memory{Total: int64(16 * gib), Used: int64(3 * gib), Free: int64(13 * gib), Usage: 18.75}, + }} + adapter.PopulateFromSnapshot(snapshot) + + client.nodeStatus = nil + second, _, _, err := mon.pollPVENode(context.Background(), "test", &mon.config.PVEInstances[0], client, node, "healthy", mon.snapshotPrevNodes("test")) + if err != nil { + t.Fatalf("second pollPVENode() error = %v", err) + } + if second.Memory.Used != first.Memory.Used || second.Memory.Cache != first.Memory.Cache || second.Memory.Free != first.Memory.Free { + t.Fatalf("second.Memory = %+v, want the poller's own reading %+v", second.Memory, first.Memory) + } + if second.Memory.Used+second.Memory.Cache+second.Memory.Free != second.Memory.Total { + t.Fatalf("second.Memory = %+v, want used + cache + free = total", second.Memory) + } + if snap := mon.nodeSnapshots[makeNodeSnapshotKey("test", "node1")]; snap.MemorySource != "previous-snapshot" { + t.Fatalf("snapshot.MemorySource = %q, want previous-snapshot", snap.MemorySource) + } +} + +// Authoritative cluster membership writes previous nodes back too. Read from +// the unified store, an offline member keeps its source ID and agent link but +// stops presenting a temperature, and a member missing from /nodes keeps a +// reading only inside the carry window. +func TestPVEMembershipReconcileKeepsPreviousNodeIdentity(t *testing.T) { + cfg := &config.Config{PVEInstances: []config.PVEInstance{{ + Name: "cluster-api", Host: "https://pve-a:8006", IsCluster: true, ClusterName: "production", + }}} + m := newUnreachableTestMonitor(t, cfg) + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m.resourceStore = adapter + instanceCfg := &cfg.PVEInstances[0] + now := time.Now() + polled := func(name string, tempAge time.Duration) models.Node { + node := m.placeholderNodeForInstance("cluster-api", instanceCfg, name) + node.Status = "online" + node.ConnectionHealth = "healthy" + node.LastSeen = now + node.Temperature = &models.Temperature{CPUPackage: 60, Available: true, HasCPU: true, LastUpdate: now.Add(-tempAge)} + return node + } + offlineMember := polled("pve-a", 0) + offlineMember.LinkedAgentID = "agent-pve-a" + m.state.UpdateNodesForInstance("cluster-api", []models.Node{ + offlineMember, polled("pve-b", 0), polled("pve-c", m.nodeTemperatureCarryWindow()+time.Minute), + }) + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + + client := &membershipPVEClient{statuses: []proxmox.ClusterStatus{ + {Type: "cluster", Name: "production", Quorate: 1}, + {Type: "node", Name: "pve-a", Online: 0}, + {Type: "node", Name: "pve-b", Online: 1}, + {Type: "node", Name: "pve-c", Online: 1}, + }} + got := pveNodeByName(m.reconcilePVENodeInventory( + context.Background(), "cluster-api", instanceCfg, client, nil, m.snapshotPrevNodes("cluster-api"), + )) + + if node := got["pve-a"]; node.ID != "production-pve-a" || node.Status != "offline" || node.Temperature != nil || node.LinkedAgentID != "agent-pve-a" { + t.Fatalf("offline member = %#v, want production-pve-a offline with its agent link and no temperature", node) + } + if node := got["pve-b"]; node.ID != "production-pve-b" || node.Status != "unknown" || node.Temperature == nil { + t.Fatalf("unobserved member = %#v, want production-pve-b unknown with its recent temperature", node) + } + if node := got["pve-c"]; node.ID != "production-pve-c" || node.Status != "unknown" || node.Temperature != nil { + t.Fatalf("unobserved member = %#v, want production-pve-c unknown without its stale temperature", node) + } + + // Writing the reconciled inventory back keeps every member the same + // unified resource. + unifiedIDs := func() map[string]string { + ids := make(map[string]string) + for _, node := range adapter.Nodes() { + ids[node.NodeName()] = node.ID() + } + return ids + } + before := unifiedIDs() + m.state.UpdateNodesForInstance("cluster-api", sortedPVENodeInventory(got)) + adapter.PopulateFromSnapshot(m.state.GetSnapshot()) + if after := unifiedIDs(); len(after) != 3 || after["pve-a"] != before["pve-a"] || after["pve-b"] != before["pve-b"] || after["pve-c"] != before["pve-c"] { + t.Fatalf("unified IDs after writeback = %v, want %v", after, before) + } +} + +// A temperature carried over from an earlier poll keeps a lapsed host agent's +// lease: a reading the agent supplied (stamped no later than its last report) +// is dropped once the agent stops reporting, even when its latest report had +// no sensors, while a reading taken after that report keeps the carry window. +// The same rule bounds what an instance outage preserves within node grace. +func TestNodeTemperatureCarryRespectsAgentLease(t *testing.T) { + now := time.Now() + silentSince := now.Add(-hostAgentHealthWindow(30) - time.Minute) + instanceCfg := config.PVEInstance{Name: "pve1", Host: "https://10.0.0.1:8006"} + base := models.Node{ID: "pve1-node1", NodeIdentity: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", Type: "node", LastSeen: now} + + for _, tc := range []struct { + name string + readingAt time.Time + agentSensors bool + wantCarried bool + }{ + {name: "reading from the silent agent", readingAt: silentSince, agentSensors: true}, + {name: "silent agent whose last report had no sensors", readingAt: silentSince.Add(-30 * time.Second)}, + {name: "SSH reading taken after the agent went silent", readingAt: now.Add(-time.Minute), agentSensors: true, wantCarried: true}, + } { + t.Run(tc.name, func(t *testing.T) { + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m := &Monitor{ + config: &config.Config{TemperatureMonitoringEnabled: true, PVEInstances: []config.PVEInstance{instanceCfg}}, + state: models.NewState(), + resourceStore: adapter, + nodeLastOnline: map[string]time.Time{base.ID: now}, + } + agent := models.Host{ID: "agent-1", Hostname: "node1", Status: "offline", LinkedNodeID: base.ID, IntervalSeconds: 30, LastSeen: silentSince} + if tc.agentSensors { + agent.Sensors = models.HostSensorSummary{TemperatureCelsius: map[string]float64{"cpu_package": 92}} + } + published := base + published.Temperature = &models.Temperature{CPUPackage: 92, CPUMin: 92, CPUMaxRecord: 92, Available: true, HasCPU: true, LastUpdate: tc.readingAt} + m.state.UpdateNodesForInstance("pve1", []models.Node{published}) + snapshot := m.state.GetSnapshot() + snapshot.Hosts = []models.Host{agent} + adapter.PopulateFromSnapshot(snapshot) + + polled := base + m.collectNodeTemperatureData(context.Background(), "pve1", &instanceCfg, proxmox.Node{Node: "node1"}, &polled, m.snapshotPrevNodes("pve1"), "online") + if carried := polled.Temperature != nil; carried != tc.wantCarried { + t.Fatalf("failed collection carried = %v (%#v), want %v", carried, polled.Temperature, tc.wantCarried) + } + + preserved := m.preserveOrExpireNodes(m.snapshotPrevNodes("pve1")) + if len(preserved) != 1 || preserved[0].Status != "online" { + t.Fatalf("preserved = %#v, want the node online within grace", preserved) + } + if kept := preserved[0].Temperature != nil; kept != tc.wantCarried { + t.Fatalf("outage preservation kept temperature = %v, want %v", kept, tc.wantCarried) + } + }) + } +} + +// A carried node reading keeps the node's CPU temperature steady through a +// failed collection, but never its disk temperatures: those also stamp the +// node's physical disks, which have their own freshness and standby state. +// Here a current agent whose only sensor is a SMART disk reports that disk in +// standby, so collection returns nothing; the disk must not be re-stamped with +// its pre-sleep reading through the carry. +func TestCarriedNodeTemperatureLeavesDiskReadingsOut(t *testing.T) { + now := time.Now() + instanceCfg := config.PVEInstance{Name: "pve1", Host: "https://10.0.0.1:8006"} + base := models.Node{ID: "pve1-node1", NodeIdentity: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", Type: "node", LastSeen: now} + disk := models.PhysicalDisk{ID: "pve1-node1-sda", Node: "node1", Instance: "pve1", DevPath: "/dev/sda", Serial: "WD-STANDBY", Type: "hdd"} + + for _, tc := range []struct { + name string + previous models.Temperature + wantCPU bool + }{ + { + name: "CPU and disk reading", + previous: models.Temperature{ + Available: true, HasCPU: true, CPUPackage: 50, HasSMART: true, HasNVMe: true, + SMART: []models.DiskTemp{{Device: "/dev/sda", Serial: "WD-STANDBY", Temperature: 38}}, + NVMe: []models.NVMeTemp{{Device: "nvme0", Temp: 44}}, + }, + wantCPU: true, + }, + { + name: "disk-only reading", + previous: models.Temperature{ + Available: true, HasSMART: true, + SMART: []models.DiskTemp{{Device: "/dev/sda", Serial: "WD-STANDBY", Temperature: 38}}, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m := &Monitor{ + config: &config.Config{TemperatureMonitoringEnabled: true, PVEInstances: []config.PVEInstance{instanceCfg}}, + state: models.NewState(), + resourceStore: adapter, + nodeLastOnline: map[string]time.Time{base.ID: now}, + } + published := base + previous := tc.previous + previous.LastUpdate = now.Add(-time.Minute) + published.Temperature = &previous + m.state.UpdateNodesForInstance("pve1", []models.Node{published}) + snapshot := m.state.GetSnapshot() + snapshot.Hosts = []models.Host{{ + ID: "agent-1", Hostname: "node1", Status: "online", LinkedNodeID: base.ID, IntervalSeconds: 30, LastSeen: now, + Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{Device: "sda", Serial: "WD-STANDBY", Standby: true}}}, + }} + adapter.PopulateFromSnapshot(snapshot) + + polled := base + m.collectNodeTemperatureData(context.Background(), "pve1", &instanceCfg, proxmox.Node{Node: "node1"}, &polled, m.snapshotPrevNodes("pve1"), "online") + if tc.wantCPU { + if polled.Temperature == nil || polled.Temperature.CPUPackage != 50 || !polled.Temperature.LastUpdate.Equal(previous.LastUpdate) { + t.Fatalf("carried temperature = %#v, want CPU 50 with its original timestamp", polled.Temperature) + } + if len(polled.Temperature.SMART) != 0 || len(polled.Temperature.NVMe) != 0 || polled.Temperature.HasSMART || polled.Temperature.HasNVMe { + t.Fatalf("carried temperature kept disk readings: %#v", polled.Temperature) + } + } else if polled.Temperature != nil { + t.Fatalf("carried temperature = %#v, want none when only disk readings were left", polled.Temperature) + } + if stamped := mergeNVMeTempsIntoDisks([]models.PhysicalDisk{disk}, []models.Node{polled}); stamped[0].Temperature != 0 { + t.Fatalf("standby disk stamped with %d°C from a carried reading, want no reading", stamped[0].Temperature) + } + + // Outage preservation follows the same rule. + preserved := m.preserveOrExpireNodes(m.snapshotPrevNodes("pve1")) + if len(preserved) != 1 { + t.Fatalf("preserved = %#v, want one node", preserved) + } + if got := preserved[0].Temperature; (got != nil) != tc.wantCPU || (got != nil && (len(got.SMART) != 0 || len(got.NVMe) != 0)) { + t.Fatalf("preserved temperature = %#v, want CPU only (%v)", got, tc.wantCPU) + } + }) + } +} diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go index 8ecdeb2250..1e84d90190 100644 --- a/internal/monitoring/monitor_alert_handling_test.go +++ b/internal/monitoring/monitor_alert_handling_test.go @@ -1547,22 +1547,24 @@ func TestSilentLinkedAgentStopsRefreshingNodeTemperatureAlert(t *testing.T) { alertConfig.NodeDefaults.Temperature = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75} manager.UpdateConfig(alertConfig) - registry := unifiedresources.NewRegistry(nil) + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) monitor := &Monitor{ config: &config.Config{TemperatureMonitoringEnabled: true}, state: models.NewState(), - resourceStore: unifiedresources.NewMonitorAdapter(registry), + resourceStore: adapter, } node := models.Node{ ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", ConnectionHealth: "healthy", LinkedAgentID: "agent-1", } - // ingest records a PVE poll that just saw the node, plus the agent as of its - // last report. + // ingest rebuilds the store, as each monitor refresh does, from the last PVE + // poll's node (with the temperature it published, which the next poll reads + // back as its previous node) plus the agent as of its last report. + lastPolled := node ingest := func(agentLastReport time.Time, agentStatus string) { - polledNode := node + polledNode := lastPolled polledNode.LastSeen = time.Now() - registry.IngestSnapshot(models.StateSnapshot{ + adapter.PopulateFromSnapshot(models.StateSnapshot{ Nodes: []models.Node{polledNode}, Hosts: []models.Host{{ ID: "agent-1", Hostname: "node1", Status: agentStatus, LinkedNodeID: node.ID, @@ -1574,13 +1576,14 @@ func TestSilentLinkedAgentStopsRefreshingNodeTemperatureAlert(t *testing.T) { // poll runs the node temperature step of a PVE poll (no SSH collector) and // evaluates the node's alerts with the result. poll := func() *models.Temperature { - _, prevNodes := monitor.snapshotPrevNodes("pve1") + prevNodes := monitor.snapshotPrevNodes("pve1") polled := node monitor.collectNodeTemperatureData( context.Background(), "pve1", &config.PVEInstance{Name: "pve1"}, proxmox.Node{Node: node.Name}, &polled, prevNodes, "online", ) manager.CheckNode(polled) + lastPolled = polled return polled.Temperature } temperatureAlert := func() alerts.Alert { @@ -1605,8 +1608,15 @@ func TestSilentLinkedAgentStopsRefreshingNodeTemperatureAlert(t *testing.T) { opened := temperatureAlert() // The agent goes silent while PVE polling continues: the merged row stays - // fresh, but the agent's retained sensors no longer count as a reading. - ingest(time.Now().Add(-hostAgentHealthWindow(30)-time.Minute), "offline") + // fresh, but the agent's retained sensors no longer count as a reading, and + // the poller does not carry the agent's last reading past its lease. With no + // clock to advance, age the agent's last report and the reading it produced + // by the same amount. + silence := hostAgentHealthWindow(30) + time.Minute + agedReading := *lastPolled.Temperature + agedReading.LastUpdate = agedReading.LastUpdate.Add(-silence) + lastPolled.Temperature = &agedReading + ingest(agentReport.Add(-silence), "offline") time.Sleep(5 * time.Millisecond) require.Nil(t, poll(), "a silent agent's retained sensors must not be presented as a current reading") held := temperatureAlert() diff --git a/internal/monitoring/monitor_extra_coverage_test.go b/internal/monitoring/monitor_extra_coverage_test.go index 4060fc88cc..23caafd87e 100644 --- a/internal/monitoring/monitor_extra_coverage_test.go +++ b/internal/monitoring/monitor_extra_coverage_test.go @@ -2132,22 +2132,32 @@ func TestMonitor_PreviousNodesForInstance_Extra(t *testing.T) { registry := unifiedresources.NewRegistry(nil) registry.IngestSnapshot(models.StateSnapshot{ Nodes: []models.Node{ - {ID: "node-1", Instance: "pve1", Name: "a", DisplayName: "a", Memory: models.Memory{Total: 100}}, + { + ID: "node-1", Instance: "pve1", Name: "a", DisplayName: "a", ClusterName: "lab", + Memory: models.Memory{Total: 100, Used: 40, Free: 60, Usage: 40}, + Temperature: &models.Temperature{CPUPackage: 52, CPUMin: 41, CPUMaxRecord: 70, Available: true, HasCPU: true}, + }, {ID: "node-2", Instance: "pve2", Name: "b", DisplayName: "b", Memory: models.Memory{Total: 200}}, }, }) m.resourceStore = unifiedresources.NewMonitorAdapter(registry) - prevNodeMemory, prevNodes := m.previousNodesForInstance("pve1") + prevNodes := m.previousNodesForInstance("pve1") if len(prevNodes) != 1 || prevNodes[0].Instance != "pve1" || prevNodes[0].DisplayName != "a" { t.Fatalf("expected only pve1 nodes, got %#v", prevNodes) } - if mem, ok := prevNodeMemory[prevNodes[0].ID]; !ok || mem.Total != 100 { - t.Fatalf("expected node memory for node-1, got %#v", prevNodeMemory) + // The poller matches previous nodes by its own source ID, never the + // registry's unified resource ID. + if prevNodes[0].ID != "node-1" || prevNodes[0].NodeIdentity != "node-1" { + t.Fatalf("previous node ID = %q / identity %q, want source ID node-1", prevNodes[0].ID, prevNodes[0].NodeIdentity) } - if len(prevNodeMemory) != 1 { - t.Fatalf("expected node-2 to be filtered out, got %#v", prevNodeMemory) + if prevNodes[0].ClusterName != "lab" || prevNodes[0].Memory.Total != 100 || prevNodes[0].Memory.Used != 40 { + t.Fatalf("previous node lost cluster or memory: %#v", prevNodes[0]) + } + temp := prevNodes[0].Temperature + if temp == nil || temp.CPUPackage != 52 || temp.CPUMin != 41 || temp.CPUMaxRecord != 70 { + t.Fatalf("previous node temperature = %#v, want the stored reading with its min/max records", temp) } } diff --git a/internal/monitoring/monitor_memory_test.go b/internal/monitoring/monitor_memory_test.go index df2a659a1b..84042d56fd 100644 --- a/internal/monitoring/monitor_memory_test.go +++ b/internal/monitoring/monitor_memory_test.go @@ -366,6 +366,11 @@ func TestPollPVEInstanceMarksStaleNodesOfflineWhenGetNodesReturnsEmpty(t *testin staleNode := first.Nodes[0] staleNode.LastSeen = time.Now().Add(-nodeOfflineGracePeriod - 2*time.Second) mon.state.UpdateNodesForInstance("test", []models.Node{staleNode}) + // The grace policy also counts this process's own online sighting, keyed + // by the node's source ID, so age that too. + mon.mu.Lock() + mon.nodeLastOnline[staleNode.ID] = staleNode.LastSeen + mon.mu.Unlock() client.nodes = nil mon.pollPVEInstance(context.Background(), "test", client) diff --git a/internal/monitoring/monitor_polling_node.go b/internal/monitoring/monitor_polling_node.go index 035897cdc3..a2b301f63a 100644 --- a/internal/monitoring/monitor_polling_node.go +++ b/internal/monitoring/monitor_polling_node.go @@ -96,26 +96,31 @@ func collectPVENodeNetworkInterfaces( return mapPVENodeNetworkInterfaces(interfaces) } -func (m *Monitor) canCarryForwardNodeMemory(instance, node string, now time.Time) bool { +// carryForwardNodeMemory returns the node's last memory reading from this +// poller's own snapshot when that reading came from a trusted source recently +// enough to stand in for one failed status read. The unified read state is not +// used here: on a node merged with a host agent it holds the agent's reading, +// not the Proxmox value this snapshot vouches for. +func (m *Monitor) carryForwardNodeMemory(instance, node string, now time.Time) (models.Memory, bool) { if m == nil { - return false + return models.Memory{}, false } m.diagMu.RLock() snapshot, ok := m.nodeSnapshots[makeNodeSnapshotKey(instance, node)] m.diagMu.RUnlock() if !ok || !snapshot.Memory.HasKnownUsage() || snapshot.RetrievedAt.IsZero() { - return false + return models.Memory{}, false } age := now.Sub(snapshot.RetrievedAt) if age < 0 || age > nodeMemoryCarryForwardMaxAge { - return false + return models.Memory{}, false } switch CanonicalMemorySource(snapshot.MemorySource) { case "available-field", "derived-free-buffers-cached", "derived-total-minus-used", "rrd-memavailable", "rrd-memused": - return true + return snapshot.Memory, true default: - return false + return models.Memory{}, false } } @@ -126,7 +131,6 @@ func (m *Monitor) pollPVENode( client PVEClientInterface, node proxmox.Node, connectionHealthStr string, - prevNodeMemory map[string]models.Memory, prevInstanceNodes []models.Node, ) (models.Node, string, string, error) { nodeStart := time.Now() @@ -314,10 +318,8 @@ func (m *Monitor) pollPVENode( } // If we couldn't update memory metrics using detailed status, preserve previous accurate values if available - if !memoryUpdated && - effectiveStatus == "online" && - m.canCarryForwardNodeMemory(instanceName, node.Node, time.Now()) { - if prevMem, exists := prevNodeMemory[modelNode.ID]; exists && prevMem.HasKnownUsage() { + if !memoryUpdated && effectiveStatus == "online" { + if prevMem, ok := m.carryForwardNodeMemory(instanceName, node.Node, time.Now()); ok { total := int64(node.MaxMem) if total == 0 { total = prevMem.Total @@ -326,14 +328,14 @@ func (m *Monitor) pollPVENode( if total > 0 && used > total { used = total } - free := total - used - if free < 0 { - free = 0 - } + // Keep the snapshot's used | cache | free split summing to total. + cache := min(max(prevMem.Cache, 0), max(total-used, 0)) + free := max(total-used-cache, 0) preserved := prevMem preserved.Total = total preserved.Used = used + preserved.Cache = cache preserved.Free = free preserved.Usage = safePercentage(float64(used), float64(total)) diff --git a/internal/monitoring/monitor_polling_node_helpers.go b/internal/monitoring/monitor_polling_node_helpers.go index ced65961a2..e8ef1e849a 100644 --- a/internal/monitoring/monitor_polling_node_helpers.go +++ b/internal/monitoring/monitor_polling_node_helpers.go @@ -331,7 +331,15 @@ func (m *Monitor) collectNodeTemperatureData( } } - if prevTemp != nil && time.Since(prevTemp.LastUpdate) > m.nodeTemperatureCarryWindow() { + if m.carriedTemperatureOutlivesAgentLease(modelNode.ID, node.Node, prevTemp, time.Now()) { + // The reading came from a host agent that has stopped reporting. + // Its reporting lease already rode out transient gaps, so + // carrying it would outlive that lease. + log.Debug(). + Str("node", node.Node). + Time("lastUpdate", prevTemp.LastUpdate). + Msg("Dropped temperature data (host agent stopped reporting)") + } else if prevTemp != nil && time.Since(prevTemp.LastUpdate) > m.nodeTemperatureCarryWindow() { // Collection has stayed broken past the carry window. Re-presenting // the old reading would let alerts, history and the UI treat it as // a live measurement forever, so the node has no reading now. @@ -340,11 +348,9 @@ func (m *Monitor) collectNodeTemperatureData( Bool("isCluster", modelNode.IsClusterMember). Time("lastUpdate", prevTemp.LastUpdate). Msg("Dropped stale temperature data (collection has not returned a reading within the carry window)") - } else if prevTemp != nil { - // Clone the previous temperature to avoid modifying historical data - preserved := *prevTemp - preserved.LastUpdate = prevTemp.LastUpdate // Keep original update time to indicate staleness - modelNode.Temperature = &preserved + } else if preserved := carriedNodeTemperature(prevTemp); preserved != nil { + // The carried copy keeps its original LastUpdate to indicate staleness. + modelNode.Temperature = preserved log.Debug(). Str("node", node.Node). Bool("isCluster", modelNode.IsClusterMember). @@ -361,6 +367,30 @@ func (m *Monitor) collectNodeTemperatureData( } } +// carriedNodeTemperature returns the part of an earlier reading a node may +// keep presenting while collection fails: its CPU and GPU temperatures, with +// the original LastUpdate. Disk temperatures (SMART, NVMe) are left out. They +// also stamp the node's physical disks, which have their own freshness and +// standby state that a carried copy cannot follow: a disk that spun down would +// be stamped, and recorded, with its pre-sleep reading. Returns nil when the +// earlier reading held nothing else. +func carriedNodeTemperature(prev *models.Temperature) *models.Temperature { + if prev == nil || !prev.Available { + return nil + } + carried := *prev + carried.Cores = append([]models.CoreTemp(nil), prev.Cores...) + carried.GPU = append([]models.GPUTemp(nil), prev.GPU...) + carried.SMART = []models.DiskTemp{} + carried.HasSMART = false + carried.NVMe = []models.NVMeTemp{} + carried.HasNVMe = false + if !hasUsableTemperatureReading(&carried) { + return nil + } + return &carried +} + func (m *Monitor) applyNodePendingUpdates(ctx context.Context, instanceName string, client PVEClientInterface, node proxmox.Node, nodeID string, effectiveStatus string, modelNode *models.Node) { if modelNode == nil { return diff --git a/internal/monitoring/monitor_previous_state.go b/internal/monitoring/monitor_previous_state.go index bfd30496c9..79568436ef 100644 --- a/internal/monitoring/monitor_previous_state.go +++ b/internal/monitoring/monitor_previous_state.go @@ -3,6 +3,7 @@ package monitoring import ( "strings" + "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" ) @@ -89,24 +90,31 @@ func (m *Monitor) previousGuestContextForInstance(instanceName string) previousG return ctx } -func (m *Monitor) previousNodesForInstance(instanceName string) (map[string]models.Memory, []models.Node) { - prevNodeMemory := make(map[string]models.Memory) +func (m *Monitor) previousNodesForInstance(instanceName string) []models.Node { prevInstanceNodes := make([]models.Node, 0) readState := m.GetUnifiedReadStateOrSnapshot() if readState == nil { - return prevNodeMemory, prevInstanceNodes + return prevInstanceNodes } + // The registry keeps no copy of the identity stamps pollPVENode derives + // from configuration, so restore them the same way. Nodes preserved through + // an instance outage are written back to state, and without these the + // registry would derive a different canonical identity for them. + instanceCfg := m.getInstanceConfig(instanceName) + providerScoped := m.pveNodeUsesProviderScopedIdentity(instanceName, instanceCfg) for _, existingNode := range readState.Nodes() { if existingNode == nil || existingNode.Instance() != instanceName { continue } modelNode := previousNodeFromView(existingNode) - prevNodeMemory[modelNode.ID] = modelNode.Memory + modelNode.ProviderScopedIdentity = providerScoped + modelNode.NativeNameAliases = config.PVEClusterNodeNativeAliases(instanceCfg, modelNode.Name) + modelNode.TLSFingerprint = pveNodeTLSFingerprint(instanceCfg, modelNode.Name) prevInstanceNodes = append(prevInstanceNodes, modelNode) } - return prevNodeMemory, prevInstanceNodes + return prevInstanceNodes } func previousVMFromView(vm *unifiedresources.VMView) models.VM { @@ -199,20 +207,33 @@ func previousNodeFromView(node *unifiedresources.NodeView) models.Node { if node == nil { return models.Node{} } + // The poller keys nodes by their Proxmox source ID (nodeLastOnline, the + // temperature carry, state writes). The unified resource ID is a separate + // registry key, so returning it would match nothing, and writing it back + // during an outage would re-key the node on every failed poll. + id := node.SourceID() + if id == "" { + id = node.ID() + } return models.Node{ - ID: node.ID(), + ID: id, + NodeIdentity: id, Name: node.NodeName(), DisplayName: node.Name(), Instance: node.Instance(), Host: node.HostURL(), Status: string(node.Status()), + Type: "node", Uptime: node.Uptime(), IsClusterMember: node.IsClusterMember(), + ClusterName: node.ClusterName(), LastSeen: node.LastSeen(), LoadAverage: node.LoadAverage(), PVEVersion: node.PVEVersion(), KernelVersion: node.KernelVersion(), NetworkInterfaces: hostNetworkInterfacesFromReadStateView(node.NetworkInterfaces()), + Temperature: node.TemperatureDetails(), + LinkedAgentID: node.LinkedAgentID(), Memory: models.Memory{ Used: node.MemoryUsed(), Total: node.MemoryTotal(), diff --git a/internal/monitoring/monitor_pve.go b/internal/monitoring/monitor_pve.go index 1394b9a20d..7f5405f896 100644 --- a/internal/monitoring/monitor_pve.go +++ b/internal/monitoring/monitor_pve.go @@ -587,7 +587,7 @@ func (m *Monitor) updatePVEConnectionHealth(ctx context.Context, instanceName st return connectionHealthStr } -func (m *Monitor) snapshotPrevNodes(instanceName string) (map[string]models.Memory, []models.Node) { +func (m *Monitor) snapshotPrevNodes(instanceName string) []models.Node { return m.previousNodesForInstance(instanceName) } @@ -598,7 +598,6 @@ func (m *Monitor) pollPVENodesParallel( client PVEClientInterface, nodes []proxmox.Node, connectionHealthStr string, - prevNodeMemory map[string]models.Memory, prevInstanceNodes []models.Node, debugEnabled bool, ) ([]models.Node, map[string]string, map[string]string) { @@ -627,7 +626,7 @@ func (m *Monitor) pollPVENodesParallel( go func(node proxmox.Node) { defer wg.Done() - modelNode, effectiveStatus, diskSource, _ := m.pollPVENode(ctx, instanceName, instanceCfg, client, node, connectionHealthStr, prevNodeMemory, prevInstanceNodes) + modelNode, effectiveStatus, diskSource, _ := m.pollPVENode(ctx, instanceName, instanceCfg, client, node, connectionHealthStr, prevInstanceNodes) resultChan <- nodePollResult{ node: modelNode, @@ -655,7 +654,7 @@ func (m *Monitor) pollPVENodesParallel( // leaves the last successful snapshot in state forever, so a shut-down host // keeps showing its final online status (#1441). func (m *Monitor) markPVEInstanceNodesUnreachable(instanceName string) { - _, prevInstanceNodes := m.snapshotPrevNodes(instanceName) + prevInstanceNodes := m.snapshotPrevNodes(instanceName) if len(prevInstanceNodes) == 0 { // Never polled successfully this process lifetime (e.g. Pulse // started while the host was already down). Synthesize offline @@ -762,6 +761,7 @@ func (m *Monitor) preserveOrExpireNodes(prevInstanceNodes []models.Node) []model if nodeCopy.ConnectionHealth == "" || strings.EqualFold(nodeCopy.ConnectionHealth, "error") { nodeCopy.ConnectionHealth = "degraded" } + m.boundCarriedNodeTemperature(&nodeCopy, now) preserved = append(preserved, nodeCopy) continue } @@ -770,11 +770,28 @@ func (m *Monitor) preserveOrExpireNodes(prevInstanceNodes []models.Node) []model nodeCopy.ConnectionHealth = "error" nodeCopy.Uptime = 0 nodeCopy.CPU = 0 + nodeCopy.Temperature = nil preserved = append(preserved, nodeCopy) } return preserved } +// boundCarriedNodeTemperature applies the rules a failed live collection's +// carry follows to a temperature a node keeps from an earlier poll: the carry +// window, a lapsed host agent's lease for a reading that agent supplied, and +// no disk temperatures (carriedNodeTemperature). +func (m *Monitor) boundCarriedNodeTemperature(node *models.Node, now time.Time) { + if node.Temperature == nil { + return + } + if now.Sub(node.Temperature.LastUpdate) > m.nodeTemperatureCarryWindow() || + m.carriedTemperatureOutlivesAgentLease(node.ID, node.Name, node.Temperature, now) { + node.Temperature = nil + return + } + node.Temperature = carriedNodeTemperature(node.Temperature) +} + func (m *Monitor) seedNodeDisplayNames(modelNodes []models.Node) { for i := range modelNodes { if modelNodes[i].DisplayName != "" { @@ -1466,8 +1483,9 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie // Check if client is a ClusterClient to determine health status connectionHealthStr := m.updatePVEConnectionHealth(ctx, instanceName, client) - // Capture previous memory metrics so we can preserve them if detailed status fails - prevNodeMemory, prevInstanceNodes := m.snapshotPrevNodes(instanceName) + // Capture the previous poll's nodes for network, temperature and + // inventory continuity when this poll cannot read them. + prevInstanceNodes := m.snapshotPrevNodes(instanceName) // Convert to models modelNodes, nodeEffectiveStatus, nodeDiskSources := m.pollPVENodesParallel( @@ -1477,7 +1495,6 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie client, nodes, connectionHealthStr, - prevNodeMemory, prevInstanceNodes, debugEnabled, ) diff --git a/internal/monitoring/monitor_pve_cluster.go b/internal/monitoring/monitor_pve_cluster.go index 083948619f..d03383f925 100644 --- a/internal/monitoring/monitor_pve_cluster.go +++ b/internal/monitoring/monitor_pve_cluster.go @@ -431,11 +431,13 @@ func (m *Monitor) reconcilePVENodeInventory( node.ConnectionHealth = "error" node.CPU = 0 node.Uptime = 0 + node.Temperature = nil } else if _, observed := currentByName[key]; !observed { node.Status = "unknown" node.ConnectionHealth = "degraded" node.CPU = 0 node.Uptime = 0 + m.boundCarriedNodeTemperature(&node, time.Now()) } reconciled[key] = node } diff --git a/internal/monitoring/node_memory_sources_test.go b/internal/monitoring/node_memory_sources_test.go index 70687c67d0..5e93730088 100644 --- a/internal/monitoring/node_memory_sources_test.go +++ b/internal/monitoring/node_memory_sources_test.go @@ -242,7 +242,6 @@ func TestPollPVENodePrefersLinkedHostDiskOverRootFS(t *testing.T) { }, "healthy", nil, - nil, ) if err != nil { t.Fatalf("pollPVENode() error = %v", err) From 474dbc4f09974154211ec75c2d4114cfb647bf97 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:05:15 +0100 Subject: [PATCH 27/27] Record Proxmox and agent alert history on the canonical resource Alert lifecycle changes carry the alert's source-native resource ID, and MonitorAdapter.RecordChange only mapped full Docker container IDs. Proxmox node, guest and storage alerts and host-agent alerts were journaled under IDs such as "instance-node", "instance:node:vmid" or "agent:", while resource facets, resource intelligence and Assistant resource context read history by canonical ID, so those surfaces never saw the alerts. Resolve such references to the canonical resource and record a history binding, so the reference keeps reaching the same history after a guest migration or inventory removal. Only durable identities bind: the canonical ID, a retired era, a source ID, a node-scoped guest reference or an agent host ID. Names and hostnames never bind, so a VM named "pulse-system" cannot capture system alerts, and a reference two resources answer to binds to neither. Docker keeps its full-ID rule. Rows already journaled under a source reference join canonical history without being rewritten: each registry generation binds the unbound alert references it can resolve, those found at startup and those written before inventory named the resource, retrying each for 30 minutes. --- .../v6/internal/subsystems/monitoring.md | 9 + .../internal/subsystems/unified-resources.md | 23 +++ .../monitoring/monitor_alert_handling_test.go | 58 ++++++ internal/unifiedresources/history_identity.go | 188 ++++++++++++++++++ .../unifiedresources/history_identity_test.go | 160 +++++++++++++++ internal/unifiedresources/monitor_adapter.go | 38 +++- 6 files changed, 469 insertions(+), 7 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 6522d39af6..5e550c5cb0 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -1043,6 +1043,15 @@ join another container's history. The real alert-manager callback path is covered by `TestDockerAlertTimelineUsesCanonicalHistoryIdentity` in `internal/monitoring/monitor_alert_handling_test.go`. +Proxmox node, guest and storage alert lifecycle events carry source-native IDs +and pass through the same writer, which resolves them to the canonical +resource. Resource facets, resource intelligence and Assistant resource context +therefore include their alert history, including events emitted after the +resource left inventory. Event IDs still hash the alert's source reference, so +replay stays idempotent. The real alert-manager callback path is covered by +`TestProxmoxAlertTimelineUsesCanonicalHistoryIdentity` in +`internal/monitoring/monitor_alert_handling_test.go`. + TrueNAS native alert projection preserves the trimmed, uppercase provider level in ResourceIncident.NativeSeverity. INFO and NOTICE retain the same canonical monitor risk; consumers must not lose their distinct actionability when projecting provider evidence. Native CRITICAL, ALERT, and EMERGENCY all project to canonical critical severity; EMERGENCY must not be discarded as unknown or make a still-active condition appear recovered. WARNING remains warning, and INFO and NOTICE remain informational at this projection boundary. Verification: `TestIncidentProjectionPreservesNativeSeverity` in `internal/truenas/provider_pool_health_contract_test.go` covers all seven native levels and case/whitespace normalization. `TestTrueNASNativeSeverityDispatch` in `internal/alerts/truenas_native_dispatch_test.go` verifies downstream INFO suppression, NOTICE preservation, notification severity, duplicate-poll retention, and confirmed recovery callback identity. The TrueNAS lifecycle tests in `internal/alerts/unified_incidents_test.go` require repeated EMERGENCY evidence to interrupt recovery confirmation. These are fixture-based projection and manager checks, not appliance ingestion or external notification-provider receipt proof. diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 5c1dfdb1bb..8b80254b40 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -5838,6 +5838,29 @@ action requests, approvals, links or exclusions. Separate monitor, API and Assistant store handles must see current persisted aliases. Missing identity storage is an error, not evidence of empty history. Retention removes an alias only after neither identity has retained journal records. +Other alert source references resolve at `MonitorAdapter.RecordChange` +through `ResourceRegistry.resolveHistoryReference`, which accepts only durable +identities: the canonical ID, a retired era, a source ID (Proxmox node, guest +and storage IDs), a node-scoped Proxmox guest reference after migration, or +`agent:`. Names and hostnames never bind history, unlike the wider +`ResolveReferenceID` used by alert policy, so a resource named like a system +reference cannot capture its events. A reference two resources answer to binds +to neither: its event keeps its own reference and creates or changes no +binding, though an earlier binding still covers that reference on read. A +reference no resource +answers to follows its retained binding after inventory removal, or keeps its +own history. Docker references never take this path. Alert journal references +without a binding (read once per process, plus any written before inventory +named their resource) are retried on each published registry generation for +`legacyHistoryBindWindow`. Bound rows keep their recorded resource ID, so a +change list may show the source reference on rows written before the binding. +Only the full-presentation `ResourceDetailDrawer` reads this history in the UI +(facets and resource intelligence); the `table-row` drawer used by Proxmox node +and Agents rows, and the guest and storage drawers, do not request it. +Assistant and Patrol resource contexts read it from the store. Proof: +`TestHistoryIdentityMonitorAdapterResolvesProxmoxAlertReferences` and +`TestHistoryIdentityBindsLegacyAlertRowsFromRegistryGenerations` in +`internal/unifiedresources/history_identity_test.go`. That same shared timeline vocabulary now includes the `activity` change kind for provider-read breadcrumbs such as VMware tasks and events, plus the `vmware_adapter` source-adapter token for canonical provenance drill-down. diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go index 8ecdeb2250..d45c8f1937 100644 --- a/internal/monitoring/monitor_alert_handling_test.go +++ b/internal/monitoring/monitor_alert_handling_test.go @@ -92,6 +92,64 @@ func TestDockerAlertTimelineUsesCanonicalHistoryIdentity(t *testing.T) { t.Logf("DOCKER_HISTORY_LIFECYCLE %s", encoded) } +// Proxmox node and guest alerts carry source-native IDs. Their lifecycle must +// reach the canonical resource history that facets, the drawer and the +// assistant read, including events emitted after the resource left inventory. +func TestProxmoxAlertTimelineUsesCanonicalHistoryIdentity(t *testing.T) { + store, err := unifiedresources.NewSQLiteResourceStore(t.TempDir(), "default") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + manager := alerts.NewManagerWithDataDir(t.TempDir()) + t.Cleanup(manager.Stop) + config := manager.GetConfig() + config.Enabled = true + config.ActivationState = alerts.ActivationPending + config.TimeThresholds = map[string]int{"node": 0, "guest": 0} + config.SuppressionWindow = 0 + manager.UpdateConfig(config) + now := time.Now() + node := models.Node{ID: "lab-pve1", Name: "pve1", Instance: "lab", Host: "https://pve1.lab:8006", Status: "online", CPU: 0.99, LastSeen: now} + vm := models.VM{ID: "lab:pve1:101", VMID: 101, Name: "web", Node: "pve1", Instance: "lab", Status: "stopped", Type: "qemu", LastSeen: now} + registry := unifiedresources.NewRegistry(store) + registry.IngestSnapshot(models.StateSnapshot{Nodes: []models.Node{node}, VMs: []models.VM{vm}}) + monitor := &Monitor{alertManager: manager, resourceStore: unifiedresources.NewMonitorAdapter(registry)} + manager.SubscribeLifecycleCallback(monitor.handleAlertLifecycleEvent) + manager.CheckNode(node) + manager.CheckGuest(vm, vm.Instance) + manager.CheckGuest(vm, vm.Instance) + + canonical := map[string]string{} + for _, resource := range registry.List() { + canonical[resource.Name] = resource.ID + } + require.NotEqual(t, node.ID, canonical["pve1"]) + require.NotEqual(t, vm.ID, canonical["web"]) + filters := unifiedresources.ResourceChangeFilters{Kinds: []unifiedresources.ChangeKind{unifiedresources.ChangeAlertFired, unifiedresources.ChangeAlertResolved}} + for name, sourceID := range map[string]string{"pve1": node.ID, "web": vm.ID} { + changes, err := store.GetRecentChangesFiltered(canonical[name], time.Time{}, 10, filters) + require.NoError(t, err) + require.Len(t, changes, 1, name) + require.Equal(t, unifiedresources.ChangeAlertFired, changes[0].Kind) + require.Equal(t, canonical[name], changes[0].ResourceID) + kinds, err := store.CountRecentChangesByKind(canonical[name], time.Time{}) + require.NoError(t, err) + require.Equal(t, 1, kinds[unifiedresources.ChangeAlertFired], name) + legacy, err := store.GetRecentChangesFiltered(sourceID, time.Time{}, 10, filters) + require.NoError(t, err) + require.Equal(t, changes, legacy, "the source reference reads the same history") + } + + // The node recovers after it left inventory; its retained binding holds. + monitor.resourceStore = unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store)) + node.CPU = 0.05 + manager.CheckNode(node) + changes, err := store.GetRecentChangesFiltered(canonical["pve1"], time.Time{}, 10, filters) + require.NoError(t, err) + require.Len(t, changes, 2) + require.Equal(t, unifiedresources.ChangeAlertResolved, changes[0].Kind) + require.Equal(t, canonical["pve1"], changes[0].ResourceID) +} + func TestMonitor_HandleAlertFired_Extra(t *testing.T) { // 1. Alert is nil m1 := &Monitor{} diff --git a/internal/unifiedresources/history_identity.go b/internal/unifiedresources/history_identity.go index e71c71a2c4..9dc69d886e 100644 --- a/internal/unifiedresources/history_identity.go +++ b/internal/unifiedresources/history_identity.go @@ -4,7 +4,10 @@ import ( "database/sql" "encoding/hex" "fmt" + "log" "strings" + "sync" + "time" ) // legacyDockerHistoryIdentity accepts the source identifier emitted by Docker @@ -29,6 +32,61 @@ func legacyDockerHistoryIdentity(ref string) (sourceID, canonicalID string, ok b return sourceID, SourceSpecificID(ResourceTypeAppContainer, SourceDocker, sourceID), true } +// isDockerHistoryReference marks Docker alert source references, which only a +// complete container ID binds (legacyDockerHistoryIdentity). A container name +// or shortened ID must never join history through general reference matching. +func isDockerHistoryReference(ref string) bool { + return strings.HasPrefix(strings.TrimSpace(ref), "docker:") +} + +// resolveHistoryReference resolves a non-Docker alert reference to the +// resource whose history it joins. Only durable identities qualify: the +// canonical ID, a retired era, a source ID (Proxmox node, guest and storage +// IDs), a node-scoped Proxmox guest reference, or an agent alert reference +// ("agent:"). Names and hostnames never bind history, so a resource +// named like a system reference cannot capture its events. claimed reports +// that inventory answers to the reference, possibly ambiguously; an ambiguous +// reference resolves to nothing and must not follow a retained binding. +func (rr *ResourceRegistry) resolveHistoryReference(ref string) (resourceID string, claimed bool) { + ref = CanonicalResourceID(ref) + if rr == nil || ref == "" || isDockerHistoryReference(ref) { + return "", false + } + rr.mu.RLock() + defer rr.mu.RUnlock() + if rr.resources[ref] != nil { + return ref, true + } + if id := rr.supersededResourceIDLocked(ref); rr.resources[id] != nil { + return id, true + } + matches := make(map[string]struct{}, 1) + for _, mapping := range rr.bySource { + if id := mapping[ref]; rr.resources[id] != nil { + matches[id] = struct{}{} + } + } + if hostID, ok := strings.CutPrefix(ref, "agent:"); ok && hostID != "" && !strings.Contains(hostID, "/") { + if id := rr.bySource[SourceAgent][hostID]; rr.resources[id] != nil { + matches[id] = struct{}{} + } + } + if instance, _, vmid, ok := ParseProxmoxGuestSourceID(ref); ok { + for _, resourceType := range []ResourceType{ResourceTypeVM, ResourceTypeSystemContainer} { + if id := ProxmoxGuestCanonicalID(resourceType, instance, vmid); rr.resources[id] != nil { + matches[id] = struct{}{} + } + } + } + if len(matches) != 1 { + return "", len(matches) > 1 + } + for id := range matches { + resourceID = id + } + return resourceID, true +} + // resourceHistoryIdentityWriter binds a source reference to the canonical // resource of one event. It affects history lookup only, never operator state, // action requests, approvals or execution identities. @@ -90,6 +148,136 @@ func (m *MemoryStore) RecordChangeWithSourceIdentity(change ResourceChange, sour return m.recordChangeLocked(change) } +// legacyHistoryBindWindow bounds how long an unresolved alert journal +// reference is retried. Inventory normally names a resource within a few poll +// cycles; a reference still unresolved after the window names a removed +// resource or a sub-resource and keeps its own history. +const legacyHistoryBindWindow = 30 * time.Minute + +// legacyHistoryBinder exposes alert journal references that no history binding +// covers. Rows recorded before their reference resolved join canonical history +// through a binding; the rows themselves are never rewritten. +type legacyHistoryBinder interface { + unboundHistoryReferences() ([]string, error) + bindHistorySourceIdentities(bindings map[string]string) error +} + +// legacyHistoryBackfill holds unbound references with their retry deadlines: +// those already journaled when the process started, and those RecordChange +// wrote before inventory could name their resource. +type legacyHistoryBackfill struct { + mu sync.Mutex + loaded bool + pending map[string]time.Time +} + +func (b *legacyHistoryBackfill) addLocked(ref string, now time.Time) { + if b.pending == nil { + b.pending = make(map[string]time.Time) + } + if _, ok := b.pending[ref]; !ok { + b.pending[ref] = now.Add(legacyHistoryBindWindow) + } +} + +func (b *legacyHistoryBackfill) add(ref string, now time.Time) { + b.mu.Lock() + defer b.mu.Unlock() + b.addLocked(ref, now) +} + +// unboundHistoryReferences lists alert journal references without a binding. +// Docker references are excluded: the store migration binds full container +// IDs, and names or shortened IDs must never bind. +func (s *SQLiteResourceStore) unboundHistoryReferences() ([]string, error) { + rows, err := s.db.Query(`SELECT DISTINCT canonical_id FROM resource_changes + WHERE kind GLOB 'alert_*' AND canonical_id NOT GLOB 'docker:*' + AND canonical_id NOT IN (SELECT source_id FROM resource_history_aliases) + ORDER BY canonical_id`) + if err != nil { + return nil, fmt.Errorf("read unbound resource history references: %w", err) + } + defer rows.Close() + var refs []string + for rows.Next() { + var ref string + if err := rows.Scan(&ref); err != nil { + return nil, err + } + refs = append(refs, ref) + } + return refs, rows.Err() +} + +func (s *SQLiteResourceStore) bindHistorySourceIdentities(bindings map[string]string) error { + if len(bindings) == 0 { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return fmt.Errorf("begin resource history identities: %w", err) + } + defer func() { _ = tx.Rollback() }() + for sourceID, canonicalID := range bindings { + if _, err := tx.Exec(`INSERT INTO resource_history_aliases (source_id, canonical_id) VALUES (?, ?) + ON CONFLICT(source_id) DO UPDATE SET canonical_id = excluded.canonical_id`, sourceID, canonicalID); err != nil { + return fmt.Errorf("bind resource history identity: %w", err) + } + } + return tx.Commit() +} + +// bindLegacyHistory binds the pending references a published registry +// generation resolves, with the same rules as RecordChange. The journal is +// scanned once per process; later references arrive from RecordChange. +func (a *MonitorAdapter) bindLegacyHistory(registry *ResourceRegistry, now time.Time) { + binder, ok := registry.store.(legacyHistoryBinder) + if !ok { + return + } + backfill := &a.legacyHistory + backfill.mu.Lock() + defer backfill.mu.Unlock() + if !backfill.loaded { + refs, err := binder.unboundHistoryReferences() + if err != nil { + log.Printf("unifiedresources: %v", err) + return + } + backfill.loaded = true + for _, ref := range refs { + backfill.addLocked(ref, now) + } + } + if len(backfill.pending) == 0 { + return + } + bindings := make(map[string]string) + var settled []string + for ref, deadline := range backfill.pending { + if resolvedID, _ := registry.resolveHistoryReference(ref); resolvedID != "" { + if resolvedID != ref { + bindings[ref] = resolvedID + } + settled = append(settled, ref) + } else if !now.Before(deadline) { + delete(backfill.pending, ref) + } + } + if err := binder.bindHistorySourceIdentities(bindings); err != nil { + log.Printf("unifiedresources: %v", err) + return + } + for _, ref := range settled { + delete(backfill.pending, ref) + } + if len(bindings) > 0 { + log.Printf("unifiedresources: bound %d alert history references to canonical resources", len(bindings)) + } +} + // migrateResourceHistoryAliases adds an index for exact legacy Docker event // identities, including resources already removed from live inventory. The // event rows and all authority-bearing tables remain unchanged. diff --git a/internal/unifiedresources/history_identity_test.go b/internal/unifiedresources/history_identity_test.go index 21c4512dbf..935e260fb4 100644 --- a/internal/unifiedresources/history_identity_test.go +++ b/internal/unifiedresources/history_identity_test.go @@ -173,3 +173,163 @@ func TestHistoryIdentityRetentionAndUnavailableLookup(t *testing.T) { _, err = store.GetRecentChanges(canonical, time.Time{}, 10) require.Error(t, err) } + +func proxmoxHistoryIdentitySnapshot(now time.Time) models.StateSnapshot { + return models.StateSnapshot{ + Nodes: []models.Node{{ID: "lab-pve1", Name: "pve1", Instance: "lab", Host: "https://pve1.lab:8006", Status: "online", LinkedAgentID: "host-pve1", LastSeen: now}}, + Hosts: []models.Host{{ID: "host-pve1", Hostname: "pve1", LinkedNodeID: "lab-pve1", MachineID: "0123456789abcdef", Status: "online", LastSeen: now}}, + VMs: []models.VM{{ID: "lab:pve1:101", VMID: 101, Name: "web", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now}}, + Containers: []models.Container{{ID: "lab-pve1-102", VMID: 102, Name: "dns", Node: "pve1", Instance: "lab", Status: "running", Type: "lxc", LastSeen: now}}, + } +} + +func historyIdentityResourceID(t *testing.T, registry *ResourceRegistry, resourceType ResourceType, name string) string { + t.Helper() + for _, resource := range registry.List() { + if resource.Type == resourceType && resource.Name == name { + return resource.ID + } + } + t.Fatalf("no %s resource named %q", resourceType, name) + return "" +} + +// Proxmox alerts carry the source-native node and guest IDs. Their lifecycle +// must land in the history the drawer, facets and assistant read by canonical ID. +func TestHistoryIdentityMonitorAdapterResolvesProxmoxAlertReferences(t *testing.T) { + store, err := NewSQLiteResourceStore(t.TempDir(), "default") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + now := time.Now().UTC().Truncate(time.Second) + snapshot := proxmoxHistoryIdentitySnapshot(now) + // A guest named like the system alert reference must not capture it. + snapshot.VMs = append(snapshot.VMs, models.VM{ID: "lab:pve1:120", VMID: 120, Name: "pulse-system", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now}) + registry := NewRegistry(store) + registry.IngestSnapshot(snapshot) + adapter := NewMonitorAdapter(registry) + nodeID := historyIdentityResourceID(t, registry, ResourceTypeAgent, "pve1") + vmID := historyIdentityResourceID(t, registry, ResourceTypeVM, "web") + ctID := historyIdentityResourceID(t, registry, ResourceTypeSystemContainer, "dns") + + // "lab:pve2:101" names a node other than the guest's current one, as an + // alert raised before a live migration does; it still names the guest. + refs := []string{"lab-pve1", "agent:host-pve1", "lab:pve2:101", "lab-pve1-102", "pulse-system", "web"} + for i, ref := range refs { + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "fired-" + ref, ResourceID: ref, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Duration(i) * time.Second)})) + } + for canonicalID, want := range map[string][]string{nodeID: {"agent:host-pve1", "lab-pve1"}, vmID: {"lab:pve2:101"}, ctID: {"lab-pve1-102"}} { + got, err := store.GetRecentChanges(canonicalID, time.Time{}, 10) + require.NoError(t, err) + require.Len(t, got, len(want), canonicalID) + for i, ref := range want { + require.Equal(t, "fired-"+ref, got[i].ID) + require.Equal(t, canonicalID, got[i].ResourceID) + bound, found, err := store.ResolveHistorySourceIdentity(ref) + require.NoError(t, err) + require.True(t, found, ref) + require.Equal(t, canonicalID, bound) + } + kinds, err := store.CountRecentChangesByKind(canonicalID, time.Time{}) + require.NoError(t, err) + require.Equal(t, len(want), kinds[ChangeAlertFired], canonicalID) + } + // Names are display identities: they keep their own history and no binding. + for _, ref := range []string{"pulse-system", "web"} { + got, err := store.GetRecentChanges(ref, time.Time{}, 10) + require.NoError(t, err) + require.Len(t, got, 1, ref) + _, found, err := store.ResolveHistorySourceIdentity(ref) + require.NoError(t, err) + require.False(t, found, ref) + } + got, err := store.GetRecentChanges(historyIdentityResourceID(t, registry, ResourceTypeVM, "pulse-system"), time.Time{}, 10) + require.NoError(t, err) + require.Empty(t, got) + + // Recovery after the node left inventory still reaches the same history. + removed := NewMonitorAdapter(NewRegistry(store)) + require.NoError(t, removed.RecordChange(ResourceChange{ID: "resolved-lab-pve1", ResourceID: "lab-pve1", Kind: ChangeAlertResolved, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Minute)})) + got, err = store.GetRecentChanges(nodeID, time.Time{}, 10) + require.NoError(t, err) + require.Len(t, got, 3) + require.Equal(t, "resolved-lab-pve1", got[0].ID) + require.Equal(t, nodeID, got[0].ResourceID) + + // When two resources answer to the reference, neither the registry nor the + // retained binding may choose: the event keeps its own reference. + registry.mu.Lock() + registry.bySource[SourcePBS] = map[string]string{"lab-pve1": vmID} + registry.mu.Unlock() + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "ambiguous-lab-pve1", ResourceID: "lab-pve1", Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(2 * time.Minute)})) + var recordedAs string + require.NoError(t, store.db.QueryRow(`SELECT canonical_id FROM resource_changes WHERE id = 'ambiguous-lab-pve1'`).Scan(&recordedAs)) + require.Equal(t, "lab-pve1", recordedAs) + bound, _, err := store.ResolveHistorySourceIdentity("lab-pve1") + require.NoError(t, err) + require.Equal(t, nodeID, bound, "an ambiguous reference never rebinds") + got, err = store.GetRecentChanges(vmID, time.Time{}, 10) + require.NoError(t, err) + require.Len(t, got, 1) +} + +// Alert rows recorded under source-native references before they resolved join +// canonical history once a registry generation can name their resource: rows +// journaled before this process started, and rows written before inventory +// knew the resource. The rows themselves are never rewritten. +func TestHistoryIdentityBindsLegacyAlertRowsFromRegistryGenerations(t *testing.T) { + store, err := NewSQLiteResourceStore(t.TempDir(), "default") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + now := time.Now().UTC().Truncate(time.Second) + for i, ref := range []string{"lab-pve1", "lab-pve1-102", "lab:pve1:103", "lab:pve1:999", "docker:tower/worker"} { + require.NoError(t, store.RecordChange(ResourceChange{ID: "legacy-" + ref, ResourceID: ref, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Duration(i) * time.Second)})) + } + adapter := NewMonitorAdapter(NewRegistry(store)) + snapshot := proxmoxHistoryIdentitySnapshot(now) + adapter.PopulateFromSnapshot(snapshot) + registry := adapter.currentRegistry() + for ref, canonicalID := range map[string]string{ + "lab-pve1": historyIdentityResourceID(t, registry, ResourceTypeAgent, "pve1"), + "lab-pve1-102": historyIdentityResourceID(t, registry, ResourceTypeSystemContainer, "dns"), + } { + got, err := store.GetRecentChangesFiltered(canonicalID, time.Time{}, 10, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertFired}}) + require.NoError(t, err) + require.Len(t, got, 1, ref) + require.Equal(t, "legacy-"+ref, got[0].ID) + require.Equal(t, ref, got[0].ResourceID) + } + + // An alert raised after the journal scan, before inventory names its guest. + require.NoError(t, adapter.RecordChange(ResourceChange{ID: "early-lab:pve1:104", ResourceID: "lab:pve1:104", Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: now.Add(time.Minute)})) + for _, ref := range []string{"lab:pve1:103", "lab:pve1:104"} { + _, found, err := store.ResolveHistorySourceIdentity(ref) + require.NoError(t, err) + require.False(t, found, ref) + } + // Both guests join inventory in a later generation and are bound then. + snapshot.VMs = append(snapshot.VMs, + models.VM{ID: "lab:pve1:103", VMID: 103, Name: "late", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now}, + models.VM{ID: "lab:pve1:104", VMID: 104, Name: "early", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now}) + adapter.PopulateFromSnapshot(snapshot) + for name, id := range map[string]string{"late": "legacy-lab:pve1:103", "early": "early-lab:pve1:104"} { + got, err := store.GetRecentChangesFiltered(historyIdentityResourceID(t, adapter.currentRegistry(), ResourceTypeVM, name), time.Time{}, 10, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertFired}}) + require.NoError(t, err) + require.Len(t, got, 1, name) + require.Equal(t, id, got[0].ID) + } + + // Unknown guests and Docker names stay unbound, and retries end at the deadline. + for _, ref := range []string{"lab:pve1:999", "docker:tower/worker"} { + _, found, err := store.ResolveHistorySourceIdentity(ref) + require.NoError(t, err) + require.False(t, found, ref) + } + require.Len(t, adapter.legacyHistory.pending, 1) + require.Contains(t, adapter.legacyHistory.pending, "lab:pve1:999") + adapter.legacyHistory.pending["lab:pve1:999"] = time.Now().Add(-time.Second) + adapter.PopulateFromSnapshot(snapshot) + require.Empty(t, adapter.legacyHistory.pending) + refs, err := store.unboundHistoryReferences() + require.NoError(t, err) + require.Equal(t, []string{"lab:pve1:999"}, refs) +} diff --git a/internal/unifiedresources/monitor_adapter.go b/internal/unifiedresources/monitor_adapter.go index 8bac41ceaa..94795237b4 100644 --- a/internal/unifiedresources/monitor_adapter.go +++ b/internal/unifiedresources/monitor_adapter.go @@ -26,6 +26,7 @@ type MonitorAdapter struct { activeAlerts []models.Alert lastRebuiltAt time.Time staleThresholds map[DataSource]time.Duration + legacyHistory legacyHistoryBackfill overlays overlayReadStateCache } @@ -195,13 +196,17 @@ func (a *MonitorAdapter) currentRegistry() *ResourceRegistry { } // RecordChange forwards canonical resource-history events into the underlying -// resource store when monitoring has a durable store attached. +// resource store when monitoring has a durable store attached. Alert lifecycle +// events carry source-native references (Proxmox node and guest IDs, agent +// IDs); they resolve here so history reads by canonical ID include them. func (a *MonitorAdapter) RecordChange(change ResourceChange) error { registry := a.currentRegistry() if registry == nil || registry.store == nil { return nil } sourceRef := change.ResourceID + history, hasHistory := registry.store.(resourceHistoryIdentityWriter) + unbound := false if sourceID, derivedID, ok := legacyDockerHistoryIdentity(sourceRef); ok { // Use exact source identity when inventory is present, including any // canonical identity merge. Full Docker IDs remain derivable after removal. @@ -212,7 +217,7 @@ func (a *MonitorAdapter) RecordChange(change ResourceChange) error { change.ResourceID = resolvedID } else { change.ResourceID = derivedID - if history, ok := registry.store.(resourceHistoryIdentityWriter); ok { + if hasHistory { id, found, err := history.ResolveHistorySourceIdentity(sourceRef) if err != nil { return err @@ -222,13 +227,31 @@ func (a *MonitorAdapter) RecordChange(change ResourceChange) error { } } } - } - if sourceRef != change.ResourceID { - if writer, ok := registry.store.(resourceHistoryIdentityWriter); ok { - return writer.RecordChangeWithSourceIdentity(change, sourceRef) + } else if resolvedID, claimed := registry.resolveHistoryReference(sourceRef); resolvedID != "" { + change.ResourceID = resolvedID + } else if hasHistory && !claimed && !isDockerHistoryReference(sourceRef) { + // No resource answers to it: keep the binding an earlier event recorded. + id, found, err := history.ResolveHistorySourceIdentity(CanonicalResourceID(sourceRef)) + if err != nil { + return err + } + if found { + change.ResourceID = id + } else { + unbound = true } } - return registry.store.RecordChange(change) + if hasHistory && CanonicalResourceID(sourceRef) != CanonicalResourceID(change.ResourceID) { + return history.RecordChangeWithSourceIdentity(change, sourceRef) + } + if err := registry.store.RecordChange(change); err != nil { + return err + } + // Inventory may name the resource in a later generation; bind the row then. + if _, ok := registry.store.(legacyHistoryBinder); unbound && ok { + a.legacyHistory.add(CanonicalResourceID(sourceRef), time.Now()) + } + return nil } // GetRecentChanges forwards canonical resource-history reads into the @@ -437,6 +460,7 @@ func (a *MonitorAdapter) replaceRegistryLocked(snapshot models.StateSnapshot, re a.activeAlerts = append([]models.Alert(nil), snapshot.ActiveAlerts...) a.lastRebuiltAt = rebuiltAt a.mu.Unlock() + a.bindLegacyHistory(rebuilt, time.Now()) } // ShouldSkipAPIPolling returns true when agent coverage indicates API