From f4c304037351fa10dd4e32598ec77fc1bde95216 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:33:08 +0100 Subject: [PATCH] Read a linked agent's own heartbeat in host snapshots A host agent linked to a Proxmox node merges into one unified host row. When the agent stops reporting but PVE polling continues, that row stays online and its LastSeen follows the PVE sighting. hostFromReadStateView copied both into models.Host, so a silent agent looked current to every consumer of the projection: - linkedHostForNode accepted it, and resolveNodeDisk kept using the agent's last filesystem summary as the node's disk, feeding disk alerts and history with a frozen reading. Its retained ZFS datasets kept merging into the node's pools too. - Agent connections built from HostsSnapshot (the Connections list and the update-readiness agent check) aged the agent from the PVE poll, so a dead agent stayed Active. The projection now takes LastSeen from the agent source's own sighting and reports the agent offline once that sighting is past its reporting lease, the same hostAgentHealthWindow evaluateHostAgents applies. Rows without an agent sighting, such as vSphere and TrueNAS hosts, keep the row's values. The node falls back to the Proxmox rootfs reading while the agent is silent and returns to the agent summary when it reports. --- .../v6/internal/subsystems/agent-lifecycle.md | 12 ++++ .../v6/internal/subsystems/monitoring.md | 19 +++++ internal/monitoring/monitor.go | 24 ++++++- .../monitoring/monitor_host_agents_test.go | 69 +++++++++++++++++++ 4 files changed, 122 insertions(+), 2 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index ea945c3572..93d863fc62 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -8797,3 +8797,15 @@ no update status, instead of reporting "no update available" (#2353). Agent registration, enrolment, install, update, removal and report identity are unchanged; the per-image memo lives in the registry checker and is pruned each collection cycle to the images in use. + +### Host snapshots report a linked agent's own heartbeat + +`internal/monitoring/monitor.go` changed only so a host produced from the read +state carries the agent source's own last report and reads offline past the +agent's reporting lease, even when the agent's row is merged with a Proxmox +node that PVE polling keeps fresh. The agent connection on the Connections +list and the update-readiness agent-continuity check therefore age a silent +agent from its last report instead of from the PVE poll, and mark it stale at +their own heartbeat cutoff (`fleethealth.AgentStaleThreshold`, which is +separate from the monitoring reporting lease). Agent registration, enrolment, +install, update, removal and report identity are unchanged. diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 17b73c02fd..0353b909c1 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -2638,6 +2638,25 @@ online/running/healthy vocabulary. The diagnostics subject set is unchanged: snapshot (real agents), which now matches what agent-only surfaces show once integration-backed ledger rows are excluded. +A host produced by `hostFromReadStateView` describes the agent, so its +`Status` and `LastSeen` are the agent's own. A host agent linked to a Proxmox +node merges into one row whose `LastSeen` follows its freshest source and +whose status stays online while PVE polling continues, so the row's values +would present a silent agent as online and current. The projection reads the +agent source's sighting from `HostView.SourceStatus(SourceAgent)`: the host +carries that report time and reads `offline` once it is past the reporting +lease (`hostAgentReportCurrent`, the `hostAgentHealthWindow` that +`evaluateHostAgents` applies to the host in state). Rows without an agent +sighting, such as vSphere and TrueNAS integration hosts, keep the row's +values. `linkedHostForNode` therefore stops handing a silent agent's retained +filesystem summary and ZFS datasets to its node, whose disk falls back to the +Proxmox rootfs or `/nodes` reading before disk alerts and history read it, and +agent connections built from `HostsSnapshot()` (the Connections list and the +update-readiness agent-continuity check) age the agent from its last report +rather than the PVE poll time, applying their own heartbeat cutoff. +`TestSilentLinkedAgentStopsFeedingNodeDisk` drives the disk and projection +path through `evaluateHostAgents` and the registry-backed store refresh. + Unified Agent host reports now make module readiness and updater/config lifecycle evidence monitoring-owned observed state. Monitoring preserves the last successful one-shot update transition across subsequent reports, forwards diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index baab181eb9..54b6e634c9 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -3577,6 +3577,7 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host { if trimmed := strings.TrimSpace(view.Name()); trimmed != "" && trimmed != view.Hostname() { displayName = trimmed } + status, lastSeen := hostAgentLivenessFromReadStateView(view, time.Now()) return models.Host{ ID: firstNonEmptyString(view.AgentID(), view.ID()), @@ -3598,10 +3599,10 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host { RAID: hostRAIDFromReadStateView(view.RAID()), Unraid: hostUnraidFromReadStateView(view.Unraid()), Ceph: hostCephFromReadStateView(view.Ceph()), - Status: string(view.Status()), + Status: status, UptimeSeconds: view.UptimeSeconds(), IntervalSeconds: view.IntervalSeconds(), - LastSeen: view.LastSeen(), + LastSeen: lastSeen, AgentVersion: view.AgentVersion(), IntegrationSource: view.IntegrationSource(), MachineID: view.MachineID(), @@ -3624,6 +3625,25 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host { } } +// hostAgentLivenessFromReadStateView returns the status and last report of the +// agent behind a read-state host row. A row merged with a Proxmox node takes +// LastSeen from its freshest source and stays online while PVE polling +// continues, so a silent agent would look current: its retained disk summary +// and datasets kept feeding the node, and its connection read as active. A +// models.Host describes the agent, so read the agent source's own sighting +// and lease, as evaluateHostAgents does for the host in state. +func hostAgentLivenessFromReadStateView(view *unifiedresources.HostView, now time.Time) (string, time.Time) { + status, lastSeen := string(view.Status()), view.LastSeen() + agentSighting, ok := view.SourceStatus(unifiedresources.SourceAgent) + if !ok || agentSighting.LastSeen.IsZero() { + return status, lastSeen + } + if !hostAgentReportCurrent(agentSighting.LastSeen, view.IntervalSeconds(), now) { + status = "offline" + } + return status, agentSighting.LastSeen +} + func vmFromReadStateView(view *unifiedresources.VMView) models.VM { if view == nil { return models.VM{} diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go index f14c2d1d55..3ef3414d7d 100644 --- a/internal/monitoring/monitor_host_agents_test.go +++ b/internal/monitoring/monitor_host_agents_test.go @@ -24,6 +24,7 @@ import ( agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker" agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" "github.com/rcourtman/pulse-go-rewrite/pkg/metrics" + "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" ) func TestNewMonitorRoutesStartupCustomSensorWarningBeforeStart(t *testing.T) { @@ -6843,3 +6844,71 @@ func TestBroadcastProjectionListsRegistryOnceAndKeepsLiveChanges(t *testing.T) { t.Fatal("removed host lifecycle surface disappeared") } } + +// A host agent linked to a Proxmox node merges into one read-state row that +// every PVE poll keeps online and fresh. Once the agent stops reporting, its +// retained filesystem summary must stop standing in for the node's disk (and +// through it disk alerts and history), and the projected agent must read as +// offline with its own last report, so its connection shows the missed +// heartbeat instead of the PVE poll time. +func TestSilentLinkedAgentStopsFeedingNodeDisk(t *testing.T) { + state := models.NewState() + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m := &Monitor{config: &config.Config{}, state: state, resourceStore: adapter} + + node := models.Node{ + ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", + ConnectionHealth: "healthy", LinkedAgentID: "agent-1", + } + rootFS := &proxmox.NodeStatus{RootFS: &proxmox.RootFS{Total: 200, Used: 50, Free: 150}} + // poll records a PVE poll that just saw the node, with the agent as of its + // last report, and resolves the node's disk the way pollPVENode does. + poll := func(agentLastReport time.Time) (models.Disk, string, models.Host) { + t.Helper() + polledNode := node + polledNode.LastSeen = time.Now() + state.UpdateNodes([]models.Node{polledNode}) + state.UpsertHost(models.Host{ + ID: "agent-1", Hostname: "node1", Status: "online", LinkedNodeID: node.ID, + IntervalSeconds: 30, LastSeen: agentLastReport, + Disks: []models.Disk{{Mountpoint: "/", Type: "ext4", Total: 1000, Used: 900, Free: 100, Usage: 90}}, + }) + m.evaluateHostAgents(time.Now()) + m.refreshUnifiedResourceStoreAfterAgentStateChange() + + disk, source := m.resolveNodeDisk("pve1", node.ID, node.Name, proxmox.Node{Node: node.Name}, rootFS) + for _, host := range m.HostsSnapshot() { + if host.ID == "agent-1" { + return disk, source, host + } + } + t.Fatal("HostsSnapshot does not list agent-1") + return disk, source, models.Host{} + } + + agentReport := time.Now().Add(-10 * time.Second) + disk, source, host := poll(agentReport) + if source != "agent" || disk.Used != 900 { + t.Fatalf("reporting agent: node disk = %+v from %q, want the agent's summary", disk, source) + } + if host.Status != "online" || !host.LastSeen.Equal(agentReport) { + t.Fatalf("reporting agent projected as %q last seen %s, want online at its report %s", host.Status, host.LastSeen, agentReport) + } + + silentSince := time.Now().Add(-hostAgentHealthWindow(30) - time.Minute) + disk, source, host = poll(silentSince) + if source != "node-status-rootfs" || disk.Used != 50 { + t.Fatalf("silent agent: node disk = %+v from %q, want Proxmox rootfs", disk, source) + } + if linked := m.linkedHostForNode("pve1", node.ID, node.Name); linked != nil { + t.Fatalf("silent agent still linked to the node for disk and dataset data: %+v", linked) + } + if host.Status != "offline" || !host.LastSeen.Equal(silentSince) { + t.Fatalf("silent agent projected as %q last seen %s, want offline at its last report %s", host.Status, host.LastSeen, silentSince) + } + + disk, source, host = poll(time.Now()) + if source != "agent" || disk.Used != 900 || host.Status != "online" { + t.Fatalf("agent reporting again: node disk = %+v from %q, agent %q; want the agent's summary and online", disk, source, host.Status) + } +}