diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index bfdbdc9577..6813b04986 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -8807,3 +8807,15 @@ no update status, instead of reporting "no update available" (#2353). Agent registration, enrolment, install, update, removal and report identity are unchanged; the per-image memo lives in the registry checker and is pruned each collection cycle to the images in use. + +### Host snapshots report a linked agent's own heartbeat + +`internal/monitoring/monitor.go` changed only so a host produced from the read +state carries the agent source's own last report and reads offline past the +agent's reporting lease, even when the agent's row is merged with a Proxmox +node that PVE polling keeps fresh. The agent connection on the Connections +list and the update-readiness agent-continuity check therefore age a silent +agent from its last report instead of from the PVE poll, and mark it stale at +their own heartbeat cutoff (`fleethealth.AgentStaleThreshold`, which is +separate from the monitoring reporting lease). Agent registration, enrolment, +install, update, removal and report identity are unchanged. diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 685049e081..6522d39af6 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -2638,6 +2638,25 @@ online/running/healthy vocabulary. The diagnostics subject set is unchanged: snapshot (real agents), which now matches what agent-only surfaces show once integration-backed ledger rows are excluded. +A host produced by `hostFromReadStateView` describes the agent, so its +`Status` and `LastSeen` are the agent's own. A host agent linked to a Proxmox +node merges into one row whose `LastSeen` follows its freshest source and +whose status stays online while PVE polling continues, so the row's values +would present a silent agent as online and current. The projection reads the +agent source's sighting from `HostView.SourceStatus(SourceAgent)`: the host +carries that report time and reads `offline` once it is past the reporting +lease (`hostAgentReportCurrent`, the `hostAgentHealthWindow` that +`evaluateHostAgents` applies to the host in state). Rows without an agent +sighting, such as vSphere and TrueNAS integration hosts, keep the row's +values. `linkedHostForNode` therefore stops handing a silent agent's retained +filesystem summary and ZFS datasets to its node, whose disk falls back to the +Proxmox rootfs or `/nodes` reading before disk alerts and history read it, and +agent connections built from `HostsSnapshot()` (the Connections list and the +update-readiness agent-continuity check) age the agent from its last report +rather than the PVE poll time, applying their own heartbeat cutoff. +`TestSilentLinkedAgentStopsFeedingNodeDisk` drives the disk and projection +path through `evaluateHostAgents` and the registry-backed store refresh. + Unified Agent host reports now make module readiness and updater/config lifecycle evidence monitoring-owned observed state. Monitoring preserves the last successful one-shot update transition across subsequent reports, forwards diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index b56678c8d9..65df2b3a2e 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -3616,6 +3616,7 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host { if trimmed := strings.TrimSpace(view.Name()); trimmed != "" && trimmed != view.Hostname() { displayName = trimmed } + status, lastSeen := hostAgentLivenessFromReadStateView(view, time.Now()) return models.Host{ ID: firstNonEmptyString(view.AgentID(), view.ID()), @@ -3637,10 +3638,10 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host { RAID: hostRAIDFromReadStateView(view.RAID()), Unraid: hostUnraidFromReadStateView(view.Unraid()), Ceph: hostCephFromReadStateView(view.Ceph()), - Status: string(view.Status()), + Status: status, UptimeSeconds: view.UptimeSeconds(), IntervalSeconds: view.IntervalSeconds(), - LastSeen: view.LastSeen(), + LastSeen: lastSeen, AgentVersion: view.AgentVersion(), IntegrationSource: view.IntegrationSource(), MachineID: view.MachineID(), @@ -3663,6 +3664,25 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host { } } +// hostAgentLivenessFromReadStateView returns the status and last report of the +// agent behind a read-state host row. A row merged with a Proxmox node takes +// LastSeen from its freshest source and stays online while PVE polling +// continues, so a silent agent would look current: its retained disk summary +// and datasets kept feeding the node, and its connection read as active. A +// models.Host describes the agent, so read the agent source's own sighting +// and lease, as evaluateHostAgents does for the host in state. +func hostAgentLivenessFromReadStateView(view *unifiedresources.HostView, now time.Time) (string, time.Time) { + status, lastSeen := string(view.Status()), view.LastSeen() + agentSighting, ok := view.SourceStatus(unifiedresources.SourceAgent) + if !ok || agentSighting.LastSeen.IsZero() { + return status, lastSeen + } + if !hostAgentReportCurrent(agentSighting.LastSeen, view.IntervalSeconds(), now) { + status = "offline" + } + return status, agentSighting.LastSeen +} + func vmFromReadStateView(view *unifiedresources.VMView) models.VM { if view == nil { return models.VM{} diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go index f14c2d1d55..3ef3414d7d 100644 --- a/internal/monitoring/monitor_host_agents_test.go +++ b/internal/monitoring/monitor_host_agents_test.go @@ -24,6 +24,7 @@ import ( agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker" agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" "github.com/rcourtman/pulse-go-rewrite/pkg/metrics" + "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" ) func TestNewMonitorRoutesStartupCustomSensorWarningBeforeStart(t *testing.T) { @@ -6843,3 +6844,71 @@ func TestBroadcastProjectionListsRegistryOnceAndKeepsLiveChanges(t *testing.T) { t.Fatal("removed host lifecycle surface disappeared") } } + +// A host agent linked to a Proxmox node merges into one read-state row that +// every PVE poll keeps online and fresh. Once the agent stops reporting, its +// retained filesystem summary must stop standing in for the node's disk (and +// through it disk alerts and history), and the projected agent must read as +// offline with its own last report, so its connection shows the missed +// heartbeat instead of the PVE poll time. +func TestSilentLinkedAgentStopsFeedingNodeDisk(t *testing.T) { + state := models.NewState() + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m := &Monitor{config: &config.Config{}, state: state, resourceStore: adapter} + + node := models.Node{ + ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", + ConnectionHealth: "healthy", LinkedAgentID: "agent-1", + } + rootFS := &proxmox.NodeStatus{RootFS: &proxmox.RootFS{Total: 200, Used: 50, Free: 150}} + // poll records a PVE poll that just saw the node, with the agent as of its + // last report, and resolves the node's disk the way pollPVENode does. + poll := func(agentLastReport time.Time) (models.Disk, string, models.Host) { + t.Helper() + polledNode := node + polledNode.LastSeen = time.Now() + state.UpdateNodes([]models.Node{polledNode}) + state.UpsertHost(models.Host{ + ID: "agent-1", Hostname: "node1", Status: "online", LinkedNodeID: node.ID, + IntervalSeconds: 30, LastSeen: agentLastReport, + Disks: []models.Disk{{Mountpoint: "/", Type: "ext4", Total: 1000, Used: 900, Free: 100, Usage: 90}}, + }) + m.evaluateHostAgents(time.Now()) + m.refreshUnifiedResourceStoreAfterAgentStateChange() + + disk, source := m.resolveNodeDisk("pve1", node.ID, node.Name, proxmox.Node{Node: node.Name}, rootFS) + for _, host := range m.HostsSnapshot() { + if host.ID == "agent-1" { + return disk, source, host + } + } + t.Fatal("HostsSnapshot does not list agent-1") + return disk, source, models.Host{} + } + + agentReport := time.Now().Add(-10 * time.Second) + disk, source, host := poll(agentReport) + if source != "agent" || disk.Used != 900 { + t.Fatalf("reporting agent: node disk = %+v from %q, want the agent's summary", disk, source) + } + if host.Status != "online" || !host.LastSeen.Equal(agentReport) { + t.Fatalf("reporting agent projected as %q last seen %s, want online at its report %s", host.Status, host.LastSeen, agentReport) + } + + silentSince := time.Now().Add(-hostAgentHealthWindow(30) - time.Minute) + disk, source, host = poll(silentSince) + if source != "node-status-rootfs" || disk.Used != 50 { + t.Fatalf("silent agent: node disk = %+v from %q, want Proxmox rootfs", disk, source) + } + if linked := m.linkedHostForNode("pve1", node.ID, node.Name); linked != nil { + t.Fatalf("silent agent still linked to the node for disk and dataset data: %+v", linked) + } + if host.Status != "offline" || !host.LastSeen.Equal(silentSince) { + t.Fatalf("silent agent projected as %q last seen %s, want offline at its last report %s", host.Status, host.LastSeen, silentSince) + } + + disk, source, host = poll(time.Now()) + if source != "agent" || disk.Used != 900 || host.Status != "online" { + t.Fatalf("agent reporting again: node disk = %+v from %q, agent %q; want the agent's summary and online", disk, source, host.Status) + } +}