diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 3e8d5f9ddc..a66c853e36 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -596,6 +596,10 @@ acknowledgement stays a success rather than a rejection so a real agent does not read a demo server as an outage and retry-storm it. Tests that assert report admission or tenant isolation must therefore not enable mock mode, or the assertion passes without exercising the boundary. +Report admission, config fetch, and continuity lookups read the live host list +through `GetLiveHostsSnapshot`, which copies only hosts (see the monitoring +contract); it must not route through a full state snapshot, which copied every +guest on each agent report. Physical-disk evidence collected by a host agent must survive projection back into monitoring's models. Absent evidence has to carry its declared sentinel diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 314c5786fe..3a8a69a77c 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -2426,6 +2426,14 @@ must apply the same continuity overlay when `HostsSnapshot()` resolves its canonical read state, so settings and other host-list consumers do not blank previously admitted Pulse Agent rows during a config-driven monitor swap while fresh reports are still in flight. +Continuity lookups must stay cheap because every canonical read-state lookup +makes one. `readStateWithStandaloneHostContinuity` takes the live host list +once per call, and `GetLiveHostsSnapshot` copies only hosts through +`State.GetHosts` rather than deep-copying every guest through `GetSnapshot`; +agent reports and config fetches share that accessor. With one offline +standalone agent, a lookup at 2,080 synthetic resources moved from 136 ms to +74 microseconds. `TestStandaloneHostContinuityReadStateReusedAcrossLookups` and +`TestGetLiveHostsSnapshotCopiesOnlyHosts` pin both. That same mock-runtime boundary also owns freshness while demos are running. The mock update loop must keep provider-backed TrueNAS and VMware records plus legacy PBS and PMG summaries on current `LastSeen` and health state each tick, diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 71e1c5d1c7..5046b9c356 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -56,6 +56,21 @@ synthetic estate of 8-node clusters the rebuild moved from 74 to 42 ms at 2,080 resources and now grows linearly; that is component evidence, not installed fleet CPU attribution. +### Host continuity overlays reuse one build per registry generation + +`ReadStateWithHostContinuity` overlays standalone agents that are offline +across a restart onto the live read state by building a registry clone. The +monitor requests that overlay on every canonical read-state lookup, which +alert evaluation makes once per resource per poll, so a single offline agent +made every poll rebuild the registry once per resource for up to 72 hours. The +adapter now keeps the last overlay keyed by registry pointer, `LastRebuiltAt`, +and a fingerprint of the requested records (excluding the build-time +`UpdatedAt` stamp), and reuses it for at most `overlayReadStateMaxAge` (2 +seconds), since the overlay evaluates source staleness when built. The live +adapter is never mutated. +`TestHostContinuityOverlayReusedWithinRegistryGeneration` pins reuse, rebuilds +on a new generation or different records, and the age bound. + ### Bounded incident-history selection Canonical frontend memory withdrawal is explicit: when a resource snapshot omits the canonical memory metric and its Proxmox memory facet marks usageUnavailable, the display merge must clear any previous metric. Plain partial omission remains compatible with richer REST state, and an incoming canonical metric (including measured zero) takes precedence over unavailable raw evidence. The adapter transition tests pin withdrawal and recovery; the hybrid-memory Chromium fixture exercises the rendered table and drawer at 1280px and 390px. Workload details remain canonical-only: withdrawal shows N/A in the table and removes the Memory section rather than manufacturing a raw-facet total; measured-zero recovery restores Total and Free, with screenshot positioning above fixed navigation. This does not change agent-only or arbitrary field-deletion semantics. diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 76c58be0f9..f3881bab8b 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -4359,7 +4359,9 @@ func (m *Monitor) GetLiveHostsSnapshot() []models.Host { if m == nil || m.state == nil { return nil } - return m.state.GetSnapshot().Hosts + // Copy only the hosts. GetSnapshot deep-copies every guest as well, and + // this runs on each agent report, config fetch, and continuity lookup. + return m.state.GetHosts() } // SetOrgID sets the organization ID for this monitor instance. diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go index db06f95852..5bfa88eb30 100644 --- a/internal/monitoring/monitor_host_agents_test.go +++ b/internal/monitoring/monitor_host_agents_test.go @@ -6484,3 +6484,58 @@ func TestAgentReportRefreshClonesRegistryOnce(t *testing.T) { t.Fatalf("registry clones for one agent report refresh = %d, want 1", counting.getAll) } } + +// A standalone agent offline across a restart is added back to every canonical +// read-state lookup from host continuity. Alert evaluation makes one lookup per +// resource per poll, so the overlay must be reused, not rebuilt per lookup. +func TestStandaloneHostContinuityReadStateReusedAcrossLookups(t *testing.T) { + now := time.Now().UTC() + state := models.NewState() + state.UpsertHost(models.Host{ID: "agent-live", Hostname: "live.example", Status: "online", LastSeen: now}) + continuity := config.NewHostContinuityStore(t.TempDir(), nil) + if err := continuity.Upsert(config.HostContinuityEntry{HostID: "agent-gone", Hostname: "gone.example", LastSeen: now.Add(-time.Hour)}); err != nil { + t.Fatalf("seed host continuity: %v", err) + } + adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)) + m := &Monitor{state: state, resourceStore: adapter, hostContinuityStore: continuity} + m.refreshUnifiedResourceStoreAfterAgentStateChange() + + first := m.GetUnifiedReadStateOrSnapshot() + if first == unifiedresources.ReadState(adapter) { + t.Fatal("read state has no continuity overlay for the offline standalone agent") + } + found := false + for _, host := range first.Hosts() { + found = found || host.Hostname() == "gone.example" + } + if !found { + t.Fatal("continuity overlay does not include the offline standalone agent") + } + if second := m.GetUnifiedReadStateOrSnapshot(); second != first { + t.Fatal("a second read-state lookup rebuilt the continuity overlay") + } +} + +// GetLiveHostsSnapshot runs on every agent report, config fetch, and host +// continuity lookup, so it must copy the hosts alone, not the whole state. +func TestGetLiveHostsSnapshotCopiesOnlyHosts(t *testing.T) { + state := models.NewState() + state.UpsertHost(models.Host{ID: "agent-1", Hostname: "pve1.example", Status: "online"}) + vms := make([]models.VM, 1000) + for i := range vms { + vms[i] = models.VM{ + ID: fmt.Sprintf("lab:pve1:%d", 100+i), VMID: 100 + i, Name: fmt.Sprintf("vm-%d", i), + Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", + Disks: []models.Disk{{Mountpoint: "/", Total: 10 << 30, Used: 1 << 30}}, + } + } + state.UpdateVMs(vms) + m := &Monitor{state: state} + + if hosts := m.GetLiveHostsSnapshot(); len(hosts) != 1 || hosts[0].ID != "agent-1" { + t.Fatalf("live hosts = %+v, want the one registered agent", hosts) + } + if allocs := testing.AllocsPerRun(10, func() { _ = m.GetLiveHostsSnapshot() }); allocs > 50 { + t.Fatalf("GetLiveHostsSnapshot allocated %.0f times with 1 host and 1000 guests; it is copying guests", allocs) + } +} diff --git a/internal/monitoring/monitored_system_usage.go b/internal/monitoring/monitored_system_usage.go index 356db4db30..6b20663ad0 100644 --- a/internal/monitoring/monitored_system_usage.go +++ b/internal/monitoring/monitored_system_usage.go @@ -76,8 +76,9 @@ func (m *Monitor) readStateWithStandaloneHostContinuity( return readState } - liveHostIDs := make(map[string]struct{}, len(m.GetLiveHostsSnapshot())) - for _, host := range m.GetLiveHostsSnapshot() { + liveHosts := m.GetLiveHostsSnapshot() + liveHostIDs := make(map[string]struct{}, len(liveHosts)) + for _, host := range liveHosts { if id := strings.TrimSpace(host.ID); id != "" { liveHostIDs[id] = struct{}{} } diff --git a/internal/unifiedresources/monitor_adapter.go b/internal/unifiedresources/monitor_adapter.go index 2f54e6e2df..f34b3ec85e 100644 --- a/internal/unifiedresources/monitor_adapter.go +++ b/internal/unifiedresources/monitor_adapter.go @@ -1,6 +1,8 @@ package unifiedresources import ( + "crypto/sha256" + "encoding/json" "fmt" "sort" "strings" @@ -24,6 +26,27 @@ type MonitorAdapter struct { activeAlerts []models.Alert lastRebuiltAt time.Time staleThresholds map[DataSource]time.Duration + + overlays overlayReadStateCache +} + +// overlayReadStateMaxAge bounds how long an overlay built from one registry +// generation is reused. The overlay evaluates source staleness when it is +// built, so reuse stays within the staleness read paths already accept. +const overlayReadStateMaxAge = 2 * time.Second + +// overlayReadStateCache holds the last overlay built from this adapter's +// registry. Host continuity overlays are requested on every canonical +// read-state lookup, which alert evaluation makes once per resource per poll; +// rebuilding the registry for each made polls quadratic while a standalone +// agent stayed offline across a restart. +type overlayReadStateCache struct { + mu sync.Mutex + registry *ResourceRegistry + rebuiltAt time.Time + key [sha256.Size]byte + builtAt time.Time + overlay ReadState } // NewMonitorAdapter creates a monitor-facing adapter around a registry. @@ -97,12 +120,67 @@ func readStateWithRecords(readState ReadState, source DataSource, records []Inge if registry == nil { return readState } + // Capture the generation before reading it, so a mutation during the build + // leaves the cached overlay keyed to the older generation. + rebuiltAt := adapter.LastRebuiltAt() + key, keyed := overlayRecordsKey(source, records, onlyMissing) + now := time.Now() + if keyed { + if overlay := adapter.overlays.lookup(registry, rebuiltAt, key, now); overlay != nil { + return overlay + } + } cloned := NewRegistry(registry.store) thresholds := adapter.currentStaleThresholds() cloned.IngestResourcesWithStaleThresholds(registry.List(), thresholds) cloned.ingestRecords(source, records, onlyMissing) - return NewMonitorAdapterWithStaleThresholds(cloned, thresholds) + overlay := NewMonitorAdapterWithStaleThresholds(cloned, thresholds) + if keyed { + adapter.overlays.store(registry, rebuiltAt, key, now, overlay) + } + return overlay +} + +// overlayRecordsKey fingerprints an overlay request. Resource.UpdatedAt is +// stamped when a record is built, not observed, so it is left out. +func overlayRecordsKey(source DataSource, records []IngestRecord, onlyMissing bool) ([sha256.Size]byte, bool) { + stable := make([]IngestRecord, len(records)) + for i, record := range records { + record.Resource.UpdatedAt = time.Time{} + stable[i] = record + } + payload, err := json.Marshal(struct { + Source DataSource + OnlyMissing bool + Records []IngestRecord + }{source, onlyMissing, stable}) + if err != nil { + return [sha256.Size]byte{}, false + } + return sha256.Sum256(payload), true +} + +func (c *overlayReadStateCache) lookup(registry *ResourceRegistry, rebuiltAt time.Time, key [sha256.Size]byte, now time.Time) ReadState { + c.mu.Lock() + defer c.mu.Unlock() + if c.overlay == nil || c.registry != registry || !c.rebuiltAt.Equal(rebuiltAt) || c.key != key { + return nil + } + if age := now.Sub(c.builtAt); age < 0 || age >= overlayReadStateMaxAge { + return nil + } + return c.overlay +} + +func (c *overlayReadStateCache) store(registry *ResourceRegistry, rebuiltAt time.Time, key [sha256.Size]byte, now time.Time, overlay ReadState) { + c.mu.Lock() + defer c.mu.Unlock() + c.registry = registry + c.rebuiltAt = rebuiltAt + c.key = key + c.builtAt = now + c.overlay = overlay } func (a *MonitorAdapter) currentRegistry() *ResourceRegistry { diff --git a/internal/unifiedresources/monitor_adapter_read_state_test.go b/internal/unifiedresources/monitor_adapter_read_state_test.go index 8d3f0daff4..eb9eeb99f8 100644 --- a/internal/unifiedresources/monitor_adapter_read_state_test.go +++ b/internal/unifiedresources/monitor_adapter_read_state_test.go @@ -892,3 +892,62 @@ func TestMonitorAdapterRebuildPersistsIdentityPins(t *testing.T) { } t.Fatalf("expected delly node resource in boot-window rebuild") } + +// Host continuity overlays are requested on every canonical read-state lookup. +// One built from a registry generation is reused until that generation, the +// requested records, or overlayReadStateMaxAge changes, and never mutates the +// live adapter. +func TestHostContinuityOverlayReusedWithinRegistryGeneration(t *testing.T) { + now := time.Now().UTC() + snapshot := models.StateSnapshot{ + LastUpdate: now, + Hosts: []models.Host{{ID: "live-1", Hostname: "live.example", Status: "online", LastSeen: now}}, + } + adapter := NewMonitorAdapter(NewRegistry(nil)) + adapter.PopulateFromSnapshot(snapshot) + offline := models.Host{ID: "offline-1", Hostname: "gone.example", Status: "offline", LastSeen: now.Add(-time.Hour)} + hasHost := func(readState ReadState, hostname string) bool { + for _, host := range readState.Hosts() { + if host.Hostname() == hostname { + return true + } + } + return false + } + + first := ReadStateWithHostContinuity(adapter, []IngestRecord{HostIngestRecord(offline)}) + if !hasHost(first, "gone.example") { + t.Fatal("continuity overlay does not include the offline host") + } + // Records are rebuilt per request and carry a fresh UpdatedAt stamp. + if again := ReadStateWithHostContinuity(adapter, []IngestRecord{HostIngestRecord(offline)}); again != first { + t.Fatal("identical continuity request rebuilt the overlay within one generation") + } + if hasHost(adapter, "gone.example") { + t.Fatal("continuity overlay mutated the live adapter") + } + + other := offline + other.ID, other.Hostname = "offline-2", "also-gone.example" + if changed := ReadStateWithHostContinuity(adapter, []IngestRecord{HostIngestRecord(other)}); changed == first || !hasHost(changed, "also-gone.example") { + t.Fatal("different continuity records reused the previous overlay") + } + + snapshot.LastUpdate = now.Add(time.Second) + adapter.PopulateFromSnapshot(snapshot) + next := ReadStateWithHostContinuity(adapter, []IngestRecord{HostIngestRecord(offline)}) + if next == first || !hasHost(next, "gone.example") { + t.Fatal("a new registry generation reused the previous generation's overlay") + } + + var cache overlayReadStateCache + registry := adapter.currentRegistry() + key, _ := overlayRecordsKey(SourceAgent, []IngestRecord{HostIngestRecord(offline)}, true) + cache.store(registry, now, key, now, next) + if cache.lookup(registry, now, key, now.Add(overlayReadStateMaxAge-time.Millisecond)) != next { + t.Fatal("overlay was not reused within its maximum age") + } + if cache.lookup(registry, now, key, now.Add(overlayReadStateMaxAge)) != nil { + t.Fatal("overlay was reused past its maximum age") + } +}