From ff5a0b495766fca24606617bc996bbd9a99a225e Mon Sep 17 00:00:00 2001 From: rcourtman Date: Thu, 11 Jun 2026 21:47:30 +0100 Subject: [PATCH] memory(cache): extend the reclaimable split to standalone host agents f62f35e24 restored the v5 used | cache | free memory split for Proxmox nodes and guests, but standalone host agents still reported a flat used/free pair, so the Machines page memory bar could not show the reclaimable segment. Flagged by the Machines page v5 parity audit. - Host agent reports cacheBytes (gopsutil Available minus Free); the ZFS ARC adjustment recomputes free so used + cache + free still covers the total. - ApplyHostReport maps the field into models.Memory.Cache and clamps inconsistent or older-agent reports so used + cache never exceeds total. - AgentMemoryMeta carries cache onto unified resources so the frontend agent payload exposes it. - Mock generic hosts split a third of non-used pages as cache, and the node-linked host conversion now holds the invariant instead of stacking the node's cache on top of a recomputed free. - Contracts: monitoring, unified-resources, and storage-recovery now document the split (also covering the f62f35e24 node/guest surface, which landed without contract deltas). --- .../v6/internal/subsystems/monitoring.md | 8 +++ .../internal/subsystems/storage-recovery.md | 8 +++ .../internal/subsystems/unified-resources.md | 7 +++ internal/hostmetrics/collector.go | 18 ++++++- internal/hostmetrics/collector_test.go | 37 ++++++++++++++ internal/mock/generator.go | 13 ++++- internal/mock/generator_test.go | 8 +++ internal/monitoring/monitor_agents.go | 8 +++ .../monitoring/monitor_host_agents_test.go | 49 +++++++++++++++++++ internal/unifiedresources/adapters.go | 1 + internal/unifiedresources/adapters_test.go | 27 ++++++++++ .../canonical_ids_types_test.go | 21 ++++++++ internal/unifiedresources/types.go | 8 +-- pkg/agents/host/report.go | 9 ++-- 14 files changed, 212 insertions(+), 10 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 03755ef4c..fed4a20c3 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -464,6 +464,14 @@ counters without requiring a parallel SMART row. Monitoring may normalize legacy statuses and filter empty slots, but it must not collapse assigned Unraid array/cache members back to generic host disks or discard native fields before unified resources builds storage and physical-disk resources. +Host-agent memory ingest carries the reclaimable page-cache split. The host +agent reports `cacheBytes` (gopsutil Available minus Free, with the ZFS ARC +adjustment recomputing free so used + cache + free still covers the total), +and `internal/monitoring/monitor_agents.go` maps it into +`models.Memory.Cache`, clamping inconsistent or older-agent reports so +used + cache never exceeds total. Mock fixtures author the same split for +generic hosts and node-linked host agents, and any mock drift updater must +hold the used + cache + free invariant as sampled usage changes. VMware vSphere now also has a locked phase-1 ingestion boundary under this lane. The admitted direction is vCenter-only in phase 1, and monitoring must stay API-first through the diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index f097bd16d..eb14f3e8a 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -965,6 +965,14 @@ recovery scope, or a storage/recovery-owned secret source. forking the toolbar. The source scope flows through `forcedSourceFilter` as a typed page input; the source filter remains available in the toolbar only when not forced. +41. Keep agent memory composition descriptive on the shared unified-resource + contract. `internal/unifiedresources/types.go` carries the reclaimable + page-cache split (`AgentMemoryMeta.cache`, holding used + cache + free + within the reported total) as host RAM description for machine surfaces. + Storage and recovery must not reinterpret that reclaimable RAM figure as + disk cache, ZFS ARC sizing, storage-tier health, or capacity-planning + evidence; disk and pool truth stays on the canonical storage and + physical-disk resources. ## Forbidden Paths diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 763e11e9d..65820e7ae 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -1262,6 +1262,13 @@ presentation and host/appliance support-floor copy. Raw appliance identity aliases such as `unraid-os` may be accepted only through the generated host-profile token projection and must resolve to a governed profile id before they reach platform filters, source IDs, or top-level resource identity. +Unified-resource `AgentData` also carries the host memory composition: +`AgentMemoryMeta` exposes the reclaimable page-cache split (`cache`) beside +used/free/swap so machine surfaces can render used | cache | free without a +parallel payload, mirroring the `proxmox.memoryCache` transport nodes and +guests use. The field is additive and omitted when an agent does not report +it; consumers must treat missing cache as zero rather than inferring it from +free space. Frontend resource identity presenters may append a runtime version to a displayed system badge only when that version is sourced from the same canonical platform or host-profile identity, such as PVE `ResourceProxmoxMeta.pveVersion` diff --git a/internal/hostmetrics/collector.go b/internal/hostmetrics/collector.go index 1ae799ce9..5c89e967c 100644 --- a/internal/hostmetrics/collector.go +++ b/internal/hostmetrics/collector.go @@ -77,6 +77,14 @@ func Collect(ctx context.Context, diskExclude []string) (Snapshot, error) { freeBytes := memStats.Free usedPercent := memStats.UsedPercent + // Reclaimable page cache: Available counts the pages the kernel would hand + // back under pressure on top of truly-free ones, so the gap is buff/cache. + // Reported separately so the memory bar can show used | cache | free. + cacheBytes := uint64(0) + if memStats.Available > memStats.Free { + cacheBytes = memStats.Available - memStats.Free + } + // ZFS ARC memory is reclaimable under pressure but is counted as "used" by // both FreeBSD (wired memory) and Linux (not in MemAvailable, openzfs/zfs#10255). // Subtract it from Used to reflect actual memory pressure. Refs: #1264/#1051 @@ -95,8 +103,13 @@ func Collect(ctx context.Context, diskExclude []string) (Snapshot, error) { usedPercent = 100 } } - if memStats.Total >= usedBytes { - freeBytes = memStats.Total - usedBytes + // Recompute free so used + cache + free still covers the total after + // the ARC pages move out of used. + if memStats.Total >= usedBytes+cacheBytes { + freeBytes = memStats.Total - usedBytes - cacheBytes + } else if memStats.Total >= usedBytes { + cacheBytes = memStats.Total - usedBytes + freeBytes = 0 } } @@ -109,6 +122,7 @@ func Collect(ctx context.Context, diskExclude []string) (Snapshot, error) { TotalBytes: int64(memStats.Total), UsedBytes: int64(usedBytes), FreeBytes: int64(freeBytes), + CacheBytes: int64(cacheBytes), Usage: usedPercent, SwapTotal: int64(memStats.SwapTotal), SwapUsed: swapUsed, diff --git a/internal/hostmetrics/collector_test.go b/internal/hostmetrics/collector_test.go index 5899ed694..4d5d3eeab 100644 --- a/internal/hostmetrics/collector_test.go +++ b/internal/hostmetrics/collector_test.go @@ -6,6 +6,7 @@ import ( "testing" godisk "github.com/shirou/gopsutil/v4/disk" + gomem "github.com/shirou/gopsutil/v4/mem" ) func TestCollectDiskIO(t *testing.T) { @@ -111,3 +112,39 @@ func TestCollectDisks_DeviceDeduplication(t *testing.T) { t.Errorf("Total storage should be %d bytes, got %d bytes", expectedTotal, total) } } + +func TestCollectSplitsReclaimableCache(t *testing.T) { + origVirtualMemory := virtualMemory + t.Cleanup(func() { virtualMemory = origVirtualMemory }) + + const gib = uint64(1024 * 1024 * 1024) + virtualMemory = func(ctx context.Context) (*gomem.VirtualMemoryStat, error) { + return &gomem.VirtualMemoryStat{ + Total: 16 * gib, + Used: 6 * gib, + Free: 4 * gib, + // Available - Free = 6 GiB of reclaimable buff/cache. + Available: 10 * gib, + UsedPercent: 37.5, + }, nil + } + + snapshot, err := Collect(context.Background(), nil) + if err != nil { + t.Fatalf("Collect failed: %v", err) + } + + if got, want := snapshot.Memory.CacheBytes, int64(6*gib); got != want { + t.Fatalf("CacheBytes = %d, want %d", got, want) + } + if got, want := snapshot.Memory.UsedBytes, int64(6*gib); got != want { + t.Fatalf("UsedBytes = %d, want %d", got, want) + } + if got, want := snapshot.Memory.FreeBytes, int64(4*gib); got != want { + t.Fatalf("FreeBytes = %d, want %d", got, want) + } + sum := snapshot.Memory.UsedBytes + snapshot.Memory.CacheBytes + snapshot.Memory.FreeBytes + if sum > snapshot.Memory.TotalBytes { + t.Fatalf("used+cache+free = %d exceeds total %d", sum, snapshot.Memory.TotalBytes) + } +} diff --git a/internal/mock/generator.go b/internal/mock/generator.go index 7c71076ef..6f5946e7a 100644 --- a/internal/mock/generator.go +++ b/internal/mock/generator.go @@ -2889,7 +2889,10 @@ func generateHosts(config MockConfig) []models.Host { memTotal := int64(memTotalGiB) << 30 memUsage := SampleMetric("agent", hostID, "memory", now) memUsed := int64(float64(memTotal) * (memUsage / 100.0)) - memFree := memTotal - memUsed + // Roughly a third of the non-used pages read as reclaimable buff/cache + // so the machine memory split is exercisable in mock mode. + memCache := (memTotal - memUsed) / 3 + memFree := memTotal - memUsed - memCache swapTotal := int64(rand.Intn(32)) << 30 swapUsed := int64(float64(swapTotal) * rand.Float64()) @@ -3051,7 +3054,7 @@ func generateHosts(config MockConfig) []models.Host { CPUCount: cpuCount, CPUUsage: cpuUsage, LoadAverage: loadAverage, - Memory: models.Memory{Total: memTotal, Used: memUsed, Free: memFree, Usage: memUsage, SwapTotal: swapTotal, SwapUsed: swapUsed}, + Memory: models.Memory{Total: memTotal, Used: memUsed, Free: memFree, Cache: memCache, Usage: memUsage, SwapTotal: swapTotal, SwapUsed: swapUsed}, Disks: disks, NetworkInterfaces: network, Sensors: sensors, @@ -3489,6 +3492,12 @@ func buildMockLinkedHostFromNode(node models.Node, hostID string, hostIndex int, if memory.Free < 0 { memory.Free = 0 } + // The node memory carries a reclaimable-cache split; keep the + // used + cache + free invariant after recomputing free. + if memory.Cache > memory.Free { + memory.Cache = memory.Free + } + memory.Free -= memory.Cache // Add swap for PVE node hosts memory.SwapTotal = int64(8+rand.Intn(24)) << 30 diff --git a/internal/mock/generator_test.go b/internal/mock/generator_test.go index 34ea41a0a..269198ee7 100644 --- a/internal/mock/generator_test.go +++ b/internal/mock/generator_test.go @@ -140,6 +140,14 @@ func TestBuildFixtureStateIncludesHostAgents(t *testing.T) { if host.Status == "" { t.Fatalf("host agent missing status: %+v", host) } + if host.Memory.Total > 0 { + if host.Memory.Cache <= 0 { + t.Fatalf("host agent %s should report reclaimable cache so the memory split is exercisable: %+v", host.ID, host.Memory) + } + if sum := host.Memory.Used + host.Memory.Cache + host.Memory.Free; sum > host.Memory.Total { + t.Fatalf("host agent %s memory used+cache+free %d exceeds total %d", host.ID, sum, host.Memory.Total) + } + } } } diff --git a/internal/monitoring/monitor_agents.go b/internal/monitoring/monitor_agents.go index f6523eed0..bffa2e61a 100644 --- a/internal/monitoring/monitor_agents.go +++ b/internal/monitoring/monitor_agents.go @@ -1759,10 +1759,18 @@ func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config. Total: report.Metrics.Memory.TotalBytes, Used: report.Metrics.Memory.UsedBytes, Free: report.Metrics.Memory.FreeBytes, + Cache: report.Metrics.Memory.CacheBytes, Usage: safeFloat(report.Metrics.Memory.Usage), SwapTotal: report.Metrics.Memory.SwapTotal, SwapUsed: report.Metrics.Memory.SwapUsed, } + // Older agents don't report cache; clamp so used + cache never exceeds total. + if memory.Cache < 0 { + memory.Cache = 0 + } + if memory.Total > 0 && memory.Used+memory.Cache > memory.Total { + memory.Cache = max(0, memory.Total-memory.Used) + } // Fallback for LXC environments: gopsutil may read Total and Free correctly // from cgroup limits but return 0 for Used. Calculate Used from Total - Free. diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go index 926836e64..fd71fd5b2 100644 --- a/internal/monitoring/monitor_host_agents_test.go +++ b/internal/monitoring/monitor_host_agents_test.go @@ -2604,3 +2604,52 @@ func TestApplyHostReportMergesAgentCephWithProxmoxAPICluster(t *testing.T) { } t.Fatalf("expected canonical API Ceph pool alert using legacy agent override, got active alerts: %#v", active) } + +func TestApplyHostReportMapsReclaimableMemoryCache(t *testing.T) { + monitor := &Monitor{ + state: models.NewState(), + alertManager: alerts.NewManager(), + hostTokenBindings: make(map[string]string), + config: &config.Config{}, + rateTracker: NewRateTracker(), + } + t.Cleanup(func() { monitor.alertManager.Stop() }) + + report := agentshost.Report{ + Agent: agentshost.AgentInfo{ID: "agent-cache", Version: "1.0.0", IntervalSeconds: 30}, + Host: agentshost.HostInfo{ + ID: "machine-cache", + Hostname: "cache-host", + Platform: "linux", + }, + Timestamp: time.Now().UTC(), + Metrics: agentshost.Metrics{ + Memory: agentshost.MemoryMetric{ + TotalBytes: 16 << 30, + UsedBytes: 6 << 30, + FreeBytes: 4 << 30, + CacheBytes: 6 << 30, + Usage: 37.5, + }, + }, + } + + host, err := monitor.ApplyHostReport(report, &config.APITokenRecord{ID: "token-cache", Name: "Token"}) + if err != nil { + t.Fatalf("ApplyHostReport: %v", err) + } + if got, want := host.Memory.Cache, int64(6<<30); got != want { + t.Fatalf("memory cache = %d, want %d", got, want) + } + + // An inconsistent report (cache pushing past total) is clamped so the + // used | cache | free split can never exceed the bar. + report.Metrics.Memory.CacheBytes = 12 << 30 + host, err = monitor.ApplyHostReport(report, &config.APITokenRecord{ID: "token-cache", Name: "Token"}) + if err != nil { + t.Fatalf("ApplyHostReport clamp: %v", err) + } + if got, want := host.Memory.Cache, int64(10<<30); got != want { + t.Fatalf("clamped memory cache = %d, want %d", got, want) + } +} diff --git a/internal/unifiedresources/adapters.go b/internal/unifiedresources/adapters.go index 6b25bb0c7..04bba818e 100644 --- a/internal/unifiedresources/adapters.go +++ b/internal/unifiedresources/adapters.go @@ -149,6 +149,7 @@ func resourceFromHost(host models.Host) (Resource, ResourceIdentity) { Total: host.Memory.Total, Used: host.Memory.Used, Free: host.Memory.Free, + Cache: host.Memory.Cache, SwapUsed: host.Memory.SwapUsed, SwapTotal: host.Memory.SwapTotal, }, diff --git a/internal/unifiedresources/adapters_test.go b/internal/unifiedresources/adapters_test.go index eecddb138..407dbb483 100644 --- a/internal/unifiedresources/adapters_test.go +++ b/internal/unifiedresources/adapters_test.go @@ -1014,3 +1014,30 @@ func TestResourceFromDockerContainerUsesHostSighting(t *testing.T) { t.Fatalf("resource.LastSeen = %s, want zero when the host has never reported", resource.LastSeen) } } + +func TestResourceFromHostCarriesReclaimableMemoryCache(t *testing.T) { + host := models.Host{ + ID: "cache-host", + Hostname: "cache-host", + Platform: "linux", + Status: "online", + Memory: models.Memory{ + Total: 16 << 30, + Used: 6 << 30, + Free: 4 << 30, + Cache: 6 << 30, + Usage: 37.5, + }, + } + + resource, _ := resourceFromHost(host) + if resource.Agent == nil || resource.Agent.Memory == nil { + t.Fatal("expected agent memory payload") + } + if got, want := resource.Agent.Memory.Cache, int64(6<<30); got != want { + t.Fatalf("agent memory cache = %d, want %d", got, want) + } + if sum := resource.Agent.Memory.Used + resource.Agent.Memory.Cache + resource.Agent.Memory.Free; sum > resource.Agent.Memory.Total { + t.Fatalf("agent memory used+cache+free %d exceeds total %d", sum, resource.Agent.Memory.Total) + } +} diff --git a/internal/unifiedresources/canonical_ids_types_test.go b/internal/unifiedresources/canonical_ids_types_test.go index 4d937c87b..45ad029a3 100644 --- a/internal/unifiedresources/canonical_ids_types_test.go +++ b/internal/unifiedresources/canonical_ids_types_test.go @@ -1,6 +1,8 @@ package unifiedresources import ( + "encoding/json" + "strings" "testing" "github.com/rcourtman/pulse-go-rewrite/internal/models" @@ -400,3 +402,22 @@ func TestIsUnsupportedLegacyResourceIDAlias(t *testing.T) { }) } } + +func TestAgentMemoryMetaSerializesReclaimableCache(t *testing.T) { + meta := AgentMemoryMeta{Total: 16, Used: 6, Free: 4, Cache: 6} + payload, err := json.Marshal(meta) + if err != nil { + t.Fatalf("marshal AgentMemoryMeta: %v", err) + } + if !strings.Contains(string(payload), `"cache":6`) { + t.Fatalf("expected cache in agent memory payload, got %s", payload) + } + + empty, err := json.Marshal(AgentMemoryMeta{Total: 16, Used: 6, Free: 10}) + if err != nil { + t.Fatalf("marshal AgentMemoryMeta without cache: %v", err) + } + if strings.Contains(string(empty), "cache") { + t.Fatalf("cache should be omitted when unreported, got %s", empty) + } +} diff --git a/internal/unifiedresources/types.go b/internal/unifiedresources/types.go index 6a94d6241..ce2418a89 100644 --- a/internal/unifiedresources/types.go +++ b/internal/unifiedresources/types.go @@ -738,9 +738,11 @@ type HostCephMeta struct { // AgentMemoryMeta describes agent-reported memory including swap. type AgentMemoryMeta struct { - Total int64 `json:"total,omitempty"` - Used int64 `json:"used,omitempty"` - Free int64 `json:"free,omitempty"` + Total int64 `json:"total,omitempty"` + Used int64 `json:"used,omitempty"` + Free int64 `json:"free,omitempty"` + // Cache is the reclaimable page cache; used + cache + free ≈ total. + Cache int64 `json:"cache,omitempty"` SwapUsed int64 `json:"swapUsed,omitempty"` SwapTotal int64 `json:"swapTotal,omitempty"` } diff --git a/pkg/agents/host/report.go b/pkg/agents/host/report.go index d8ff89b46..f552ef082 100644 --- a/pkg/agents/host/report.go +++ b/pkg/agents/host/report.go @@ -67,9 +67,12 @@ type Metrics struct { // MemoryMetric captures memory usage statistics in bytes. type MemoryMetric struct { - TotalBytes int64 `json:"totalBytes,omitempty"` - UsedBytes int64 `json:"usedBytes,omitempty"` - FreeBytes int64 `json:"freeBytes,omitempty"` + TotalBytes int64 `json:"totalBytes,omitempty"` + UsedBytes int64 `json:"usedBytes,omitempty"` + FreeBytes int64 `json:"freeBytes,omitempty"` + // CacheBytes is the reclaimable page cache (Available - Free); + // used + cache + free ≈ total. + CacheBytes int64 `json:"cacheBytes,omitempty"` Usage float64 `json:"usage,omitempty"` SwapTotal int64 `json:"swapTotalBytes,omitempty"` SwapUsed int64 `json:"swapUsedBytes,omitempty"`