diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 30312f3b0c..20e0678d14 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -25,6 +25,35 @@ that same result. Successful reads retain their content and execution provenance ## Purpose +### Patrol guest memory is qualified evidence, not an unlabelled percentage + +Patrol consumes the selected guest memory metric's original observation, +including absence, last-known state, source and time. It does not borrow a +different raw Proxmox facet, a refreshed resource timestamp or disk readiness. +Current, dated cache-aware Proxmox/QGA/Pulse-agent readings can support guest +pressure flags, anomalies, forecasts, actionability and recovery verification. +Cache-inclusive status/cluster/total-minus-used estimates, unknown sources, +undated/future origins and retained readings remain visible but cannot establish +current pressure or recovery. Missing memory is N/A, not measured zero; real +current zero and real high pressure both retain their meaning. Other platforms +without Proxmox observations keep their existing selected readings. + +Full, quiet and condensed inventory and actual triage seeds preserve these +qualifications, including unflagged running guests. The no-flag rollup is not a +health verdict. Native guest runtime status, rather than canonical collection +status, owns running/paused/stopped eligibility. A snapshot-only alert review +cannot auto-resolve a memory alert from unknown guest pressure; independently +verified current recovery and removed-resource handling remain separate. + +`TestPatrolGuestMemoryEvidence`, `TestPatrolGuestMemoryForecastsAndAnomalies`, +`TestPatrolGuestMemoryAlertRecovery` and +`TestPatrolGuestMemorySelectedMetricAndOtherPlatforms` exercise these connected +paths. `TestIssue2762NamedMemoryFields` checks the existing named MemAvailable +parser against synthetic cache-heavy/reordered evidence, not a new free-table +parser. These controls do not establish issue #2762's installed input or cause, +a cleared finding, or a containing release. Monitoring collection, guest-agent +admission, configured alert thresholds, history storage and access are unchanged. + ### Assistant action plans do not guess an installation from session placement An explicit current canonical resource ID takes precedence over stale session diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 7ec82afccf..219fe2b0b0 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -17,6 +17,18 @@ ## Purpose +### Guest memory evidence survives typed consumers + +The monitor's existing guest memory observation remains source-owned when +Patrol reads it through VM/system-container views. Selected canonical memory, +not a retained raw Proxmox facet, supplies provenance; absent memory is distinct +from measured zero. `TestGuestViewMemoryObservationKeepsSelectedOrigin` checks +that read boundary and `TestPatrolGuestMemoryEvidence` checks the consumer's +qualifications through ordinary snapshot ingestion. This changes no memory +collector/parser, guest command, TTL, shared safety pause, alert threshold or +History writer. Synthetic named-field checks and qualified summaries are not +native acceptance of #2762's reported input or guest recovery. + ### Cluster recovery preserves established certificate pins Startup and unhealthy-endpoint recovery must not replace the configured primary diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 1973469f3c..b2c41cb7f2 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -23,6 +23,19 @@ and sort the complete canonical change table while startup and ingestion wait. ## Purpose +### Selected guest memory observation in typed read state + +VM and system-container views expose a value-copy of the selected memory +metric's observation and an explicit presence bit. A selected measured zero +has presence; a missing selected metric does not. Empty metadata remains +unknown and cannot be supplied by a conflicting raw platform memory facet or +resource LastSeen. No wire, metric-priority, identity, history or correlation +contract changes. `TestGuestViewMemoryObservationKeepsSelectedOrigin` in +`views_test.go` covers selected-agent versus raw-Proxmox disagreement, absent +metrics, legacy zero and value-copy isolation. Patrol owns interpretation of +this evidence; the accessor does not establish pressure, recovery or permission +to run a guest command. + ### Internal QGA query-eligibility evidence The native Proxmox guest facet carries `GuestAgentEvidence` internally. A full diff --git a/internal/ai/patrol_ai.go b/internal/ai/patrol_ai.go index 6ff9a5932b..35d1f75d0e 100644 --- a/internal/ai/patrol_ai.go +++ b/internal/ai/patrol_ai.go @@ -1153,6 +1153,7 @@ func (p *PatrolService) buildTriageSeedSectionsState( {priority: 0, name: "findings", content: findingsCtx}, {priority: 0, name: "health_alerts", content: p.seedHealthAndAlertsState(snap, seedSet, cfg, now)}, {priority: 0, name: "scope", content: buildScopeSection(scope, sortedScopedIDs(seedSet))}, + {priority: 1, name: "guest_memory_evidence", content: patrolGuestMemoryEvidenceSection(snap, p.buildScopedSetForRuntime(scope, snap))}, // P2 — triage already preserves the flagged set, so these sections can // summarize under tighter provider-derived retry budgets. @@ -1210,6 +1211,7 @@ func (p *PatrolService) buildSeedSectionsState(snap patrolRuntimeState, scope *P // P1 — always include (typically compact). {priority: 1, name: "previous_run", content: p.seedPreviousRun(now)}, + {priority: 1, name: "guest_memory_evidence", content: patrolGuestMemoryEvidenceSection(snap, scopedSet)}, // P2 — summarize when needed. { @@ -1693,7 +1695,10 @@ func (p *PatrolService) seedPrecomputeIntelligenceState(snap patrolRuntimeState, if g.template || g.status != "running" { continue } - metrics := map[string]float64{"memory": g.memPercent, "disk": g.diskPercent} + metrics := map[string]float64{"disk": g.diskPercent} + if g.memoryKnown { + metrics["memory"] = g.memPercent + } if g.cpuFraction > 0 { metrics["cpu"] = g.cpuFraction } @@ -1762,8 +1767,10 @@ func (p *PatrolService) seedPrecomputeIntelligenceState(snap patrolRuntimeState, if g.template || g.status != "running" { continue } - if pts := mh.GetGuestMetrics(g.id, "memory", 48*time.Hour); len(pts) >= 5 { - addForecast(g.id, g.name, "memory", pts, g.memPercent) + if g.memoryKnown { + if pts := mh.GetGuestMetrics(g.id, "memory", 48*time.Hour); len(pts) >= 5 { + addForecast(g.id, g.name, "memory", pts, g.memPercent) + } } if pts := mh.GetGuestMetrics(g.id, "disk", 48*time.Hour); len(pts) >= 5 { addForecast(g.id, g.name, "disk", pts, g.diskPercent) @@ -1893,6 +1900,7 @@ type patrolGuestInventoryRow struct { id string name, gType, node, status string cpu, mem, disk float64 + memory patrolGuestMemoryReading vmid int ip string lastBackup time.Time @@ -1955,6 +1963,7 @@ type patrolPrecomputeGuestSource struct { status string cpuFraction float64 memPercent float64 + memoryKnown bool diskPercent float64 } @@ -2020,14 +2029,17 @@ func patrolGuestInventoryRows(snap patrolRuntimeState, scopedSet map[string]bool continue } gi := guestIntel[vmv.ID()] + observation, hasMemory := vmv.MemoryObservation() + _, hasProxmox := vmv.SourceStatus(unifiedresources.SourceProxmox) guests = append(guests, patrolGuestInventoryRow{ id: vmv.ID(), name: vmv.Name(), gType: "VM", node: vmv.Node(), - status: string(vmv.Status()), + status: patrolGuestRuntimeStatus(vmv.RuntimeStatus(), vmv.Status()), cpu: vmv.CPUPercent(), mem: vmv.MemoryPercent(), + memory: readPatrolGuestMemory(vmv.MemoryPercent(), observation, hasMemory, hasProxmox || vmv.VMID() > 0), disk: vmv.DiskPercent(), vmid: vmv.VMID(), ip: patrolFirstIP(vmv.IPAddresses()), @@ -2041,14 +2053,17 @@ func patrolGuestInventoryRows(snap patrolRuntimeState, scopedSet map[string]bool continue } gi := guestIntel[ctv.ID()] + observation, hasMemory := ctv.MemoryObservation() + _, hasProxmox := ctv.SourceStatus(unifiedresources.SourceProxmox) guests = append(guests, patrolGuestInventoryRow{ id: ctv.ID(), name: ctv.Name(), gType: "Container", node: ctv.Node(), - status: string(ctv.Status()), + status: patrolGuestRuntimeStatus(ctv.RuntimeStatus(), ctv.Status()), cpu: ctv.CPUPercent(), mem: ctv.MemoryPercent(), + memory: readPatrolGuestMemory(ctv.MemoryPercent(), observation, hasMemory, hasProxmox || ctv.VMID() > 0), disk: ctv.DiskPercent(), vmid: ctv.VMID(), ip: patrolFirstIP(ctv.IPAddresses()), @@ -2074,6 +2089,7 @@ func patrolGuestInventoryRows(snap patrolRuntimeState, scopedSet map[string]bool status: vm.Status, cpu: unifiedresources.ProxmoxGuestCPUPercent(vm.CPU), mem: vm.Memory.Usage, + memory: readPatrolGuestMemory(vm.Memory.Usage, vm.Memory.Observation, vm.Memory.HasKnownUsage(), vm.VMID > 0 || vm.Type == "qemu"), disk: vm.Disk.Usage, vmid: vm.VMID, ip: patrolFirstIP(vm.IPAddresses), @@ -2095,6 +2111,7 @@ func patrolGuestInventoryRows(snap patrolRuntimeState, scopedSet map[string]bool status: ct.Status, cpu: unifiedresources.ProxmoxGuestCPUPercent(ct.CPU), mem: ct.Memory.Usage, + memory: readPatrolGuestMemory(ct.Memory.Usage, ct.Memory.Observation, ct.Memory.HasKnownUsage(), ct.VMID > 0 || ct.Type == "lxc"), disk: ct.Disk.Usage, vmid: ct.VMID, ip: patrolFirstIP(ct.IPAddresses), @@ -2626,6 +2643,7 @@ func patrolPrecomputeGuestSources(snap patrolRuntimeState, scopedSet map[string] status: guest.status, cpuFraction: guest.cpu / 100, memPercent: guest.mem, + memoryKnown: guest.memory.pressureKnown, diskPercent: guest.disk, }) } @@ -2767,6 +2785,7 @@ func (p *PatrolService) seedResourceInventoryState(snap patrolRuntimeState, scop } if len(guests) > 0 { + memoryGaps := patrolGuestMemoryGaps(guests) if isQuiet && scopedSet == nil { running, stopped := 0, 0 var unreachableNames []string @@ -2792,11 +2811,14 @@ func (p *PatrolService) seedResourceInventoryState(snap patrolRuntimeState, scop } } if hasReachabilityData { - sb.WriteString(fmt.Sprintf("# Guests: %d running, %d stopped, no issues detected. All reachable.\n\n", running, stopped)) + sb.WriteString(fmt.Sprintf("# Guests: %d running, %d stopped, no issues detected in available readings. All reachable.\n\n", running, stopped)) } else { - sb.WriteString(fmt.Sprintf("# Guests: %d running, %d stopped, no issues detected.\n\n", running, stopped)) + sb.WriteString(fmt.Sprintf("# Guests: %d running, %d stopped, no issues detected in available readings.\n\n", running, stopped)) } } + if memoryGaps != "" { + sb.WriteString(memoryGaps + "\n\n") + } } else { sb.WriteString("# Guest Metrics\n") sb.WriteString("| Name | Type | Node | Service | CPU | Mem | Disk | Status | Reachable | Last Backup |\n") @@ -2806,8 +2828,8 @@ func (p *PatrolService) seedResourceInventoryState(snap patrolRuntimeState, scop if !g.lastBackup.IsZero() { backup = seedFormatTimeAgo(now, g.lastBackup) } - sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s | %.0f%% | %.0f%% | %.0f%% | %s | %s | %s |\n", - g.name, g.gType, g.node, g.service, g.cpu, g.mem, g.disk, g.status, g.reachable, backup)) + sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s | %.0f%% | %s | %.0f%% | %s | %s | %s |\n", + g.name, g.gType, g.node, g.service, g.cpu, g.memory.display(), g.disk, g.status, g.reachable, backup)) } sb.WriteString("\n") @@ -3103,11 +3125,15 @@ func (p *PatrolService) seedResourceInventorySummaryState(snap patrolRuntimeStat guestRows := patrolGuestInventoryRows(snap, scopedSet, guestIntel) guests := make([]compactResource, 0, len(guestRows)) for _, g := range guestRows { + memory := g.mem + if !g.memory.pressureKnown { + memory = -1 // Unknown pressure cannot be a high-usage outlier. + } guests = append(guests, compactResource{ name: g.name, status: g.status, cpu: g.cpu, - mem: g.mem, + mem: memory, disk: g.disk, }) } @@ -3141,6 +3167,9 @@ func (p *PatrolService) seedResourceInventorySummaryState(snap patrolRuntimeStat } lines = append(lines, line) + if gaps := patrolGuestMemoryGaps(guestRows); gaps != "" { + lines = append(lines, gaps) + } } } diff --git a/internal/ai/patrol_findings.go b/internal/ai/patrol_findings.go index d4f5fbc2b7..b286ce17cc 100644 --- a/internal/ai/patrol_findings.go +++ b/internal/ai/patrol_findings.go @@ -2575,11 +2575,11 @@ func patrolVisitGuestMetrics(snap patrolRuntimeState, guestType string, visit pa continue } count++ - if !visit([]string{guest.id, guest.name}, map[string]float64{ - "cpu": guest.cpu, - "memory": guest.mem, - "disk": guest.disk, - }) { + metrics := map[string]float64{"cpu": guest.cpu, "disk": guest.disk} + if guest.memory.pressureKnown { + metrics["memory"] = guest.mem + } + if !visit([]string{guest.id, guest.name}, metrics) { return true } } diff --git a/internal/ai/patrol_guest_memory.go b/internal/ai/patrol_guest_memory.go new file mode 100644 index 0000000000..87282b73c6 --- /dev/null +++ b/internal/ai/patrol_guest_memory.go @@ -0,0 +1,119 @@ +package ai + +import ( + "fmt" + "math" + "strings" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +func patrolGuestRuntimeStatus(runtime string, status unifiedresources.ResourceStatus) string { + if runtime != "" { + return runtime + } + // Canonical online is compatible with a running guest; warning/offline + // collection state is not evidence of a different native power state. + if status == unifiedresources.StatusOnline { + return "running" + } + return string(status) +} + +func patrolGuestMemoryEvidenceSection(snap patrolRuntimeState, scopedSet map[string]bool) string { + if gaps := patrolGuestMemoryGaps(patrolGuestInventoryRows(snap, scopedSet, nil)); gaps != "" { + return "# Guest memory evidence\n" + gaps + "\n\n" + } + return "" +} + +// A hypervisor's cache-inclusive footprint is not guest memory pressure. Keep +// it visible as evidence, but do not use it (or a retained/undated reading) to +// manufacture a pressure flag, forecast, anomaly or verified recovery. +type patrolGuestMemoryReading struct { + percent float64 + available bool + pressureKnown bool + state string + source string + observedAt time.Time +} + +func readPatrolGuestMemory(percent float64, observation models.MemoryObservation, available, proxmoxGuest bool) patrolGuestMemoryReading { + r := patrolGuestMemoryReading{percent: percent, available: available && !math.IsNaN(percent) && !math.IsInf(percent, 0) && percent >= 0 && percent <= 100} + if !r.available || observation.State == "unavailable" { + r.available = false + r.state = "unavailable" + return r + } + // Other platforms need not implement Proxmox's memory evidence contract. + // Preserve their existing readings, including a selected measured zero. + if !proxmoxGuest && observation == (models.MemoryObservation{}) { + r.pressureKnown = true + return r + } + r.state = "unknown" + if observation.ObservedAt.After(time.Unix(0, 0)) && !observation.ObservedAt.After(time.Now()) { + r.observedAt = observation.ObservedAt + } + if observation.State == "last-known" { + r.state = "last-known" + } else if observation.State == "current" && !r.observedAt.IsZero() { + r.state = "current" + } + // Only known source names are put in the seed. A future/unknown source is + // not an assertion of either cache awareness or current guest pressure. + switch observation.Source { + case "available-field", "derived-free-buffers-cached", "guest-agent-meminfo", "guest-agent-meminfo-derived", "agent": + r.source = observation.Source + r.pressureKnown = r.state == "current" + case "status-mem", "status-freemem", "cluster-resources", "derived-total-minus-used", "previous-snapshot": + r.source = observation.Source + } + return r +} + +func (r patrolGuestMemoryReading) display() string { + if !r.available { + return "N/A (guest memory unavailable; not evidence of recovery)" + } + if r.state == "" { + return fmt.Sprintf("%.0f%%", r.percent) + } + source := r.source + if source == "" { + source = "unknown" + } + observed := "time unknown" + if !r.observedAt.IsZero() { + observed = "observed " + r.observedAt.UTC().Format(time.RFC3339) + } + qualification := "cache-aware guest usage" + if !r.pressureKnown { + qualification = "guest pressure unknown" + if r.source == "status-mem" || r.source == "status-freemem" || r.source == "cluster-resources" || r.source == "derived-total-minus-used" { + qualification += "; may include reclaimable cache" + } + } + return fmt.Sprintf("%.0f%% (%s; source %s; %s; %s)", r.percent, r.state, source, observed, qualification) +} + +func patrolGuestMemoryGaps(rows []patrolGuestInventoryRow) string { + count := 0 + var examples []string + for _, row := range rows { + if row.memory.pressureKnown || row.status != "running" { + continue + } + count++ + if len(examples) < 5 { + examples = append(examples, row.name+": "+row.memory.display()) + } + } + if count == 0 { + return "" + } + return fmt.Sprintf("Guest memory pressure unknown for %d running guests (not evidence of health or recovery): %s", count, strings.Join(examples, "; ")) +} diff --git a/internal/ai/patrol_guest_memory_reading_test.go b/internal/ai/patrol_guest_memory_reading_test.go new file mode 100644 index 0000000000..bc3cb82973 --- /dev/null +++ b/internal/ai/patrol_guest_memory_reading_test.go @@ -0,0 +1,45 @@ +package ai + +import ( + "math" + "strings" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +func TestPatrolGuestMemoryReadingBoundaries(t *testing.T) { + at := time.Now().UTC().Add(-time.Minute) + obs := models.MemoryObservation{State: "current", Source: "available-field", ObservedAt: at} + for _, percent := range []float64{math.NaN(), math.Inf(1), math.Inf(-1), -1, 101} { + reading := readPatrolGuestMemory(percent, obs, true, true) + if reading.pressureKnown || reading.available || !strings.HasPrefix(reading.display(), "N/A") { + t.Errorf("invalid percent became live pressure: %+v", reading) + } + } + for _, source := range []string{"available-field", "derived-free-buffers-cached", "guest-agent-meminfo", "guest-agent-meminfo-derived", "agent"} { + obs.Source = source + if r := readPatrolGuestMemory(96, obs, true, true); !r.pressureKnown { + t.Errorf("current cache-aware positive rejected: %s", source) + } + } + for _, state := range []string{"", "future", "last-known", "unavailable"} { + obs.State = state + if r := readPatrolGuestMemory(96, obs, true, true); r.pressureKnown { + t.Errorf("non-current state became current: %s", state) + } + } + obs.State = "current" + obs.ObservedAt = time.Unix(0, 0) + if r := readPatrolGuestMemory(96, obs, true, true); r.pressureKnown || !r.observedAt.IsZero() { + t.Fatal("invalid observation origin became current") + } + if status := patrolGuestRuntimeStatus("paused", unifiedresources.StatusOffline); status != "paused" { + t.Fatal("native pause became canonical collection status") + } + if status := patrolGuestRuntimeStatus("running", unifiedresources.StatusWarning); status != "running" { + t.Fatal("collection warning changed native runtime state") + } +} diff --git a/internal/ai/patrol_guest_memory_test.go b/internal/ai/patrol_guest_memory_test.go new file mode 100644 index 0000000000..8294d26c85 --- /dev/null +++ b/internal/ai/patrol_guest_memory_test.go @@ -0,0 +1,232 @@ +package ai + +import ( + "context" + "errors" + "math" + "strings" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline" + "github.com/rcourtman/pulse-go-rewrite/internal/config" + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts" + "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" +) + +func patrolMemoryFixture(percent float64, state, source string, at time.Time) models.Memory { + const total = 20 * 1024 * 1024 * 1024 + used := int64(float64(total) * percent / 100) + return models.Memory{Total: total, Used: used, Free: total - used, Usage: percent, + Observation: models.MemoryObservation{State: state, Source: source, ObservedAt: at}} +} + +// These connected controls use only existing APIs so the same tests can be +// applied to the exact parent. No guest command, provider or workload is used. +func TestPatrolGuestMemoryEvidence(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + at := now.Add(-2 * time.Minute) + cases := []struct { + name, state, source, label string + percent float64 + at time.Time + unavailable, pressure bool + }{ + {"healthy-cache-aware", "current", "guest-agent-meminfo", "cache-aware guest usage", 24, at, false, true}, + {"real-pressure", "current", "available-field", "cache-aware guest usage", 96, at, false, true}, + {"measured-zero", "current", "agent", "cache-aware guest usage", 0, at, false, true}, + {"zero-availability", "current", "guest-agent-meminfo", "cache-aware guest usage", 100, at, false, true}, + {"hypervisor", "current", "status-mem", "guest pressure unknown", 96, at, false, false}, + {"free-not-available", "current", "status-freemem", "guest pressure unknown", 96, at, false, false}, + {"cluster-cache", "current", "cluster-resources", "guest pressure unknown", 96, at, false, false}, + {"total-minus-used", "current", "derived-total-minus-used", "guest pressure unknown", 96, at, false, false}, + {"last-known-high", "last-known", "guest-agent-meminfo", "last-known", 96, at, false, false}, + {"last-known-low", "last-known", "agent", "last-known", 24, at, false, false}, + {"missing-time", "current", "available-field", "time unknown", 96, time.Time{}, false, false}, + {"future-time", "current", "agent", "time unknown", 96, now.Add(time.Hour), false, false}, + {"legacy", "", "", "guest pressure unknown", 96, time.Time{}, false, false}, + {"future-source", "current", "future-source", "source unknown", 96, at, false, false}, + {"unavailable", "unavailable", "unavailable", "N/A", 0, time.Time{}, true, false}, + } + for _, tc := range cases { + for _, kind := range []string{"vm", "system-container"} { + for _, path := range []string{"legacy", "canonical"} { + t.Run(tc.name+"/"+kind+"/"+path, func(t *testing.T) { + mem := patrolMemoryFixture(tc.percent, tc.state, tc.source, tc.at) + mem.UsageUnavailable = tc.unavailable + snapshot := models.StateSnapshot{} + if kind == "vm" { + snapshot.VMs = []models.VM{{ID: "guest", Name: "guest", VMID: 100, Type: "qemu", Instance: "pve", Node: "node", Status: "running", LastSeen: now, LastBackup: now, Memory: mem}} + } else { + snapshot.Containers = []models.Container{{ID: "guest", Name: "guest", VMID: 100, Type: "lxc", Instance: "pve", Node: "node", Status: "running", LastSeen: now, LastBackup: now, Memory: mem}} + } + snap := newPatrolRuntimeState(snapshot) + if path == "canonical" { + registry := unifiedresources.NewRegistry(nil) + registry.IngestSnapshot(snapshot) + snap = patrolRuntimeState{readState: registry} + } + ps := NewPatrolService(nil, nil) + cfg := PatrolConfig{AnalyzeGuests: true} + full := ps.seedResourceInventoryState(snap, nil, cfg, now, false, nil) + if !strings.Contains(full, tc.label) { + t.Errorf("memory qualification %q missing from actual inventory:\n%s", tc.label, full) + } + if tc.unavailable && strings.Contains(full, "| 0% | 0% |") { + t.Error("unavailable memory was shown as measured zero") + } + if !tc.unavailable && !tc.at.IsZero() && !tc.at.After(now) && !strings.Contains(full, tc.at.Format(time.RFC3339)) { + t.Error("original memory observation time was lost or renewed") + } + flags := triageThresholdChecksState(snap, nil, DefaultPatrolThresholds()) + memoryFlag := triageFindFlag(flags, func(f TriageFlag) bool { return f.Metric == "memory" }) + if (memoryFlag != nil) != (tc.pressure && tc.percent > DefaultPatrolThresholds().GuestMemWatch) { + t.Errorf("pressure flag mismatch: known=%t percent=%v flag=%+v", tc.pressure, tc.percent, memoryFlag) + } + metrics, found := patrolLookupResourceMetricsForType(snap, "guest", kind) + if !found { + t.Fatal("guest identity disappeared from metric verification") + } + _, hasMemory := metrics["memory"] + if hasMemory != tc.pressure { + t.Errorf("unqualified memory entered verification: %+v", metrics) + } + ok, err := verifyMetricRecoveredState(snap, DefaultPatrolThresholds(), "memory-high", "guest", kind) + if !tc.pressure && (ok || !errors.Is(err, aicontracts.ErrVerificationUnknown)) { + t.Errorf("unknown memory decided recovery: ok=%t err=%v", ok, err) + } + if tc.pressure && (err != nil || ok != (tc.percent < DefaultPatrolThresholds().GuestMemWarning*0.95)) { + t.Errorf("measured recovery control failed: ok=%t err=%v", ok, err) + } + if !tc.pressure { + compact := ps.seedResourceInventorySummaryState(snap, nil, cfg, now, nil) + quiet := ps.seedResourceInventoryState(snap, nil, cfg, now, true, nil) + for _, output := range []string{compact, quiet} { + if !strings.Contains(output, "Guest memory pressure unknown") || strings.Contains(output, "guest (Mem 96%)") { + t.Errorf("condensed seed lost uncertainty or declared pressure:\n%s", output) + } + } + ps.SetConfig(cfg) + triage := ps.runDeterministicTriageState(context.Background(), snap, nil, nil) + seed, _ := ps.buildTriageSeedContextState(triage, snap, nil, nil) + if !strings.Contains(seed, "Guest memory pressure unknown") || strings.Contains(seed, "## Healthy Resources") { + t.Errorf("actual triage seed lost unflagged memory uncertainty:\n%s", seed) + } + } + }) + } + } + } +} + +func TestPatrolGuestMemoryForecastsAndAnomalies(t *testing.T) { + now := time.Now().UTC() + for _, source := range []string{"status-mem", "guest-agent-meminfo"} { + t.Run(source, func(t *testing.T) { + ps := NewPatrolService(nil, nil) + bs := baseline.NewStore(baseline.StoreConfig{MinSamples: 1}) + var baselinePoints []baseline.MetricPoint + var points []models.MetricPoint + for i := 0; i < 10; i++ { + at := now.Add(-time.Duration(10-i) * time.Hour) + baselinePoints = append(baselinePoints, baseline.MetricPoint{Value: 24, Timestamp: at}) + points = append(points, models.MetricPoint{Value: 96, Timestamp: at}) + } + if err := bs.Learn("guest", "vm", "memory", baselinePoints); err != nil { + t.Fatal(err) + } + ps.SetBaselineStore(bs) + ps.SetMetricsHistoryProvider(&precomputeMetricsHistoryProvider{metrics: map[string][]models.MetricPoint{"guest:memory": points, "guest:disk": points}}) + snap := newPatrolRuntimeState(models.StateSnapshot{VMs: []models.VM{{ID: "guest", VMID: 100, Type: "qemu", Status: "running", Memory: patrolMemoryFixture(96, "current", source, now.Add(-time.Minute)), Disk: models.Disk{Usage: 96}}}}) + intel := ps.seedPrecomputeIntelligenceState(snap, nil, now) + memoryForecast, diskForecast, memoryAnomaly := false, false, false + for _, f := range intel.forecasts { + memoryForecast = memoryForecast || f.metric == "memory" + diskForecast = diskForecast || f.metric == "disk" + } + for _, a := range intel.anomalies { + memoryAnomaly = memoryAnomaly || a.Metric == "memory" + } + known := source == "guest-agent-meminfo" + if memoryForecast != known || memoryAnomaly != known || !diskForecast { + t.Errorf("memory knowledge did not gate derived signals independently of disk: forecast=%t anomaly=%t disk=%t known=%t", memoryForecast, memoryAnomaly, diskForecast, known) + } + }) + } +} + +func TestPatrolGuestMemoryAlertRecovery(t *testing.T) { + now := time.Now().UTC() + for _, state := range []string{"unavailable", "last-known", "current"} { + t.Run(state, func(t *testing.T) { + ps := NewPatrolService(nil, nil) + resolver := &stubAlertResolver{alerts: []AlertInfo{{ID: "a", Type: "memory", ResourceID: "guest", ResourceType: "vm", Threshold: 80, StartTime: now.Add(-time.Hour)}}} + ps.alertResolver = resolver + provider := &mockPatrolProvider{response: "ALERT 1: RESOLVE: memory is now low"} + ps.aiService = &Service{provider: provider, cfg: &config.AIConfig{Enabled: true, PatrolModel: "mock:model"}} + mem := patrolMemoryFixture(0, state, "agent", now.Add(-time.Minute)) + mem.UsageUnavailable = state == "unavailable" + snap := newPatrolRuntimeState(models.StateSnapshot{VMs: []models.VM{{ID: "guest", Name: "guest", VMID: 100, Type: "qemu", Status: "running", Memory: mem}}}) + resolved := ps.reviewAndResolveAlertsState(context.Background(), snap, true, "") + if provider.calls != 1 || (resolved == 1) != (state == "current") || len(resolver.clears) != resolved { + t.Errorf("unknown memory falsely resolved, or measured zero could not recover: calls=%d resolved=%d clears=%v", provider.calls, resolved, resolver.clears) + } + prompt := ps.buildAlertReviewPromptState(resolver.alerts, snap) + if state != "current" && !strings.Contains(prompt, map[string]string{"unavailable": "N/A", "last-known": "guest pressure unknown"}[state]) { + t.Errorf("alert review prompt claims current guest memory: %s", prompt) + } + }) + } +} + +func TestPatrolGuestMemorySelectedMetricAndOtherPlatforms(t *testing.T) { + now := time.Now().UTC().Add(-time.Minute) + r := &unifiedresources.Resource{ID: "guest", Name: "guest", Type: unifiedresources.ResourceTypeVM, Status: unifiedresources.StatusOnline, + Proxmox: &unifiedresources.ProxmoxData{VMID: 100, RuntimeStatus: "running", Memory: func() *models.Memory { + m := patrolMemoryFixture(96, "last-known", "status-mem", now.Add(-time.Hour)) + return &m + }()}, + Metrics: &unifiedresources.ResourceMetrics{Memory: &unifiedresources.MetricValue{Percent: 24, Source: unifiedresources.SourceAgent, Observation: models.MemoryObservation{State: "current", Source: "agent", ObservedAt: now}}}} + vm := unifiedresources.NewVMView(r) + snap := patrolRuntimeState{readState: &mockReadState{vms: []*unifiedresources.VMView{&vm}}} + ps := NewPatrolService(nil, nil) + output := ps.seedResourceInventoryState(snap, nil, PatrolConfig{AnalyzeGuests: true}, time.Now(), false, nil) + if !strings.Contains(output, "24% (current; source agent") || strings.Contains(output, "status-mem") || strings.Contains(output, "96%") { + t.Errorf("raw platform memory replaced selected agent evidence: %s", output) + } + // A non-Proxmox VM keeps its existing selected reading without being + // required to invent the Proxmox observation metadata. + r.Proxmox = nil + r.Metrics.Memory = &unifiedresources.MetricValue{Percent: 96, Source: unifiedresources.SourceVMware} + flags := triageThresholdChecksState(snap, nil, DefaultPatrolThresholds()) + if triageFindFlag(flags, func(f TriageFlag) bool { return f.Metric == "memory" }) == nil { + t.Error("unrelated platform's real pressure was silenced") + } +} + +func TestIssue2762NamedMemoryFields(t *testing.T) { + // A synthetic 20 GiB guest with 15.2 GiB available and little truly free + // memory is not at 96% pressure. Field order and reclaimable cache do not + // override MemAvailable, and missing/truncated evidence stays unknown. + fields := []string{"MemTotal: 20971520 kB", "MemFree: 838860 kB", "MemAvailable: 15938355 kB", "Buffers: 1048576 kB", "Cached: 14942208 kB"} + for _, reverse := range []bool{false, true} { + if reverse { + for i, j := 0, len(fields)-1; i < j; i, j = i+1, j-1 { + fields[i], fields[j] = fields[j], fields[i] + } + } + got, err := proxmox.ParseLinuxMemoryAvailability(strings.Join(fields, "\n"), false) + if err != nil { + t.Fatal(err) + } + pressure := float64(got.Total-got.EffectiveAvailable) / float64(got.Total) * 100 + if math.Abs(pressure-24) > 0.01 || got.Source != "meminfo-available" { + t.Errorf("named available field was mistaken for cache or free: pressure=%v evidence=%+v", pressure, got) + } + } + if _, err := proxmox.ParseLinuxMemoryAvailability("MemTotal: 20971520 kB\nMemFree: 838860 kB", true); err == nil { + t.Error("partial free-only payload manufactured available memory") + } +} diff --git a/internal/ai/patrol_run.go b/internal/ai/patrol_run.go index 63250c668e..17f68eed3a 100644 --- a/internal/ai/patrol_run.go +++ b/internal/ai/patrol_run.go @@ -2536,6 +2536,7 @@ type patrolAlertResourceState struct { status string cpu float64 memory float64 + guestMemory *patrolGuestMemoryReading disk float64 found bool } @@ -2653,6 +2654,7 @@ func patrolLookupGuestAlertResourceState(alert AlertInfo, snap patrolRuntimeStat status: guest.status, cpu: guest.cpu, memory: guest.mem, + guestMemory: &guest.memory, found: true, }, true } @@ -2732,6 +2734,9 @@ func (p *PatrolService) getCurrentMetricValueState(alert AlertInfo, snap patrolR case "cpu": return resource.cpu case "memory": + if resource.guestMemory != nil && !resource.guestMemory.pressureKnown { + return -1 + } return resource.memory default: if resource.resourceType == "storage" { @@ -2815,6 +2820,19 @@ func (p *PatrolService) reviewAlertBatchState(ctx context.Context, batch []Alert verdicts[0].reason = patrolAlertResolveReason(strings.TrimPrefix(trimmed[len("RESOLVE:"):], " ")) } } + // This quick review has only the supplied snapshot, not an independent + // guest measurement. Missing/stale/cache-inclusive memory cannot prove a + // memory alert recovered, even if the model asks to resolve it. + for i, alert := range batch { + if alert.Type != "memory" { + continue + } + resource := lookupPatrolAlertResourceState(alert, snap) + if resource.guestMemory != nil && !resource.guestMemory.pressureKnown { + verdicts[i].resolve = false + verdicts[i].reason = "" + } + } return verdicts } @@ -2917,11 +2935,11 @@ func (p *PatrolService) getResourceCurrentStateState(alert AlertInfo, snap patro return fmt.Sprintf("Agent host '%s': CPU %.1f%%, Memory %.1f%%, Status: %s", resource.name, resource.cpu, resource.memory, resource.status) case "vm": - return fmt.Sprintf("VM '%s': CPU %.1f%%, Memory %.1f%%, Status: %s", - resource.name, resource.cpu, resource.memory, resource.status) + return fmt.Sprintf("VM '%s': CPU %.1f%%, Memory %s, Status: %s", + resource.name, resource.cpu, resource.guestMemory.display(), resource.status) case "system-container": - return fmt.Sprintf("Container '%s': CPU %.1f%%, Memory %.1f%%, Status: %s", - resource.name, resource.cpu, resource.memory, resource.status) + return fmt.Sprintf("Container '%s': CPU %.1f%%, Memory %s, Status: %s", + resource.name, resource.cpu, resource.guestMemory.display(), resource.status) case "app-container": return fmt.Sprintf("Docker container '%s': CPU %.1f%%, Memory %.1f%%, State: %s", resource.name, resource.cpu, resource.memory, resource.status) diff --git a/internal/ai/patrol_triage.go b/internal/ai/patrol_triage.go index 04822aa4f6..5e78b3eb39 100644 --- a/internal/ai/patrol_triage.go +++ b/internal/ai/patrol_triage.go @@ -207,7 +207,7 @@ func triageThresholdChecksState(snap patrolRuntimeState, scopedSet map[string]bo if guest.gType == "VM" { resourceType = "vm" } - if sev, threshold := triageWarnWatchSeverity(guest.mem, thresholds.GuestMemWarning, thresholds.GuestMemWatch); sev != "" { + if sev, threshold := triageWarnWatchSeverity(guest.mem, thresholds.GuestMemWarning, thresholds.GuestMemWatch); guest.memory.pressureKnown && sev != "" { flags = append(flags, TriageFlag{ ResourceID: guest.id, ResourceName: guest.name, @@ -653,8 +653,9 @@ func formatTriageHealthySummarySection(triage *TriageResult) string { totalHealthy := healthyNodes + healthyGuests + healthyStorage + healthyDocker + healthyTrueNAS + healthyPBS + healthyPMG var sb strings.Builder - sb.WriteString(fmt.Sprintf("## Healthy Resources (%d)\n", totalHealthy)) - sb.WriteString(fmt.Sprintf("Nodes: %d healthy\n", healthyNodes)) + sb.WriteString(fmt.Sprintf("## Resources without triage flags (%d)\n", totalHealthy)) + sb.WriteString("No flag is not proof of health; missing or uncertain readings remain unknown.\n") + sb.WriteString(fmt.Sprintf("Nodes: %d unflagged\n", healthyNodes)) sb.WriteString(fmt.Sprintf("Guests: %d running, %d stopped\n", triage.Summary.RunningGuests, triage.Summary.StoppedGuests)) sb.WriteString(fmt.Sprintf("Storage: %d resources monitored (%d pools, %d physical disks)\n", healthyStorage, diff --git a/internal/ai/patrol_triage_test.go b/internal/ai/patrol_triage_test.go index fe3c471563..6d0dbe0f43 100644 --- a/internal/ai/patrol_triage_test.go +++ b/internal/ai/patrol_triage_test.go @@ -762,7 +762,7 @@ func TestFormatTriageBriefing(t *testing.T) { if !strings.Contains(out, "web-01") || !strings.Contains(out, "High Memory") { t.Fatalf("expected resource row in output, got:\n%s", out) } - if !strings.Contains(out, "## Healthy Resources") { + if !strings.Contains(out, "## Resources without triage flags") { t.Fatalf("expected healthy summary section, got:\n%s", out) } if !strings.Contains(out, "Scanned 7 resources: 1 nodes, 2 guests, 1 storage resources (1 pools, 0 physical disks), 1 docker hosts, 1 TrueNAS systems, 1 PBS, 0 PMG.") { diff --git a/internal/unifiedresources/views.go b/internal/unifiedresources/views.go index f15e604c11..09dcbebb94 100644 --- a/internal/unifiedresources/views.go +++ b/internal/unifiedresources/views.go @@ -383,6 +383,20 @@ func (v VMView) MemoryPercent() float64 { return viewMetricPercent(v.r.Metrics, selectMetricsMemory) } +// MemoryObservation returns provenance of the selected memory metric, not a +// possibly conflicting platform facet. The bool distinguishes absent memory +// from a measured zero; an empty observation still has unknown provenance. +func (v VMView) MemoryObservation() (models.MemoryObservation, bool) { + return selectedMemoryObservation(v.r) +} + +func selectedMemoryObservation(r *Resource) (models.MemoryObservation, bool) { + if r == nil || r.Metrics == nil || r.Metrics.Memory == nil { + return models.MemoryObservation{}, false + } + return r.Metrics.Memory.Observation, true +} + func (v VMView) DiskUsed() int64 { if v.r == nil { return 0 @@ -709,6 +723,10 @@ func (v ContainerView) MemoryPercent() float64 { return viewMetricPercent(v.r.Metrics, selectMetricsMemory) } +func (v ContainerView) MemoryObservation() (models.MemoryObservation, bool) { + return selectedMemoryObservation(v.r) +} + func (v ContainerView) DiskUsed() int64 { if v.r == nil { return 0 diff --git a/internal/unifiedresources/views_test.go b/internal/unifiedresources/views_test.go index 87b7a9ddeb..1e8abe363c 100644 --- a/internal/unifiedresources/views_test.go +++ b/internal/unifiedresources/views_test.go @@ -2551,3 +2551,39 @@ func TestGuestViewGovernanceIdentity(t *testing.T) { } } } + +func TestGuestViewMemoryObservationKeepsSelectedOrigin(t *testing.T) { + at := time.Now().UTC().Add(-time.Minute) + selected := models.MemoryObservation{State: "current", Source: "agent", ObservedAt: at} + r := &Resource{ + Metrics: &ResourceMetrics{Memory: &MetricValue{Percent: 0, Observation: selected}}, + Proxmox: &ProxmoxData{Memory: &models.Memory{Usage: 99, Observation: models.MemoryObservation{State: "last-known", Source: "status-mem", ObservedAt: at.Add(-time.Hour)}}}, + } + for _, kind := range []string{"vm", "system-container"} { + t.Run(kind, func(t *testing.T) { + read := func(r *Resource) (models.MemoryObservation, bool) { + if kind == "vm" { + return NewVMView(r).MemoryObservation() + } + return NewContainerView(r).MemoryObservation() + } + got, ok := read(r) + if !ok || got != selected { + t.Fatalf("selected zero or its source was replaced by the raw facet: %+v present=%t", got, ok) + } + got.Source = "changed" + if r.Metrics.Memory.Observation != selected { + t.Fatal("view returned mutable source evidence") + } + for _, absent := range []*Resource{nil, {}, {Metrics: &ResourceMetrics{}}, {Proxmox: r.Proxmox}} { + if got, ok := read(absent); ok || got != (models.MemoryObservation{}) { + t.Fatalf("absent selected memory borrowed platform provenance: %+v present=%t", got, ok) + } + } + legacy := &Resource{Metrics: &ResourceMetrics{Memory: &MetricValue{Percent: 0}}} + if got, ok := read(legacy); !ok || got != (models.MemoryObservation{}) { + t.Fatalf("unannotated zero was mistaken for missing memory: %+v present=%t", got, ok) + } + }) + } +}