From 5ce63f140022fa85169c7f64c3ff2eae7cae1e2b Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 21:39:16 +0100 Subject: [PATCH] Keep retained guest readings out of alert decisions Use selected memory observation state and filesystem read validity at both threshold and filter boundaries. Retained or failed inventory cannot open or resolve incidents, refresh their evidence or remove per-filesystem alerts. Independent current metrics, legacy readings and deliberate disablement retain their existing policy. Change-source: pulse-maintainer --- .../v6/internal/subsystems/alerts.md | 28 ++++++++++++ internal/alerts/filter_evaluation.go | 32 +++++++++----- internal/alerts/guest.go | 15 ++++++- internal/alerts/guest_snapshot.go | 44 ++++++++++++------- internal/alerts/guest_snapshot_test.go | 44 +++++++++++++++++++ 5 files changed, 133 insertions(+), 30 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 3c89b76df0..5f7c630e4f 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -15,6 +15,34 @@ ## Purpose +### Retained guest observations are not alert evidence + +Guest memory marked `last-known`, `unavailable` or an unknown observation state +cannot start, advance, refresh or resolve a memory incident. A VM filesystem +read with a nonempty `diskStatusReason`, including retained `prev-*` values, +or unavailable aggregate usage is likewise not current disk evidence. It cannot +evaluate either aggregate or per-filesystem thresholds, replace an aggregate +incident with filesystem incidents, or clear filesystem alerts as if a failed +or expired inventory were authoritative removal. Metric filters do not match +these unavailable memory/disk values, including a fabricated healthy zero. + +Existing incidents retain their last trusted value, occurrence, acknowledgement, +observation time and lifecycle/delivery evidence until a current observation +resumes evaluation. Successful current inventory still resolves removed +filesystems. Explicit disablement, suppression and stopped-guest behavior remain +effective independently of telemetry. Legacy unannotated memory and independent +current PVE/Pulse-agent memory/disks keep their existing policy, even while QGA +is deferred; current CPU and I/O continue to evaluate. Producers own freshness; +no new age threshold, QGA request, payload, alert family or UI surface is added. + +`guest_observation_test.go` pins typed/pointer VM/LXC observation admission, +delayed breaches, canonical incident/acknowledgement/ledger continuity, filters, +fresh recovery and removal, independent metrics and deliberate disablement. +`TestGuestBackupDeferralAlertObservationLifecycle` connects actual HTTP client, +poll builder, retained snapshot, unified view, History and the production alert +call. These are synthetic source proofs, not native backup/thaw, containing +release, installed delivery or external recipient acceptance. + ### Pulse system alerts retain their service scope in the existing UI The `pulse-system-` identity prefix or explicit `systemAlert: true` metadata diff --git a/internal/alerts/filter_evaluation.go b/internal/alerts/filter_evaluation.go index 5741e176a1..d8c0954130 100644 --- a/internal/alerts/filter_evaluation.go +++ b/internal/alerts/filter_evaluation.go @@ -19,18 +19,20 @@ func (m *Manager) evaluateFilterCondition(guest any, condition FilterCondition) // guestMetrics holds common metrics for filter evaluation type guestMetrics struct { - CPU float64 // CPU usage as percentage (0-100) - MemUsage float64 // Memory usage percentage - DiskUsage float64 // Disk usage percentage - DiskRead int64 // Bytes/s - DiskWrite int64 // Bytes/s - NetworkIn int64 // Bytes/s - NetworkOut int64 // Bytes/s - IORateValidity models.IORateValidity - Name string - Node string - ID string - Status string + CPU float64 // CPU usage as percentage (0-100) + MemUsage float64 // Memory usage percentage + DiskUsage float64 // Disk usage percentage + MemoryUnavailable bool + DiskUnavailable bool + DiskRead int64 // Bytes/s + DiskWrite int64 // Bytes/s + NetworkIn int64 // Bytes/s + NetworkOut int64 // Bytes/s + IORateValidity models.IORateValidity + Name string + Node string + ID string + Status string } // extractGuestMetrics extracts common metrics from a VM or Container @@ -57,8 +59,14 @@ func evaluateGuestCondition(metrics guestMetrics, condition FilterCondition) boo case "cpu": value = metrics.CPU case "memory": + if metrics.MemoryUnavailable { + return false + } value = metrics.MemUsage case "disk": + if metrics.DiskUnavailable { + return false + } value = metrics.DiskUsage case "diskread": if !validity.DiskRead { diff --git a/internal/alerts/guest.go b/internal/alerts/guest.go index b185ea1d30..6bc19205af 100644 --- a/internal/alerts/guest.go +++ b/internal/alerts/guest.go @@ -259,8 +259,11 @@ func (m *Manager) CheckGuest(guest any, instanceName string) { memoryMetric = &UnifiedResourceMetric{Percent: memUsage} } diskReadMetric, diskWriteMetric, networkInMetric, networkOutMetric := guestIORateMetrics(snapshot) - diskMetric := &UnifiedResourceMetric{Percent: diskUsage} - if len(disks) > 0 { + var diskMetric *UnifiedResourceMetric + if !snapshot.DiskUnavailable { + diskMetric = &UnifiedResourceMetric{Percent: diskUsage} + } + if !snapshot.DiskUnavailable && len(disks) > 0 { seenAggregateDiskKeys := make(map[string]struct{}) for idx, disk := range disks { if disk.Total <= 0 || disk.Usage < 0 { @@ -295,6 +298,14 @@ func (m *Manager) CheckGuest(guest any, instanceName string) { NetworkOut: networkOutMetric, }, thresholds, evalOpts) + // Retained/failed filesystem evidence is useful to display, but neither + // its values nor an empty result can establish a breach, recovery or disk + // removal. CPU/I/O and independent current memory were evaluated above; + // explicit suppression/disablement and stopped-guest handling stay earlier. + if snapshot.DiskUnavailable { + return + } + if len(disks) > 0 { seenDiskKeys := make(map[string]struct{}) seenDiskResources := make(map[string]struct{}) diff --git a/internal/alerts/guest_snapshot.go b/internal/alerts/guest_snapshot.go index f9741f9647..9cbcc805e0 100644 --- a/internal/alerts/guest_snapshot.go +++ b/internal/alerts/guest_snapshot.go @@ -28,9 +28,10 @@ type guestSnapshot struct { CPUPercent float64 MemUsage float64 - // MemoryUnavailable prevents a missing cache-aware sample from clearing a - // real alert or starting a false one. + // Unavailable observations include retained display values. They cannot + // supply a new breach, recovery, filter match or filesystem removal. MemoryUnavailable bool + DiskUnavailable bool DiskUsage float64 DiskRead int64 DiskWrite int64 @@ -81,18 +82,20 @@ func (g guestSnapshot) resourceType() string { func (g guestSnapshot) metrics() guestMetrics { return guestMetrics{ - CPU: g.CPUPercent, - MemUsage: g.MemUsage, - DiskUsage: g.DiskUsage, - DiskRead: g.DiskRead, - DiskWrite: g.DiskWrite, - NetworkIn: g.NetworkIn, - NetworkOut: g.NetworkOut, - IORateValidity: g.IORateValidity, - Name: g.Name, - Node: g.Node, - ID: g.ID, - Status: g.Status, + CPU: g.CPUPercent, + MemUsage: g.MemUsage, + DiskUsage: g.DiskUsage, + MemoryUnavailable: g.MemoryUnavailable, + DiskUnavailable: g.DiskUnavailable, + DiskRead: g.DiskRead, + DiskWrite: g.DiskWrite, + NetworkIn: g.NetworkIn, + NetworkOut: g.NetworkOut, + IORateValidity: g.IORateValidity, + Name: g.Name, + Node: g.Node, + ID: g.ID, + Status: g.Status, } } @@ -108,7 +111,8 @@ func guestSnapshotFromVM(vm models.VM) guestSnapshot { Lock: vm.Lock, CPUPercent: unifiedresources.ProxmoxGuestCPUPercent(vm.CPU), MemUsage: vm.Memory.Usage, - MemoryUnavailable: vm.Memory.UsageUnavailable, + MemoryUnavailable: guestMemoryUnavailableForAlerts(vm.Memory), + DiskUnavailable: vm.DiskStatusReason != "" || vm.Disk.Usage < 0, DiskUsage: vm.Disk.Usage, DiskRead: vm.DiskRead, DiskWrite: vm.DiskWrite, @@ -133,7 +137,8 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot { Lock: container.Lock, CPUPercent: unifiedresources.ProxmoxGuestCPUPercent(container.CPU), MemUsage: container.Memory.Usage, - MemoryUnavailable: container.Memory.UsageUnavailable, + MemoryUnavailable: guestMemoryUnavailableForAlerts(container.Memory), + DiskUnavailable: container.Disk.Usage < 0, DiskUsage: container.Disk.Usage, DiskRead: container.DiskRead, DiskWrite: container.DiskWrite, @@ -146,6 +151,13 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot { }.normalizeCollections() } +func guestMemoryUnavailableForAlerts(memory models.Memory) bool { + // The producer owns freshness and source selection. Legacy values without + // an observation retain their existing behavior; do not infer freshness + // from disk/QGA state, since PVE or a linked Pulse agent can be independent. + return memory.UsageUnavailable || (memory.Observation.State != "" && memory.Observation.State != "current") +} + func guestKindFromType(guestType string) guestKind { switch strings.ToLower(strings.TrimSpace(guestType)) { case "qemu", "vm": diff --git a/internal/alerts/guest_snapshot_test.go b/internal/alerts/guest_snapshot_test.go index 28c7e4e65f..863dcec16a 100644 --- a/internal/alerts/guest_snapshot_test.go +++ b/internal/alerts/guest_snapshot_test.go @@ -19,6 +19,50 @@ func TestGuestSnapshotResourceTypeUsesCanonicalSystemContainer(t *testing.T) { } } +func TestGuestSnapshotObservationAvailability(t *testing.T) { + for _, test := range []struct { + name string + memory models.Memory + disk models.Disk + reason string + wantMemory bool + wantDisk bool + }{ + {name: "legacy measured zero", memory: models.Memory{}, disk: models.Disk{}}, + {name: "current independent read", memory: models.Memory{Usage: 96, Observation: models.MemoryObservation{State: "current", Source: "agent"}}, disk: models.Disk{Usage: 98}}, + {name: "retained memory only", memory: models.Memory{Usage: 96, Observation: models.MemoryObservation{State: "last-known"}}, disk: models.Disk{Usage: 98}, wantMemory: true}, + {name: "unavailable numeric placeholder", memory: models.Memory{Usage: 0, Observation: models.MemoryObservation{State: "unavailable"}}, wantMemory: true}, + {name: "unknown observation version", memory: models.Memory{Usage: 96, Observation: models.MemoryObservation{State: "future-version"}}, wantMemory: true}, + {name: "explicit unavailable overrides current", memory: models.Memory{Usage: 96, UsageUnavailable: true, Observation: models.MemoryObservation{State: "current"}}, wantMemory: true}, + {name: "retained filesystem only", memory: models.Memory{Usage: 96}, disk: models.Disk{Usage: 98}, reason: "prev-vm-locked", wantDisk: true}, + {name: "failed empty filesystem inventory", memory: models.Memory{Usage: 96}, disk: models.Disk{Usage: -1}, reason: "agent-error", wantDisk: true}, + {name: "legacy unavailable aggregate", memory: models.Memory{Usage: 96}, disk: models.Disk{Usage: -1}, wantDisk: true}, + } { + t.Run(test.name, func(t *testing.T) { + // A deferred QGA session does not by itself make independent + // selected memory or Pulse-agent filesystems unavailable. + vm := models.VM{Memory: test.memory, Disk: test.disk, DiskStatusReason: test.reason, GuestAgentStatus: "deferred"} + for _, guest := range []any{vm, &vm} { + snapshot, ok := extractGuestSnapshot(guest) + if !ok || snapshot.MemoryUnavailable != test.wantMemory || snapshot.DiskUnavailable != test.wantDisk { + t.Fatalf("observation flags lost: %+v", snapshot) + } + metrics := snapshot.metrics() + if metrics.MemoryUnavailable != test.wantMemory || metrics.DiskUnavailable != test.wantDisk || metrics.MemUsage != test.memory.Usage || metrics.DiskUsage != test.disk.Usage { + t.Fatal("filter projection lost availability or rewrote retained values") + } + } + ct := models.Container{Memory: test.memory, Disk: test.disk} + for _, guest := range []any{ct, &ct} { + snapshot, _ := extractGuestSnapshot(guest) + if snapshot.MemoryUnavailable != test.wantMemory || snapshot.DiskUnavailable != (test.disk.Usage < 0) { + t.Fatal("LXC observation availability changed with an unrelated VM disk failure") + } + } + }) + } +} + func TestGuestSnapshotUsesCanonicalProxmoxCPUPercent(t *testing.T) { for _, tc := range []struct { name string