diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 3c89b76df0..5f7c630e4f 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -15,6 +15,34 @@ ## Purpose +### Retained guest observations are not alert evidence + +Guest memory marked `last-known`, `unavailable` or an unknown observation state +cannot start, advance, refresh or resolve a memory incident. A VM filesystem +read with a nonempty `diskStatusReason`, including retained `prev-*` values, +or unavailable aggregate usage is likewise not current disk evidence. It cannot +evaluate either aggregate or per-filesystem thresholds, replace an aggregate +incident with filesystem incidents, or clear filesystem alerts as if a failed +or expired inventory were authoritative removal. Metric filters do not match +these unavailable memory/disk values, including a fabricated healthy zero. + +Existing incidents retain their last trusted value, occurrence, acknowledgement, +observation time and lifecycle/delivery evidence until a current observation +resumes evaluation. Successful current inventory still resolves removed +filesystems. Explicit disablement, suppression and stopped-guest behavior remain +effective independently of telemetry. Legacy unannotated memory and independent +current PVE/Pulse-agent memory/disks keep their existing policy, even while QGA +is deferred; current CPU and I/O continue to evaluate. Producers own freshness; +no new age threshold, QGA request, payload, alert family or UI surface is added. + +`guest_observation_test.go` pins typed/pointer VM/LXC observation admission, +delayed breaches, canonical incident/acknowledgement/ledger continuity, filters, +fresh recovery and removal, independent metrics and deliberate disablement. +`TestGuestBackupDeferralAlertObservationLifecycle` connects actual HTTP client, +poll builder, retained snapshot, unified view, History and the production alert +call. These are synthetic source proofs, not native backup/thaw, containing +release, installed delivery or external recipient acceptance. + ### Pulse system alerts retain their service scope in the existing UI The `pulse-system-` identity prefix or explicit `systemAlert: true` metadata diff --git a/internal/alerts/filter_evaluation.go b/internal/alerts/filter_evaluation.go index 5741e176a1..d8c0954130 100644 --- a/internal/alerts/filter_evaluation.go +++ b/internal/alerts/filter_evaluation.go @@ -19,18 +19,20 @@ func (m *Manager) evaluateFilterCondition(guest any, condition FilterCondition) // guestMetrics holds common metrics for filter evaluation type guestMetrics struct { - CPU float64 // CPU usage as percentage (0-100) - MemUsage float64 // Memory usage percentage - DiskUsage float64 // Disk usage percentage - DiskRead int64 // Bytes/s - DiskWrite int64 // Bytes/s - NetworkIn int64 // Bytes/s - NetworkOut int64 // Bytes/s - IORateValidity models.IORateValidity - Name string - Node string - ID string - Status string + CPU float64 // CPU usage as percentage (0-100) + MemUsage float64 // Memory usage percentage + DiskUsage float64 // Disk usage percentage + MemoryUnavailable bool + DiskUnavailable bool + DiskRead int64 // Bytes/s + DiskWrite int64 // Bytes/s + NetworkIn int64 // Bytes/s + NetworkOut int64 // Bytes/s + IORateValidity models.IORateValidity + Name string + Node string + ID string + Status string } // extractGuestMetrics extracts common metrics from a VM or Container @@ -57,8 +59,14 @@ func evaluateGuestCondition(metrics guestMetrics, condition FilterCondition) boo case "cpu": value = metrics.CPU case "memory": + if metrics.MemoryUnavailable { + return false + } value = metrics.MemUsage case "disk": + if metrics.DiskUnavailable { + return false + } value = metrics.DiskUsage case "diskread": if !validity.DiskRead { diff --git a/internal/alerts/guest.go b/internal/alerts/guest.go index b185ea1d30..6bc19205af 100644 --- a/internal/alerts/guest.go +++ b/internal/alerts/guest.go @@ -259,8 +259,11 @@ func (m *Manager) CheckGuest(guest any, instanceName string) { memoryMetric = &UnifiedResourceMetric{Percent: memUsage} } diskReadMetric, diskWriteMetric, networkInMetric, networkOutMetric := guestIORateMetrics(snapshot) - diskMetric := &UnifiedResourceMetric{Percent: diskUsage} - if len(disks) > 0 { + var diskMetric *UnifiedResourceMetric + if !snapshot.DiskUnavailable { + diskMetric = &UnifiedResourceMetric{Percent: diskUsage} + } + if !snapshot.DiskUnavailable && len(disks) > 0 { seenAggregateDiskKeys := make(map[string]struct{}) for idx, disk := range disks { if disk.Total <= 0 || disk.Usage < 0 { @@ -295,6 +298,14 @@ func (m *Manager) CheckGuest(guest any, instanceName string) { NetworkOut: networkOutMetric, }, thresholds, evalOpts) + // Retained/failed filesystem evidence is useful to display, but neither + // its values nor an empty result can establish a breach, recovery or disk + // removal. CPU/I/O and independent current memory were evaluated above; + // explicit suppression/disablement and stopped-guest handling stay earlier. + if snapshot.DiskUnavailable { + return + } + if len(disks) > 0 { seenDiskKeys := make(map[string]struct{}) seenDiskResources := make(map[string]struct{}) diff --git a/internal/alerts/guest_snapshot.go b/internal/alerts/guest_snapshot.go index f9741f9647..9cbcc805e0 100644 --- a/internal/alerts/guest_snapshot.go +++ b/internal/alerts/guest_snapshot.go @@ -28,9 +28,10 @@ type guestSnapshot struct { CPUPercent float64 MemUsage float64 - // MemoryUnavailable prevents a missing cache-aware sample from clearing a - // real alert or starting a false one. + // Unavailable observations include retained display values. They cannot + // supply a new breach, recovery, filter match or filesystem removal. MemoryUnavailable bool + DiskUnavailable bool DiskUsage float64 DiskRead int64 DiskWrite int64 @@ -81,18 +82,20 @@ func (g guestSnapshot) resourceType() string { func (g guestSnapshot) metrics() guestMetrics { return guestMetrics{ - CPU: g.CPUPercent, - MemUsage: g.MemUsage, - DiskUsage: g.DiskUsage, - DiskRead: g.DiskRead, - DiskWrite: g.DiskWrite, - NetworkIn: g.NetworkIn, - NetworkOut: g.NetworkOut, - IORateValidity: g.IORateValidity, - Name: g.Name, - Node: g.Node, - ID: g.ID, - Status: g.Status, + CPU: g.CPUPercent, + MemUsage: g.MemUsage, + DiskUsage: g.DiskUsage, + MemoryUnavailable: g.MemoryUnavailable, + DiskUnavailable: g.DiskUnavailable, + DiskRead: g.DiskRead, + DiskWrite: g.DiskWrite, + NetworkIn: g.NetworkIn, + NetworkOut: g.NetworkOut, + IORateValidity: g.IORateValidity, + Name: g.Name, + Node: g.Node, + ID: g.ID, + Status: g.Status, } } @@ -108,7 +111,8 @@ func guestSnapshotFromVM(vm models.VM) guestSnapshot { Lock: vm.Lock, CPUPercent: unifiedresources.ProxmoxGuestCPUPercent(vm.CPU), MemUsage: vm.Memory.Usage, - MemoryUnavailable: vm.Memory.UsageUnavailable, + MemoryUnavailable: guestMemoryUnavailableForAlerts(vm.Memory), + DiskUnavailable: vm.DiskStatusReason != "" || vm.Disk.Usage < 0, DiskUsage: vm.Disk.Usage, DiskRead: vm.DiskRead, DiskWrite: vm.DiskWrite, @@ -133,7 +137,8 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot { Lock: container.Lock, CPUPercent: unifiedresources.ProxmoxGuestCPUPercent(container.CPU), MemUsage: container.Memory.Usage, - MemoryUnavailable: container.Memory.UsageUnavailable, + MemoryUnavailable: guestMemoryUnavailableForAlerts(container.Memory), + DiskUnavailable: container.Disk.Usage < 0, DiskUsage: container.Disk.Usage, DiskRead: container.DiskRead, DiskWrite: container.DiskWrite, @@ -146,6 +151,13 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot { }.normalizeCollections() } +func guestMemoryUnavailableForAlerts(memory models.Memory) bool { + // The producer owns freshness and source selection. Legacy values without + // an observation retain their existing behavior; do not infer freshness + // from disk/QGA state, since PVE or a linked Pulse agent can be independent. + return memory.UsageUnavailable || (memory.Observation.State != "" && memory.Observation.State != "current") +} + func guestKindFromType(guestType string) guestKind { switch strings.ToLower(strings.TrimSpace(guestType)) { case "qemu", "vm": diff --git a/internal/alerts/guest_snapshot_test.go b/internal/alerts/guest_snapshot_test.go index 28c7e4e65f..863dcec16a 100644 --- a/internal/alerts/guest_snapshot_test.go +++ b/internal/alerts/guest_snapshot_test.go @@ -19,6 +19,50 @@ func TestGuestSnapshotResourceTypeUsesCanonicalSystemContainer(t *testing.T) { } } +func TestGuestSnapshotObservationAvailability(t *testing.T) { + for _, test := range []struct { + name string + memory models.Memory + disk models.Disk + reason string + wantMemory bool + wantDisk bool + }{ + {name: "legacy measured zero", memory: models.Memory{}, disk: models.Disk{}}, + {name: "current independent read", memory: models.Memory{Usage: 96, Observation: models.MemoryObservation{State: "current", Source: "agent"}}, disk: models.Disk{Usage: 98}}, + {name: "retained memory only", memory: models.Memory{Usage: 96, Observation: models.MemoryObservation{State: "last-known"}}, disk: models.Disk{Usage: 98}, wantMemory: true}, + {name: "unavailable numeric placeholder", memory: models.Memory{Usage: 0, Observation: models.MemoryObservation{State: "unavailable"}}, wantMemory: true}, + {name: "unknown observation version", memory: models.Memory{Usage: 96, Observation: models.MemoryObservation{State: "future-version"}}, wantMemory: true}, + {name: "explicit unavailable overrides current", memory: models.Memory{Usage: 96, UsageUnavailable: true, Observation: models.MemoryObservation{State: "current"}}, wantMemory: true}, + {name: "retained filesystem only", memory: models.Memory{Usage: 96}, disk: models.Disk{Usage: 98}, reason: "prev-vm-locked", wantDisk: true}, + {name: "failed empty filesystem inventory", memory: models.Memory{Usage: 96}, disk: models.Disk{Usage: -1}, reason: "agent-error", wantDisk: true}, + {name: "legacy unavailable aggregate", memory: models.Memory{Usage: 96}, disk: models.Disk{Usage: -1}, wantDisk: true}, + } { + t.Run(test.name, func(t *testing.T) { + // A deferred QGA session does not by itself make independent + // selected memory or Pulse-agent filesystems unavailable. + vm := models.VM{Memory: test.memory, Disk: test.disk, DiskStatusReason: test.reason, GuestAgentStatus: "deferred"} + for _, guest := range []any{vm, &vm} { + snapshot, ok := extractGuestSnapshot(guest) + if !ok || snapshot.MemoryUnavailable != test.wantMemory || snapshot.DiskUnavailable != test.wantDisk { + t.Fatalf("observation flags lost: %+v", snapshot) + } + metrics := snapshot.metrics() + if metrics.MemoryUnavailable != test.wantMemory || metrics.DiskUnavailable != test.wantDisk || metrics.MemUsage != test.memory.Usage || metrics.DiskUsage != test.disk.Usage { + t.Fatal("filter projection lost availability or rewrote retained values") + } + } + ct := models.Container{Memory: test.memory, Disk: test.disk} + for _, guest := range []any{ct, &ct} { + snapshot, _ := extractGuestSnapshot(guest) + if snapshot.MemoryUnavailable != test.wantMemory || snapshot.DiskUnavailable != (test.disk.Usage < 0) { + t.Fatal("LXC observation availability changed with an unrelated VM disk failure") + } + } + }) + } +} + func TestGuestSnapshotUsesCanonicalProxmoxCPUPercent(t *testing.T) { for _, tc := range []struct { name string