gsadmin/pulse
1
0
Fork 0
mirror of https://github.com/rcourtman/Pulse.git synced 2026-10-23 20:25:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Keep retained guest readings out of alert decisions

Browse Source 
Use selected memory observation state and filesystem read validity at both threshold and filter boundaries. Retained or failed inventory cannot open or resolve incidents, refresh their evidence or remove per-filesystem alerts. Independent current metrics, legacy readings and deliberate disablement retain their existing policy.

Change-source: pulse-maintainer
This commit is contained in:
pulse-triage[bot]
2026-10-04 21:39:16 +01:00
parent b0f8b96c8a
commit 5ce63f1400
5 changed files with 133 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/release-control/v6/internal/subsystems/alerts.md
+28
View File
@@ -15,6 +15,34 @@
## Purpose
### Retained guest observations are not alert evidence
Guest memory marked `last-known`, `unavailable` or an unknown observation state
cannot start, advance, refresh or resolve a memory incident. A VM filesystem
read with a nonempty `diskStatusReason`, including retained `prev-*` values,
or unavailable aggregate usage is likewise not current disk evidence. It cannot
evaluate either aggregate or per-filesystem thresholds, replace an aggregate
incident with filesystem incidents, or clear filesystem alerts as if a failed
or expired inventory were authoritative removal. Metric filters do not match
these unavailable memory/disk values, including a fabricated healthy zero.
Existing incidents retain their last trusted value, occurrence, acknowledgement,
observation time and lifecycle/delivery evidence until a current observation
resumes evaluation. Successful current inventory still resolves removed
filesystems. Explicit disablement, suppression and stopped-guest behavior remain
effective independently of telemetry. Legacy unannotated memory and independent
current PVE/Pulse-agent memory/disks keep their existing policy, even while QGA
is deferred; current CPU and I/O continue to evaluate. Producers own freshness;
no new age threshold, QGA request, payload, alert family or UI surface is added.
`guest_observation_test.go` pins typed/pointer VM/LXC observation admission,
delayed breaches, canonical incident/acknowledgement/ledger continuity, filters,
fresh recovery and removal, independent metrics and deliberate disablement.
`TestGuestBackupDeferralAlertObservationLifecycle` connects actual HTTP client,
poll builder, retained snapshot, unified view, History and the production alert
call. These are synthetic source proofs, not native backup/thaw, containing
release, installed delivery or external recipient acceptance.
### Pulse system alerts retain their service scope in the existing UI
The `pulse-system-` identity prefix or explicit `systemAlert: true` metadata
internal/alerts/filter_evaluation.go
+20 -12
View File
@@ -19,18 +19,20 @@ func (m *Manager) evaluateFilterCondition(guest any, condition FilterCondition)
// guestMetrics holds common metrics for filter evaluation
type guestMetrics struct {
CPU float64 // CPU usage as percentage (0-100)
MemUsage float64 // Memory usage percentage
DiskUsage float64 // Disk usage percentage
DiskRead int64 // Bytes/s
DiskWrite int64 // Bytes/s
NetworkIn int64 // Bytes/s
NetworkOut int64 // Bytes/s
IORateValidity models.IORateValidity
Name string
Node string
ID string
Status string
CPU float64 // CPU usage as percentage (0-100)
MemUsage float64 // Memory usage percentage
DiskUsage float64 // Disk usage percentage
MemoryUnavailable bool
DiskUnavailable bool
DiskRead int64 // Bytes/s
DiskWrite int64 // Bytes/s
NetworkIn int64 // Bytes/s
NetworkOut int64 // Bytes/s
IORateValidity models.IORateValidity
Name string
Node string
ID string
Status string
}
// extractGuestMetrics extracts common metrics from a VM or Container
@@ -57,8 +59,14 @@ func evaluateGuestCondition(metrics guestMetrics, condition FilterCondition) boo
case "cpu":
value = metrics.CPU
case "memory":
if metrics.MemoryUnavailable {
return false
}
value = metrics.MemUsage
case "disk":
if metrics.DiskUnavailable {
return false
}
value = metrics.DiskUsage
case "diskread":
if !validity.DiskRead {
internal/alerts/guest.go
+13 -2
View File
@@ -259,8 +259,11 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {
memoryMetric = &UnifiedResourceMetric{Percent: memUsage}
}
diskReadMetric, diskWriteMetric, networkInMetric, networkOutMetric := guestIORateMetrics(snapshot)
diskMetric := &UnifiedResourceMetric{Percent: diskUsage}
if len(disks) > 0 {
var diskMetric *UnifiedResourceMetric
if !snapshot.DiskUnavailable {
diskMetric = &UnifiedResourceMetric{Percent: diskUsage}
}
if !snapshot.DiskUnavailable && len(disks) > 0 {
seenAggregateDiskKeys := make(map[string]struct{})
for idx, disk := range disks {
if disk.Total <= 0 || disk.Usage < 0 {
@@ -295,6 +298,14 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {
NetworkOut: networkOutMetric,
}, thresholds, evalOpts)
// Retained/failed filesystem evidence is useful to display, but neither
// its values nor an empty result can establish a breach, recovery or disk
// removal. CPU/I/O and independent current memory were evaluated above;
// explicit suppression/disablement and stopped-guest handling stay earlier.
if snapshot.DiskUnavailable {
return
}
if len(disks) > 0 {
seenDiskKeys := make(map[string]struct{})
seenDiskResources := make(map[string]struct{})
internal/alerts/guest_snapshot.go
+28 -16
View File
@@ -28,9 +28,10 @@ type guestSnapshot struct {
CPUPercent float64
MemUsage float64
// MemoryUnavailable prevents a missing cache-aware sample from clearing a
// real alert or starting a false one.
// Unavailable observations include retained display values. They cannot
// supply a new breach, recovery, filter match or filesystem removal.
MemoryUnavailable bool
DiskUnavailable bool
DiskUsage float64
DiskRead int64
DiskWrite int64
@@ -81,18 +82,20 @@ func (g guestSnapshot) resourceType() string {
func (g guestSnapshot) metrics() guestMetrics {
return guestMetrics{
CPU: g.CPUPercent,
MemUsage: g.MemUsage,
DiskUsage: g.DiskUsage,
DiskRead: g.DiskRead,
DiskWrite: g.DiskWrite,
NetworkIn: g.NetworkIn,
NetworkOut: g.NetworkOut,
IORateValidity: g.IORateValidity,
Name: g.Name,
Node: g.Node,
ID: g.ID,
Status: g.Status,
CPU: g.CPUPercent,
MemUsage: g.MemUsage,
DiskUsage: g.DiskUsage,
MemoryUnavailable: g.MemoryUnavailable,
DiskUnavailable: g.DiskUnavailable,
DiskRead: g.DiskRead,
DiskWrite: g.DiskWrite,
NetworkIn: g.NetworkIn,
NetworkOut: g.NetworkOut,
IORateValidity: g.IORateValidity,
Name: g.Name,
Node: g.Node,
ID: g.ID,
Status: g.Status,
}
}
@@ -108,7 +111,8 @@ func guestSnapshotFromVM(vm models.VM) guestSnapshot {
Lock: vm.Lock,
CPUPercent: unifiedresources.ProxmoxGuestCPUPercent(vm.CPU),
MemUsage: vm.Memory.Usage,
MemoryUnavailable: vm.Memory.UsageUnavailable,
MemoryUnavailable: guestMemoryUnavailableForAlerts(vm.Memory),
DiskUnavailable: vm.DiskStatusReason != "" || vm.Disk.Usage < 0,
DiskUsage: vm.Disk.Usage,
DiskRead: vm.DiskRead,
DiskWrite: vm.DiskWrite,
@@ -133,7 +137,8 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot {
Lock: container.Lock,
CPUPercent: unifiedresources.ProxmoxGuestCPUPercent(container.CPU),
MemUsage: container.Memory.Usage,
MemoryUnavailable: container.Memory.UsageUnavailable,
MemoryUnavailable: guestMemoryUnavailableForAlerts(container.Memory),
DiskUnavailable: container.Disk.Usage < 0,
DiskUsage: container.Disk.Usage,
DiskRead: container.DiskRead,
DiskWrite: container.DiskWrite,
@@ -146,6 +151,13 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot {
}.normalizeCollections()
}
func guestMemoryUnavailableForAlerts(memory models.Memory) bool {
// The producer owns freshness and source selection. Legacy values without
// an observation retain their existing behavior; do not infer freshness
// from disk/QGA state, since PVE or a linked Pulse agent can be independent.
return memory.UsageUnavailable || (memory.Observation.State != "" && memory.Observation.State != "current")
}
func guestKindFromType(guestType string) guestKind {
switch strings.ToLower(strings.TrimSpace(guestType)) {
case "qemu", "vm":
internal/alerts/guest_snapshot_test.go
+44
View File
@@ -19,6 +19,50 @@ func TestGuestSnapshotResourceTypeUsesCanonicalSystemContainer(t *testing.T) {
}
}
func TestGuestSnapshotObservationAvailability(t *testing.T) {
for _, test := range []struct {
name string
memory models.Memory
disk models.Disk
reason string
wantMemory bool
wantDisk bool
}{
{name: "legacy measured zero", memory: models.Memory{}, disk: models.Disk{}},
{name: "current independent read", memory: models.Memory{Usage: 96, Observation: models.MemoryObservation{State: "current", Source: "agent"}}, disk: models.Disk{Usage: 98}},
{name: "retained memory only", memory: models.Memory{Usage: 96, Observation: models.MemoryObservation{State: "last-known"}}, disk: models.Disk{Usage: 98}, wantMemory: true},
{name: "unavailable numeric placeholder", memory: models.Memory{Usage: 0, Observation: models.MemoryObservation{State: "unavailable"}}, wantMemory: true},
{name: "unknown observation version", memory: models.Memory{Usage: 96, Observation: models.MemoryObservation{State: "future-version"}}, wantMemory: true},
{name: "explicit unavailable overrides current", memory: models.Memory{Usage: 96, UsageUnavailable: true, Observation: models.MemoryObservation{State: "current"}}, wantMemory: true},
{name: "retained filesystem only", memory: models.Memory{Usage: 96}, disk: models.Disk{Usage: 98}, reason: "prev-vm-locked", wantDisk: true},
{name: "failed empty filesystem inventory", memory: models.Memory{Usage: 96}, disk: models.Disk{Usage: -1}, reason: "agent-error", wantDisk: true},
{name: "legacy unavailable aggregate", memory: models.Memory{Usage: 96}, disk: models.Disk{Usage: -1}, wantDisk: true},
} {
t.Run(test.name, func(t *testing.T) {
// A deferred QGA session does not by itself make independent
// selected memory or Pulse-agent filesystems unavailable.
vm := models.VM{Memory: test.memory, Disk: test.disk, DiskStatusReason: test.reason, GuestAgentStatus: "deferred"}
for _, guest := range []any{vm, &vm} {
snapshot, ok := extractGuestSnapshot(guest)
if !ok || snapshot.MemoryUnavailable != test.wantMemory || snapshot.DiskUnavailable != test.wantDisk {
t.Fatalf("observation flags lost: %+v", snapshot)
}
metrics := snapshot.metrics()
if metrics.MemoryUnavailable != test.wantMemory || metrics.DiskUnavailable != test.wantDisk || metrics.MemUsage != test.memory.Usage || metrics.DiskUsage != test.disk.Usage {
t.Fatal("filter projection lost availability or rewrote retained values")
}
}
ct := models.Container{Memory: test.memory, Disk: test.disk}
for _, guest := range []any{ct, &ct} {
snapshot, _ := extractGuestSnapshot(guest)
if snapshot.MemoryUnavailable != test.wantMemory || snapshot.DiskUnavailable != (test.disk.Usage < 0) {
t.Fatal("LXC observation availability changed with an unrelated VM disk failure")
}
}
})
}
}
func TestGuestSnapshotUsesCanonicalProxmoxCPUPercent(t *testing.T) {
for _, tc := range []struct {
name string