From b02f9659c0341620f6ce7bb0521f8eec314d3ec1 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 23:24:06 +0100 Subject: [PATCH 1/3] Keep unavailable Proxmox node disks from resolving warnings Reuse canonical disk observation admission before node alert evaluation. Missing capacity and contradictory readings preserve trusted occurrences and restart activation timing; measured zero, explicit policy and real linked-agent handover keep their semantics. Contract-Neutral: Reuse existing disk observation and alert lifecycle contracts; no API, schema or ownership policy shape changes. Change-source: pulse-maintainer --- .../v6/internal/subsystems/alerts.md | 18 ++ internal/alerts/node.go | 9 +- internal/alerts/node_disk_observation_test.go | 260 ++++++++++++++++++ 3 files changed, 285 insertions(+), 2 deletions(-) create mode 100644 internal/alerts/node_disk_observation_test.go diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 591e36a1c5..f04ad7c148 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -15,6 +15,24 @@ ## Purpose +### Unavailable Proxmox node disks are not recovered alerts + +The node evaluator admits the same disk observations as the canonical node +metric: positive capacity, finite percentage in range, and non-negative used +space within capacity. Missing or contradictory readings hold an open disk +occurrence with its acknowledgement and trusted evidence, and interrupt pending +activation and explicit intent grace. An observed zero still recovers normally. +Explicit disabled/off policy and an actually covering linked agent still release +the node alert; a missing reading does not claim another owner. Independent CPU, +memory and temperature evaluation and the existing offline policy are unchanged. + +`node_disk_observation_test.go` exercises CheckNode, lifecycle events, measured +zero, both grace paths, exact-node isolation, explicit policy and real linked-agent +handover. Its optional manager frames feed the production Alerts API adapter and +Overview at desktop/narrow viewports. This aligns observation admission only: no +collector, guest operation, schema, historical rewrite or notification transport +changes, and no installed recovery or inbox claim. + ### Unavailable agent filesystems are not recovered disk alerts Host filesystem thresholds use the canonical disk-observation admission: a diff --git a/internal/alerts/node.go b/internal/alerts/node.go index 00db8a2ba2..d187fd2da7 100644 --- a/internal/alerts/node.go +++ b/internal/alerts/node.go @@ -119,8 +119,13 @@ func (m *Manager) CheckNode(node models.Node) { } if owners.disk.disk { released = append(released, "disk") - } else { - input.Disk = &UnifiedResourceMetric{Percent: node.Disk.Usage} + } else if percent, observed := unifiedresources.DiskUsagePercent(node.Disk); observed { + input.Disk = &UnifiedResourceMetric{Percent: percent} + } else if !liveHysteresisThreshold(thresholds.Disk) { + // Explicit policy still releases the alert, even without a + // reading. Otherwise leave it unknown: missing capacity or + // contradictory used space is not measured zero/recovery. + input.Disk = &UnifiedResourceMetric{} } m.releaseNodeMetricAlerts(node, owners, released...) m.evaluateUnifiedMetrics(input, thresholds, nil, "temperature") // checkNodeTemperature owns its observation gaps. diff --git a/internal/alerts/node_disk_observation_test.go b/internal/alerts/node_disk_observation_test.go new file mode 100644 index 0000000000..9ed9289b1a --- /dev/null +++ b/internal/alerts/node_disk_observation_test.go @@ -0,0 +1,260 @@ +package alerts + +import ( + "bytes" + "encoding/json" + "fmt" + "math" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog" + "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer" + "github.com/rcourtman/pulse-go-rewrite/internal/models" +) + +func nodeDiskManager(t *testing.T, intent bool) (*Manager, *atomic.Int64) { + t.Helper() + m, clock := continuityManager(t, intent) + m.mu.Lock() + m.config.NodeDefaults = ThresholdConfig{Disk: &HysteresisThreshold{Trigger: 80, Clear: 70}} + if !intent { + m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 0}} + } + m.mu.Unlock() + return m, clock +} + +func nodeDiskSample(id string, usage float64) models.Node { + return models.Node{ID: id, Name: "pve", Instance: "site", Status: "online", Disk: models.Disk{Total: 1000, Used: int64(usage * 10), Usage: usage}} +} + +func nodeDiskGap(node models.Node, gap string) models.Node { + switch gap { + case "missing": + node.Disk = models.Disk{} + case "capacity-only": + node.Disk = models.Disk{Total: 1000, Used: -1} + case "no-capacity": + node.Disk.Total = 0 + case "negative-capacity": + node.Disk.Total = -1 + case "over-capacity": + node.Disk.Used = 1001 + case "negative": + node.Disk.Usage = -1 + case "over-100": + node.Disk.Usage = 101 + case "NaN": + node.Disk.Usage = math.NaN() + case "+Inf": + node.Disk.Usage = math.Inf(1) + } + return node +} + +func nodeDiskJSON(t *testing.T, value any) []byte { + t.Helper() + b, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return b +} + +func TestNodeDiskUnknownPreservesOccurrence(t *testing.T) { + for _, gap := range []string{"missing", "capacity-only", "no-capacity", "negative-capacity", "over-capacity", "negative", "over-100", "NaN", "+Inf"} { + t.Run(gap, func(t *testing.T) { + m, clock := nodeDiskManager(t, false) + node := nodeDiskSample("site-pve", 95) + m.CheckNode(node) + if len(m.GetActiveAlerts()) != 1 { + t.Fatal("known high disk must fire") + } + id := m.GetActiveAlerts()[0].ID + if err := m.AcknowledgeAlert(id, "operator"); err != nil { + t.Fatal(err) + } + before := nodeDiskJSON(t, m.GetActiveAlerts()) + events := len(queryAlertEvents(t, m, eventlog.Filter{})) + clock.Store(int64(2 * time.Minute)) + for range 3 { + m.CheckNode(nodeDiskGap(node, gap)) + } + if !bytes.Equal(before, nodeDiskJSON(t, m.GetActiveAlerts())) || len(queryAlertEvents(t, m, eventlog.Filter{})) != events { + t.Fatal("unknown node disk changed occurrence, acknowledgement, trusted evidence or lifecycle") + } + m.CheckNode(nodeDiskSample(node.ID, 0)) + if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{AlertID: id, Types: []string{eventlog.TypeResolved}})) != 1 { + t.Fatal("measured zero must resolve the original once") + } + }) + } +} + +func TestNodeDiskUnknownNeverStartsAnIncident(t *testing.T) { + for _, gap := range []string{"missing", "capacity-only", "no-capacity", "negative-capacity", "over-capacity", "negative", "over-100", "NaN", "+Inf"} { + t.Run(gap, func(t *testing.T) { + m, clock := nodeDiskManager(t, false) + node := nodeDiskGap(nodeDiskSample("site-pve", 95), gap) + m.CheckNode(node) + clock.Store(int64(time.Hour)) + m.CheckNode(node) + if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{})) != 0 { + t.Fatal("unavailable node disk invented an incident") + } + }) + } +} + +func TestNodeDiskGapRestartsActivation(t *testing.T) { + for _, intent := range []bool{false, true} { + for _, gap := range []string{"missing", "no-capacity", "over-capacity"} { + t.Run(fmt.Sprintf("intent=%v/%s", intent, gap), func(t *testing.T) { + m, clock := nodeDiskManager(t, intent) + m.mu.Lock() + m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 60}} + m.mu.Unlock() + node := nodeDiskSample("site-pve", 85) + other := nodeDiskSample("other-pve", 85) + m.CheckNode(node) + m.CheckNode(other) + incident, exists := continuityIncident(m, node.ID, "disk") + if !exists || incident.State != reducer.StatePending { + t.Fatal("control must create timed activation") + } + clock.Store(int64(2 * time.Minute)) + m.CheckNode(nodeDiskGap(node, gap)) + if _, exists := continuityIncident(m, node.ID, "disk"); exists { + t.Fatal("unknown retained timed disk evidence") + } + if _, exists := continuityIncident(m, other.ID, "disk"); !exists { + t.Fatal("gap crossed exact node identity") + } + m.CheckNode(node) + clock.Store(int64(2*time.Minute + 59*time.Second)) + m.CheckNode(node) + if len(m.GetActiveAlerts()) != 0 { + t.Fatal("post-gap run completed early") + } + clock.Store(int64(3 * time.Minute)) + m.CheckNode(node) + if len(m.GetActiveAlerts()) != 1 { + t.Fatal("full fresh run did not fire") + } + if m.ShadowDivergences() != 0 { + t.Fatal("core and shadow diverged") + } + }) + } + } +} + +func TestNodeDiskUnknownHonoursExplicitPolicy(t *testing.T) { + for _, policy := range []string{"nil-threshold", "zero-threshold", "override", "node-disabled", "all-nodes-disabled"} { + for _, gap := range []string{"missing", "NaN"} { + t.Run(policy+"/"+gap, func(t *testing.T) { + m, _ := nodeDiskManager(t, false) + node := nodeDiskSample("site-pve", 95) + m.CheckNode(node) + if len(m.GetActiveAlerts()) != 1 { + t.Fatal("known disk control must fire") + } + // Direct changes exercise CheckNode's policy handling, not UpdateConfig cleanup. + m.mu.Lock() + switch policy { + case "nil-threshold": + m.config.NodeDefaults.Disk = nil + case "zero-threshold": + m.config.NodeDefaults.Disk = &HysteresisThreshold{} + case "override": + m.config.Overrides[node.ID] = ThresholdConfig{Disk: &HysteresisThreshold{}} + case "node-disabled": + m.config.Overrides[node.ID] = ThresholdConfig{Disabled: true} + case "all-nodes-disabled": + m.config.DisableAllNodes = true + } + m.mu.Unlock() + m.CheckNode(nodeDiskGap(node, gap)) + if len(m.GetActiveAlerts()) != 0 { + t.Fatal("unavailable disk overrode explicit alert policy") + } + }) + } + } +} + +func TestNodeDiskUnknownLeavesIndependentMetricsAndAgentHandover(t *testing.T) { + m, _ := nodeDiskManager(t, false) + m.mu.Lock() + m.config.NodeDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 70} + m.config.NodeDefaults.Memory = &HysteresisThreshold{Trigger: 80, Clear: 70} + m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 0, "cpu": 0, "memory": 0}} + m.mu.Unlock() + node := nodeDiskSample("site-pve", 95) + m.CheckNode(node) + gap := nodeDiskGap(node, "missing") + gap.CPU = .95 + gap.Memory = models.Memory{Total: 1000, Used: 950, Usage: 95} + m.CheckNode(gap) + if len(m.GetActiveAlerts()) != 3 { + t.Fatal("disk gap masked independently observed CPU or memory") + } + // A real linked agent still takes over its disk; absence is not another owner. + host := hostFilesystemSample("host-node-disk", 95) + host.LinkedNodeID = node.ID + m.mu.Lock() + m.config.AgentDefaults = ThresholdConfig{Disk: &HysteresisThreshold{Trigger: 80, Clear: 70}} + m.mu.Unlock() + m.CheckHost(host) + m.CheckNode(gap) + if testHasActiveAlert(t, m, canonicalMetricStateID(node.ID, "disk")) { + t.Fatal("real agent handover left duplicate node disk") + } + if !testHasActiveAlert(t, m, canonicalMetricStateID(hostFilesystemID(host), "disk")) { + t.Fatal("handover lost agent disk incident") + } + m.HandleHostRemoved(host) + m.CheckNode(gap) + if testHasActiveAlert(t, m, canonicalMetricStateID(node.ID, "disk")) { + t.Fatal("removing agent invented a node reading") + } + m.CheckNode(node) + if !testHasActiveAlert(t, m, canonicalMetricStateID(node.ID, "disk")) { + t.Fatal("fresh node disk did not regain ownership") + } +} + +// Export only opt-in synthetic manager frames for the real consuming UI proof. +func TestNodeDiskBrowserProducer(t *testing.T) { + m, clock := nodeDiskManager(t, false) + node := nodeDiskSample("site-browser-pve", 95) + frames := map[string]json.RawMessage{} + for _, phase := range []string{"breach", "missing", "over-capacity", "recovered"} { + sample := node + if phase == "recovered" { + sample = nodeDiskSample(node.ID, 0) + } else if phase != "breach" { + sample = nodeDiskGap(node, phase) + } + clock.Add(int64(time.Second)) + m.CheckNode(sample) + active := m.GetActiveAlerts() + want := 1 + if phase == "recovered" { + want = 0 + } + if len(active) != want { + t.Fatalf("phase %s: %d active, want %d", phase, len(active), want) + } + frames[phase] = nodeDiskJSON(t, active) + } + if !bytes.Equal(frames["breach"], frames["missing"]) || !bytes.Equal(frames["breach"], frames["over-capacity"]) { + t.Fatal("unknown changed exported occurrence") + } + if os.Getenv("PULSE_NODE_DISK_FRAMES") == "1" { + fmt.Printf("NODE_DISK_FRAMES=%s\n", nodeDiskJSON(t, frames)) + } +} From a940a089d81564885e732cdbdbbfba29f9a04c9f Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 23:35:03 +0100 Subject: [PATCH 2/3] Use observed capacity and forward report time in node disk controls The broad race proof found a legacy disk fixture without capacity and a new handover control whose fixed observation clock predated the real release. Supply measured bytes and advance report time; do not change alert recovery or handover behavior. Contract-Neutral: Correct node disk regression fixtures without changing runtime contracts or verification thresholds. Change-source: pulse-maintainer --- internal/alerts/alerts_test.go | 2 ++ internal/alerts/node_disk_observation_test.go | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go index a78ad933bd..5f5bff62a6 100644 --- a/internal/alerts/alerts_test.go +++ b/internal/alerts/alerts_test.go @@ -15870,6 +15870,8 @@ func TestCheckNode(t *testing.T) { Instance: "pve1", Status: "online", Disk: models.Disk{ + Total: 100, + Used: 95, Usage: 95.0, // Above trigger }, } diff --git a/internal/alerts/node_disk_observation_test.go b/internal/alerts/node_disk_observation_test.go index 9ed9289b1a..76f340b695 100644 --- a/internal/alerts/node_disk_observation_test.go +++ b/internal/alerts/node_disk_observation_test.go @@ -187,7 +187,7 @@ func TestNodeDiskUnknownHonoursExplicitPolicy(t *testing.T) { } func TestNodeDiskUnknownLeavesIndependentMetricsAndAgentHandover(t *testing.T) { - m, _ := nodeDiskManager(t, false) + m, clock := nodeDiskManager(t, false) m.mu.Lock() m.config.NodeDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 70} m.config.NodeDefaults.Memory = &HysteresisThreshold{Trigger: 80, Clear: 70} @@ -217,10 +217,14 @@ func TestNodeDiskUnknownLeavesIndependentMetricsAndAgentHandover(t *testing.T) { t.Fatal("handover lost agent disk incident") } m.HandleHostRemoved(host) + // Handover release uses its real observation time. Later reports must + // advance the synthetic clock too, rather than predate that release. + clock.Store(int64(time.Second)) m.CheckNode(gap) if testHasActiveAlert(t, m, canonicalMetricStateID(node.ID, "disk")) { t.Fatal("removing agent invented a node reading") } + clock.Add(int64(time.Second)) m.CheckNode(node) if !testHasActiveAlert(t, m, canonicalMetricStateID(node.ID, "disk")) { t.Fatal("fresh node disk did not regain ownership") From da7b4707180b98ae0a108a5573d709c12babdc41 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 23:45:47 +0100 Subject: [PATCH 3/3] Respect existing duplicate suppression in node handover control The unchanged two-percent/five-minute duplicate rule prevents an immediate repeat at 95 percent. Use a changed measured 98 percent to prove node evaluation resumes after agent removal. The earlier clock-only explanation was not causal; no production behavior changes. Contract-Neutral: Correct the handover control's observation without changing alert thresholds, suppression policy or runtime contracts. Change-source: pulse-maintainer --- internal/alerts/node_disk_observation_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/alerts/node_disk_observation_test.go b/internal/alerts/node_disk_observation_test.go index 76f340b695..99f888dfb8 100644 --- a/internal/alerts/node_disk_observation_test.go +++ b/internal/alerts/node_disk_observation_test.go @@ -192,6 +192,8 @@ func TestNodeDiskUnknownLeavesIndependentMetricsAndAgentHandover(t *testing.T) { m.config.NodeDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 70} m.config.NodeDefaults.Memory = &HysteresisThreshold{Trigger: 80, Clear: 70} m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 0, "cpu": 0, "memory": 0}} + m.config.MinimumDelta = 2 + m.config.SuppressionWindow = 5 m.mu.Unlock() node := nodeDiskSample("site-pve", 95) m.CheckNode(node) @@ -217,13 +219,16 @@ func TestNodeDiskUnknownLeavesIndependentMetricsAndAgentHandover(t *testing.T) { t.Fatal("handover lost agent disk incident") } m.HandleHostRemoved(host) - // Handover release uses its real observation time. Later reports must - // advance the synthetic clock too, rather than predate that release. + // These are later reports, not a replay of the original observation. clock.Store(int64(time.Second)) m.CheckNode(gap) if testHasActiveAlert(t, m, canonicalMetricStateID(node.ID, "disk")) { t.Fatal("removing agent invented a node reading") } + // The existing 2% minimum delta suppresses a new occurrence at the same + // 95% within five minutes. Use a genuinely changed measured value; this + // control tests restored node evidence, not bypassing duplicate policy. + node.Disk = nodeDiskSample(node.ID, 98).Disk clock.Add(int64(time.Second)) m.CheckNode(node) if !testHasActiveAlert(t, m, canonicalMetricStateID(node.ID, "disk")) {