From db0c4fd3bb0a0fad94fb85782bc43629fef73f43 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:10:03 +0100 Subject: [PATCH] fix(alerts): notify on health assessment escalation A running custom sensor updated its canonical incident from warning to critical without dispatching the new severity (#1801). Enable upward transitions in the shared health adapter while keeping unchanged observations and downgrades quiet. Use the existing hourly budget and dispatch suppression rather than bypassing policy. Pin CheckHost callback delivery, acknowledgement, snooze, inactive and flapping suppression, and shared ZFS rate exhaustion with focused regressions. Change-source: pulse-maintainer --- .../v6/internal/subsystems/alerts.md | 17 ++++ internal/alerts/canonical_stateful_test.go | 44 +++++++++ internal/alerts/health_assessment.go | 7 +- internal/alerts/host_unraid_lifecycle_test.go | 98 +++++++++++++++++++ 4 files changed, 165 insertions(+), 1 deletion(-) diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 4f3717b24..49dc1123c 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -813,6 +813,23 @@ local command or REST URL. `TestHostCustomSensorAlertLifecycle` and `internal/alerts/host_unraid_lifecycle_test.go` pin creation, recovery, opt-out, and cleanup. +Health-assessment warning→critical transitions dispatch the updated incident +through the normal acknowledgement, snooze, activation and flapping policy; +unchanged severity and critical→warning transitions do not dispatch. The shared +health-assessment caller opts into the configured per-incident hourly limit for +initial, refired and escalation notifications, so repeated severity oscillations +cannot bypass that budget. Incident severity still updates when delivery is +suppressed. This applies to all callers of the shared assessment adapter +(custom sensors, host storage/RAID and storage ZFS pool/device health), not other +stateful alert families. No delivery-policy bypass or separate sensor callback +is introduced. +`TestHostCustomSensorEscalationDelivery` in +`internal/alerts/host_unraid_lifecycle_test.go` pins the running-host callback, +stable incident identity and suppression/no-noise cases; +`TestHealthAssessmentEscalationDelivery` in +`internal/alerts/canonical_stateful_test.go` pins shared ZFS escalation and +hourly-budget exhaustion across warning/critical oscillation. + The alert resource-incident panel (`frontend-modern/src/features/alerts/AlertResourceIncidentsPanel.tsx`) dropped its "Open in Infrastructure / Workloads / Storage / Recovery" diff --git a/internal/alerts/canonical_stateful_test.go b/internal/alerts/canonical_stateful_test.go index 7920c9563..8bf50a00e 100644 --- a/internal/alerts/canonical_stateful_test.go +++ b/internal/alerts/canonical_stateful_test.go @@ -279,3 +279,47 @@ func TestStoragePolicyAliasesLegacyIdentity(t *testing.T) { }) } } + +// The shared assessment path also owns ZFS health, not only custom sensors. +func TestHealthAssessmentEscalationDelivery(t *testing.T) { + m := newTestManager(t) + cfg := m.GetConfig() + cfg.Enabled = true + cfg.ActivationState = ActivationActive + cfg.FlappingEnabled = false + cfg.Schedule.MaxAlertsHour = 2 + m.UpdateConfig(cfg) + var delivered []AlertLevel + m.SetAlertCallback(func(a *Alert) { delivered = append(delivered, a.Level) }) + resourceID := "storage-1/zfs-pool:tank" + params := canonicalHealthAssessmentAlertParams{ + SpecID: resourceID + "-health", Signal: "zfs_pool", Codes: zfsPoolAssessmentCodes, + AlertID: buildCanonicalStateID(resourceID, resourceID+"-health"), + AlertType: "zfs-pool-state", SpecResourceID: resourceID, ResourceID: resourceID, + ResourceName: "tank", ResourceType: unifiedresources.ResourceTypeStorage, + } + observe := func(severity storagehealth.RiskLevel) { + t.Helper() + params.Reasons = []storagehealth.Reason{{Code: "zfs_pool_state", Severity: severity, Summary: "pool health"}} + if _, ok := m.syncCanonicalHealthAssessmentAlert(params); !ok { + t.Fatal("assessment rejected") + } + } + observe(storagehealth.RiskWarning) + observe(storagehealth.RiskWarning) + observe(storagehealth.RiskCritical) + observe(storagehealth.RiskCritical) + observe(storagehealth.RiskWarning) + if len(delivered) != 2 || delivered[0] != AlertLevelWarning || delivered[1] != AlertLevelCritical { + t.Fatalf("initial/escalated deliveries = %v", delivered) + } + // The two admitted notifications exhaust the configured hourly budget; + // oscillating back to critical must not bypass it. + observe(storagehealth.RiskCritical) + if len(delivered) != 2 { + t.Fatalf("rate limit bypassed: %v", delivered) + } + if a := testRequireActiveAlert(t, m, params.AlertID); a.Level != AlertLevelCritical { + t.Fatalf("rate-limited incident failed to update: %v", a.Level) + } +} diff --git a/internal/alerts/health_assessment.go b/internal/alerts/health_assessment.go index 852c27efc..b25cddcc1 100644 --- a/internal/alerts/health_assessment.go +++ b/internal/alerts/health_assessment.go @@ -187,6 +187,7 @@ func (m *Manager) syncCanonicalHealthAssessmentAlert(params canonicalHealthAsses return alertspecs.EvaluationResult{}, false } + severity := storageHealthAssessmentSeverity(params.Reasons) now := time.Now() return m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{ Spec: spec, @@ -194,7 +195,7 @@ func (m *Manager) syncCanonicalHealthAssessmentAlert(params canonicalHealthAsses ObservedAt: now, HealthAssessment: &alertspecs.HealthAssessmentEvidence{ Signal: params.Signal, - Severity: storageHealthAssessmentSeverity(params.Reasons), + Severity: severity, Codes: storageHealthReasonCodes(params.Reasons), }, }, @@ -209,5 +210,9 @@ func (m *Manager) syncCanonicalHealthAssessmentAlert(params canonicalHealthAsses AddToRecent: true, AddToHistory: true, MessageBuilder: params.MessageBuilder, + // Health assessments have warning/critical firing levels. Only upward + // transitions notify; unchanged observations and downgrades stay quiet. + NotifyOnSeverityChange: severity == alertspecs.AlertSeverityCritical, + RateLimit: true, }) } diff --git a/internal/alerts/host_unraid_lifecycle_test.go b/internal/alerts/host_unraid_lifecycle_test.go index 14b244e6d..3388516e3 100644 --- a/internal/alerts/host_unraid_lifecycle_test.go +++ b/internal/alerts/host_unraid_lifecycle_test.go @@ -207,3 +207,101 @@ func hasAlertType(alerts []Alert, alertType string) bool { } return false } + +func TestHostCustomSensorEscalationDelivery(t *testing.T) { + for _, policy := range []string{"ready", "acknowledged", "snoozed", "rate-limited", "flapping", "inactive"} { + t.Run(policy, func(t *testing.T) { + m := newTestManager(t) + cfg := m.GetConfig() + cfg.Enabled = true + cfg.ActivationState = ActivationActive + cfg.FlappingEnabled = false + cfg.Schedule.MaxAlertsHour = 0 + m.UpdateConfig(cfg) + var delivered []AlertLevel + m.SetAlertCallback(func(a *Alert) { + if a.Type == "custom-sensor" { + delivered = append(delivered, a.Level) + } + }) + host := models.Host{ID: "sensor-host", Hostname: "sensor-host", Sensors: models.HostSensorSummary{ + Custom: []models.HostCustomSensorMetric{{ID: "probe", Name: "Probe", Status: "warning", ObservedAt: time.Now()}}, + }} + m.CheckHost(host) + if len(delivered) != 1 || delivered[0] != AlertLevelWarning { + t.Fatalf("initial delivery = %v", delivered) + } + var id string + for _, a := range m.GetActiveAlerts() { + if a.Type == "custom-sensor" { + id = a.ID + } + } + if id == "" { + t.Fatal("missing custom sensor incident") + } + switch policy { + case "acknowledged": + if err := m.AcknowledgeAlert(id, "tester"); err != nil { + t.Fatal(err) + } + case "snoozed": + if err := m.SnoozeAlert(id, "tester", time.Now().Add(time.Hour)); err != nil { + t.Fatal(err) + } + case "rate-limited": + a := testRequireActiveAlert(t, m, id) + m.mu.Lock() + m.config.Schedule.MaxAlertsHour = 1 + m.alertRateLimit[canonicalTrackingKeyForAlert(a)] = []time.Time{time.Now()} + m.mu.Unlock() + case "flapping": + a := testRequireActiveAlert(t, m, id) + m.mu.Lock() + m.config.FlappingEnabled = true + m.suppressedUntil[canonicalTrackingKeyForAlert(a)] = time.Now().Add(time.Hour) + m.mu.Unlock() + case "inactive": + cfg.ActivationState = ActivationPending + m.UpdateConfig(cfg) + } + // Repeated warning observations must not resend. + m.CheckHost(host) + host.Sensors.Custom[0].Status = "critical" + m.CheckHost(host) + want := 1 + if policy == "ready" { + want = 2 + } + if len(delivered) != want { + t.Fatalf("warning->critical deliveries = %v, want %d callbacks", delivered, want) + } + if policy == "ready" && delivered[1] != AlertLevelCritical { + t.Fatalf("escalation = %v", delivered) + } + active := m.GetActiveAlerts() + found := false + for _, a := range active { + if a.Type == "custom-sensor" { + found = true + if a.ID != id || a.Level != AlertLevelCritical { + t.Fatalf("updated incident = %+v", a) + } + if policy == "acknowledged" && !a.Acknowledged { + t.Fatal("acknowledgement lost") + } + } + } + if !found { + t.Fatal("critical incident missing") + } + m.CheckHost(host) + host.Sensors.Custom[0].Status = "warning" + m.CheckHost(host) + m.CheckHost(host) + if len(delivered) != want { + t.Fatalf("unchanged/downgrade noise: %v", delivered) + } + }) + } +}