fix(alerts): notify on health assessment escalation

A running custom sensor updated its canonical incident from warning to critical without dispatching the new severity (#1801). Enable upward transitions in the shared health adapter while keeping unchanged observations and downgrades quiet. Use the existing hourly budget and dispatch suppression rather than bypassing policy.

Pin CheckHost callback delivery, acknowledgement, snooze, inactive and flapping suppression, and shared ZFS rate exhaustion with focused regressions.

Change-source: pulse-maintainer

docs/release-control/v6/internal/subsystems/alerts.md

@@ -813,6 +813,23 @@ local command or REST URL. `TestHostCustomSensorAlertLifecycle` and
 `internal/alerts/host_unraid_lifecycle_test.go` pin creation, recovery,
 opt-out, and cleanup.
 
+Health-assessment warning→critical transitions dispatch the updated incident
+through the normal acknowledgement, snooze, activation and flapping policy;
+unchanged severity and critical→warning transitions do not dispatch. The shared
+health-assessment caller opts into the configured per-incident hourly limit for
+initial, refired and escalation notifications, so repeated severity oscillations
+cannot bypass that budget. Incident severity still updates when delivery is
+suppressed. This applies to all callers of the shared assessment adapter
+(custom sensors, host storage/RAID and storage ZFS pool/device health), not other
+stateful alert families. No delivery-policy bypass or separate sensor callback
+is introduced.
+`TestHostCustomSensorEscalationDelivery` in
+`internal/alerts/host_unraid_lifecycle_test.go` pins the running-host callback,
+stable incident identity and suppression/no-noise cases;
+`TestHealthAssessmentEscalationDelivery` in
+`internal/alerts/canonical_stateful_test.go` pins shared ZFS escalation and
+hourly-budget exhaustion across warning/critical oscillation.
+
 The alert resource-incident panel
 (`frontend-modern/src/features/alerts/AlertResourceIncidentsPanel.tsx`)
 dropped its "Open in Infrastructure / Workloads / Storage / Recovery"

internal/alerts/canonical_stateful_test.go

@@ -279,3 +279,47 @@ func TestStoragePolicyAliasesLegacyIdentity(t *testing.T) {
 		})
 	}
 }
+
+// The shared assessment path also owns ZFS health, not only custom sensors.
+func TestHealthAssessmentEscalationDelivery(t *testing.T) {
+	m := newTestManager(t)
+	cfg := m.GetConfig()
+	cfg.Enabled = true
+	cfg.ActivationState = ActivationActive
+	cfg.FlappingEnabled = false
+	cfg.Schedule.MaxAlertsHour = 2
+	m.UpdateConfig(cfg)
+	var delivered []AlertLevel
+	m.SetAlertCallback(func(a *Alert) { delivered = append(delivered, a.Level) })
+	resourceID := "storage-1/zfs-pool:tank"
+	params := canonicalHealthAssessmentAlertParams{
+		SpecID: resourceID + "-health", Signal: "zfs_pool", Codes: zfsPoolAssessmentCodes,
+		AlertID:   buildCanonicalStateID(resourceID, resourceID+"-health"),
+		AlertType: "zfs-pool-state", SpecResourceID: resourceID, ResourceID: resourceID,
+		ResourceName: "tank", ResourceType: unifiedresources.ResourceTypeStorage,
+	}
+	observe := func(severity storagehealth.RiskLevel) {
+		t.Helper()
+		params.Reasons = []storagehealth.Reason{{Code: "zfs_pool_state", Severity: severity, Summary: "pool health"}}
+		if _, ok := m.syncCanonicalHealthAssessmentAlert(params); !ok {
+			t.Fatal("assessment rejected")
+		}
+	}
+	observe(storagehealth.RiskWarning)
+	observe(storagehealth.RiskWarning)
+	observe(storagehealth.RiskCritical)
+	observe(storagehealth.RiskCritical)
+	observe(storagehealth.RiskWarning)
+	if len(delivered) != 2 || delivered[0] != AlertLevelWarning || delivered[1] != AlertLevelCritical {
+		t.Fatalf("initial/escalated deliveries = %v", delivered)
+	}
+	// The two admitted notifications exhaust the configured hourly budget;
+	// oscillating back to critical must not bypass it.
+	observe(storagehealth.RiskCritical)
+	if len(delivered) != 2 {
+		t.Fatalf("rate limit bypassed: %v", delivered)
+	}
+	if a := testRequireActiveAlert(t, m, params.AlertID); a.Level != AlertLevelCritical {
+		t.Fatalf("rate-limited incident failed to update: %v", a.Level)
+	}
+}

internal/alerts/health_assessment.go

@@ -187,6 +187,7 @@ func (m *Manager) syncCanonicalHealthAssessmentAlert(params canonicalHealthAsses
 		return alertspecs.EvaluationResult{}, false
 	}
 
+	severity := storageHealthAssessmentSeverity(params.Reasons)
 	now := time.Now()
 	return m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
 		Spec: spec,
@@ -194,7 +195,7 @@ func (m *Manager) syncCanonicalHealthAssessmentAlert(params canonicalHealthAsses
 			ObservedAt: now,
 			HealthAssessment: &alertspecs.HealthAssessmentEvidence{
 				Signal:   params.Signal,
-				Severity: storageHealthAssessmentSeverity(params.Reasons),
+				Severity: severity,
 				Codes:    storageHealthReasonCodes(params.Reasons),
 			},
 		},
@@ -209,5 +210,9 @@ func (m *Manager) syncCanonicalHealthAssessmentAlert(params canonicalHealthAsses
 		AddToRecent:    true,
 		AddToHistory:   true,
 		MessageBuilder: params.MessageBuilder,
+		// Health assessments have warning/critical firing levels. Only upward
+		// transitions notify; unchanged observations and downgrades stay quiet.
+		NotifyOnSeverityChange: severity == alertspecs.AlertSeverityCritical,
+		RateLimit:              true,
 	})
 }

internal/alerts/host_unraid_lifecycle_test.go

@@ -207,3 +207,101 @@ func hasAlertType(alerts []Alert, alertType string) bool {
 	}
 	return false
 }
+
+func TestHostCustomSensorEscalationDelivery(t *testing.T) {
+	for _, policy := range []string{"ready", "acknowledged", "snoozed", "rate-limited", "flapping", "inactive"} {
+		t.Run(policy, func(t *testing.T) {
+			m := newTestManager(t)
+			cfg := m.GetConfig()
+			cfg.Enabled = true
+			cfg.ActivationState = ActivationActive
+			cfg.FlappingEnabled = false
+			cfg.Schedule.MaxAlertsHour = 0
+			m.UpdateConfig(cfg)
+			var delivered []AlertLevel
+			m.SetAlertCallback(func(a *Alert) {
+				if a.Type == "custom-sensor" {
+					delivered = append(delivered, a.Level)
+				}
+			})
+			host := models.Host{ID: "sensor-host", Hostname: "sensor-host", Sensors: models.HostSensorSummary{
+				Custom: []models.HostCustomSensorMetric{{ID: "probe", Name: "Probe", Status: "warning", ObservedAt: time.Now()}},
+			}}
+			m.CheckHost(host)
+			if len(delivered) != 1 || delivered[0] != AlertLevelWarning {
+				t.Fatalf("initial delivery = %v", delivered)
+			}
+			var id string
+			for _, a := range m.GetActiveAlerts() {
+				if a.Type == "custom-sensor" {
+					id = a.ID
+				}
+			}
+			if id == "" {
+				t.Fatal("missing custom sensor incident")
+			}
+			switch policy {
+			case "acknowledged":
+				if err := m.AcknowledgeAlert(id, "tester"); err != nil {
+					t.Fatal(err)
+				}
+			case "snoozed":
+				if err := m.SnoozeAlert(id, "tester", time.Now().Add(time.Hour)); err != nil {
+					t.Fatal(err)
+				}
+			case "rate-limited":
+				a := testRequireActiveAlert(t, m, id)
+				m.mu.Lock()
+				m.config.Schedule.MaxAlertsHour = 1
+				m.alertRateLimit[canonicalTrackingKeyForAlert(a)] = []time.Time{time.Now()}
+				m.mu.Unlock()
+			case "flapping":
+				a := testRequireActiveAlert(t, m, id)
+				m.mu.Lock()
+				m.config.FlappingEnabled = true
+				m.suppressedUntil[canonicalTrackingKeyForAlert(a)] = time.Now().Add(time.Hour)
+				m.mu.Unlock()
+			case "inactive":
+				cfg.ActivationState = ActivationPending
+				m.UpdateConfig(cfg)
+			}
+			// Repeated warning observations must not resend.
+			m.CheckHost(host)
+			host.Sensors.Custom[0].Status = "critical"
+			m.CheckHost(host)
+			want := 1
+			if policy == "ready" {
+				want = 2
+			}
+			if len(delivered) != want {
+				t.Fatalf("warning->critical deliveries = %v, want %d callbacks", delivered, want)
+			}
+			if policy == "ready" && delivered[1] != AlertLevelCritical {
+				t.Fatalf("escalation = %v", delivered)
+			}
+			active := m.GetActiveAlerts()
+			found := false
+			for _, a := range active {
+				if a.Type == "custom-sensor" {
+					found = true
+					if a.ID != id || a.Level != AlertLevelCritical {
+						t.Fatalf("updated incident = %+v", a)
+					}
+					if policy == "acknowledged" && !a.Acknowledged {
+						t.Fatal("acknowledgement lost")
+					}
+				}
+			}
+			if !found {
+				t.Fatal("critical incident missing")
+			}
+			m.CheckHost(host)
+			host.Sensors.Custom[0].Status = "warning"
+			m.CheckHost(host)
+			m.CheckHost(host)
+			if len(delivered) != want {
+				t.Fatalf("unchanged/downgrade noise: %v", delivered)
+			}
+		})
+	}
+}