Merge pull request #1902 from rcourtman/maintainer/20260905T040121Z-release-v6.4

Notify when provider incidents become critical

docs/release-control/v6/internal/subsystems/alerts.md

@@ -29,6 +29,17 @@ and preservation of provider resource evidence.
 native projection, NOTICE activation without severity inflation, and confirmed
 recovery to INFO.
 
+Existing provider incidents that transition to critical severity dispatch again
+through the normal alert delivery path, preserving incident ID and start time.
+Acknowledgement and the canonical incident notification-rate budget still gate
+that dispatch; repeated critical observations and severity downgrades do not
+notify again. Confirmed provider recovery retains the same lifecycle identity.
+`TestTrueNASNativeCriticalTransition` in
+`internal/alerts/unified_incidents_test.go` pins the native TrueNAS
+WARNING → EMERGENCY → WARNING → INFO sequence, including repeated critical
+observations, acknowledgement suppression, and exhausted-budget suppression.
+This is callback-level regression proof, not external notification receipt.
+
 Own alert identity, alert specs, evaluation, persistence semantics, and
 operator-facing alert routing behavior for live runtime alerts.
 Alert-history investigations on narrow layouts render in a shared drawer

internal/alerts/unified_incidents.go

@@ -254,6 +254,7 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso
 
 	for storageKey, alert := range desired {
 		if existing, exists := m.getActiveAlertNoLock(storageKey); exists && existing != nil {
+			oldLevel := existing.Level
 			existing.LastSeen = alert.LastSeen
 			existing.Level = alert.Level
 			existing.ResourceID = alert.ResourceID
@@ -281,6 +282,14 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso
 			}
 			applyCanonicalIdentity(existing, alert.CanonicalSpecID, alert.CanonicalKind)
 			m.setActiveAlertNoLock(storageKey, existing)
+			// Like metric alerts, an existing provider incident becoming critical
+			// must notify again. Keep its lifecycle and all delivery policy gates;
+			// unchanged severity and downgrades must not create notification noise.
+			if oldLevel != existing.Level && existing.Level == AlertLevelCritical &&
+				!existing.Acknowledged &&
+				m.allowNotificationByRateLimit(storageKey, existing, "critical-escalation") {
+				m.dispatchAlert(existing, false)
+			}
 			continue
 		}
 

internal/alerts/unified_incidents_test.go

@@ -1253,3 +1253,82 @@ func TestSyncUnifiedResourceIncidentsTrueNASNativeRecoveryStreak(t *testing.T) {
 		}
 	}
 }
+
+// A severity change must retain the incident identity without silently losing
+// the critical notification or dispatching on every subsequent observation.
+func TestTrueNASNativeCriticalTransition(t *testing.T) {
+	for _, gate := range []string{"active", "acknowledged", "rate-limited"} {
+		t.Run(gate, func(t *testing.T) {
+			m := newTestManager(t)
+			configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
+			var dispatched []Alert
+			m.SetAlertCallback(func(a *Alert) { dispatched = append(dispatched, *a) })
+			resolved := make(chan string, 4)
+			m.SetResolvedCallback(func(id string) { resolved <- id })
+			syncLevel := func(level string) {
+				records := truenas.FixtureRecords(truenas.FixtureSnapshot{
+					CollectedAt: time.Now(),
+					System:      truenas.SystemInfo{Hostname: "native-transition", Healthy: true},
+					Alerts:      []truenas.Alert{{ID: "native-1", Level: level, Message: "Provider condition"}},
+				})
+				var resources []unifiedresources.Resource
+				for _, record := range records {
+					record.Resource.ID = "host:" + record.SourceID
+					resources = append(resources, record.Resource)
+				}
+				m.SyncUnifiedResourceIncidents(resources)
+			}
+			syncLevel("WARNING")
+			if len(dispatched) != 1 {
+				t.Fatalf("initial dispatches = %d", len(dispatched))
+			}
+			initial := dispatched[0]
+			if gate != "active" {
+				m.mu.Lock()
+				for key, a := range m.activeAlerts {
+					if gate == "acknowledged" {
+						a.Acknowledged = true
+					} else {
+						// Exhaust the canonical incident budget before escalation.
+						m.config.Schedule.MaxAlertsHour = 1
+						m.alertRateLimit[key] = []time.Time{time.Now()}
+					}
+				}
+				m.mu.Unlock()
+			}
+			syncLevel("EMERGENCY")
+			syncLevel("EMERGENCY")
+			active := m.GetActiveAlerts()
+			if len(active) != 1 || active[0].ID != initial.ID || active[0].Level != AlertLevelCritical || !active[0].StartTime.Equal(initial.StartTime) {
+				t.Fatalf("critical transition lost lifecycle identity: %+v", active)
+			}
+			want := 2
+			if gate != "active" {
+				want = 1
+			}
+			if len(dispatched) != want {
+				t.Fatalf("dispatches = %d, want %d", len(dispatched), want)
+			}
+			if gate == "active" && (dispatched[1].ID != initial.ID || dispatched[1].Level != AlertLevelCritical) {
+				t.Fatal("critical notification lost severity or identity")
+			}
+			syncLevel("WARNING")
+			if len(dispatched) != want {
+				t.Fatal("downgrade dispatched")
+			}
+			syncLevel("INFO")
+			syncLevel("INFO")
+			if len(m.GetActiveAlerts()) != 0 {
+				t.Fatal("confirmed recovery retained incident")
+			}
+			select {
+			case id := <-resolved:
+				if id != initial.ID {
+					t.Fatalf("recovery ID = %s, want %s", id, initial.ID)
+				}
+			case <-time.After(2 * time.Second):
+				t.Fatal("missing recovery callback")
+			}
+		})
+	}
+}