diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index bb592212c..9049c7dcf 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -29,6 +29,17 @@ and preservation of provider resource evidence. native projection, NOTICE activation without severity inflation, and confirmed recovery to INFO. +Existing provider incidents that transition to critical severity dispatch again +through the normal alert delivery path, preserving incident ID and start time. +Acknowledgement and the canonical incident notification-rate budget still gate +that dispatch; repeated critical observations and severity downgrades do not +notify again. Confirmed provider recovery retains the same lifecycle identity. +`TestTrueNASNativeCriticalTransition` in +`internal/alerts/unified_incidents_test.go` pins the native TrueNAS +WARNING → EMERGENCY → WARNING → INFO sequence, including repeated critical +observations, acknowledgement suppression, and exhausted-budget suppression. +This is callback-level regression proof, not external notification receipt. + Own alert identity, alert specs, evaluation, persistence semantics, and operator-facing alert routing behavior for live runtime alerts. Alert-history investigations on narrow layouts render in a shared drawer diff --git a/internal/alerts/unified_incidents.go b/internal/alerts/unified_incidents.go index 93e1aabe6..4130235da 100644 --- a/internal/alerts/unified_incidents.go +++ b/internal/alerts/unified_incidents.go @@ -254,6 +254,7 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso for storageKey, alert := range desired { if existing, exists := m.getActiveAlertNoLock(storageKey); exists && existing != nil { + oldLevel := existing.Level existing.LastSeen = alert.LastSeen existing.Level = alert.Level existing.ResourceID = alert.ResourceID @@ -281,6 +282,14 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso } applyCanonicalIdentity(existing, alert.CanonicalSpecID, alert.CanonicalKind) m.setActiveAlertNoLock(storageKey, existing) + // Like metric alerts, an existing provider incident becoming critical + // must notify again. Keep its lifecycle and all delivery policy gates; + // unchanged severity and downgrades must not create notification noise. + if oldLevel != existing.Level && existing.Level == AlertLevelCritical && + !existing.Acknowledged && + m.allowNotificationByRateLimit(storageKey, existing, "critical-escalation") { + m.dispatchAlert(existing, false) + } continue } diff --git a/internal/alerts/unified_incidents_test.go b/internal/alerts/unified_incidents_test.go index 7eb2b75d3..8bcac0c40 100644 --- a/internal/alerts/unified_incidents_test.go +++ b/internal/alerts/unified_incidents_test.go @@ -1253,3 +1253,82 @@ func TestSyncUnifiedResourceIncidentsTrueNASNativeRecoveryStreak(t *testing.T) { } } } + +// A severity change must retain the incident identity without silently losing +// the critical notification or dispatching on every subsequent observation. +func TestTrueNASNativeCriticalTransition(t *testing.T) { + for _, gate := range []string{"active", "acknowledged", "rate-limited"} { + t.Run(gate, func(t *testing.T) { + m := newTestManager(t) + configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig()) + var dispatched []Alert + m.SetAlertCallback(func(a *Alert) { dispatched = append(dispatched, *a) }) + resolved := make(chan string, 4) + m.SetResolvedCallback(func(id string) { resolved <- id }) + syncLevel := func(level string) { + records := truenas.FixtureRecords(truenas.FixtureSnapshot{ + CollectedAt: time.Now(), + System: truenas.SystemInfo{Hostname: "native-transition", Healthy: true}, + Alerts: []truenas.Alert{{ID: "native-1", Level: level, Message: "Provider condition"}}, + }) + var resources []unifiedresources.Resource + for _, record := range records { + record.Resource.ID = "host:" + record.SourceID + resources = append(resources, record.Resource) + } + m.SyncUnifiedResourceIncidents(resources) + } + syncLevel("WARNING") + if len(dispatched) != 1 { + t.Fatalf("initial dispatches = %d", len(dispatched)) + } + initial := dispatched[0] + if gate != "active" { + m.mu.Lock() + for key, a := range m.activeAlerts { + if gate == "acknowledged" { + a.Acknowledged = true + } else { + // Exhaust the canonical incident budget before escalation. + m.config.Schedule.MaxAlertsHour = 1 + m.alertRateLimit[key] = []time.Time{time.Now()} + } + } + m.mu.Unlock() + } + syncLevel("EMERGENCY") + syncLevel("EMERGENCY") + active := m.GetActiveAlerts() + if len(active) != 1 || active[0].ID != initial.ID || active[0].Level != AlertLevelCritical || !active[0].StartTime.Equal(initial.StartTime) { + t.Fatalf("critical transition lost lifecycle identity: %+v", active) + } + want := 2 + if gate != "active" { + want = 1 + } + if len(dispatched) != want { + t.Fatalf("dispatches = %d, want %d", len(dispatched), want) + } + if gate == "active" && (dispatched[1].ID != initial.ID || dispatched[1].Level != AlertLevelCritical) { + t.Fatal("critical notification lost severity or identity") + } + syncLevel("WARNING") + if len(dispatched) != want { + t.Fatal("downgrade dispatched") + } + syncLevel("INFO") + syncLevel("INFO") + if len(m.GetActiveAlerts()) != 0 { + t.Fatal("confirmed recovery retained incident") + } + select { + case id := <-resolved: + if id != initial.ID { + t.Fatalf("recovery ID = %s, want %s", id, initial.ID) + } + case <-time.After(2 * time.Second): + t.Fatal("missing recovery callback") + } + }) + } +}