diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 4f3717b24..49dc1123c 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -813,6 +813,23 @@ local command or REST URL. `TestHostCustomSensorAlertLifecycle` and `internal/alerts/host_unraid_lifecycle_test.go` pin creation, recovery, opt-out, and cleanup. +Health-assessment warning→critical transitions dispatch the updated incident +through the normal acknowledgement, snooze, activation and flapping policy; +unchanged severity and critical→warning transitions do not dispatch. The shared +health-assessment caller opts into the configured per-incident hourly limit for +initial, refired and escalation notifications, so repeated severity oscillations +cannot bypass that budget. Incident severity still updates when delivery is +suppressed. This applies to all callers of the shared assessment adapter +(custom sensors, host storage/RAID and storage ZFS pool/device health), not other +stateful alert families. No delivery-policy bypass or separate sensor callback +is introduced. +`TestHostCustomSensorEscalationDelivery` in +`internal/alerts/host_unraid_lifecycle_test.go` pins the running-host callback, +stable incident identity and suppression/no-noise cases; +`TestHealthAssessmentEscalationDelivery` in +`internal/alerts/canonical_stateful_test.go` pins shared ZFS escalation and +hourly-budget exhaustion across warning/critical oscillation. + The alert resource-incident panel (`frontend-modern/src/features/alerts/AlertResourceIncidentsPanel.tsx`) dropped its "Open in Infrastructure / Workloads / Storage / Recovery" diff --git a/internal/alerts/canonical_stateful_test.go b/internal/alerts/canonical_stateful_test.go index 7920c9563..8bf50a00e 100644 --- a/internal/alerts/canonical_stateful_test.go +++ b/internal/alerts/canonical_stateful_test.go @@ -279,3 +279,47 @@ func TestStoragePolicyAliasesLegacyIdentity(t *testing.T) { }) } } + +// The shared assessment path also owns ZFS health, not only custom sensors. +func TestHealthAssessmentEscalationDelivery(t *testing.T) { + m := newTestManager(t) + cfg := m.GetConfig() + cfg.Enabled = true + cfg.ActivationState = ActivationActive + cfg.FlappingEnabled = false + cfg.Schedule.MaxAlertsHour = 2 + m.UpdateConfig(cfg) + var delivered []AlertLevel + m.SetAlertCallback(func(a *Alert) { delivered = append(delivered, a.Level) }) + resourceID := "storage-1/zfs-pool:tank" + params := canonicalHealthAssessmentAlertParams{ + SpecID: resourceID + "-health", Signal: "zfs_pool", Codes: zfsPoolAssessmentCodes, + AlertID: buildCanonicalStateID(resourceID, resourceID+"-health"), + AlertType: "zfs-pool-state", SpecResourceID: resourceID, ResourceID: resourceID, + ResourceName: "tank", ResourceType: unifiedresources.ResourceTypeStorage, + } + observe := func(severity storagehealth.RiskLevel) { + t.Helper() + params.Reasons = []storagehealth.Reason{{Code: "zfs_pool_state", Severity: severity, Summary: "pool health"}} + if _, ok := m.syncCanonicalHealthAssessmentAlert(params); !ok { + t.Fatal("assessment rejected") + } + } + observe(storagehealth.RiskWarning) + observe(storagehealth.RiskWarning) + observe(storagehealth.RiskCritical) + observe(storagehealth.RiskCritical) + observe(storagehealth.RiskWarning) + if len(delivered) != 2 || delivered[0] != AlertLevelWarning || delivered[1] != AlertLevelCritical { + t.Fatalf("initial/escalated deliveries = %v", delivered) + } + // The two admitted notifications exhaust the configured hourly budget; + // oscillating back to critical must not bypass it. + observe(storagehealth.RiskCritical) + if len(delivered) != 2 { + t.Fatalf("rate limit bypassed: %v", delivered) + } + if a := testRequireActiveAlert(t, m, params.AlertID); a.Level != AlertLevelCritical { + t.Fatalf("rate-limited incident failed to update: %v", a.Level) + } +} diff --git a/internal/alerts/health_assessment.go b/internal/alerts/health_assessment.go index 852c27efc..b25cddcc1 100644 --- a/internal/alerts/health_assessment.go +++ b/internal/alerts/health_assessment.go @@ -187,6 +187,7 @@ func (m *Manager) syncCanonicalHealthAssessmentAlert(params canonicalHealthAsses return alertspecs.EvaluationResult{}, false } + severity := storageHealthAssessmentSeverity(params.Reasons) now := time.Now() return m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{ Spec: spec, @@ -194,7 +195,7 @@ func (m *Manager) syncCanonicalHealthAssessmentAlert(params canonicalHealthAsses ObservedAt: now, HealthAssessment: &alertspecs.HealthAssessmentEvidence{ Signal: params.Signal, - Severity: storageHealthAssessmentSeverity(params.Reasons), + Severity: severity, Codes: storageHealthReasonCodes(params.Reasons), }, }, @@ -209,5 +210,9 @@ func (m *Manager) syncCanonicalHealthAssessmentAlert(params canonicalHealthAsses AddToRecent: true, AddToHistory: true, MessageBuilder: params.MessageBuilder, + // Health assessments have warning/critical firing levels. Only upward + // transitions notify; unchanged observations and downgrades stay quiet. + NotifyOnSeverityChange: severity == alertspecs.AlertSeverityCritical, + RateLimit: true, }) } diff --git a/internal/alerts/host_unraid_lifecycle_test.go b/internal/alerts/host_unraid_lifecycle_test.go index 14b244e6d..3388516e3 100644 --- a/internal/alerts/host_unraid_lifecycle_test.go +++ b/internal/alerts/host_unraid_lifecycle_test.go @@ -207,3 +207,101 @@ func hasAlertType(alerts []Alert, alertType string) bool { } return false } + +func TestHostCustomSensorEscalationDelivery(t *testing.T) { + for _, policy := range []string{"ready", "acknowledged", "snoozed", "rate-limited", "flapping", "inactive"} { + t.Run(policy, func(t *testing.T) { + m := newTestManager(t) + cfg := m.GetConfig() + cfg.Enabled = true + cfg.ActivationState = ActivationActive + cfg.FlappingEnabled = false + cfg.Schedule.MaxAlertsHour = 0 + m.UpdateConfig(cfg) + var delivered []AlertLevel + m.SetAlertCallback(func(a *Alert) { + if a.Type == "custom-sensor" { + delivered = append(delivered, a.Level) + } + }) + host := models.Host{ID: "sensor-host", Hostname: "sensor-host", Sensors: models.HostSensorSummary{ + Custom: []models.HostCustomSensorMetric{{ID: "probe", Name: "Probe", Status: "warning", ObservedAt: time.Now()}}, + }} + m.CheckHost(host) + if len(delivered) != 1 || delivered[0] != AlertLevelWarning { + t.Fatalf("initial delivery = %v", delivered) + } + var id string + for _, a := range m.GetActiveAlerts() { + if a.Type == "custom-sensor" { + id = a.ID + } + } + if id == "" { + t.Fatal("missing custom sensor incident") + } + switch policy { + case "acknowledged": + if err := m.AcknowledgeAlert(id, "tester"); err != nil { + t.Fatal(err) + } + case "snoozed": + if err := m.SnoozeAlert(id, "tester", time.Now().Add(time.Hour)); err != nil { + t.Fatal(err) + } + case "rate-limited": + a := testRequireActiveAlert(t, m, id) + m.mu.Lock() + m.config.Schedule.MaxAlertsHour = 1 + m.alertRateLimit[canonicalTrackingKeyForAlert(a)] = []time.Time{time.Now()} + m.mu.Unlock() + case "flapping": + a := testRequireActiveAlert(t, m, id) + m.mu.Lock() + m.config.FlappingEnabled = true + m.suppressedUntil[canonicalTrackingKeyForAlert(a)] = time.Now().Add(time.Hour) + m.mu.Unlock() + case "inactive": + cfg.ActivationState = ActivationPending + m.UpdateConfig(cfg) + } + // Repeated warning observations must not resend. + m.CheckHost(host) + host.Sensors.Custom[0].Status = "critical" + m.CheckHost(host) + want := 1 + if policy == "ready" { + want = 2 + } + if len(delivered) != want { + t.Fatalf("warning->critical deliveries = %v, want %d callbacks", delivered, want) + } + if policy == "ready" && delivered[1] != AlertLevelCritical { + t.Fatalf("escalation = %v", delivered) + } + active := m.GetActiveAlerts() + found := false + for _, a := range active { + if a.Type == "custom-sensor" { + found = true + if a.ID != id || a.Level != AlertLevelCritical { + t.Fatalf("updated incident = %+v", a) + } + if policy == "acknowledged" && !a.Acknowledged { + t.Fatal("acknowledgement lost") + } + } + } + if !found { + t.Fatal("critical incident missing") + } + m.CheckHost(host) + host.Sensors.Custom[0].Status = "warning" + m.CheckHost(host) + m.CheckHost(host) + if len(delivered) != want { + t.Fatalf("unchanged/downgrade noise: %v", delivered) + } + }) + } +}