diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index da1a40fbfc..60d37911dd 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -2833,11 +2833,16 @@ notification delivery, with the typed `correlated_primary` reason and primary alert ID in the event log. The primary remains the one delivery owner. Supporting members also skip escalation levels and critical repeats while that supported cause is current; an already queued escalation is revalidated before delivery. +If the supported primary recovers or the causal link is lost while its symptom +still fails, the symptom regains independent delivery without a new occurrence. +It receives its first notification only if it was never dispatched before +grouping; an already-notified symptom does not duplicate on group dissolution. +Acknowledgement, rate limits and other ordinary delivery gates still apply. `observation-set` members continue to notify independently because Pulse has not established causality. `incident_synthesis_test.go` pins classification, contradiction downgrade, bounded evidence, duplicate-delivery suppression, -partial-recovery behavior, escalation suppression, and queued-callback -revalidation without muting observation sets. +partial-recovery delivery and no duplicate after prior dispatch, escalation +suppression, and queued-callback revalidation without muting observation sets. ### Alert hydration is not resource admission diff --git a/internal/alerts/incident_synthesis_test.go b/internal/alerts/incident_synthesis_test.go index 33cc09d830..e1db7511ea 100644 --- a/internal/alerts/incident_synthesis_test.go +++ b/internal/alerts/incident_synthesis_test.go @@ -235,6 +235,113 @@ func TestInfrastructureIncidentSynthesisShowsPartialRecovery(t *testing.T) { } } +func TestSupportedSymptomNotifiesWhenPrimaryRecoversButSymptomPersists(t *testing.T) { + manager := newTestManager(t) + configureUnifiedEvalManager(t, manager, unifiedEvalBaseConfig()) + startedAt := time.Date(2026, 9, 29, 18, 0, 0, 0, time.UTC) + hostID := "agent:edge-1" + endpointID := "availability:checkout" + host := unifiedresources.Resource{ + ID: hostID, Type: unifiedresources.ResourceTypeAgent, Name: "Edge host", Status: unifiedresources.StatusOffline, + Incidents: []unifiedresources.ResourceIncident{{ + Provider: "agent", NativeID: "edge-1", Code: "agent_offline", Severity: storagehealth.RiskCritical, + Source: "agent.heartbeat", Summary: "Edge host is offline", StartedAt: startedAt, + ConfirmationsRequired: 1, RecoveryConfirmationsRequired: 1, + }}, + } + endpoint := unifiedresources.Resource{ + ID: endpointID, Type: unifiedresources.ResourceTypeNetworkEndpoint, Name: "Checkout", Status: unifiedresources.StatusOffline, + Availability: &unifiedresources.AvailabilityData{AggregateState: "unavailable", TransportOutcome: "unreachable"}, + Relationships: []unifiedresources.ResourceRelationship{{ + SourceID: endpointID, TargetID: hostID, Type: unifiedresources.RelChecks, Confidence: 1, Active: true, + }}, + Incidents: []unifiedresources.ResourceIncident{{ + Provider: "availability", NativeID: "checkout", Code: "availability_unreachable", Severity: storagehealth.RiskCritical, + Source: "availability.probe", Summary: "Checkout is unreachable", StartedAt: startedAt.Add(time.Second), + ConfirmationsRequired: 1, RecoveryConfirmationsRequired: 1, + }}, + } + var delivered []string + manager.SetAlertCallback(func(alert *Alert) { delivered = append(delivered, alert.ResourceID) }) + + manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint}) + if len(delivered) != 1 || delivered[0] != hostID { + t.Fatalf("grouped delivery = %v, want only the primary host", delivered) + } + for _, alert := range manager.GetActiveAlerts() { + if alert.ResourceID == endpointID && (!isSupportedInfrastructureSymptom(&alert) || alert.LastNotified != nil) { + t.Fatalf("grouped symptom = %+v, want supported and not separately dispatched", alert) + } + } + + // The host is healthy again, but the endpoint still fails. It must regain + // its own notification path without requiring a new alert occurrence. + host.Status = unifiedresources.StatusOnline + host.Incidents = nil + manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint}) + if len(delivered) != 2 || delivered[1] != endpointID { + t.Fatalf("after primary recovery delivery = %v, want continuing symptom", delivered) + } + manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint}) + if len(delivered) != 2 { + t.Fatalf("unchanged independent symptom sent again: %v", delivered) + } +} + +func TestPreviouslyNotifiedSymptomDoesNotDuplicateWhenPrimaryRecovers(t *testing.T) { + manager := newTestManager(t) + configureUnifiedEvalManager(t, manager, unifiedEvalBaseConfig()) + startedAt := time.Date(2026, 9, 29, 18, 0, 0, 0, time.UTC) + hostID := "agent:edge-1" + endpointID := "availability:checkout" + host := unifiedresources.Resource{ + ID: hostID, Type: unifiedresources.ResourceTypeAgent, Status: unifiedresources.StatusOnline, + } + endpoint := unifiedresources.Resource{ + ID: endpointID, Type: unifiedresources.ResourceTypeNetworkEndpoint, Status: unifiedresources.StatusOffline, + Availability: &unifiedresources.AvailabilityData{AggregateState: "unavailable", TransportOutcome: "unreachable"}, + Relationships: []unifiedresources.ResourceRelationship{{ + SourceID: endpointID, TargetID: hostID, Type: unifiedresources.RelChecks, Confidence: 1, Active: true, + }}, + Incidents: []unifiedresources.ResourceIncident{{ + Provider: "availability", NativeID: "checkout", Code: "availability_unreachable", Severity: storagehealth.RiskCritical, + Source: "availability.probe", StartedAt: startedAt.Add(time.Second), ConfirmationsRequired: 1, RecoveryConfirmationsRequired: 1, + }}, + } + var delivered []string + manager.SetAlertCallback(func(alert *Alert) { delivered = append(delivered, alert.ResourceID) }) + + manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint}) + if len(delivered) != 1 || delivered[0] != endpointID { + t.Fatalf("standalone symptom delivery = %v, want endpoint", delivered) + } + host.Status = unifiedresources.StatusOffline + host.Incidents = []unifiedresources.ResourceIncident{{ + Provider: "agent", NativeID: "edge-1", Code: "agent_offline", Severity: storagehealth.RiskCritical, + Source: "agent.heartbeat", StartedAt: startedAt, ConfirmationsRequired: 1, RecoveryConfirmationsRequired: 1, + }} + manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint}) + if len(delivered) != 2 || delivered[1] != hostID { + t.Fatalf("new primary delivery = %v, want endpoint then host", delivered) + } + var grouped bool + for _, alert := range manager.GetActiveAlerts() { + if alert.ResourceID == endpointID { + grouped = isSupportedInfrastructureSymptom(&alert) + } + } + if !grouped { + t.Fatal("endpoint did not gain a supported primary") + } + + host.Status = unifiedresources.StatusOnline + host.Incidents = nil + manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint}) + if len(delivered) != 2 { + t.Fatalf("already-notified endpoint duplicated after primary recovery: %v", delivered) + } +} + func TestClassifyIncidentFailureSeparatesApplicationCertificateAndCoverage(t *testing.T) { applicationResource := unifiedresources.Resource{Availability: &unifiedresources.AvailabilityData{ AggregateState: "unavailable", TransportOutcome: "reachable", ApplicationOutcome: "failed", diff --git a/internal/alerts/unified_incidents.go b/internal/alerts/unified_incidents.go index da4fd8f9d3..b27011dffc 100644 --- a/internal/alerts/unified_incidents.go +++ b/internal/alerts/unified_incidents.go @@ -260,6 +260,17 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso m.clearAlertNoLock(storageKey) } + // Synthesis replaces its previous links in place. Remember which retained + // incidents had a primary delivery owner before that information is cleared. + // If the primary recovers while a symptom remains active, the symptom needs + // its own first notification even though its occurrence did not restart. + previouslySupported := make(map[string]bool) + for storageKey := range desired { + if existing, exists := m.getActiveAlertNoLock(storageKey); exists && isSupportedInfrastructureSymptom(existing) { + previouslySupported[storageKey] = true + } + } + // Synthesize over the post-recovery live set before updating or dispatching // any detector. Supporting symptoms retain their independent lifecycle but // can share one evidence-backed incident presentation and delivery decision. @@ -297,12 +308,19 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso if !m.setActiveAlertNoLock(storageKey, existing) { continue } - // Like metric alerts, an existing provider incident becoming critical - // must notify again. Keep its lifecycle and all delivery policy gates; - // unchanged severity and downgrades must not create notification noise. - if oldLevel != existing.Level && existing.Level == AlertLevelCritical && - !existing.Acknowledged && - m.allowNotificationByRateLimit(storageKey, existing, "critical-escalation") { + // A still-failing symptom which loses its supported primary needs its + // first notification. An alert already dispatched before grouping must + // not be duplicated when the group dissolves. As with metric alerts, a + // separate increase to critical may also notify through normal gates. + newlyIndependent := previouslySupported[storageKey] && + !isSupportedInfrastructureSymptom(existing) && existing.LastNotified == nil + becameCritical := oldLevel != existing.Level && existing.Level == AlertLevelCritical + notificationReason := "critical-escalation" + if newlyIndependent { + notificationReason = "supported-primary-ended" + } + if (newlyIndependent || becameCritical) && !existing.Acknowledged && + m.allowNotificationByRateLimit(storageKey, existing, notificationReason) { m.dispatchAlert(existing, false) } continue