Notify continuing symptoms after primary recovery

A supported endpoint failure may outlive its primary host incident. When synthesis removes the supported cause, dispatch its first independent notification without reopening the occurrence or duplicating a symptom previously notified before grouping.

Change-source: pulse-maintainer
This commit is contained in:
pulse-triage[bot]
2026-09-29 21:18:19 +01:00
parent 90556cc1fc
commit fd9e836ca4
3 changed files with 138 additions and 8 deletions
@@ -2833,11 +2833,16 @@ notification delivery, with the typed `correlated_primary` reason and primary
alert ID in the event log. The primary remains the one delivery owner. Supporting
members also skip escalation levels and critical repeats while that supported
cause is current; an already queued escalation is revalidated before delivery.
If the supported primary recovers or the causal link is lost while its symptom
still fails, the symptom regains independent delivery without a new occurrence.
It receives its first notification only if it was never dispatched before
grouping; an already-notified symptom does not duplicate on group dissolution.
Acknowledgement, rate limits and other ordinary delivery gates still apply.
`observation-set` members continue to notify independently because Pulse has
not established causality. `incident_synthesis_test.go` pins classification,
contradiction downgrade, bounded evidence, duplicate-delivery suppression,
partial-recovery behavior, escalation suppression, and queued-callback
revalidation without muting observation sets.
partial-recovery delivery and no duplicate after prior dispatch, escalation
suppression, and queued-callback revalidation without muting observation sets.
### Alert hydration is not resource admission
+107
View File
@@ -235,6 +235,113 @@ func TestInfrastructureIncidentSynthesisShowsPartialRecovery(t *testing.T) {
}
}
func TestSupportedSymptomNotifiesWhenPrimaryRecoversButSymptomPersists(t *testing.T) {
manager := newTestManager(t)
configureUnifiedEvalManager(t, manager, unifiedEvalBaseConfig())
startedAt := time.Date(2026, 9, 29, 18, 0, 0, 0, time.UTC)
hostID := "agent:edge-1"
endpointID := "availability:checkout"
host := unifiedresources.Resource{
ID: hostID, Type: unifiedresources.ResourceTypeAgent, Name: "Edge host", Status: unifiedresources.StatusOffline,
Incidents: []unifiedresources.ResourceIncident{{
Provider: "agent", NativeID: "edge-1", Code: "agent_offline", Severity: storagehealth.RiskCritical,
Source: "agent.heartbeat", Summary: "Edge host is offline", StartedAt: startedAt,
ConfirmationsRequired: 1, RecoveryConfirmationsRequired: 1,
}},
}
endpoint := unifiedresources.Resource{
ID: endpointID, Type: unifiedresources.ResourceTypeNetworkEndpoint, Name: "Checkout", Status: unifiedresources.StatusOffline,
Availability: &unifiedresources.AvailabilityData{AggregateState: "unavailable", TransportOutcome: "unreachable"},
Relationships: []unifiedresources.ResourceRelationship{{
SourceID: endpointID, TargetID: hostID, Type: unifiedresources.RelChecks, Confidence: 1, Active: true,
}},
Incidents: []unifiedresources.ResourceIncident{{
Provider: "availability", NativeID: "checkout", Code: "availability_unreachable", Severity: storagehealth.RiskCritical,
Source: "availability.probe", Summary: "Checkout is unreachable", StartedAt: startedAt.Add(time.Second),
ConfirmationsRequired: 1, RecoveryConfirmationsRequired: 1,
}},
}
var delivered []string
manager.SetAlertCallback(func(alert *Alert) { delivered = append(delivered, alert.ResourceID) })
manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint})
if len(delivered) != 1 || delivered[0] != hostID {
t.Fatalf("grouped delivery = %v, want only the primary host", delivered)
}
for _, alert := range manager.GetActiveAlerts() {
if alert.ResourceID == endpointID && (!isSupportedInfrastructureSymptom(&alert) || alert.LastNotified != nil) {
t.Fatalf("grouped symptom = %+v, want supported and not separately dispatched", alert)
}
}
// The host is healthy again, but the endpoint still fails. It must regain
// its own notification path without requiring a new alert occurrence.
host.Status = unifiedresources.StatusOnline
host.Incidents = nil
manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint})
if len(delivered) != 2 || delivered[1] != endpointID {
t.Fatalf("after primary recovery delivery = %v, want continuing symptom", delivered)
}
manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint})
if len(delivered) != 2 {
t.Fatalf("unchanged independent symptom sent again: %v", delivered)
}
}
func TestPreviouslyNotifiedSymptomDoesNotDuplicateWhenPrimaryRecovers(t *testing.T) {
manager := newTestManager(t)
configureUnifiedEvalManager(t, manager, unifiedEvalBaseConfig())
startedAt := time.Date(2026, 9, 29, 18, 0, 0, 0, time.UTC)
hostID := "agent:edge-1"
endpointID := "availability:checkout"
host := unifiedresources.Resource{
ID: hostID, Type: unifiedresources.ResourceTypeAgent, Status: unifiedresources.StatusOnline,
}
endpoint := unifiedresources.Resource{
ID: endpointID, Type: unifiedresources.ResourceTypeNetworkEndpoint, Status: unifiedresources.StatusOffline,
Availability: &unifiedresources.AvailabilityData{AggregateState: "unavailable", TransportOutcome: "unreachable"},
Relationships: []unifiedresources.ResourceRelationship{{
SourceID: endpointID, TargetID: hostID, Type: unifiedresources.RelChecks, Confidence: 1, Active: true,
}},
Incidents: []unifiedresources.ResourceIncident{{
Provider: "availability", NativeID: "checkout", Code: "availability_unreachable", Severity: storagehealth.RiskCritical,
Source: "availability.probe", StartedAt: startedAt.Add(time.Second), ConfirmationsRequired: 1, RecoveryConfirmationsRequired: 1,
}},
}
var delivered []string
manager.SetAlertCallback(func(alert *Alert) { delivered = append(delivered, alert.ResourceID) })
manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint})
if len(delivered) != 1 || delivered[0] != endpointID {
t.Fatalf("standalone symptom delivery = %v, want endpoint", delivered)
}
host.Status = unifiedresources.StatusOffline
host.Incidents = []unifiedresources.ResourceIncident{{
Provider: "agent", NativeID: "edge-1", Code: "agent_offline", Severity: storagehealth.RiskCritical,
Source: "agent.heartbeat", StartedAt: startedAt, ConfirmationsRequired: 1, RecoveryConfirmationsRequired: 1,
}}
manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint})
if len(delivered) != 2 || delivered[1] != hostID {
t.Fatalf("new primary delivery = %v, want endpoint then host", delivered)
}
var grouped bool
for _, alert := range manager.GetActiveAlerts() {
if alert.ResourceID == endpointID {
grouped = isSupportedInfrastructureSymptom(&alert)
}
}
if !grouped {
t.Fatal("endpoint did not gain a supported primary")
}
host.Status = unifiedresources.StatusOnline
host.Incidents = nil
manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint})
if len(delivered) != 2 {
t.Fatalf("already-notified endpoint duplicated after primary recovery: %v", delivered)
}
}
func TestClassifyIncidentFailureSeparatesApplicationCertificateAndCoverage(t *testing.T) {
applicationResource := unifiedresources.Resource{Availability: &unifiedresources.AvailabilityData{
AggregateState: "unavailable", TransportOutcome: "reachable", ApplicationOutcome: "failed",
+24 -6
View File
@@ -260,6 +260,17 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso
m.clearAlertNoLock(storageKey)
}
// Synthesis replaces its previous links in place. Remember which retained
// incidents had a primary delivery owner before that information is cleared.
// If the primary recovers while a symptom remains active, the symptom needs
// its own first notification even though its occurrence did not restart.
previouslySupported := make(map[string]bool)
for storageKey := range desired {
if existing, exists := m.getActiveAlertNoLock(storageKey); exists && isSupportedInfrastructureSymptom(existing) {
previouslySupported[storageKey] = true
}
}
// Synthesize over the post-recovery live set before updating or dispatching
// any detector. Supporting symptoms retain their independent lifecycle but
// can share one evidence-backed incident presentation and delivery decision.
@@ -297,12 +308,19 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso
if !m.setActiveAlertNoLock(storageKey, existing) {
continue
}
// Like metric alerts, an existing provider incident becoming critical
// must notify again. Keep its lifecycle and all delivery policy gates;
// unchanged severity and downgrades must not create notification noise.
if oldLevel != existing.Level && existing.Level == AlertLevelCritical &&
!existing.Acknowledged &&
m.allowNotificationByRateLimit(storageKey, existing, "critical-escalation") {
// A still-failing symptom which loses its supported primary needs its
// first notification. An alert already dispatched before grouping must
// not be duplicated when the group dissolves. As with metric alerts, a
// separate increase to critical may also notify through normal gates.
newlyIndependent := previouslySupported[storageKey] &&
!isSupportedInfrastructureSymptom(existing) && existing.LastNotified == nil
becameCritical := oldLevel != existing.Level && existing.Level == AlertLevelCritical
notificationReason := "critical-escalation"
if newlyIndependent {
notificationReason = "supported-primary-ended"
}
if (newlyIndependent || becameCritical) && !existing.Acknowledged &&
m.allowNotificationByRateLimit(storageKey, existing, notificationReason) {
m.dispatchAlert(existing, false)
}
continue