mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-23 20:25:56 +00:00
Notify continuing symptoms after primary recovery
A supported endpoint failure may outlive its primary host incident. When synthesis removes the supported cause, dispatch its first independent notification without reopening the occurrence or duplicating a symptom previously notified before grouping. Change-source: pulse-maintainer
This commit is contained in:
@@ -2833,11 +2833,16 @@ notification delivery, with the typed `correlated_primary` reason and primary
|
||||
alert ID in the event log. The primary remains the one delivery owner. Supporting
|
||||
members also skip escalation levels and critical repeats while that supported
|
||||
cause is current; an already queued escalation is revalidated before delivery.
|
||||
If the supported primary recovers or the causal link is lost while its symptom
|
||||
still fails, the symptom regains independent delivery without a new occurrence.
|
||||
It receives its first notification only if it was never dispatched before
|
||||
grouping; an already-notified symptom does not duplicate on group dissolution.
|
||||
Acknowledgement, rate limits and other ordinary delivery gates still apply.
|
||||
`observation-set` members continue to notify independently because Pulse has
|
||||
not established causality. `incident_synthesis_test.go` pins classification,
|
||||
contradiction downgrade, bounded evidence, duplicate-delivery suppression,
|
||||
partial-recovery behavior, escalation suppression, and queued-callback
|
||||
revalidation without muting observation sets.
|
||||
partial-recovery delivery and no duplicate after prior dispatch, escalation
|
||||
suppression, and queued-callback revalidation without muting observation sets.
|
||||
|
||||
### Alert hydration is not resource admission
|
||||
|
||||
|
||||
@@ -235,6 +235,113 @@ func TestInfrastructureIncidentSynthesisShowsPartialRecovery(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupportedSymptomNotifiesWhenPrimaryRecoversButSymptomPersists(t *testing.T) {
|
||||
manager := newTestManager(t)
|
||||
configureUnifiedEvalManager(t, manager, unifiedEvalBaseConfig())
|
||||
startedAt := time.Date(2026, 9, 29, 18, 0, 0, 0, time.UTC)
|
||||
hostID := "agent:edge-1"
|
||||
endpointID := "availability:checkout"
|
||||
host := unifiedresources.Resource{
|
||||
ID: hostID, Type: unifiedresources.ResourceTypeAgent, Name: "Edge host", Status: unifiedresources.StatusOffline,
|
||||
Incidents: []unifiedresources.ResourceIncident{{
|
||||
Provider: "agent", NativeID: "edge-1", Code: "agent_offline", Severity: storagehealth.RiskCritical,
|
||||
Source: "agent.heartbeat", Summary: "Edge host is offline", StartedAt: startedAt,
|
||||
ConfirmationsRequired: 1, RecoveryConfirmationsRequired: 1,
|
||||
}},
|
||||
}
|
||||
endpoint := unifiedresources.Resource{
|
||||
ID: endpointID, Type: unifiedresources.ResourceTypeNetworkEndpoint, Name: "Checkout", Status: unifiedresources.StatusOffline,
|
||||
Availability: &unifiedresources.AvailabilityData{AggregateState: "unavailable", TransportOutcome: "unreachable"},
|
||||
Relationships: []unifiedresources.ResourceRelationship{{
|
||||
SourceID: endpointID, TargetID: hostID, Type: unifiedresources.RelChecks, Confidence: 1, Active: true,
|
||||
}},
|
||||
Incidents: []unifiedresources.ResourceIncident{{
|
||||
Provider: "availability", NativeID: "checkout", Code: "availability_unreachable", Severity: storagehealth.RiskCritical,
|
||||
Source: "availability.probe", Summary: "Checkout is unreachable", StartedAt: startedAt.Add(time.Second),
|
||||
ConfirmationsRequired: 1, RecoveryConfirmationsRequired: 1,
|
||||
}},
|
||||
}
|
||||
var delivered []string
|
||||
manager.SetAlertCallback(func(alert *Alert) { delivered = append(delivered, alert.ResourceID) })
|
||||
|
||||
manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint})
|
||||
if len(delivered) != 1 || delivered[0] != hostID {
|
||||
t.Fatalf("grouped delivery = %v, want only the primary host", delivered)
|
||||
}
|
||||
for _, alert := range manager.GetActiveAlerts() {
|
||||
if alert.ResourceID == endpointID && (!isSupportedInfrastructureSymptom(&alert) || alert.LastNotified != nil) {
|
||||
t.Fatalf("grouped symptom = %+v, want supported and not separately dispatched", alert)
|
||||
}
|
||||
}
|
||||
|
||||
// The host is healthy again, but the endpoint still fails. It must regain
|
||||
// its own notification path without requiring a new alert occurrence.
|
||||
host.Status = unifiedresources.StatusOnline
|
||||
host.Incidents = nil
|
||||
manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint})
|
||||
if len(delivered) != 2 || delivered[1] != endpointID {
|
||||
t.Fatalf("after primary recovery delivery = %v, want continuing symptom", delivered)
|
||||
}
|
||||
manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint})
|
||||
if len(delivered) != 2 {
|
||||
t.Fatalf("unchanged independent symptom sent again: %v", delivered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviouslyNotifiedSymptomDoesNotDuplicateWhenPrimaryRecovers(t *testing.T) {
|
||||
manager := newTestManager(t)
|
||||
configureUnifiedEvalManager(t, manager, unifiedEvalBaseConfig())
|
||||
startedAt := time.Date(2026, 9, 29, 18, 0, 0, 0, time.UTC)
|
||||
hostID := "agent:edge-1"
|
||||
endpointID := "availability:checkout"
|
||||
host := unifiedresources.Resource{
|
||||
ID: hostID, Type: unifiedresources.ResourceTypeAgent, Status: unifiedresources.StatusOnline,
|
||||
}
|
||||
endpoint := unifiedresources.Resource{
|
||||
ID: endpointID, Type: unifiedresources.ResourceTypeNetworkEndpoint, Status: unifiedresources.StatusOffline,
|
||||
Availability: &unifiedresources.AvailabilityData{AggregateState: "unavailable", TransportOutcome: "unreachable"},
|
||||
Relationships: []unifiedresources.ResourceRelationship{{
|
||||
SourceID: endpointID, TargetID: hostID, Type: unifiedresources.RelChecks, Confidence: 1, Active: true,
|
||||
}},
|
||||
Incidents: []unifiedresources.ResourceIncident{{
|
||||
Provider: "availability", NativeID: "checkout", Code: "availability_unreachable", Severity: storagehealth.RiskCritical,
|
||||
Source: "availability.probe", StartedAt: startedAt.Add(time.Second), ConfirmationsRequired: 1, RecoveryConfirmationsRequired: 1,
|
||||
}},
|
||||
}
|
||||
var delivered []string
|
||||
manager.SetAlertCallback(func(alert *Alert) { delivered = append(delivered, alert.ResourceID) })
|
||||
|
||||
manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint})
|
||||
if len(delivered) != 1 || delivered[0] != endpointID {
|
||||
t.Fatalf("standalone symptom delivery = %v, want endpoint", delivered)
|
||||
}
|
||||
host.Status = unifiedresources.StatusOffline
|
||||
host.Incidents = []unifiedresources.ResourceIncident{{
|
||||
Provider: "agent", NativeID: "edge-1", Code: "agent_offline", Severity: storagehealth.RiskCritical,
|
||||
Source: "agent.heartbeat", StartedAt: startedAt, ConfirmationsRequired: 1, RecoveryConfirmationsRequired: 1,
|
||||
}}
|
||||
manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint})
|
||||
if len(delivered) != 2 || delivered[1] != hostID {
|
||||
t.Fatalf("new primary delivery = %v, want endpoint then host", delivered)
|
||||
}
|
||||
var grouped bool
|
||||
for _, alert := range manager.GetActiveAlerts() {
|
||||
if alert.ResourceID == endpointID {
|
||||
grouped = isSupportedInfrastructureSymptom(&alert)
|
||||
}
|
||||
}
|
||||
if !grouped {
|
||||
t.Fatal("endpoint did not gain a supported primary")
|
||||
}
|
||||
|
||||
host.Status = unifiedresources.StatusOnline
|
||||
host.Incidents = nil
|
||||
manager.SyncUnifiedResourceIncidents([]unifiedresources.Resource{host, endpoint})
|
||||
if len(delivered) != 2 {
|
||||
t.Fatalf("already-notified endpoint duplicated after primary recovery: %v", delivered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyIncidentFailureSeparatesApplicationCertificateAndCoverage(t *testing.T) {
|
||||
applicationResource := unifiedresources.Resource{Availability: &unifiedresources.AvailabilityData{
|
||||
AggregateState: "unavailable", TransportOutcome: "reachable", ApplicationOutcome: "failed",
|
||||
|
||||
@@ -260,6 +260,17 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso
|
||||
m.clearAlertNoLock(storageKey)
|
||||
}
|
||||
|
||||
// Synthesis replaces its previous links in place. Remember which retained
|
||||
// incidents had a primary delivery owner before that information is cleared.
|
||||
// If the primary recovers while a symptom remains active, the symptom needs
|
||||
// its own first notification even though its occurrence did not restart.
|
||||
previouslySupported := make(map[string]bool)
|
||||
for storageKey := range desired {
|
||||
if existing, exists := m.getActiveAlertNoLock(storageKey); exists && isSupportedInfrastructureSymptom(existing) {
|
||||
previouslySupported[storageKey] = true
|
||||
}
|
||||
}
|
||||
|
||||
// Synthesize over the post-recovery live set before updating or dispatching
|
||||
// any detector. Supporting symptoms retain their independent lifecycle but
|
||||
// can share one evidence-backed incident presentation and delivery decision.
|
||||
@@ -297,12 +308,19 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso
|
||||
if !m.setActiveAlertNoLock(storageKey, existing) {
|
||||
continue
|
||||
}
|
||||
// Like metric alerts, an existing provider incident becoming critical
|
||||
// must notify again. Keep its lifecycle and all delivery policy gates;
|
||||
// unchanged severity and downgrades must not create notification noise.
|
||||
if oldLevel != existing.Level && existing.Level == AlertLevelCritical &&
|
||||
!existing.Acknowledged &&
|
||||
m.allowNotificationByRateLimit(storageKey, existing, "critical-escalation") {
|
||||
// A still-failing symptom which loses its supported primary needs its
|
||||
// first notification. An alert already dispatched before grouping must
|
||||
// not be duplicated when the group dissolves. As with metric alerts, a
|
||||
// separate increase to critical may also notify through normal gates.
|
||||
newlyIndependent := previouslySupported[storageKey] &&
|
||||
!isSupportedInfrastructureSymptom(existing) && existing.LastNotified == nil
|
||||
becameCritical := oldLevel != existing.Level && existing.Level == AlertLevelCritical
|
||||
notificationReason := "critical-escalation"
|
||||
if newlyIndependent {
|
||||
notificationReason = "supported-primary-ended"
|
||||
}
|
||||
if (newlyIndependent || becameCritical) && !existing.Acknowledged &&
|
||||
m.allowNotificationByRateLimit(storageKey, existing, notificationReason) {
|
||||
m.dispatchAlert(existing, false)
|
||||
}
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user