From e37637c6447aa631051da40059e7ea43f3ef9611 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Fri, 20 Mar 2026 11:51:44 +0000 Subject: [PATCH] Thin incident shell state --- .../v6/internal/subsystems/ai-runtime.md | 5 ++ internal/ai/memory/incidents.go | 79 +++++++++++++------ internal/ai/memory/incidents_test.go | 64 +++++++++++++++ 3 files changed, 122 insertions(+), 26 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 4a2d295fe..25e197867 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -206,6 +206,11 @@ exist as compatibility-era shell state for segmentation and fallback, but the projected incident returned to runtime consumers must rebuild those fields from canonical resource changes and preserve only annotation-local entries such as analysis and notes. +The remaining shell should stay as narrow as possible: alert occurrence +boundaries and annotation anchors may remain private implementation state, but +public incident status, acknowledgement, and remediation entries should be +treated as read-model output rebuilt from canonical history whenever that +history exists. The AI correlation root-cause engine also consumes the canonical unified- resource relationship model directly, so cross-resource reasoning stays aligned with the same relationship edges that back the resource API instead of diff --git a/internal/ai/memory/incidents.go b/internal/ai/memory/incidents.go index 47c765838..199a94e41 100644 --- a/internal/ai/memory/incidents.go +++ b/internal/ai/memory/incidents.go @@ -67,6 +67,8 @@ type Incident struct { AckUser string `json:"ackUser,omitempty"` AckTime *time.Time `json:"ackTime,omitempty"` Events []IncidentEvent `json:"events,omitempty"` + + occurrenceClosedAt *time.Time } type incidentJSON struct { @@ -268,17 +270,15 @@ func (s *IncidentStore) RecordAlertAcknowledged(alert *alerts.Alert, user string defer s.mu.Unlock() incident := s.ensureIncidentForAlertLocked(alert) - - incident.Acknowledged = true - if alert.AckTime != nil { - incident.AckTime = alert.AckTime - } else { - now := time.Now() - incident.AckTime = &now - } - incident.AckUser = user - if !s.projectsFromCanonicalLocked() { + incident.Acknowledged = true + if alert.AckTime != nil { + incident.AckTime = alert.AckTime + } else { + now := time.Now() + incident.AckTime = &now + } + incident.AckUser = user s.addEventLocked(incident, IncidentEventAlertAcknowledged, "Alert acknowledged", map[string]interface{}{ "user": user, }) @@ -298,12 +298,10 @@ func (s *IncidentStore) RecordAlertUnacknowledged(alert *alerts.Alert, user stri defer s.mu.Unlock() incident := s.ensureIncidentForAlertLocked(alert) - - incident.Acknowledged = false - incident.AckTime = nil - incident.AckUser = "" - if !s.projectsFromCanonicalLocked() { + incident.Acknowledged = false + incident.AckTime = nil + incident.AckUser = "" s.addEventLocked(incident, IncidentEventAlertUnacknowledged, "Alert unacknowledged", map[string]interface{}{ "user": user, }) @@ -328,14 +326,15 @@ func (s *IncidentStore) RecordAlertResolved(alert *alerts.Alert, resolvedAt time s.incidents = append(s.incidents, incident) } - incident.Status = IncidentStatusResolved if resolvedAt.IsZero() { now := time.Now() resolvedAt = now } - incident.ClosedAt = &resolvedAt + incident.occurrenceClosedAt = cloneTime(resolvedAt) if !s.projectsFromCanonicalLocked() { + incident.Status = IncidentStatusResolved + incident.ClosedAt = cloneTime(resolvedAt) s.addEventLocked(incident, IncidentEventAlertResolved, "Alert resolved", map[string]interface{}{ "resolved_at": resolvedAt.Format(time.RFC3339), }) @@ -1077,9 +1076,6 @@ func newIncidentShellFromAlert(alert *alerts.Alert) *Incident { Message: alert.Message, Status: IncidentStatusOpen, OpenedAt: openedAt, - Acknowledged: alert.Acknowledged, - AckUser: alert.AckUser, - AckTime: alert.AckTime, Events: make([]IncidentEvent, 0), } } @@ -1095,9 +1091,6 @@ func updateIncidentShellFromAlert(incident *Incident, alert *alerts.Alert) { incident.Node = alert.Node incident.Instance = alert.Instance incident.Message = alert.Message - incident.Acknowledged = alert.Acknowledged - incident.AckUser = alert.AckUser - incident.AckTime = alert.AckTime } func (s *IncidentStore) ensureIncidentForAlertLocked(alert *alerts.Alert) *Incident { @@ -1148,7 +1141,7 @@ func (s *IncidentStore) findOpenIncidentByAlertIdentifierLocked(alertIdentifier } for i := len(s.incidents) - 1; i >= 0; i-- { incident := s.incidents[i] - if incident != nil && incident.AlertIdentifier == alertIdentifier && incident.Status == IncidentStatusOpen { + if incident != nil && incident.AlertIdentifier == alertIdentifier && incidentOccurrenceClosedAt(incident) == nil { return incident } } @@ -1190,8 +1183,8 @@ func (s *IncidentStore) trimLocked() { continue } compareTime := incident.OpenedAt - if incident.ClosedAt != nil { - compareTime = *incident.ClosedAt + if closedAt := incidentOccurrenceClosedAt(incident); closedAt != nil { + compareTime = *closedAt } if compareTime.After(cutoff) { filtered = append(filtered, incident) @@ -1280,6 +1273,9 @@ func (s *IncidentStore) loadFromDisk() error { return err } + for _, incident := range incidents { + normalizeIncidentShellState(incident) + } s.incidents = incidents s.trimLocked() return nil @@ -1298,6 +1294,10 @@ func cloneIncident(src *Incident) *Incident { t := *src.ClosedAt clone.ClosedAt = &t } + if src.occurrenceClosedAt != nil { + t := *src.occurrenceClosedAt + clone.occurrenceClosedAt = &t + } if len(src.Events) > 0 { clone.Events = make([]IncidentEvent, len(src.Events)) for i, event := range src.Events { @@ -1315,6 +1315,33 @@ func cloneIncident(src *Incident) *Incident { return &clone } +func incidentOccurrenceClosedAt(incident *Incident) *time.Time { + if incident == nil { + return nil + } + if incident.occurrenceClosedAt != nil { + return incident.occurrenceClosedAt + } + return incident.ClosedAt +} + +func normalizeIncidentShellState(incident *Incident) { + if incident == nil { + return + } + if incident.occurrenceClosedAt == nil && incident.ClosedAt != nil { + incident.occurrenceClosedAt = cloneTime(*incident.ClosedAt) + } +} + +func cloneTime(value time.Time) *time.Time { + if value.IsZero() { + return nil + } + cloned := value + return &cloned +} + var incidentCounter int64 func generateIncidentID() string { diff --git a/internal/ai/memory/incidents_test.go b/internal/ai/memory/incidents_test.go index 51eb43e3e..7b471c889 100644 --- a/internal/ai/memory/incidents_test.go +++ b/internal/ai/memory/incidents_test.go @@ -243,6 +243,70 @@ func TestIncidentStore_CanonicalProjectionOverridesLegacyLifecycleState(t *testi } } +func TestIncidentStore_CanonicalModeKeepsShellAsOccurrenceAndAnnotationsOnly(t *testing.T) { + store := NewIncidentStore(IncidentStoreConfig{ + MaxIncidents: 10, + MaxEventsPerIncident: 20, + MaxAgeDays: 30, + }) + store.SetResourceTimelineStore(unifiedresources.NewMemoryStore()) + + firstStart := time.Now().UTC().Add(-25 * time.Minute).Truncate(time.Second) + first := &alerts.Alert{ + ID: "alert-shell-1", + Type: "cpu", + Level: alerts.AlertLevelCritical, + ResourceID: "res-shell-1", + ResourceName: "vm-shell-1", + StartTime: firstStart, + } + + store.RecordAlertFired(first) + store.RecordAlertAcknowledged(first, "operator") + store.RecordRunbook(first.ID, "rb-1", "Restart service", "resolved", true, "Recovered") + store.RecordAlertResolved(first, firstStart.Add(4*time.Minute)) + + if len(store.incidents) != 1 { + t.Fatalf("expected one shell incident, got %d", len(store.incidents)) + } + firstShell := store.incidents[0] + if firstShell.Acknowledged { + t.Fatal("expected canonical shell to avoid persisted acknowledgement state") + } + if firstShell.AckUser != "" || firstShell.AckTime != nil { + t.Fatal("expected canonical shell to avoid persisted acknowledgement metadata") + } + if firstShell.ClosedAt != nil { + t.Fatal("expected canonical shell to avoid persisted resolved timestamp") + } + if firstShell.occurrenceClosedAt == nil { + t.Fatal("expected canonical shell to preserve private occurrence closure boundary") + } + for _, event := range firstShell.Events { + switch event.Type { + case IncidentEventAlertAcknowledged, IncidentEventAlertResolved, IncidentEventRunbook: + t.Fatalf("unexpected derived event %q stored in canonical incident shell", event.Type) + } + } + + second := &alerts.Alert{ + ID: first.ID, + Type: first.Type, + Level: first.Level, + ResourceID: first.ResourceID, + ResourceName: first.ResourceName, + StartTime: firstStart.Add(10 * time.Minute), + } + store.RecordAlertFired(second) + + if len(store.incidents) != 2 { + t.Fatalf("expected second alert occurrence to create a new shell, got %d incidents", len(store.incidents)) + } + if store.incidents[1].occurrenceClosedAt != nil { + t.Fatal("expected new occurrence shell to remain open") + } +} + func TestIncidentStore_GetTimelineByAlertAt(t *testing.T) { store := NewIncidentStore(IncidentStoreConfig{ MaxIncidents: 10,