Thin incident shell state

This commit is contained in:
rcourtman
2026-03-20 11:51:44 +00:00
parent 81cddc10f8
commit e37637c644
3 changed files with 122 additions and 26 deletions
@@ -206,6 +206,11 @@ exist as compatibility-era shell state for segmentation and fallback, but the
projected incident returned to runtime consumers must rebuild those fields from
canonical resource changes and preserve only annotation-local entries such as
analysis and notes.
The remaining shell should stay as narrow as possible: alert occurrence
boundaries and annotation anchors may remain private implementation state, but
public incident status, acknowledgement, and remediation entries should be
treated as read-model output rebuilt from canonical history whenever that
history exists.
The AI correlation root-cause engine also consumes the canonical unified-
resource relationship model directly, so cross-resource reasoning stays aligned
with the same relationship edges that back the resource API instead of
+53 -26
View File
@@ -67,6 +67,8 @@ type Incident struct {
AckUser string `json:"ackUser,omitempty"`
AckTime *time.Time `json:"ackTime,omitempty"`
Events []IncidentEvent `json:"events,omitempty"`
occurrenceClosedAt *time.Time
}
type incidentJSON struct {
@@ -268,17 +270,15 @@ func (s *IncidentStore) RecordAlertAcknowledged(alert *alerts.Alert, user string
defer s.mu.Unlock()
incident := s.ensureIncidentForAlertLocked(alert)
incident.Acknowledged = true
if alert.AckTime != nil {
incident.AckTime = alert.AckTime
} else {
now := time.Now()
incident.AckTime = &now
}
incident.AckUser = user
if !s.projectsFromCanonicalLocked() {
incident.Acknowledged = true
if alert.AckTime != nil {
incident.AckTime = alert.AckTime
} else {
now := time.Now()
incident.AckTime = &now
}
incident.AckUser = user
s.addEventLocked(incident, IncidentEventAlertAcknowledged, "Alert acknowledged", map[string]interface{}{
"user": user,
})
@@ -298,12 +298,10 @@ func (s *IncidentStore) RecordAlertUnacknowledged(alert *alerts.Alert, user stri
defer s.mu.Unlock()
incident := s.ensureIncidentForAlertLocked(alert)
incident.Acknowledged = false
incident.AckTime = nil
incident.AckUser = ""
if !s.projectsFromCanonicalLocked() {
incident.Acknowledged = false
incident.AckTime = nil
incident.AckUser = ""
s.addEventLocked(incident, IncidentEventAlertUnacknowledged, "Alert unacknowledged", map[string]interface{}{
"user": user,
})
@@ -328,14 +326,15 @@ func (s *IncidentStore) RecordAlertResolved(alert *alerts.Alert, resolvedAt time
s.incidents = append(s.incidents, incident)
}
incident.Status = IncidentStatusResolved
if resolvedAt.IsZero() {
now := time.Now()
resolvedAt = now
}
incident.ClosedAt = &resolvedAt
incident.occurrenceClosedAt = cloneTime(resolvedAt)
if !s.projectsFromCanonicalLocked() {
incident.Status = IncidentStatusResolved
incident.ClosedAt = cloneTime(resolvedAt)
s.addEventLocked(incident, IncidentEventAlertResolved, "Alert resolved", map[string]interface{}{
"resolved_at": resolvedAt.Format(time.RFC3339),
})
@@ -1077,9 +1076,6 @@ func newIncidentShellFromAlert(alert *alerts.Alert) *Incident {
Message: alert.Message,
Status: IncidentStatusOpen,
OpenedAt: openedAt,
Acknowledged: alert.Acknowledged,
AckUser: alert.AckUser,
AckTime: alert.AckTime,
Events: make([]IncidentEvent, 0),
}
}
@@ -1095,9 +1091,6 @@ func updateIncidentShellFromAlert(incident *Incident, alert *alerts.Alert) {
incident.Node = alert.Node
incident.Instance = alert.Instance
incident.Message = alert.Message
incident.Acknowledged = alert.Acknowledged
incident.AckUser = alert.AckUser
incident.AckTime = alert.AckTime
}
func (s *IncidentStore) ensureIncidentForAlertLocked(alert *alerts.Alert) *Incident {
@@ -1148,7 +1141,7 @@ func (s *IncidentStore) findOpenIncidentByAlertIdentifierLocked(alertIdentifier
}
for i := len(s.incidents) - 1; i >= 0; i-- {
incident := s.incidents[i]
if incident != nil && incident.AlertIdentifier == alertIdentifier && incident.Status == IncidentStatusOpen {
if incident != nil && incident.AlertIdentifier == alertIdentifier && incidentOccurrenceClosedAt(incident) == nil {
return incident
}
}
@@ -1190,8 +1183,8 @@ func (s *IncidentStore) trimLocked() {
continue
}
compareTime := incident.OpenedAt
if incident.ClosedAt != nil {
compareTime = *incident.ClosedAt
if closedAt := incidentOccurrenceClosedAt(incident); closedAt != nil {
compareTime = *closedAt
}
if compareTime.After(cutoff) {
filtered = append(filtered, incident)
@@ -1280,6 +1273,9 @@ func (s *IncidentStore) loadFromDisk() error {
return err
}
for _, incident := range incidents {
normalizeIncidentShellState(incident)
}
s.incidents = incidents
s.trimLocked()
return nil
@@ -1298,6 +1294,10 @@ func cloneIncident(src *Incident) *Incident {
t := *src.ClosedAt
clone.ClosedAt = &t
}
if src.occurrenceClosedAt != nil {
t := *src.occurrenceClosedAt
clone.occurrenceClosedAt = &t
}
if len(src.Events) > 0 {
clone.Events = make([]IncidentEvent, len(src.Events))
for i, event := range src.Events {
@@ -1315,6 +1315,33 @@ func cloneIncident(src *Incident) *Incident {
return &clone
}
func incidentOccurrenceClosedAt(incident *Incident) *time.Time {
if incident == nil {
return nil
}
if incident.occurrenceClosedAt != nil {
return incident.occurrenceClosedAt
}
return incident.ClosedAt
}
func normalizeIncidentShellState(incident *Incident) {
if incident == nil {
return
}
if incident.occurrenceClosedAt == nil && incident.ClosedAt != nil {
incident.occurrenceClosedAt = cloneTime(*incident.ClosedAt)
}
}
func cloneTime(value time.Time) *time.Time {
if value.IsZero() {
return nil
}
cloned := value
return &cloned
}
var incidentCounter int64
func generateIncidentID() string {
+64
View File
@@ -243,6 +243,70 @@ func TestIncidentStore_CanonicalProjectionOverridesLegacyLifecycleState(t *testi
}
}
func TestIncidentStore_CanonicalModeKeepsShellAsOccurrenceAndAnnotationsOnly(t *testing.T) {
store := NewIncidentStore(IncidentStoreConfig{
MaxIncidents: 10,
MaxEventsPerIncident: 20,
MaxAgeDays: 30,
})
store.SetResourceTimelineStore(unifiedresources.NewMemoryStore())
firstStart := time.Now().UTC().Add(-25 * time.Minute).Truncate(time.Second)
first := &alerts.Alert{
ID: "alert-shell-1",
Type: "cpu",
Level: alerts.AlertLevelCritical,
ResourceID: "res-shell-1",
ResourceName: "vm-shell-1",
StartTime: firstStart,
}
store.RecordAlertFired(first)
store.RecordAlertAcknowledged(first, "operator")
store.RecordRunbook(first.ID, "rb-1", "Restart service", "resolved", true, "Recovered")
store.RecordAlertResolved(first, firstStart.Add(4*time.Minute))
if len(store.incidents) != 1 {
t.Fatalf("expected one shell incident, got %d", len(store.incidents))
}
firstShell := store.incidents[0]
if firstShell.Acknowledged {
t.Fatal("expected canonical shell to avoid persisted acknowledgement state")
}
if firstShell.AckUser != "" || firstShell.AckTime != nil {
t.Fatal("expected canonical shell to avoid persisted acknowledgement metadata")
}
if firstShell.ClosedAt != nil {
t.Fatal("expected canonical shell to avoid persisted resolved timestamp")
}
if firstShell.occurrenceClosedAt == nil {
t.Fatal("expected canonical shell to preserve private occurrence closure boundary")
}
for _, event := range firstShell.Events {
switch event.Type {
case IncidentEventAlertAcknowledged, IncidentEventAlertResolved, IncidentEventRunbook:
t.Fatalf("unexpected derived event %q stored in canonical incident shell", event.Type)
}
}
second := &alerts.Alert{
ID: first.ID,
Type: first.Type,
Level: first.Level,
ResourceID: first.ResourceID,
ResourceName: first.ResourceName,
StartTime: firstStart.Add(10 * time.Minute),
}
store.RecordAlertFired(second)
if len(store.incidents) != 2 {
t.Fatalf("expected second alert occurrence to create a new shell, got %d incidents", len(store.incidents))
}
if store.incidents[1].occurrenceClosedAt != nil {
t.Fatal("expected new occurrence shell to remain open")
}
}
func TestIncidentStore_GetTimelineByAlertAt(t *testing.T) {
store := NewIncidentStore(IncidentStoreConfig{
MaxIncidents: 10,