mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-23 03:33:53 +00:00
Thin incident shell state
This commit is contained in:
@@ -206,6 +206,11 @@ exist as compatibility-era shell state for segmentation and fallback, but the
|
||||
projected incident returned to runtime consumers must rebuild those fields from
|
||||
canonical resource changes and preserve only annotation-local entries such as
|
||||
analysis and notes.
|
||||
The remaining shell should stay as narrow as possible: alert occurrence
|
||||
boundaries and annotation anchors may remain private implementation state, but
|
||||
public incident status, acknowledgement, and remediation entries should be
|
||||
treated as read-model output rebuilt from canonical history whenever that
|
||||
history exists.
|
||||
The AI correlation root-cause engine also consumes the canonical unified-
|
||||
resource relationship model directly, so cross-resource reasoning stays aligned
|
||||
with the same relationship edges that back the resource API instead of
|
||||
|
||||
@@ -67,6 +67,8 @@ type Incident struct {
|
||||
AckUser string `json:"ackUser,omitempty"`
|
||||
AckTime *time.Time `json:"ackTime,omitempty"`
|
||||
Events []IncidentEvent `json:"events,omitempty"`
|
||||
|
||||
occurrenceClosedAt *time.Time
|
||||
}
|
||||
|
||||
type incidentJSON struct {
|
||||
@@ -268,17 +270,15 @@ func (s *IncidentStore) RecordAlertAcknowledged(alert *alerts.Alert, user string
|
||||
defer s.mu.Unlock()
|
||||
|
||||
incident := s.ensureIncidentForAlertLocked(alert)
|
||||
|
||||
incident.Acknowledged = true
|
||||
if alert.AckTime != nil {
|
||||
incident.AckTime = alert.AckTime
|
||||
} else {
|
||||
now := time.Now()
|
||||
incident.AckTime = &now
|
||||
}
|
||||
incident.AckUser = user
|
||||
|
||||
if !s.projectsFromCanonicalLocked() {
|
||||
incident.Acknowledged = true
|
||||
if alert.AckTime != nil {
|
||||
incident.AckTime = alert.AckTime
|
||||
} else {
|
||||
now := time.Now()
|
||||
incident.AckTime = &now
|
||||
}
|
||||
incident.AckUser = user
|
||||
s.addEventLocked(incident, IncidentEventAlertAcknowledged, "Alert acknowledged", map[string]interface{}{
|
||||
"user": user,
|
||||
})
|
||||
@@ -298,12 +298,10 @@ func (s *IncidentStore) RecordAlertUnacknowledged(alert *alerts.Alert, user stri
|
||||
defer s.mu.Unlock()
|
||||
|
||||
incident := s.ensureIncidentForAlertLocked(alert)
|
||||
|
||||
incident.Acknowledged = false
|
||||
incident.AckTime = nil
|
||||
incident.AckUser = ""
|
||||
|
||||
if !s.projectsFromCanonicalLocked() {
|
||||
incident.Acknowledged = false
|
||||
incident.AckTime = nil
|
||||
incident.AckUser = ""
|
||||
s.addEventLocked(incident, IncidentEventAlertUnacknowledged, "Alert unacknowledged", map[string]interface{}{
|
||||
"user": user,
|
||||
})
|
||||
@@ -328,14 +326,15 @@ func (s *IncidentStore) RecordAlertResolved(alert *alerts.Alert, resolvedAt time
|
||||
s.incidents = append(s.incidents, incident)
|
||||
}
|
||||
|
||||
incident.Status = IncidentStatusResolved
|
||||
if resolvedAt.IsZero() {
|
||||
now := time.Now()
|
||||
resolvedAt = now
|
||||
}
|
||||
incident.ClosedAt = &resolvedAt
|
||||
incident.occurrenceClosedAt = cloneTime(resolvedAt)
|
||||
|
||||
if !s.projectsFromCanonicalLocked() {
|
||||
incident.Status = IncidentStatusResolved
|
||||
incident.ClosedAt = cloneTime(resolvedAt)
|
||||
s.addEventLocked(incident, IncidentEventAlertResolved, "Alert resolved", map[string]interface{}{
|
||||
"resolved_at": resolvedAt.Format(time.RFC3339),
|
||||
})
|
||||
@@ -1077,9 +1076,6 @@ func newIncidentShellFromAlert(alert *alerts.Alert) *Incident {
|
||||
Message: alert.Message,
|
||||
Status: IncidentStatusOpen,
|
||||
OpenedAt: openedAt,
|
||||
Acknowledged: alert.Acknowledged,
|
||||
AckUser: alert.AckUser,
|
||||
AckTime: alert.AckTime,
|
||||
Events: make([]IncidentEvent, 0),
|
||||
}
|
||||
}
|
||||
@@ -1095,9 +1091,6 @@ func updateIncidentShellFromAlert(incident *Incident, alert *alerts.Alert) {
|
||||
incident.Node = alert.Node
|
||||
incident.Instance = alert.Instance
|
||||
incident.Message = alert.Message
|
||||
incident.Acknowledged = alert.Acknowledged
|
||||
incident.AckUser = alert.AckUser
|
||||
incident.AckTime = alert.AckTime
|
||||
}
|
||||
|
||||
func (s *IncidentStore) ensureIncidentForAlertLocked(alert *alerts.Alert) *Incident {
|
||||
@@ -1148,7 +1141,7 @@ func (s *IncidentStore) findOpenIncidentByAlertIdentifierLocked(alertIdentifier
|
||||
}
|
||||
for i := len(s.incidents) - 1; i >= 0; i-- {
|
||||
incident := s.incidents[i]
|
||||
if incident != nil && incident.AlertIdentifier == alertIdentifier && incident.Status == IncidentStatusOpen {
|
||||
if incident != nil && incident.AlertIdentifier == alertIdentifier && incidentOccurrenceClosedAt(incident) == nil {
|
||||
return incident
|
||||
}
|
||||
}
|
||||
@@ -1190,8 +1183,8 @@ func (s *IncidentStore) trimLocked() {
|
||||
continue
|
||||
}
|
||||
compareTime := incident.OpenedAt
|
||||
if incident.ClosedAt != nil {
|
||||
compareTime = *incident.ClosedAt
|
||||
if closedAt := incidentOccurrenceClosedAt(incident); closedAt != nil {
|
||||
compareTime = *closedAt
|
||||
}
|
||||
if compareTime.After(cutoff) {
|
||||
filtered = append(filtered, incident)
|
||||
@@ -1280,6 +1273,9 @@ func (s *IncidentStore) loadFromDisk() error {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, incident := range incidents {
|
||||
normalizeIncidentShellState(incident)
|
||||
}
|
||||
s.incidents = incidents
|
||||
s.trimLocked()
|
||||
return nil
|
||||
@@ -1298,6 +1294,10 @@ func cloneIncident(src *Incident) *Incident {
|
||||
t := *src.ClosedAt
|
||||
clone.ClosedAt = &t
|
||||
}
|
||||
if src.occurrenceClosedAt != nil {
|
||||
t := *src.occurrenceClosedAt
|
||||
clone.occurrenceClosedAt = &t
|
||||
}
|
||||
if len(src.Events) > 0 {
|
||||
clone.Events = make([]IncidentEvent, len(src.Events))
|
||||
for i, event := range src.Events {
|
||||
@@ -1315,6 +1315,33 @@ func cloneIncident(src *Incident) *Incident {
|
||||
return &clone
|
||||
}
|
||||
|
||||
func incidentOccurrenceClosedAt(incident *Incident) *time.Time {
|
||||
if incident == nil {
|
||||
return nil
|
||||
}
|
||||
if incident.occurrenceClosedAt != nil {
|
||||
return incident.occurrenceClosedAt
|
||||
}
|
||||
return incident.ClosedAt
|
||||
}
|
||||
|
||||
func normalizeIncidentShellState(incident *Incident) {
|
||||
if incident == nil {
|
||||
return
|
||||
}
|
||||
if incident.occurrenceClosedAt == nil && incident.ClosedAt != nil {
|
||||
incident.occurrenceClosedAt = cloneTime(*incident.ClosedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneTime(value time.Time) *time.Time {
|
||||
if value.IsZero() {
|
||||
return nil
|
||||
}
|
||||
cloned := value
|
||||
return &cloned
|
||||
}
|
||||
|
||||
var incidentCounter int64
|
||||
|
||||
func generateIncidentID() string {
|
||||
|
||||
@@ -243,6 +243,70 @@ func TestIncidentStore_CanonicalProjectionOverridesLegacyLifecycleState(t *testi
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncidentStore_CanonicalModeKeepsShellAsOccurrenceAndAnnotationsOnly(t *testing.T) {
|
||||
store := NewIncidentStore(IncidentStoreConfig{
|
||||
MaxIncidents: 10,
|
||||
MaxEventsPerIncident: 20,
|
||||
MaxAgeDays: 30,
|
||||
})
|
||||
store.SetResourceTimelineStore(unifiedresources.NewMemoryStore())
|
||||
|
||||
firstStart := time.Now().UTC().Add(-25 * time.Minute).Truncate(time.Second)
|
||||
first := &alerts.Alert{
|
||||
ID: "alert-shell-1",
|
||||
Type: "cpu",
|
||||
Level: alerts.AlertLevelCritical,
|
||||
ResourceID: "res-shell-1",
|
||||
ResourceName: "vm-shell-1",
|
||||
StartTime: firstStart,
|
||||
}
|
||||
|
||||
store.RecordAlertFired(first)
|
||||
store.RecordAlertAcknowledged(first, "operator")
|
||||
store.RecordRunbook(first.ID, "rb-1", "Restart service", "resolved", true, "Recovered")
|
||||
store.RecordAlertResolved(first, firstStart.Add(4*time.Minute))
|
||||
|
||||
if len(store.incidents) != 1 {
|
||||
t.Fatalf("expected one shell incident, got %d", len(store.incidents))
|
||||
}
|
||||
firstShell := store.incidents[0]
|
||||
if firstShell.Acknowledged {
|
||||
t.Fatal("expected canonical shell to avoid persisted acknowledgement state")
|
||||
}
|
||||
if firstShell.AckUser != "" || firstShell.AckTime != nil {
|
||||
t.Fatal("expected canonical shell to avoid persisted acknowledgement metadata")
|
||||
}
|
||||
if firstShell.ClosedAt != nil {
|
||||
t.Fatal("expected canonical shell to avoid persisted resolved timestamp")
|
||||
}
|
||||
if firstShell.occurrenceClosedAt == nil {
|
||||
t.Fatal("expected canonical shell to preserve private occurrence closure boundary")
|
||||
}
|
||||
for _, event := range firstShell.Events {
|
||||
switch event.Type {
|
||||
case IncidentEventAlertAcknowledged, IncidentEventAlertResolved, IncidentEventRunbook:
|
||||
t.Fatalf("unexpected derived event %q stored in canonical incident shell", event.Type)
|
||||
}
|
||||
}
|
||||
|
||||
second := &alerts.Alert{
|
||||
ID: first.ID,
|
||||
Type: first.Type,
|
||||
Level: first.Level,
|
||||
ResourceID: first.ResourceID,
|
||||
ResourceName: first.ResourceName,
|
||||
StartTime: firstStart.Add(10 * time.Minute),
|
||||
}
|
||||
store.RecordAlertFired(second)
|
||||
|
||||
if len(store.incidents) != 2 {
|
||||
t.Fatalf("expected second alert occurrence to create a new shell, got %d incidents", len(store.incidents))
|
||||
}
|
||||
if store.incidents[1].occurrenceClosedAt != nil {
|
||||
t.Fatal("expected new occurrence shell to remain open")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncidentStore_GetTimelineByAlertAt(t *testing.T) {
|
||||
store := NewIncidentStore(IncidentStoreConfig{
|
||||
MaxIncidents: 10,
|
||||
|
||||
Reference in New Issue
Block a user