From 9bb421f230feaf137048eebae06d0cbad96e98a1 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:30:17 +0100 Subject: [PATCH] alerts: keep only the newest evidence envelope in history entries Most of the alert history fold was evidence nothing reads. On the dogfood install a resolved snapshot averaged 21 KB, 83% of it the evidence trail of up to 64 envelopes (35 on average), across 1,895 occurrences kept for 90 days (#1803). No Pulse UI, AI tool or API handler reads Alert.evidence from history: the incident panel uses the incident timeline, and the AI uses recently resolved alerts. The fold now trims each decoded snapshot to its newest envelope, by observedAt and then position (an updated envelope keeps its slot, so the last one is not always the newest). History output applies the same rule to live overlays and the JSON fallback, so every /api/alerts/history entry carries at most one envelope. operationalRecord.evidenceIds and transitions keep their full lists, the event log keeps every snapshot whole, and /api/alerts/active still returns an open alert's full evidence. On a log shaped like the dogfood install's, the history fold went from 27.2 MB to 2.1 MB of retained heap. Negative controls: keeping full evidence in the fold, in the output, or choosing the newest by position each fail their test. The alerts tree passes on pulse-dev. Related to #1803 --- docs/API.md | 2 +- .../v6/internal/subsystems/alerts.md | 7 + .../v6/internal/subsystems/api-contracts.md | 9 +- frontend-modern/public/docs/API.md | 2 +- .../src/utils/__tests__/docsLinks.test.ts | 13 ++ internal/alerts/canonical_identity.go | 3 + internal/alerts/history_projection.go | 23 ++++ .../alerts/history_projection_parity_test.go | 128 ++++++++++++++++++ internal/alerts/operational_contract_test.go | 36 +++++ 9 files changed, 220 insertions(+), 3 deletions(-) diff --git a/docs/API.md b/docs/API.md index 432aaadcf8..570a8e053e 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1070,7 +1070,7 @@ it, use the snooze and unsnooze endpoints below. - `GET /api/alerts/active` - `GET /api/alerts/delivery-diagnosis?alertIdentifier=` (omit `alertIdentifier` to get the diagnosis array for every active alert) - `GET /api/alerts/events?alertIdentifier=&type=&since=&limit=` — append-only alert event log: lifecycle transitions and notification decisions, including suppressions with reasons; all parameters optional, newest first -- `GET /api/alerts/history` +- `GET /api/alerts/history` — each entry carries at most one `evidence` envelope, the newest; `operationalRecord.evidenceIds` keeps the full list, and `GET /api/alerts/active` returns an open alert's full evidence - `DELETE /api/alerts/history` - `GET /api/alerts/incidents` - `POST /api/alerts/incidents/note` diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 8388d6fad9..4c9d934dec 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -1041,6 +1041,13 @@ independent walk of only the window, and `TestWindowedHistoryReusesTheFullHistoryFold` and `TestHistoryReadResultsDoNotAliasTheSharedFold` pin the copy, reuse and isolation rules. +The fold keeps only each snapshot's newest evidence envelope +(`newestHistoryEvidence`, by `observedAt` then position), and history output +applies the same rule to live overlays and the JSON fallback. A resolved +alert's trail of up to 64 envelopes was about 83% of its snapshot. The event log +still stores every snapshot whole. `TestHistoryEntriesKeepOnlyTheNewestEvidence`, +`TestHistoryOverlayKeepsOnlyTheNewestEvidence` and +`TestHistoryFallbackKeepsOnlyTheNewestEvidence` pin this. PBS node-status availability is distinct from connectivity. While a connected diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 3e09582e45..279f7d8518 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -636,7 +636,14 @@ grouping-key fields while silently ignoring `enabled`. `warning`, and `critical` (plus omitted/`all` for no severity filter) and returns only matching rows after the time and resource filters. Alert payloads use the same closed three-level vocabulary; legacy `error` data is normalized -to `critical` before it crosses the API boundary. Notification configuration +to `critical` before it crosses the API boundary. +Each history entry carries at most one `evidence` envelope, the newest by +`observedAt`, whether it comes from the event-log projection, a live overlay or +the JSON fallback (#1803). `operationalRecord.evidenceIds` and +`transitions[].evidenceIds` keep their full lists, so some identifiers name +envelopes the entry omits. The alert event log keeps every snapshot whole, and +`GET /api/alerts/active` returns an open alert's full evidence. +Notification configuration round-trips destination minimum severity as `all`, `warning`, or `critical`. The Relay adapter has a deliberately narrower persisted floor of `all` or `critical`, but an all-alert mobile payload still preserves canonical `info` diff --git a/frontend-modern/public/docs/API.md b/frontend-modern/public/docs/API.md index 432aaadcf8..570a8e053e 100644 --- a/frontend-modern/public/docs/API.md +++ b/frontend-modern/public/docs/API.md @@ -1070,7 +1070,7 @@ it, use the snooze and unsnooze endpoints below. - `GET /api/alerts/active` - `GET /api/alerts/delivery-diagnosis?alertIdentifier=` (omit `alertIdentifier` to get the diagnosis array for every active alert) - `GET /api/alerts/events?alertIdentifier=&type=&since=&limit=` — append-only alert event log: lifecycle transitions and notification decisions, including suppressions with reasons; all parameters optional, newest first -- `GET /api/alerts/history` +- `GET /api/alerts/history` — each entry carries at most one `evidence` envelope, the newest; `operationalRecord.evidenceIds` keeps the full list, and `GET /api/alerts/active` returns an open alert's full evidence - `DELETE /api/alerts/history` - `GET /api/alerts/incidents` - `POST /api/alerts/incidents/note` diff --git a/frontend-modern/src/utils/__tests__/docsLinks.test.ts b/frontend-modern/src/utils/__tests__/docsLinks.test.ts index 811b8a6fa0..efeae304bd 100644 --- a/frontend-modern/src/utils/__tests__/docsLinks.test.ts +++ b/frontend-modern/src/utils/__tests__/docsLinks.test.ts @@ -220,6 +220,19 @@ describe('docsLinks', () => { expect(handler).toContain('agentID != "" && len(*compatibility.ObservationLocationIDs) == 1'); }); + it('documents alert history entries carrying only the newest evidence envelope', () => { + const api = readFileSync(path.join(repoRoot, 'docs', 'API.md'), 'utf8'); + expect(api).toContain( + '`GET /api/alerts/history` — each entry carries at most one `evidence` envelope, the newest', + ); + expect(api).toContain('`operationalRecord.evidenceIds` keeps the full list'); + const projection = readFileSync( + path.join(repoRoot, 'internal', 'alerts', 'history_projection.go'), + 'utf8', + ); + expect(projection).toContain('snapshot.Evidence = newestHistoryEvidence(snapshot.Evidence)'); + }); + it('documents alert configuration saves as replacing only the keys they send', () => { const api = readFileSync(path.join(repoRoot, 'docs', 'API.md'), 'utf8'); expect(api).toContain('`PUT /api/alerts/config` — replaces only the top-level keys the body'); diff --git a/internal/alerts/canonical_identity.go b/internal/alerts/canonical_identity.go index e6365029d8..4b070a4f8c 100644 --- a/internal/alerts/canonical_identity.go +++ b/internal/alerts/canonical_identity.go @@ -407,6 +407,9 @@ func canonicalizeAlertHistoryForOutput(history []Alert) []Alert { if exportedAlert == nil { continue } + // Live overlays and the JSON fallback follow the projection's rule: + // a history entry carries only its newest evidence envelope. + exportedAlert.Evidence = newestHistoryEvidence(exportedAlert.Evidence) exported = append(exported, *exportedAlert) } return exported diff --git a/internal/alerts/history_projection.go b/internal/alerts/history_projection.go index 1e83218a7a..11e4a9cf3b 100644 --- a/internal/alerts/history_projection.go +++ b/internal/alerts/history_projection.go @@ -15,8 +15,28 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog" + "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" ) +// newestHistoryEvidence keeps the newest evidence envelope, by ObservedAt and +// then position, in a fresh slice. A history entry carries at most this one: +// a resolved alert's full trail of up to maxOperationalEvidencePerAlert +// envelopes was most of each entry's size, and nothing reads it from history +// (#1803). The event log keeps every snapshot whole, and operationalRecord and +// transitions keep their full evidence ID lists. +func newestHistoryEvidence(evidence []operationaltrust.EvidenceEnvelope) []operationaltrust.EvidenceEnvelope { + if len(evidence) == 0 { + return nil + } + newest := 0 + for i := range evidence { + if !evidence[i].ObservedAt.Before(evidence[newest].ObservedAt) { + newest = i + } + } + return []operationaltrust.EvidenceEnvelope{evidence[newest].Clone()} +} + // historyEventMark is what one folded lifecycle event contributed to its // occurrence. The marks let a Since window be answered from the full fold // instead of decoding every snapshot in the window again (#1803). @@ -182,6 +202,9 @@ func (p *historyProjection) fold(throughID int64) error { if err := json.Unmarshal(event.Snapshot, &snapshot); err != nil { return nil } + // History keeps only the newest evidence envelope. Merging takes the + // incoming snapshot's evidence, so trimming here is all the fold holds. + snapshot.Evidence = newestHistoryEvidence(snapshot.Evidence) key := historyOccurrenceKey(event.AlertID, &snapshot) occ, exists := p.occurrences[key] if !exists { diff --git a/internal/alerts/history_projection_parity_test.go b/internal/alerts/history_projection_parity_test.go index b028865299..4fb7ab6d55 100644 --- a/internal/alerts/history_projection_parity_test.go +++ b/internal/alerts/history_projection_parity_test.go @@ -19,6 +19,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog" + "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" ) func TestHistoryProjectionIncrementalReadersPreserveChronologyAndIsolation(t *testing.T) { @@ -554,3 +555,130 @@ func TestLimitedHistoryReadCopiesOnlyReturnedRows(t *testing.T) { t.Fatalf("a 10-row read allocated %d bytes against %d for all 2000 rows; it must not copy the whole fold", ten, everything) } } + +// historyEvidence builds n envelopes whose ObservedAt order differs from their +// position, so "newest" must be chosen by time, not by slice order. +func historyEvidence(base time.Time, n int, newestIndex int) []operationaltrust.EvidenceEnvelope { + evidence := make([]operationaltrust.EvidenceEnvelope, 0, n) + for i := 0; i < n; i++ { + observed := base.Add(time.Duration(n-i) * time.Second) + if i == newestIndex { + observed = base.Add(time.Hour) + } + evidence = append(evidence, operationaltrust.EvidenceEnvelope{ID: fmt.Sprintf("ev-%02d", i), ObservedAt: observed}) + } + return evidence +} + +func TestNewestHistoryEvidenceChoosesByObservationTime(t *testing.T) { + base := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + if got := newestHistoryEvidence(nil); got != nil { + t.Fatalf("no evidence must stay nil, got %+v", got) + } + got := newestHistoryEvidence(historyEvidence(base, 64, 17)) + if len(got) != 1 || got[0].ID != "ev-17" { + t.Fatalf("newest by ObservedAt = %+v, want ev-17", got) + } + tied := []operationaltrust.EvidenceEnvelope{{ID: "first", ObservedAt: base}, {ID: "second", ObservedAt: base}} + if got := newestHistoryEvidence(tied); len(got) != 1 || got[0].ID != "second" { + t.Fatalf("a tie must keep the later envelope, got %+v", got) + } +} + +// A history entry carries only its newest evidence envelope, in the shared +// fold as well as the response; the event log and the record keep everything. +func TestHistoryEntriesKeepOnlyTheNewestEvidence(t *testing.T) { + m := newHistoryParityManager(t) + store := m.eventLogStore() + start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + ids := make([]string, 0, 64) + for i := 0; i < 64; i++ { + ids = append(ids, fmt.Sprintf("ev-%02d", i)) + } + write := func(kind string, occurred time.Time, evidence []operationaltrust.EvidenceEnvelope) { + t.Helper() + snapshot, err := json.Marshal(Alert{ID: "disk-1", ResourceID: "vm-1", StartTime: start, LastSeen: occurred, + Evidence: evidence, OperationalRecord: &operationaltrust.OperationalRecord{ID: "record-1", EvidenceIDs: ids}}) + if err != nil { + t.Fatal(err) + } + if err := store.AppendDurable(eventlog.Event{Type: kind, AlertID: "disk-1", OccurredAt: occurred, Snapshot: snapshot}); err != nil { + t.Fatal(err) + } + } + write(eventlog.TypeFired, start, historyEvidence(start, 5, 2)) + write(eventlog.TypeResolved, start.Add(time.Minute), historyEvidence(start, 64, 40)) + + for _, since := range []time.Time{{}, start.Add(30 * time.Second)} { + history, ok := m.AlertHistoryFromEvents(since, 0) + if !ok || len(history) != 1 { + t.Fatalf("since=%s: history = %d entries, ok=%v", since, len(history), ok) + } + if got := history[0].Evidence; len(got) != 1 || got[0].ID != "ev-40" { + t.Fatalf("since=%s: evidence = %+v, want only the newest envelope ev-40", since, got) + } + if got := history[0].OperationalRecord; got == nil || len(got.EvidenceIDs) != 64 { + t.Fatalf("since=%s: the record must keep its full evidence ID list, got %+v", since, got) + } + } + m.historyProjectionMu.Lock() + for _, occ := range m.historyProjection.occurrences { + if len(occ.alert.Evidence) != 1 { + t.Errorf("the shared fold holds %d evidence envelopes; it must hold only the newest", len(occ.alert.Evidence)) + } + } + m.historyProjectionMu.Unlock() + + var logged []int + if err := store.WalkOldest(eventlog.Filter{AlertID: "disk-1"}, func(event eventlog.Event) error { + var snapshot Alert + if err := json.Unmarshal(event.Snapshot, &snapshot); err != nil { + return err + } + logged = append(logged, len(snapshot.Evidence)) + return nil + }); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(logged, []int{5, 64}) { + t.Fatalf("the event log must keep every snapshot whole, evidence lengths = %v", logged) + } +} + +func TestHistoryOverlayKeepsOnlyTheNewestEvidence(t *testing.T) { + start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + m := newHistoryParityManager(t) + m.mu.Lock() + m.activeAlerts["live"] = &Alert{ID: "cpu-1", ResourceID: "vm-2", StartTime: start, LastSeen: start, + Evidence: historyEvidence(start, 12, 3)} + m.mu.Unlock() + history, ok := m.AlertHistoryFromEvents(time.Time{}, 0) + if !ok || len(history) != 1 { + t.Fatalf("overlay history = %d entries, ok=%v", len(history), ok) + } + if got := history[0].Evidence; len(got) != 1 || got[0].ID != "ev-03" { + t.Fatalf("a live overlay entry carries %+v, want only ev-03", got) + } + m.mu.RLock() + live := len(m.activeAlerts["live"].Evidence) + m.mu.RUnlock() + if live != 12 { + t.Fatalf("trimming history output changed the live alert's evidence to %d envelopes", live) + } +} + +// A separate test: two newTestManager calls on one t deadlock. +func TestHistoryFallbackKeepsOnlyTheNewestEvidence(t *testing.T) { + start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + fallback := newTestManager(t) + fallback.SetEventLog(nil) // read the JSON-backed history, not a projection + fallback.historyManager.AddAlert(Alert{ID: "mem-1", ResourceID: "vm-3", StartTime: start, LastSeen: start, + Evidence: historyEvidence(start, 9, 6)}) + entries := fallback.GetAlertHistory(0) + if len(entries) != 1 { + t.Fatalf("fallback history = %d entries", len(entries)) + } + if got := entries[0].Evidence; len(got) != 1 || got[0].ID != "ev-06" { + t.Fatalf("a JSON-history entry carries %+v, want only ev-06", got) + } +} diff --git a/internal/alerts/operational_contract_test.go b/internal/alerts/operational_contract_test.go index c15a8b48f5..abdef56a18 100644 --- a/internal/alerts/operational_contract_test.go +++ b/internal/alerts/operational_contract_test.go @@ -980,3 +980,39 @@ func TestCapacityForecastAlertNamesItsPlatform(t *testing.T) { t.Fatalf("platformType metadata = %v, want vmware-vsphere", got) } } + +// History output keeps an alert's operational record and transitions whole but +// only its newest evidence envelope, and never changes the alert it copies +// (#1803). +func TestHistoryOutputKeepsTheRecordAndOnlyTheNewestEvidence(t *testing.T) { + observed := time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC) + alert := Alert{ + ID: "disk-1", + ResourceID: "vm-1", + Level: AlertLevelWarning, + StartTime: observed, + LastSeen: observed, + Evidence: []operationaltrust.EvidenceEnvelope{ + {ID: "ev-new", ObservedAt: observed.Add(time.Hour)}, + {ID: "ev-old", ObservedAt: observed}, + }, + OperationalRecord: &operationaltrust.OperationalRecord{ID: "record-1", EvidenceIDs: []string{"ev-old", "ev-new"}}, + Transitions: []operationaltrust.LifecycleTransition{{ID: "transition-1", EvidenceIDs: []string{"ev-old"}}}, + } + output := canonicalizeAlertHistoryForOutput([]Alert{alert}) + if len(output) != 1 { + t.Fatalf("output = %d entries", len(output)) + } + if got := output[0].Evidence; len(got) != 1 || got[0].ID != "ev-new" { + t.Fatalf("history evidence = %+v, want only ev-new", got) + } + if got := output[0].OperationalRecord; got == nil || len(got.EvidenceIDs) != 2 { + t.Fatalf("the operational record must keep both evidence IDs, got %+v", got) + } + if got := output[0].Transitions; len(got) != 1 || len(got[0].EvidenceIDs) != 1 { + t.Fatalf("transitions must be kept whole, got %+v", got) + } + if len(alert.Evidence) != 2 { + t.Fatalf("shaping history output changed its input alert to %d envelopes", len(alert.Evidence)) + } +}