feat(alerts): make the event log the alert history authority

The retirement scoped by docs/ALERT_ENGINE_EVOLUTION.md: with an event
log enabled, GetAlertHistory and GetAlertHistorySince serve the
projection from the log's lifecycle snapshots; the JSON snapshot file
stops being written and is renamed to *.imported after a one-time
migration. The #1693 class — fire/resolve churn rewriting the JSON file
and deleting history — cannot recur against an append-only store.

Migration: on the first EnableEventLog with a JSON history file
present, every entry is written synchronously into the log as a
history_imported event (ImportEvents bypasses the droppable append
buffer — losing an entry to a full buffer would silently lose user
data), then the files retire to *.imported as backups. The file's
presence is the migration marker, so the import is idempotent and a
failed import leaves the JSON authoritative for the next attempt.
History retention (30 days) sits inside log retention (90 days), so no
entry can be pruned out from under the migration.

Clearing history keeps the log append-only: the clear is a
history_cleared tombstone and the projection ignores lifecycle events
that precede it. One deliberate difference from the JSON behavior:
still-active alerts reappear in history immediately after a clear —
they are current state, not cleared history.

The in-memory history manager remains as the fallback read model for
managers without an event log, and the parity suite now reads it
directly so it keeps characterizing the projection instead of comparing
the projection with itself.
This commit is contained in:
rcourtman
2026-08-27 09:22:05 +01:00
parent 871f2348bd
commit 2dc2b72759
8 changed files with 294 additions and 3 deletions
+1
View File
@@ -31,6 +31,7 @@ func (m *Manager) EnableEventLog() {
return
}
m.SetEventLog(store)
m.importLegacyHistoryIntoEventLog(store)
}
// SetEventLog installs an event log store. Passing nil disables recording.
+23
View File
@@ -44,6 +44,14 @@ const (
// live manager on an alert's state — the always-on parity signal
// (docs/ALERT_ENGINE_EVOLUTION.md, Phase 1).
TypeShadowDivergence = "shadow_divergence"
// TypeHistoryImported carries one legacy JSON-history entry migrated
// into the log when the log becomes the history authority. Its
// snapshot is the entry's final state.
TypeHistoryImported = "history_imported"
// TypeHistoryCleared is the user's clear-history action. The log stays
// append-only; the history projection ignores lifecycle events that
// precede the newest tombstone.
TypeHistoryCleared = "history_cleared"
)
// Event is one immutable alert event.
@@ -239,6 +247,21 @@ func (s *Store) Append(event Event) {
}
}
// ImportEvents writes events synchronously, bypassing the droppable append
// buffer. It exists for the one-time legacy-history migration, where losing
// an entry to a full buffer would silently lose user data.
func (s *Store) ImportEvents(events []Event) error {
if s == nil {
return fmt.Errorf("event log is not enabled")
}
for i := range events {
if events[i].OccurredAt.IsZero() {
events[i].OccurredAt = time.Now()
}
}
return s.insertBatch(events)
}
// Dropped reports how many events were discarded because the append buffer
// was full. A non-zero value means the log is incomplete for that window.
func (s *Store) Dropped() int64 {
+56
View File
@@ -57,6 +57,9 @@ type HistoryManager struct {
saveTicker *time.Ticker
workerWG sync.WaitGroup
callbacks []AlertCallback // Called when alerts are added
// storageRetired marks the JSON files migrated into the event log:
// in-memory history continues, disk writes stop.
storageRetired bool
}
func historyIdentityKey(alert *Alert) string {
@@ -235,6 +238,53 @@ func mergeHistoryAlertSnapshots(existing, incoming Alert) Alert {
return merged
}
// SnapshotEntries returns a copy of every history entry, oldest first, for
// the one-time migration into the event log.
func (hm *HistoryManager) SnapshotEntries() []HistoryEntry {
hm.mu.RLock()
defer hm.mu.RUnlock()
entries := make([]HistoryEntry, 0, len(hm.history))
for _, entry := range hm.history {
entries = append(entries, HistoryEntry{Alert: *entry.Alert.Clone(), Timestamp: entry.Timestamp})
}
return entries
}
// RetireStorage renames the JSON history files out of the load path after a
// successful migration into the event log, and stops further disk writes.
// The in-memory entries stay available as the fallback read model for the
// rest of the process lifetime.
func (hm *HistoryManager) RetireStorage() error {
hm.saveMu.Lock()
defer hm.saveMu.Unlock()
hm.mu.Lock()
hm.storageRetired = true
hm.mu.Unlock()
for _, path := range []string{hm.historyFile, hm.backupFile} {
if path == "" {
continue
}
if _, err := os.Stat(path); err != nil {
continue
}
if err := os.Rename(path, path+".imported"); err != nil {
return fmt.Errorf("retire history file %s: %w", path, err)
}
}
return nil
}
// StorageFileExists reports whether the JSON history file is still present —
// the self-describing marker that the legacy history has not been migrated
// into the event log yet.
func (hm *HistoryManager) StorageFileExists() bool {
if hm.historyFile == "" {
return false
}
_, err := os.Stat(hm.historyFile)
return err == nil
}
// UpdateAlertLastSeen updates the LastSeen timestamp on the most recent
// history entry matching the given alert ID. This is called when an alert is
// resolved so that the stored history reflects the true duration of the alert,
@@ -463,6 +513,12 @@ func (hm *HistoryManager) saveHistory() error {
// saveHistoryWithRetry saves history with exponential backoff retry
func (hm *HistoryManager) saveHistoryWithRetry(maxRetries int) error {
hm.mu.RLock()
retired := hm.storageRetired
hm.mu.RUnlock()
if retired {
return nil
}
if maxRetries < 1 {
maxRetries = 1
}
+73
View File
@@ -0,0 +1,73 @@
package alerts
// One-time migration of the legacy JSON alert history into the event log
// (docs/ALERT_ENGINE_EVOLUTION.md — the log becomes the sole history
// authority). The JSON file's continued presence is the migration marker:
// import runs when the file exists, and a successful import renames it to
// *.imported, so the migration is idempotent and the original data survives
// as a backup.
import (
"encoding/json"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
"github.com/rs/zerolog/log"
)
// importLegacyHistoryIntoEventLog migrates the loaded JSON history entries
// into the event log as history_imported events and retires the JSON files.
// A failed import leaves the files in place so the next startup retries;
// history reads fall back to the in-memory entries either way.
func (m *Manager) importLegacyHistoryIntoEventLog(store *eventlog.Store) {
if m == nil || store == nil || m.historyManager == nil {
return
}
if !m.historyManager.StorageFileExists() {
return
}
entries := m.historyManager.SnapshotEntries()
events := make([]eventlog.Event, 0, len(entries))
for i := range entries {
entry := &entries[i]
exported := cloneAlertForOutput(&entry.Alert)
if exported == nil || exported.ID == "" {
continue
}
snapshot, err := json.Marshal(exported)
if err != nil {
continue
}
occurredAt := entry.Timestamp
if entry.Alert.LastSeen.After(occurredAt) {
occurredAt = entry.Alert.LastSeen
}
events = append(events, eventlog.Event{
OccurredAt: occurredAt,
Type: eventlog.TypeHistoryImported,
AlertID: exported.ID,
ResourceID: exported.ResourceID,
ResourceName: exported.ResourceName,
AlertType: exported.Type,
Level: string(exported.Level),
Message: "Imported from the legacy alert history file.",
Snapshot: snapshot,
})
}
if len(events) > 0 {
if err := store.ImportEvents(events); err != nil {
log.Error().Err(err).
Int("entries", len(events)).
Msg("legacy alert history import failed; JSON history stays authoritative until the next attempt")
return
}
}
if err := m.historyManager.RetireStorage(); err != nil {
log.Error().Err(err).Msg("legacy alert history files could not be retired after import")
return
}
log.Info().
Int("entries", len(events)).
Msg("legacy alert history imported into the event log; JSON history files retired")
}
+105
View File
@@ -0,0 +1,105 @@
package alerts
// The one-time legacy-history migration: JSON entries become
// history_imported events, the files retire to *.imported, and the
// projection serves the imported entries. The file's presence is the
// marker, so a second startup imports nothing.
import (
"os"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
)
func TestLegacyHistoryImportRetiresJSONAndServesEntries(t *testing.T) {
m := newTestManager(t)
m.UpdateConfig(contractTestConfig(m))
// Seed a legacy occurrence and persist it the way the JSON manager did.
past := time.Now().Add(-2 * time.Hour)
legacyAlert := Alert{
ID: "legacy-vm::metric-threshold:cpu",
Type: "cpu",
Level: AlertLevelWarning,
ResourceID: "legacy-vm",
ResourceName: "Legacy VM",
Value: 91,
StartTime: past,
LastSeen: past.Add(30 * time.Minute),
}
m.historyManager.AddAlert(legacyAlert)
if err := m.historyManager.saveHistory(); err != nil {
t.Fatalf("persist legacy history: %v", err)
}
if !m.historyManager.StorageFileExists() {
t.Fatal("expected the JSON history file on disk before migration")
}
historyFile := m.historyManager.historyFile
store, err := eventlog.OpenInMemory()
if err != nil {
t.Fatal(err)
}
m.SetEventLog(store)
t.Cleanup(func() { m.SetEventLog(nil) })
m.importLegacyHistoryIntoEventLog(store)
if m.historyManager.StorageFileExists() {
t.Fatal("JSON history file still present after import")
}
if _, err := os.Stat(historyFile + ".imported"); err != nil {
t.Fatalf("retired history backup missing: %v", err)
}
projected, ok := m.AlertHistoryFromEvents(time.Time{}, 0)
if !ok {
t.Fatal("projection unavailable")
}
found := false
for _, entry := range projected {
if entry.ResourceID == "legacy-vm" {
found = true
if !entry.StartTime.Truncate(time.Second).Equal(past.Truncate(time.Second)) {
t.Errorf("imported entry StartTime = %v, want %v", entry.StartTime, past)
}
}
}
if !found {
t.Fatalf("imported legacy entry missing from projection: %+v", summarizeHistory(projected))
}
// Second run: the marker file is gone, so nothing imports again.
before, _ := store.Query(eventlog.Filter{Types: []string{eventlog.TypeHistoryImported}, Limit: 1000})
m.importLegacyHistoryIntoEventLog(store)
store.Flush()
after, _ := store.Query(eventlog.Filter{Types: []string{eventlog.TypeHistoryImported}, Limit: 1000})
if len(after) != len(before) {
t.Fatalf("re-running the import duplicated entries: %d -> %d", len(before), len(after))
}
}
func TestClearAlertHistoryTombstonesTheProjection(t *testing.T) {
m := newHistoryParityManager(t)
cfg := m.GetConfig()
contractRaiseGuestCPUAlert(t, m, "clear-vm-1", 95)
m.checkMetric("clear-vm-1", "Contract VM clear-vm-1", "node1", "inst1", "guest", "cpu", 40, cfg.GuestDefaults.CPU, nil)
if entries := m.GetAlertHistory(0); len(entries) == 0 {
t.Fatal("expected history before the clear")
}
if err := m.ClearAlertHistory(); err != nil {
t.Fatalf("clear history: %v", err)
}
if entries := m.GetAlertHistory(0); len(entries) != 0 {
t.Fatalf("expected empty history after the clear, got %d entries: %v", len(entries), summarizeHistory(entries))
}
// Lifecycle after the tombstone repopulates normally.
contractRaiseGuestCPUAlert(t, m, "clear-vm-2", 96)
entries := m.GetAlertHistory(0)
if len(entries) != 1 || entries[0].ResourceID != "clear-vm-2" {
t.Fatalf("expected exactly the post-clear occurrence, got %v", summarizeHistory(entries))
}
}
+9
View File
@@ -46,6 +46,8 @@ func (m *Manager) AlertHistoryFromEvents(since time.Time, limit int) ([]Alert, b
eventlog.TypeAcknowledged,
eventlog.TypeUnacknowledged,
eventlog.TypeEscalated,
eventlog.TypeHistoryImported,
eventlog.TypeHistoryCleared,
},
Since: since,
Limit: 1000,
@@ -60,6 +62,13 @@ func (m *Manager) AlertHistoryFromEvents(since time.Time, limit int) ([]Alert, b
order := make([]string, 0, len(events))
for i := len(events) - 1; i >= 0; i-- {
event := events[i]
if event.Type == eventlog.TypeHistoryCleared {
// The user cleared history: everything before the tombstone
// leaves the projection. The log itself stays append-only.
occurrences = make(map[string]*historyOccurrence)
order = order[:0]
continue
}
if len(event.Snapshot) == 0 {
continue
}
@@ -31,7 +31,10 @@ func newHistoryParityManager(t *testing.T) *Manager {
// the fields the history API exposes.
func assertHistoryParity(t *testing.T, m *Manager) {
t.Helper()
legacy := m.GetAlertHistory(0)
// Read the JSON-backed side directly: GetAlertHistory itself now
// returns the projection, so going through it would compare the
// projection with itself.
legacy := m.applyCurrentNodeDisplayNames(canonicalizeAlertHistoryForOutput(m.historyManager.GetAllHistory(0)))
projected, ok := m.AlertHistoryFromEvents(time.Time{}, 0)
if !ok {
t.Fatal("projection unavailable despite an enabled event log")
+23 -2
View File
@@ -3,6 +3,7 @@ package alerts
import (
"encoding/json"
"fmt"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
"sort"
"strconv"
"strings"
@@ -330,8 +331,14 @@ func (m *Manager) GetResolvedAlert(alertID string) *ResolvedAlert {
return result
}
// GetAlertHistory returns alert history
// GetAlertHistory returns alert history. With an event log enabled the
// list is projected from the log's lifecycle snapshots (the authority);
// the in-memory JSON-loaded history is the fallback for managers without
// a log.
func (m *Manager) GetAlertHistory(limit int) []Alert {
if projected, ok := m.AlertHistoryFromEvents(time.Time{}, limit); ok {
return projected
}
return m.applyCurrentNodeDisplayNames(canonicalizeAlertHistoryForOutput(m.historyManager.GetAllHistory(limit)))
}
@@ -341,6 +348,9 @@ func (m *Manager) GetAlertHistorySince(since time.Time, limit int) []Alert {
return m.GetAlertHistory(limit)
}
if projected, ok := m.AlertHistoryFromEvents(since, limit); ok {
return projected
}
return m.applyCurrentNodeDisplayNames(canonicalizeAlertHistoryForOutput(m.historyManager.GetHistory(since, limit)))
}
@@ -355,8 +365,19 @@ func (m *Manager) applyCurrentNodeDisplayNames(alerts []Alert) []Alert {
return alerts
}
// ClearAlertHistory clears all alert history
// ClearAlertHistory clears all alert history. The event log stays
// append-only: the clear is a tombstone event, and the projection ignores
// lifecycle events that precede it.
func (m *Manager) ClearAlertHistory() error {
if store := m.eventLogStore(); store != nil {
if err := store.ImportEvents([]eventlog.Event{{
Type: eventlog.TypeHistoryCleared,
AlertID: "history",
Message: "Alert history cleared by the user.",
}}); err != nil {
return err
}
}
return m.historyManager.ClearAllHistory()
}