mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-11 14:00:29 +00:00
feat(alerts): make the event log the alert history authority
The retirement scoped by docs/ALERT_ENGINE_EVOLUTION.md: with an event log enabled, GetAlertHistory and GetAlertHistorySince serve the projection from the log's lifecycle snapshots; the JSON snapshot file stops being written and is renamed to *.imported after a one-time migration. The #1693 class — fire/resolve churn rewriting the JSON file and deleting history — cannot recur against an append-only store. Migration: on the first EnableEventLog with a JSON history file present, every entry is written synchronously into the log as a history_imported event (ImportEvents bypasses the droppable append buffer — losing an entry to a full buffer would silently lose user data), then the files retire to *.imported as backups. The file's presence is the migration marker, so the import is idempotent and a failed import leaves the JSON authoritative for the next attempt. History retention (30 days) sits inside log retention (90 days), so no entry can be pruned out from under the migration. Clearing history keeps the log append-only: the clear is a history_cleared tombstone and the projection ignores lifecycle events that precede it. One deliberate difference from the JSON behavior: still-active alerts reappear in history immediately after a clear — they are current state, not cleared history. The in-memory history manager remains as the fallback read model for managers without an event log, and the parity suite now reads it directly so it keeps characterizing the projection instead of comparing the projection with itself.
This commit is contained in:
@@ -31,6 +31,7 @@ func (m *Manager) EnableEventLog() {
|
||||
return
|
||||
}
|
||||
m.SetEventLog(store)
|
||||
m.importLegacyHistoryIntoEventLog(store)
|
||||
}
|
||||
|
||||
// SetEventLog installs an event log store. Passing nil disables recording.
|
||||
|
||||
@@ -44,6 +44,14 @@ const (
|
||||
// live manager on an alert's state — the always-on parity signal
|
||||
// (docs/ALERT_ENGINE_EVOLUTION.md, Phase 1).
|
||||
TypeShadowDivergence = "shadow_divergence"
|
||||
// TypeHistoryImported carries one legacy JSON-history entry migrated
|
||||
// into the log when the log becomes the history authority. Its
|
||||
// snapshot is the entry's final state.
|
||||
TypeHistoryImported = "history_imported"
|
||||
// TypeHistoryCleared is the user's clear-history action. The log stays
|
||||
// append-only; the history projection ignores lifecycle events that
|
||||
// precede the newest tombstone.
|
||||
TypeHistoryCleared = "history_cleared"
|
||||
)
|
||||
|
||||
// Event is one immutable alert event.
|
||||
@@ -239,6 +247,21 @@ func (s *Store) Append(event Event) {
|
||||
}
|
||||
}
|
||||
|
||||
// ImportEvents writes events synchronously, bypassing the droppable append
|
||||
// buffer. It exists for the one-time legacy-history migration, where losing
|
||||
// an entry to a full buffer would silently lose user data.
|
||||
func (s *Store) ImportEvents(events []Event) error {
|
||||
if s == nil {
|
||||
return fmt.Errorf("event log is not enabled")
|
||||
}
|
||||
for i := range events {
|
||||
if events[i].OccurredAt.IsZero() {
|
||||
events[i].OccurredAt = time.Now()
|
||||
}
|
||||
}
|
||||
return s.insertBatch(events)
|
||||
}
|
||||
|
||||
// Dropped reports how many events were discarded because the append buffer
|
||||
// was full. A non-zero value means the log is incomplete for that window.
|
||||
func (s *Store) Dropped() int64 {
|
||||
|
||||
@@ -57,6 +57,9 @@ type HistoryManager struct {
|
||||
saveTicker *time.Ticker
|
||||
workerWG sync.WaitGroup
|
||||
callbacks []AlertCallback // Called when alerts are added
|
||||
// storageRetired marks the JSON files migrated into the event log:
|
||||
// in-memory history continues, disk writes stop.
|
||||
storageRetired bool
|
||||
}
|
||||
|
||||
func historyIdentityKey(alert *Alert) string {
|
||||
@@ -235,6 +238,53 @@ func mergeHistoryAlertSnapshots(existing, incoming Alert) Alert {
|
||||
return merged
|
||||
}
|
||||
|
||||
// SnapshotEntries returns a copy of every history entry, oldest first, for
|
||||
// the one-time migration into the event log.
|
||||
func (hm *HistoryManager) SnapshotEntries() []HistoryEntry {
|
||||
hm.mu.RLock()
|
||||
defer hm.mu.RUnlock()
|
||||
entries := make([]HistoryEntry, 0, len(hm.history))
|
||||
for _, entry := range hm.history {
|
||||
entries = append(entries, HistoryEntry{Alert: *entry.Alert.Clone(), Timestamp: entry.Timestamp})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
// RetireStorage renames the JSON history files out of the load path after a
|
||||
// successful migration into the event log, and stops further disk writes.
|
||||
// The in-memory entries stay available as the fallback read model for the
|
||||
// rest of the process lifetime.
|
||||
func (hm *HistoryManager) RetireStorage() error {
|
||||
hm.saveMu.Lock()
|
||||
defer hm.saveMu.Unlock()
|
||||
hm.mu.Lock()
|
||||
hm.storageRetired = true
|
||||
hm.mu.Unlock()
|
||||
for _, path := range []string{hm.historyFile, hm.backupFile} {
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
continue
|
||||
}
|
||||
if err := os.Rename(path, path+".imported"); err != nil {
|
||||
return fmt.Errorf("retire history file %s: %w", path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StorageFileExists reports whether the JSON history file is still present —
|
||||
// the self-describing marker that the legacy history has not been migrated
|
||||
// into the event log yet.
|
||||
func (hm *HistoryManager) StorageFileExists() bool {
|
||||
if hm.historyFile == "" {
|
||||
return false
|
||||
}
|
||||
_, err := os.Stat(hm.historyFile)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// UpdateAlertLastSeen updates the LastSeen timestamp on the most recent
|
||||
// history entry matching the given alert ID. This is called when an alert is
|
||||
// resolved so that the stored history reflects the true duration of the alert,
|
||||
@@ -463,6 +513,12 @@ func (hm *HistoryManager) saveHistory() error {
|
||||
|
||||
// saveHistoryWithRetry saves history with exponential backoff retry
|
||||
func (hm *HistoryManager) saveHistoryWithRetry(maxRetries int) error {
|
||||
hm.mu.RLock()
|
||||
retired := hm.storageRetired
|
||||
hm.mu.RUnlock()
|
||||
if retired {
|
||||
return nil
|
||||
}
|
||||
if maxRetries < 1 {
|
||||
maxRetries = 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package alerts
|
||||
|
||||
// One-time migration of the legacy JSON alert history into the event log
|
||||
// (docs/ALERT_ENGINE_EVOLUTION.md — the log becomes the sole history
|
||||
// authority). The JSON file's continued presence is the migration marker:
|
||||
// import runs when the file exists, and a successful import renames it to
|
||||
// *.imported, so the migration is idempotent and the original data survives
|
||||
// as a backup.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// importLegacyHistoryIntoEventLog migrates the loaded JSON history entries
|
||||
// into the event log as history_imported events and retires the JSON files.
|
||||
// A failed import leaves the files in place so the next startup retries;
|
||||
// history reads fall back to the in-memory entries either way.
|
||||
func (m *Manager) importLegacyHistoryIntoEventLog(store *eventlog.Store) {
|
||||
if m == nil || store == nil || m.historyManager == nil {
|
||||
return
|
||||
}
|
||||
if !m.historyManager.StorageFileExists() {
|
||||
return
|
||||
}
|
||||
|
||||
entries := m.historyManager.SnapshotEntries()
|
||||
events := make([]eventlog.Event, 0, len(entries))
|
||||
for i := range entries {
|
||||
entry := &entries[i]
|
||||
exported := cloneAlertForOutput(&entry.Alert)
|
||||
if exported == nil || exported.ID == "" {
|
||||
continue
|
||||
}
|
||||
snapshot, err := json.Marshal(exported)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
occurredAt := entry.Timestamp
|
||||
if entry.Alert.LastSeen.After(occurredAt) {
|
||||
occurredAt = entry.Alert.LastSeen
|
||||
}
|
||||
events = append(events, eventlog.Event{
|
||||
OccurredAt: occurredAt,
|
||||
Type: eventlog.TypeHistoryImported,
|
||||
AlertID: exported.ID,
|
||||
ResourceID: exported.ResourceID,
|
||||
ResourceName: exported.ResourceName,
|
||||
AlertType: exported.Type,
|
||||
Level: string(exported.Level),
|
||||
Message: "Imported from the legacy alert history file.",
|
||||
Snapshot: snapshot,
|
||||
})
|
||||
}
|
||||
|
||||
if len(events) > 0 {
|
||||
if err := store.ImportEvents(events); err != nil {
|
||||
log.Error().Err(err).
|
||||
Int("entries", len(events)).
|
||||
Msg("legacy alert history import failed; JSON history stays authoritative until the next attempt")
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := m.historyManager.RetireStorage(); err != nil {
|
||||
log.Error().Err(err).Msg("legacy alert history files could not be retired after import")
|
||||
return
|
||||
}
|
||||
log.Info().
|
||||
Int("entries", len(events)).
|
||||
Msg("legacy alert history imported into the event log; JSON history files retired")
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package alerts
|
||||
|
||||
// The one-time legacy-history migration: JSON entries become
|
||||
// history_imported events, the files retire to *.imported, and the
|
||||
// projection serves the imported entries. The file's presence is the
|
||||
// marker, so a second startup imports nothing.
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
|
||||
)
|
||||
|
||||
func TestLegacyHistoryImportRetiresJSONAndServesEntries(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
m.UpdateConfig(contractTestConfig(m))
|
||||
|
||||
// Seed a legacy occurrence and persist it the way the JSON manager did.
|
||||
past := time.Now().Add(-2 * time.Hour)
|
||||
legacyAlert := Alert{
|
||||
ID: "legacy-vm::metric-threshold:cpu",
|
||||
Type: "cpu",
|
||||
Level: AlertLevelWarning,
|
||||
ResourceID: "legacy-vm",
|
||||
ResourceName: "Legacy VM",
|
||||
Value: 91,
|
||||
StartTime: past,
|
||||
LastSeen: past.Add(30 * time.Minute),
|
||||
}
|
||||
m.historyManager.AddAlert(legacyAlert)
|
||||
if err := m.historyManager.saveHistory(); err != nil {
|
||||
t.Fatalf("persist legacy history: %v", err)
|
||||
}
|
||||
if !m.historyManager.StorageFileExists() {
|
||||
t.Fatal("expected the JSON history file on disk before migration")
|
||||
}
|
||||
historyFile := m.historyManager.historyFile
|
||||
|
||||
store, err := eventlog.OpenInMemory()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.SetEventLog(store)
|
||||
t.Cleanup(func() { m.SetEventLog(nil) })
|
||||
m.importLegacyHistoryIntoEventLog(store)
|
||||
|
||||
if m.historyManager.StorageFileExists() {
|
||||
t.Fatal("JSON history file still present after import")
|
||||
}
|
||||
if _, err := os.Stat(historyFile + ".imported"); err != nil {
|
||||
t.Fatalf("retired history backup missing: %v", err)
|
||||
}
|
||||
|
||||
projected, ok := m.AlertHistoryFromEvents(time.Time{}, 0)
|
||||
if !ok {
|
||||
t.Fatal("projection unavailable")
|
||||
}
|
||||
found := false
|
||||
for _, entry := range projected {
|
||||
if entry.ResourceID == "legacy-vm" {
|
||||
found = true
|
||||
if !entry.StartTime.Truncate(time.Second).Equal(past.Truncate(time.Second)) {
|
||||
t.Errorf("imported entry StartTime = %v, want %v", entry.StartTime, past)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("imported legacy entry missing from projection: %+v", summarizeHistory(projected))
|
||||
}
|
||||
|
||||
// Second run: the marker file is gone, so nothing imports again.
|
||||
before, _ := store.Query(eventlog.Filter{Types: []string{eventlog.TypeHistoryImported}, Limit: 1000})
|
||||
m.importLegacyHistoryIntoEventLog(store)
|
||||
store.Flush()
|
||||
after, _ := store.Query(eventlog.Filter{Types: []string{eventlog.TypeHistoryImported}, Limit: 1000})
|
||||
if len(after) != len(before) {
|
||||
t.Fatalf("re-running the import duplicated entries: %d -> %d", len(before), len(after))
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearAlertHistoryTombstonesTheProjection(t *testing.T) {
|
||||
m := newHistoryParityManager(t)
|
||||
cfg := m.GetConfig()
|
||||
contractRaiseGuestCPUAlert(t, m, "clear-vm-1", 95)
|
||||
m.checkMetric("clear-vm-1", "Contract VM clear-vm-1", "node1", "inst1", "guest", "cpu", 40, cfg.GuestDefaults.CPU, nil)
|
||||
|
||||
if entries := m.GetAlertHistory(0); len(entries) == 0 {
|
||||
t.Fatal("expected history before the clear")
|
||||
}
|
||||
if err := m.ClearAlertHistory(); err != nil {
|
||||
t.Fatalf("clear history: %v", err)
|
||||
}
|
||||
if entries := m.GetAlertHistory(0); len(entries) != 0 {
|
||||
t.Fatalf("expected empty history after the clear, got %d entries: %v", len(entries), summarizeHistory(entries))
|
||||
}
|
||||
|
||||
// Lifecycle after the tombstone repopulates normally.
|
||||
contractRaiseGuestCPUAlert(t, m, "clear-vm-2", 96)
|
||||
entries := m.GetAlertHistory(0)
|
||||
if len(entries) != 1 || entries[0].ResourceID != "clear-vm-2" {
|
||||
t.Fatalf("expected exactly the post-clear occurrence, got %v", summarizeHistory(entries))
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,8 @@ func (m *Manager) AlertHistoryFromEvents(since time.Time, limit int) ([]Alert, b
|
||||
eventlog.TypeAcknowledged,
|
||||
eventlog.TypeUnacknowledged,
|
||||
eventlog.TypeEscalated,
|
||||
eventlog.TypeHistoryImported,
|
||||
eventlog.TypeHistoryCleared,
|
||||
},
|
||||
Since: since,
|
||||
Limit: 1000,
|
||||
@@ -60,6 +62,13 @@ func (m *Manager) AlertHistoryFromEvents(since time.Time, limit int) ([]Alert, b
|
||||
order := make([]string, 0, len(events))
|
||||
for i := len(events) - 1; i >= 0; i-- {
|
||||
event := events[i]
|
||||
if event.Type == eventlog.TypeHistoryCleared {
|
||||
// The user cleared history: everything before the tombstone
|
||||
// leaves the projection. The log itself stays append-only.
|
||||
occurrences = make(map[string]*historyOccurrence)
|
||||
order = order[:0]
|
||||
continue
|
||||
}
|
||||
if len(event.Snapshot) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -31,7 +31,10 @@ func newHistoryParityManager(t *testing.T) *Manager {
|
||||
// the fields the history API exposes.
|
||||
func assertHistoryParity(t *testing.T, m *Manager) {
|
||||
t.Helper()
|
||||
legacy := m.GetAlertHistory(0)
|
||||
// Read the JSON-backed side directly: GetAlertHistory itself now
|
||||
// returns the projection, so going through it would compare the
|
||||
// projection with itself.
|
||||
legacy := m.applyCurrentNodeDisplayNames(canonicalizeAlertHistoryForOutput(m.historyManager.GetAllHistory(0)))
|
||||
projected, ok := m.AlertHistoryFromEvents(time.Time{}, 0)
|
||||
if !ok {
|
||||
t.Fatal("projection unavailable despite an enabled event log")
|
||||
|
||||
@@ -3,6 +3,7 @@ package alerts
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -330,8 +331,14 @@ func (m *Manager) GetResolvedAlert(alertID string) *ResolvedAlert {
|
||||
return result
|
||||
}
|
||||
|
||||
// GetAlertHistory returns alert history
|
||||
// GetAlertHistory returns alert history. With an event log enabled the
|
||||
// list is projected from the log's lifecycle snapshots (the authority);
|
||||
// the in-memory JSON-loaded history is the fallback for managers without
|
||||
// a log.
|
||||
func (m *Manager) GetAlertHistory(limit int) []Alert {
|
||||
if projected, ok := m.AlertHistoryFromEvents(time.Time{}, limit); ok {
|
||||
return projected
|
||||
}
|
||||
return m.applyCurrentNodeDisplayNames(canonicalizeAlertHistoryForOutput(m.historyManager.GetAllHistory(limit)))
|
||||
}
|
||||
|
||||
@@ -341,6 +348,9 @@ func (m *Manager) GetAlertHistorySince(since time.Time, limit int) []Alert {
|
||||
return m.GetAlertHistory(limit)
|
||||
}
|
||||
|
||||
if projected, ok := m.AlertHistoryFromEvents(since, limit); ok {
|
||||
return projected
|
||||
}
|
||||
return m.applyCurrentNodeDisplayNames(canonicalizeAlertHistoryForOutput(m.historyManager.GetHistory(since, limit)))
|
||||
}
|
||||
|
||||
@@ -355,8 +365,19 @@ func (m *Manager) applyCurrentNodeDisplayNames(alerts []Alert) []Alert {
|
||||
return alerts
|
||||
}
|
||||
|
||||
// ClearAlertHistory clears all alert history
|
||||
// ClearAlertHistory clears all alert history. The event log stays
|
||||
// append-only: the clear is a tombstone event, and the projection ignores
|
||||
// lifecycle events that precede it.
|
||||
func (m *Manager) ClearAlertHistory() error {
|
||||
if store := m.eventLogStore(); store != nil {
|
||||
if err := store.ImportEvents([]eventlog.Event{{
|
||||
Type: eventlog.TypeHistoryCleared,
|
||||
AlertID: "history",
|
||||
Message: "Alert history cleared by the user.",
|
||||
}}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return m.historyManager.ClearAllHistory()
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user