From 2e93644991341fe3257589af1221fb0946b4fa60 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 12:24:27 +0100 Subject: [PATCH] test(alerts): protect storage connectivity recovery across restart Exercise mirror and SQLite restoration through missing connectivity, independent healthy capacity, confirmed recovery and a second restart. Negative control fails in both modes with the old missing-status recovery behaviour. Change-source: pulse-maintainer --- internal/alerts/storage_restart_test.go | 82 +++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/internal/alerts/storage_restart_test.go b/internal/alerts/storage_restart_test.go index 5e1a7ae71..85f96d202 100644 --- a/internal/alerts/storage_restart_test.go +++ b/internal/alerts/storage_restart_test.go @@ -124,3 +124,85 @@ func TestStorageRecoveryAcrossRestart(t *testing.T) { }) } } + +// Connectivity uses the discrete recovery gate, not the capacity evaluator. +// A restored incident must still require observed recovery, even when capacity +// telemetry is healthy and connectivity is absent. +func TestStorageConnectivityRecoveryAcrossRestart(t *testing.T) { + for _, sqlite := range []bool{false, true} { + name := "recovery mirror" + if sqlite { + name = "SQLite authority" + } + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + start := func() *Manager { + m := NewManagerWithDataDir(dir) + t.Cleanup(m.Stop) + if sqlite { + m.EnableEventLog() + if !m.activeStateAuthoritative.Load() { + t.Fatal("SQLite authority not enabled") + } + } + m.UpdateConfig(AlertConfig{Enabled: true, ActivationState: ActivationActive, + TimeThresholds: map[string]int{"storage": 0}, + StorageDefault: HysteresisThreshold{Trigger: 80, Clear: 70}}) + return m + } + checkpoint := func(m *Manager) { + t.Helper() + if err := m.SaveActiveAlerts(); err != nil { + t.Fatal(err) + } + m.Stop() + } + s := models.Storage{ID: "storage-connectivity-restart", Name: "backups", + Status: "unavailable", Total: 1000, Free: 1000} + id := canonicalConnectivityStateID(s.ID) + m := start() + for range 3 { + m.CheckStorage(s) + } + original := *testRequireActiveAlert(t, m, id) + checkpoint(m) + + m = start() + for _, status := range []string{"", "unknown", " UNKNOWN "} { + s.Status = status + for range 5 { + m.CheckStorage(s) + } + retained := testRequireActiveAlert(t, m, id) + if !retained.StartTime.Equal(original.StartTime) || m.GetResolvedAlert(id) != nil { + t.Fatal("missing connectivity changed the restored incident") + } + } + s.Status = "available" + for i := 1; i < offlineRecoveryConfirmationsStorage; i++ { + m.CheckStorage(s) + testRequireActiveAlert(t, m, id) + if m.GetResolvedAlert(id) != nil { + t.Fatal("restored connectivity incident recovered before confirmation") + } + } + m.CheckStorage(s) + if testHasActiveAlert(t, m, id) { + t.Fatal("confirmed connectivity did not recover") + } + if resolved := m.GetResolvedAlert(id); resolved == nil || !resolved.StartTime.Equal(original.StartTime) { + t.Fatalf("recovery lost original incident identity: %+v", resolved) + } + checkpoint(m) + + m = start() + s.Status = "unknown" + for range 5 { + m.CheckStorage(s) + } + if testHasActiveAlert(t, m, id) { + t.Fatal("restart or missing connectivity resurrected resolved incident") + } + }) + } +}