test(alerts): protect storage connectivity recovery across restart

Exercise mirror and SQLite restoration through missing connectivity, independent healthy capacity, confirmed recovery and a second restart. Negative control fails in both modes with the old missing-status recovery behaviour.

Change-source: pulse-maintainer
This commit is contained in:
pulse-triage[bot]
2026-09-06 12:24:27 +01:00
parent a8c1b067d6
commit 2e93644991
+82
View File
@@ -124,3 +124,85 @@ func TestStorageRecoveryAcrossRestart(t *testing.T) {
})
}
}
// Connectivity uses the discrete recovery gate, not the capacity evaluator.
// A restored incident must still require observed recovery, even when capacity
// telemetry is healthy and connectivity is absent.
func TestStorageConnectivityRecoveryAcrossRestart(t *testing.T) {
for _, sqlite := range []bool{false, true} {
name := "recovery mirror"
if sqlite {
name = "SQLite authority"
}
t.Run(name, func(t *testing.T) {
dir := t.TempDir()
start := func() *Manager {
m := NewManagerWithDataDir(dir)
t.Cleanup(m.Stop)
if sqlite {
m.EnableEventLog()
if !m.activeStateAuthoritative.Load() {
t.Fatal("SQLite authority not enabled")
}
}
m.UpdateConfig(AlertConfig{Enabled: true, ActivationState: ActivationActive,
TimeThresholds: map[string]int{"storage": 0},
StorageDefault: HysteresisThreshold{Trigger: 80, Clear: 70}})
return m
}
checkpoint := func(m *Manager) {
t.Helper()
if err := m.SaveActiveAlerts(); err != nil {
t.Fatal(err)
}
m.Stop()
}
s := models.Storage{ID: "storage-connectivity-restart", Name: "backups",
Status: "unavailable", Total: 1000, Free: 1000}
id := canonicalConnectivityStateID(s.ID)
m := start()
for range 3 {
m.CheckStorage(s)
}
original := *testRequireActiveAlert(t, m, id)
checkpoint(m)
m = start()
for _, status := range []string{"", "unknown", " UNKNOWN "} {
s.Status = status
for range 5 {
m.CheckStorage(s)
}
retained := testRequireActiveAlert(t, m, id)
if !retained.StartTime.Equal(original.StartTime) || m.GetResolvedAlert(id) != nil {
t.Fatal("missing connectivity changed the restored incident")
}
}
s.Status = "available"
for i := 1; i < offlineRecoveryConfirmationsStorage; i++ {
m.CheckStorage(s)
testRequireActiveAlert(t, m, id)
if m.GetResolvedAlert(id) != nil {
t.Fatal("restored connectivity incident recovered before confirmation")
}
}
m.CheckStorage(s)
if testHasActiveAlert(t, m, id) {
t.Fatal("confirmed connectivity did not recover")
}
if resolved := m.GetResolvedAlert(id); resolved == nil || !resolved.StartTime.Equal(original.StartTime) {
t.Fatalf("recovery lost original incident identity: %+v", resolved)
}
checkpoint(m)
m = start()
s.Status = "unknown"
for range 5 {
m.CheckStorage(s)
}
if testHasActiveAlert(t, m, id) {
t.Fatal("restart or missing connectivity resurrected resolved incident")
}
})
}
}