mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
test(alerts): protect storage connectivity recovery across restart
Exercise mirror and SQLite restoration through missing connectivity, independent healthy capacity, confirmed recovery and a second restart. Negative control fails in both modes with the old missing-status recovery behaviour. Change-source: pulse-maintainer
This commit is contained in:
@@ -124,3 +124,85 @@ func TestStorageRecoveryAcrossRestart(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Connectivity uses the discrete recovery gate, not the capacity evaluator.
|
||||
// A restored incident must still require observed recovery, even when capacity
|
||||
// telemetry is healthy and connectivity is absent.
|
||||
func TestStorageConnectivityRecoveryAcrossRestart(t *testing.T) {
|
||||
for _, sqlite := range []bool{false, true} {
|
||||
name := "recovery mirror"
|
||||
if sqlite {
|
||||
name = "SQLite authority"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
start := func() *Manager {
|
||||
m := NewManagerWithDataDir(dir)
|
||||
t.Cleanup(m.Stop)
|
||||
if sqlite {
|
||||
m.EnableEventLog()
|
||||
if !m.activeStateAuthoritative.Load() {
|
||||
t.Fatal("SQLite authority not enabled")
|
||||
}
|
||||
}
|
||||
m.UpdateConfig(AlertConfig{Enabled: true, ActivationState: ActivationActive,
|
||||
TimeThresholds: map[string]int{"storage": 0},
|
||||
StorageDefault: HysteresisThreshold{Trigger: 80, Clear: 70}})
|
||||
return m
|
||||
}
|
||||
checkpoint := func(m *Manager) {
|
||||
t.Helper()
|
||||
if err := m.SaveActiveAlerts(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Stop()
|
||||
}
|
||||
s := models.Storage{ID: "storage-connectivity-restart", Name: "backups",
|
||||
Status: "unavailable", Total: 1000, Free: 1000}
|
||||
id := canonicalConnectivityStateID(s.ID)
|
||||
m := start()
|
||||
for range 3 {
|
||||
m.CheckStorage(s)
|
||||
}
|
||||
original := *testRequireActiveAlert(t, m, id)
|
||||
checkpoint(m)
|
||||
|
||||
m = start()
|
||||
for _, status := range []string{"", "unknown", " UNKNOWN "} {
|
||||
s.Status = status
|
||||
for range 5 {
|
||||
m.CheckStorage(s)
|
||||
}
|
||||
retained := testRequireActiveAlert(t, m, id)
|
||||
if !retained.StartTime.Equal(original.StartTime) || m.GetResolvedAlert(id) != nil {
|
||||
t.Fatal("missing connectivity changed the restored incident")
|
||||
}
|
||||
}
|
||||
s.Status = "available"
|
||||
for i := 1; i < offlineRecoveryConfirmationsStorage; i++ {
|
||||
m.CheckStorage(s)
|
||||
testRequireActiveAlert(t, m, id)
|
||||
if m.GetResolvedAlert(id) != nil {
|
||||
t.Fatal("restored connectivity incident recovered before confirmation")
|
||||
}
|
||||
}
|
||||
m.CheckStorage(s)
|
||||
if testHasActiveAlert(t, m, id) {
|
||||
t.Fatal("confirmed connectivity did not recover")
|
||||
}
|
||||
if resolved := m.GetResolvedAlert(id); resolved == nil || !resolved.StartTime.Equal(original.StartTime) {
|
||||
t.Fatalf("recovery lost original incident identity: %+v", resolved)
|
||||
}
|
||||
checkpoint(m)
|
||||
|
||||
m = start()
|
||||
s.Status = "unknown"
|
||||
for range 5 {
|
||||
m.CheckStorage(s)
|
||||
}
|
||||
if testHasActiveAlert(t, m, id) {
|
||||
t.Fatal("restart or missing connectivity resurrected resolved incident")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user