mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-11 14:00:29 +00:00
ee35d178b5
The last uncharacterized behavior: while Pulse has fresh evidence a Proxmox backup caused an offline state, activation defers — bounded by the max-deferral cap on total condition-active time — and after the backup ends the grace extends to the backup's end plus the post-grace, still capped. The discrete activation path is unified so the gate always operates on a tracked pending incident, and the shadow feed now models the deferral independently from the manager's decision instead of echoing its hold as operator suppression. Unit tests cover deferral, post-grace release, the never-ending-backup cap, and post-grace capping; parity runs the real LoadIntentPolicies/IntentBackup composition with simulated clocks. Fourth consecutive parity slice with no manager defect. Every discrete-family and metric-family behavior is now pinned.
189 lines
8.1 KiB
Go
189 lines
8.1 KiB
Go
package reducer
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestIntentOperatorSuppressionHoldsActivation(t *testing.T) {
|
|
state := NewState()
|
|
suppressed := DiscreteRule{Confirmations: 1, Intent: &DiscreteIntent{
|
|
OperatorSuppressed: true, OperatorReason: "operator_expected_offline",
|
|
}}
|
|
|
|
events := state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 0), suppressed)
|
|
if len(events) != 1 || events[0].Type != EventPending {
|
|
t.Fatalf("events = %+v, want pending while operator-suppressed", events)
|
|
}
|
|
if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, time.Minute), suppressed); len(events) != 0 {
|
|
t.Fatalf("held observation emitted %+v", events)
|
|
}
|
|
|
|
// Suppression lifts: next observation activates with the run's first
|
|
// active observation as the start.
|
|
released := DiscreteRule{Confirmations: 1, Intent: &DiscreteIntent{}}
|
|
events = state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 2*time.Minute), released)
|
|
if len(events) != 1 || events[0].Type != EventFired {
|
|
t.Fatalf("events = %+v, want fired after suppression lifts", events)
|
|
}
|
|
incident, _ := state.Incident("node-1", "connectivity")
|
|
if !incident.StartedAt.Equal(t0) {
|
|
t.Fatalf("StartedAt = %v, want first active observation %v", incident.StartedAt, t0)
|
|
}
|
|
}
|
|
|
|
func TestIntentGraceHoldsUntilElapsed(t *testing.T) {
|
|
state := NewState()
|
|
rule := DiscreteRule{Confirmations: 1, Intent: &DiscreteIntent{Explicit: true, GraceSeconds: 120}}
|
|
|
|
state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
|
|
if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, time.Minute), rule); len(events) != 0 {
|
|
t.Fatalf("inside grace emitted %+v", events)
|
|
}
|
|
events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 2*time.Minute), rule)
|
|
if len(events) != 1 || events[0].Type != EventFired {
|
|
t.Fatalf("events = %+v, want fired at grace boundary", events)
|
|
}
|
|
incident, _ := state.Incident("node-1", "connectivity")
|
|
if !incident.StartedAt.Equal(t0) {
|
|
t.Fatalf("StartedAt = %v, want grace-run start %v", incident.StartedAt, t0)
|
|
}
|
|
}
|
|
|
|
func TestIntentDipDuringGraceResetsRun(t *testing.T) {
|
|
state := NewState()
|
|
rule := DiscreteRule{Confirmations: 1, Intent: &DiscreteIntent{Explicit: true, GraceSeconds: 120}}
|
|
|
|
state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
|
|
events := state.ApplyDiscrete(discreteSignalAt(false, "", time.Minute), rule)
|
|
if len(events) != 1 || events[0].Type != EventPendingCleared {
|
|
t.Fatalf("events = %+v, want pending cleared on dip", events)
|
|
}
|
|
|
|
// Grace restarts from the re-entry.
|
|
state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 2*time.Minute), rule)
|
|
if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 3*time.Minute), rule); len(events) != 0 {
|
|
t.Fatalf("restarted grace emitted %+v", events)
|
|
}
|
|
events = state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 4*time.Minute), rule)
|
|
if len(events) != 1 || events[0].Type != EventFired {
|
|
t.Fatalf("events = %+v, want fired after restarted grace", events)
|
|
}
|
|
incident, _ := state.Incident("node-1", "connectivity")
|
|
if !incident.StartedAt.Equal(t0.Add(2 * time.Minute)) {
|
|
t.Fatalf("StartedAt = %v, want restarted run start", incident.StartedAt)
|
|
}
|
|
}
|
|
|
|
func TestIntentDoesNotSuppressFiringIncident(t *testing.T) {
|
|
state := NewState()
|
|
state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), DiscreteRule{Confirmations: 1})
|
|
|
|
// Maintenance starting after activation does not clear or hold the
|
|
// already-firing incident.
|
|
held := DiscreteRule{Confirmations: 1, Intent: &DiscreteIntent{
|
|
OperatorSuppressed: true, OperatorReason: "operator_maintenance",
|
|
}}
|
|
if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, time.Minute), held); len(events) != 0 {
|
|
t.Fatalf("firing incident emitted %+v under maintenance", events)
|
|
}
|
|
incident, _ := state.Incident("node-1", "connectivity")
|
|
if incident.State != StateFiring {
|
|
t.Fatalf("incident = %+v, want still firing", incident)
|
|
}
|
|
}
|
|
|
|
func TestIntentGraceComposesWithConfirmations(t *testing.T) {
|
|
state := NewState()
|
|
// Three confirmations AND a 5-minute grace: activation needs both.
|
|
rule := DiscreteRule{Confirmations: 3, Intent: &DiscreteIntent{Explicit: true, GraceSeconds: 300}}
|
|
|
|
state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
|
|
state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, time.Minute), rule)
|
|
// Third confirmation reached at 2m, but grace holds until 5m.
|
|
if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 2*time.Minute), rule); len(events) != 0 {
|
|
t.Fatalf("confirmed-but-in-grace emitted %+v", events)
|
|
}
|
|
events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 5*time.Minute), rule)
|
|
if len(events) != 1 || events[0].Type != EventFired {
|
|
t.Fatalf("events = %+v, want fired once grace also elapses", events)
|
|
}
|
|
incident, _ := state.Incident("node-1", "connectivity")
|
|
if !incident.StartedAt.Equal(t0) {
|
|
t.Fatalf("StartedAt = %v, want run start %v", incident.StartedAt, t0)
|
|
}
|
|
}
|
|
|
|
func backupIntent(grace, post, cap int, backupActive bool) *DiscreteIntent {
|
|
return &DiscreteIntent{
|
|
Explicit: true,
|
|
GraceSeconds: grace,
|
|
BackupEnabled: true,
|
|
BackupActive: backupActive,
|
|
BackupPostGraceSeconds: post,
|
|
BackupMaxDeferralSeconds: cap,
|
|
}
|
|
}
|
|
|
|
func TestBackupDeferralHoldsWhileBackupRuns(t *testing.T) {
|
|
state := NewState()
|
|
rule := func(active bool) DiscreteRule {
|
|
return DiscreteRule{Confirmations: 1, Intent: backupIntent(60, 120, 600, active)}
|
|
}
|
|
|
|
// Offline with a backup running: held well past the plain grace.
|
|
state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 0), rule(true))
|
|
if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 3*time.Minute), rule(true)); len(events) != 0 {
|
|
t.Fatalf("backup-deferred observation emitted %+v", events)
|
|
}
|
|
|
|
// Backup ends at 4m: eligibility extends to 4m + 120s post-grace.
|
|
if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 4*time.Minute), rule(false)); len(events) != 0 {
|
|
t.Fatalf("post-backup grace emitted %+v", events)
|
|
}
|
|
if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 5*time.Minute), rule(false)); len(events) != 0 {
|
|
t.Fatalf("still inside post-grace emitted %+v", events)
|
|
}
|
|
events := state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 6*time.Minute), rule(false))
|
|
if len(events) != 1 || events[0].Type != EventFired {
|
|
t.Fatalf("events = %+v, want fired at backup end + post-grace", events)
|
|
}
|
|
incident, _ := state.Incident("node-1", "connectivity")
|
|
if !incident.StartedAt.Equal(t0) {
|
|
t.Fatalf("StartedAt = %v, want run start %v", incident.StartedAt, t0)
|
|
}
|
|
}
|
|
|
|
func TestBackupDeferralCapBoundsTheHold(t *testing.T) {
|
|
state := NewState()
|
|
rule := DiscreteRule{Confirmations: 1, Intent: backupIntent(60, 120, 300, true)}
|
|
|
|
// Backup never ends: activation at the max-deferral cap (5m).
|
|
state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
|
|
if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 4*time.Minute), rule); len(events) != 0 {
|
|
t.Fatalf("under cap emitted %+v", events)
|
|
}
|
|
events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 5*time.Minute), rule)
|
|
if len(events) != 1 || events[0].Type != EventFired {
|
|
t.Fatalf("events = %+v, want fired at deferral cap", events)
|
|
}
|
|
}
|
|
|
|
func TestBackupPostGraceCappedByMaxDeferral(t *testing.T) {
|
|
state := NewState()
|
|
// Backup ends at 4m; post-grace would extend to 4m+180s=7m, but the cap
|
|
// is 5m — activation at 5m.
|
|
rule := func(active bool) DiscreteRule {
|
|
return DiscreteRule{Confirmations: 1, Intent: backupIntent(60, 180, 300, active)}
|
|
}
|
|
state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule(true))
|
|
state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 4*time.Minute), rule(false))
|
|
if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 4*time.Minute+30*time.Second), rule(false)); len(events) != 0 {
|
|
t.Fatalf("under cap emitted %+v", events)
|
|
}
|
|
events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 5*time.Minute), rule(false))
|
|
if len(events) != 1 || events[0].Type != EventFired {
|
|
t.Fatalf("events = %+v, want fired at cap despite longer post-grace", events)
|
|
}
|
|
}
|