mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
add hourly sweep for overdue will_fix_later commitments
This commit is contained in:
@@ -1027,6 +1027,58 @@ func legacyAlertMirrorShape(f *Finding) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// SweepWillFixLaterReminders proactively wakes will_fix_later commitments
|
||||
// whose RemindAt deadline has passed without waiting for re-detection. The
|
||||
// existing wake path in Add() only fires on re-detection — if patrol's
|
||||
// detection is sparse or the operational issue transiently clears between
|
||||
// runs, an overdue commitment can sit forever without surfacing. The sweep
|
||||
// closes that gap by walking all dismissed-will_fix_later findings on a
|
||||
// timer and converging them onto the same in-memory state the re-detection
|
||||
// path would have produced.
|
||||
//
|
||||
// Mirrors the field-clearing of the will_fix_later branch in Add (around
|
||||
// findings.go line 1366): clear DismissedReason, Suppressed, RemindAt,
|
||||
// AcknowledgedAt; keep UserNote because the operator's promise is still
|
||||
// relevant context; bump RemindCount; emit a "reminded" lifecycle event.
|
||||
//
|
||||
// Returns the number of findings woken. Caller is patrol_run.remindSweepLoop.
|
||||
func (s *FindingsStore) SweepWillFixLaterReminders(now time.Time) int {
|
||||
s.mu.Lock()
|
||||
swept := 0
|
||||
for _, f := range s.findings {
|
||||
if f == nil {
|
||||
continue
|
||||
}
|
||||
if f.DismissedReason != DismissReasonWillFixLater {
|
||||
continue
|
||||
}
|
||||
if f.RemindAt == nil {
|
||||
continue
|
||||
}
|
||||
if !now.After(*f.RemindAt) {
|
||||
continue
|
||||
}
|
||||
meta := map[string]string{
|
||||
"reason_was": f.DismissedReason,
|
||||
"remind_at": f.RemindAt.Format(time.RFC3339),
|
||||
"trigger": "sweep",
|
||||
}
|
||||
f.RemindCount++
|
||||
s.appendLifecycleLocked(f, "reminded", "will_fix_later remind-at deadline passed (sweep); re-surfacing finding", string(FindingLoopStateDismissed), string(FindingLoopStateDetected), meta)
|
||||
f.DismissedReason = ""
|
||||
f.Suppressed = false
|
||||
f.RemindAt = nil
|
||||
f.AcknowledgedAt = nil
|
||||
f.LoopState = string(FindingLoopStateDetected)
|
||||
swept++
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if swept > 0 {
|
||||
s.scheduleSave()
|
||||
}
|
||||
return swept
|
||||
}
|
||||
|
||||
// scheduleSave schedules a debounced save operation
|
||||
// This method is lock-safe and can be called without holding the store lock.
|
||||
func (s *FindingsStore) scheduleSave() {
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestSweepWillFixLaterReminders_PromotesPastDeadline verifies the sweep
|
||||
// converges an overdue will_fix_later finding onto the same in-memory
|
||||
// state the Add() re-detection path produces when the deadline passes:
|
||||
// dismissal cleared, RemindAt cleared, RemindCount bumped, LoopState
|
||||
// back to detected, and a "reminded" lifecycle event recorded. Future-
|
||||
// dated commitments must stay untouched.
|
||||
func TestSweepWillFixLaterReminders_PromotesPastDeadline(t *testing.T) {
|
||||
store := NewFindingsStore()
|
||||
now := time.Now()
|
||||
past := now.Add(-30 * time.Minute)
|
||||
future := now.Add(2 * time.Hour)
|
||||
|
||||
overdue := &Finding{
|
||||
ID: "overdue-1",
|
||||
Severity: FindingSeverityWarning,
|
||||
Category: FindingCategoryCapacity,
|
||||
ResourceID: "node1-storage",
|
||||
ResourceName: "tank",
|
||||
ResourceType: "storage",
|
||||
Title: "Storage growth trending toward full",
|
||||
DetectedAt: now.Add(-3 * time.Hour),
|
||||
LastSeenAt: now.Add(-3 * time.Hour),
|
||||
DismissedReason: DismissReasonWillFixLater,
|
||||
UserNote: "Will run cleanup next maintenance window",
|
||||
LoopState: string(FindingLoopStateDismissed),
|
||||
RemindAt: &past,
|
||||
RemindCount: 0,
|
||||
}
|
||||
pending := &Finding{
|
||||
ID: "pending-1",
|
||||
Severity: FindingSeverityInfo,
|
||||
Category: FindingCategoryPerformance,
|
||||
ResourceID: "node1-vm-200",
|
||||
ResourceName: "appserver",
|
||||
ResourceType: "vm",
|
||||
Title: "Elevated CPU on appserver",
|
||||
DetectedAt: now.Add(-1 * time.Hour),
|
||||
LastSeenAt: now.Add(-1 * time.Hour),
|
||||
DismissedReason: DismissReasonWillFixLater,
|
||||
LoopState: string(FindingLoopStateDismissed),
|
||||
RemindAt: &future,
|
||||
RemindCount: 0,
|
||||
}
|
||||
|
||||
store.findings[overdue.ID] = overdue
|
||||
store.findings[pending.ID] = pending
|
||||
|
||||
swept := store.SweepWillFixLaterReminders(now)
|
||||
if swept != 1 {
|
||||
t.Fatalf("swept count: got %d, want 1", swept)
|
||||
}
|
||||
|
||||
got := store.findings[overdue.ID]
|
||||
if got.DismissedReason != "" {
|
||||
t.Errorf("DismissedReason: got %q, want \"\"", got.DismissedReason)
|
||||
}
|
||||
if got.RemindAt != nil {
|
||||
t.Errorf("RemindAt: got %v, want nil", got.RemindAt)
|
||||
}
|
||||
if got.RemindCount != 1 {
|
||||
t.Errorf("RemindCount: got %d, want 1", got.RemindCount)
|
||||
}
|
||||
if got.LoopState != string(FindingLoopStateDetected) {
|
||||
t.Errorf("LoopState: got %q, want %q", got.LoopState, FindingLoopStateDetected)
|
||||
}
|
||||
if got.UserNote != "Will run cleanup next maintenance window" {
|
||||
t.Errorf("UserNote should be preserved on remind wake, got %q", got.UserNote)
|
||||
}
|
||||
if len(got.Lifecycle) == 0 {
|
||||
t.Fatal("expected a lifecycle event to be appended")
|
||||
}
|
||||
last := got.Lifecycle[len(got.Lifecycle)-1]
|
||||
if last.Type != "reminded" {
|
||||
t.Errorf("last lifecycle event Type: got %q, want \"reminded\"", last.Type)
|
||||
}
|
||||
if last.From != string(FindingLoopStateDismissed) || last.To != string(FindingLoopStateDetected) {
|
||||
t.Errorf("lifecycle transition: from=%q to=%q, want dismissed->detected", last.From, last.To)
|
||||
}
|
||||
if last.Metadata["trigger"] != "sweep" {
|
||||
t.Errorf("lifecycle metadata trigger: got %q, want \"sweep\"", last.Metadata["trigger"])
|
||||
}
|
||||
|
||||
pendingAfter := store.findings[pending.ID]
|
||||
if pendingAfter.DismissedReason != DismissReasonWillFixLater {
|
||||
t.Errorf("pending finding should still be dismissed, got %q", pendingAfter.DismissedReason)
|
||||
}
|
||||
if pendingAfter.RemindAt == nil || !pendingAfter.RemindAt.Equal(future) {
|
||||
t.Errorf("pending RemindAt should be untouched, got %v", pendingAfter.RemindAt)
|
||||
}
|
||||
if pendingAfter.RemindCount != 0 {
|
||||
t.Errorf("pending RemindCount should be 0, got %d", pendingAfter.RemindCount)
|
||||
}
|
||||
if len(pendingAfter.Lifecycle) != 0 {
|
||||
t.Errorf("pending finding should not have a new lifecycle event, got %d", len(pendingAfter.Lifecycle))
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,8 @@ const (
|
||||
scopedPatrolRetryBackoff2 = 15 * time.Second // Second retry backoff for dropped scoped patrols
|
||||
scopedPatrolMaxRetries = 2 // Maximum re-queue attempts for dropped scoped patrols
|
||||
scopedPatrolLogIDLimit = 10 // Maximum number of effective scope IDs to log inline
|
||||
remindSweepInterval = time.Hour // Cadence for the proactive will_fix_later remind sweep
|
||||
remindSweepInitialDelay = 30 * time.Second // First sweep runs shortly after start so a restart-with-overdue does not have to wait a full hour
|
||||
)
|
||||
|
||||
// Start begins the background patrol loop
|
||||
@@ -46,6 +48,7 @@ func (p *PatrolService) Start(ctx context.Context) {
|
||||
Msg("Starting AI Patrol Service")
|
||||
|
||||
go p.patrolLoop(ctx)
|
||||
go p.remindSweepLoop(ctx)
|
||||
}
|
||||
|
||||
// Stop stops the patrol service. It signals the patrol loop to exit, then
|
||||
@@ -197,6 +200,47 @@ func (p *PatrolService) patrolLoop(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// remindSweepLoop fires SweepWillFixLaterReminders on a 1h cadence so
|
||||
// will_fix_later commitments past their RemindAt deadline surface even
|
||||
// when patrol detection is sparse or the operational issue transiently
|
||||
// clears between runs. Runs an initial sweep ~30s after start so a
|
||||
// restart-with-overdue does not have to wait a full hour.
|
||||
func (p *PatrolService) remindSweepLoop(ctx context.Context) {
|
||||
sweep := func() {
|
||||
if p.findings == nil {
|
||||
return
|
||||
}
|
||||
swept := p.findings.SweepWillFixLaterReminders(time.Now())
|
||||
if swept > 0 {
|
||||
log.Info().Int("swept", swept).Msg("Patrol remind-sweep promoted overdue will_fix_later commitments")
|
||||
}
|
||||
}
|
||||
|
||||
initialTimer := time.NewTimer(remindSweepInitialDelay)
|
||||
defer initialTimer.Stop()
|
||||
select {
|
||||
case <-initialTimer.C:
|
||||
sweep()
|
||||
case <-p.stopCh:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(remindSweepInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
sweep()
|
||||
case <-p.stopCh:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func shouldSkipInitialFullPatrol(runHistory []PatrolRunRecord, now time.Time) bool {
|
||||
for _, run := range runHistory {
|
||||
if run.CompletedAt.IsZero() {
|
||||
|
||||
Reference in New Issue
Block a user