add hourly sweep for overdue will_fix_later commitments

This commit is contained in:
rcourtman
2026-05-13 00:54:48 +01:00
parent 183bf5b617
commit d4f84e69e0
3 changed files with 199 additions and 0 deletions
+52
View File
@@ -1027,6 +1027,58 @@ func legacyAlertMirrorShape(f *Finding) bool {
return true
}
// SweepWillFixLaterReminders proactively wakes will_fix_later commitments
// whose RemindAt deadline has passed without waiting for re-detection. The
// existing wake path in Add() only fires on re-detection — if patrol's
// detection is sparse or the operational issue transiently clears between
// runs, an overdue commitment can sit forever without surfacing. The sweep
// closes that gap by walking all dismissed-will_fix_later findings on a
// timer and converging them onto the same in-memory state the re-detection
// path would have produced.
//
// Mirrors the field-clearing of the will_fix_later branch in Add (around
// findings.go line 1366): clear DismissedReason, Suppressed, RemindAt,
// AcknowledgedAt; keep UserNote because the operator's promise is still
// relevant context; bump RemindCount; emit a "reminded" lifecycle event.
//
// Returns the number of findings woken. Caller is patrol_run.remindSweepLoop.
func (s *FindingsStore) SweepWillFixLaterReminders(now time.Time) int {
s.mu.Lock()
swept := 0
for _, f := range s.findings {
if f == nil {
continue
}
if f.DismissedReason != DismissReasonWillFixLater {
continue
}
if f.RemindAt == nil {
continue
}
if !now.After(*f.RemindAt) {
continue
}
meta := map[string]string{
"reason_was": f.DismissedReason,
"remind_at": f.RemindAt.Format(time.RFC3339),
"trigger": "sweep",
}
f.RemindCount++
s.appendLifecycleLocked(f, "reminded", "will_fix_later remind-at deadline passed (sweep); re-surfacing finding", string(FindingLoopStateDismissed), string(FindingLoopStateDetected), meta)
f.DismissedReason = ""
f.Suppressed = false
f.RemindAt = nil
f.AcknowledgedAt = nil
f.LoopState = string(FindingLoopStateDetected)
swept++
}
s.mu.Unlock()
if swept > 0 {
s.scheduleSave()
}
return swept
}
// scheduleSave schedules a debounced save operation
// This method is lock-safe and can be called without holding the store lock.
func (s *FindingsStore) scheduleSave() {
+103
View File
@@ -0,0 +1,103 @@
package ai
import (
"testing"
"time"
)
// TestSweepWillFixLaterReminders_PromotesPastDeadline verifies the sweep
// converges an overdue will_fix_later finding onto the same in-memory
// state the Add() re-detection path produces when the deadline passes:
// dismissal cleared, RemindAt cleared, RemindCount bumped, LoopState
// back to detected, and a "reminded" lifecycle event recorded. Future-
// dated commitments must stay untouched.
func TestSweepWillFixLaterReminders_PromotesPastDeadline(t *testing.T) {
store := NewFindingsStore()
now := time.Now()
past := now.Add(-30 * time.Minute)
future := now.Add(2 * time.Hour)
overdue := &Finding{
ID: "overdue-1",
Severity: FindingSeverityWarning,
Category: FindingCategoryCapacity,
ResourceID: "node1-storage",
ResourceName: "tank",
ResourceType: "storage",
Title: "Storage growth trending toward full",
DetectedAt: now.Add(-3 * time.Hour),
LastSeenAt: now.Add(-3 * time.Hour),
DismissedReason: DismissReasonWillFixLater,
UserNote: "Will run cleanup next maintenance window",
LoopState: string(FindingLoopStateDismissed),
RemindAt: &past,
RemindCount: 0,
}
pending := &Finding{
ID: "pending-1",
Severity: FindingSeverityInfo,
Category: FindingCategoryPerformance,
ResourceID: "node1-vm-200",
ResourceName: "appserver",
ResourceType: "vm",
Title: "Elevated CPU on appserver",
DetectedAt: now.Add(-1 * time.Hour),
LastSeenAt: now.Add(-1 * time.Hour),
DismissedReason: DismissReasonWillFixLater,
LoopState: string(FindingLoopStateDismissed),
RemindAt: &future,
RemindCount: 0,
}
store.findings[overdue.ID] = overdue
store.findings[pending.ID] = pending
swept := store.SweepWillFixLaterReminders(now)
if swept != 1 {
t.Fatalf("swept count: got %d, want 1", swept)
}
got := store.findings[overdue.ID]
if got.DismissedReason != "" {
t.Errorf("DismissedReason: got %q, want \"\"", got.DismissedReason)
}
if got.RemindAt != nil {
t.Errorf("RemindAt: got %v, want nil", got.RemindAt)
}
if got.RemindCount != 1 {
t.Errorf("RemindCount: got %d, want 1", got.RemindCount)
}
if got.LoopState != string(FindingLoopStateDetected) {
t.Errorf("LoopState: got %q, want %q", got.LoopState, FindingLoopStateDetected)
}
if got.UserNote != "Will run cleanup next maintenance window" {
t.Errorf("UserNote should be preserved on remind wake, got %q", got.UserNote)
}
if len(got.Lifecycle) == 0 {
t.Fatal("expected a lifecycle event to be appended")
}
last := got.Lifecycle[len(got.Lifecycle)-1]
if last.Type != "reminded" {
t.Errorf("last lifecycle event Type: got %q, want \"reminded\"", last.Type)
}
if last.From != string(FindingLoopStateDismissed) || last.To != string(FindingLoopStateDetected) {
t.Errorf("lifecycle transition: from=%q to=%q, want dismissed->detected", last.From, last.To)
}
if last.Metadata["trigger"] != "sweep" {
t.Errorf("lifecycle metadata trigger: got %q, want \"sweep\"", last.Metadata["trigger"])
}
pendingAfter := store.findings[pending.ID]
if pendingAfter.DismissedReason != DismissReasonWillFixLater {
t.Errorf("pending finding should still be dismissed, got %q", pendingAfter.DismissedReason)
}
if pendingAfter.RemindAt == nil || !pendingAfter.RemindAt.Equal(future) {
t.Errorf("pending RemindAt should be untouched, got %v", pendingAfter.RemindAt)
}
if pendingAfter.RemindCount != 0 {
t.Errorf("pending RemindCount should be 0, got %d", pendingAfter.RemindCount)
}
if len(pendingAfter.Lifecycle) != 0 {
t.Errorf("pending finding should not have a new lifecycle event, got %d", len(pendingAfter.Lifecycle))
}
}
+44
View File
@@ -27,6 +27,8 @@ const (
scopedPatrolRetryBackoff2 = 15 * time.Second // Second retry backoff for dropped scoped patrols
scopedPatrolMaxRetries = 2 // Maximum re-queue attempts for dropped scoped patrols
scopedPatrolLogIDLimit = 10 // Maximum number of effective scope IDs to log inline
remindSweepInterval = time.Hour // Cadence for the proactive will_fix_later remind sweep
remindSweepInitialDelay = 30 * time.Second // First sweep runs shortly after start so a restart-with-overdue does not have to wait a full hour
)
// Start begins the background patrol loop
@@ -46,6 +48,7 @@ func (p *PatrolService) Start(ctx context.Context) {
Msg("Starting AI Patrol Service")
go p.patrolLoop(ctx)
go p.remindSweepLoop(ctx)
}
// Stop stops the patrol service. It signals the patrol loop to exit, then
@@ -197,6 +200,47 @@ func (p *PatrolService) patrolLoop(ctx context.Context) {
}
}
// remindSweepLoop fires SweepWillFixLaterReminders on a 1h cadence so
// will_fix_later commitments past their RemindAt deadline surface even
// when patrol detection is sparse or the operational issue transiently
// clears between runs. Runs an initial sweep ~30s after start so a
// restart-with-overdue does not have to wait a full hour.
func (p *PatrolService) remindSweepLoop(ctx context.Context) {
sweep := func() {
if p.findings == nil {
return
}
swept := p.findings.SweepWillFixLaterReminders(time.Now())
if swept > 0 {
log.Info().Int("swept", swept).Msg("Patrol remind-sweep promoted overdue will_fix_later commitments")
}
}
initialTimer := time.NewTimer(remindSweepInitialDelay)
defer initialTimer.Stop()
select {
case <-initialTimer.C:
sweep()
case <-p.stopCh:
return
case <-ctx.Done():
return
}
ticker := time.NewTicker(remindSweepInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
sweep()
case <-p.stopCh:
return
case <-ctx.Done():
return
}
}
}
func shouldSkipInitialFullPatrol(runHistory []PatrolRunRecord, now time.Time) bool {
for _, run := range runHistory {
if run.CompletedAt.IsZero() {