Deliver alerts from first startup reports

Wire outbound alert callbacks before a monitor is published so immediately reconnecting agents cannot create active warnings in the constructor-to-Start gap. Add a pre-Start custom-sensor regression and bind the ordering in monitoring and agent lifecycle contracts.

Change-source: pulse-maintainer
This commit is contained in:
pulse-triage[bot]
2026-08-29 16:18:40 +01:00
parent f29e7397b1
commit 3de3cd320d
4 changed files with 117 additions and 19 deletions
+39 -19
View File
@@ -1857,6 +1857,13 @@ func New(cfg *config.Config) (*Monitor, error) {
Version: "2.0.0-go",
}
// Agent endpoints can receive reports as soon as New returns, before the
// monitoring loop goroutine reaches Start. Wire external alert delivery now
// so a first startup observation cannot create an active alert without also
// reaching notifications. Start rewires the escalation callback with its
// WebSocket hub and installs lifecycle projection handling.
m.wireExternalAlertCallbacks(nil)
return m, nil
}
@@ -1991,25 +1998,7 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
// Set up alert callbacks
m.alertManager.SubscribeLifecycleCallback(m.handleAlertLifecycleEvent)
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
m.handleAlertFired(alert)
})
// Set up AI analysis callback - this bypasses activation state and other notification suppression
// so AI can analyze alerts even during pending_review setup phase
m.alertManager.SetAlertForAICallback(func(alert *alerts.Alert) {
log.Debug().Str("alertID", alert.ID).Msg("AI alert callback invoked (bypassing notification suppression)")
if m.alertTriggeredAICallback != nil {
m.alertTriggeredAICallback(alert)
}
})
m.alertManager.SetResolvedCallback(func(alertID string) {
m.handleAlertResolved(alertID)
// Don't broadcast full state here - it causes a cascade with many guests.
// The frontend will get the updated alerts through the regular broadcast ticker.
})
m.alertManager.SetEscalateCallback(func(alert *alerts.Alert, level int) {
m.handleAlertEscalated(wsHub, alert, level)
})
m.wireExternalAlertCallbacks(wsHub)
m.replayAlertLifecycleProjections()
m.reconcileActiveAlertTimelines()
m.markDeadManMonitoringProgress(time.Now().UTC())
@@ -2142,6 +2131,37 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
}
}
// wireExternalAlertCallbacks installs callbacks whose effects leave the alert
// manager. It is safe to call again: these are single callback slots, so Start
// replaces the constructor-time escalation closure with one carrying its hub.
func (m *Monitor) wireExternalAlertCallbacks(wsHub *websocket.Hub) {
if m == nil || m.alertManager == nil {
return
}
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
m.handleAlertFired(alert)
})
// AI analysis bypasses activation and notification suppression so findings
// can be prepared while alert delivery is still pending review.
m.alertManager.SetAlertForAICallback(func(alert *alerts.Alert) {
log.Debug().Str("alertID", alert.ID).Msg("AI alert callback invoked (bypassing notification suppression)")
m.mu.RLock()
callback := m.alertTriggeredAICallback
m.mu.RUnlock()
if callback != nil {
callback(alert)
}
})
m.alertManager.SetResolvedCallback(func(alertID string) {
m.handleAlertResolved(alertID)
// Don't broadcast full state here - it causes a cascade with many guests.
// The frontend gets updated alerts through the regular broadcast ticker.
})
m.alertManager.SetEscalateCallback(func(alert *alerts.Alert, level int) {
m.handleAlertEscalated(wsHub, alert, level)
})
}
// poll fetches data from all configured instances
func (m *Monitor) poll(_ context.Context, wsHub *websocket.Hub) {
defer recoverFromPanic("poll")
@@ -24,6 +24,48 @@ import (
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
)
func TestNewMonitorRoutesStartupCustomSensorWarningBeforeStart(t *testing.T) {
monitor, err := New(&config.Config{DataPath: t.TempDir()})
if err != nil {
t.Fatalf("New: %v", err)
}
t.Cleanup(monitor.Stop)
delivered := make(chan *alerts.Alert, 1)
monitor.SetAlertPushCallback(func(alert *alerts.Alert) {
delivered <- alert
})
manager := monitor.GetAlertManager()
alertConfig := manager.GetConfig()
alertConfig.Enabled = true
alertConfig.ActivationState = alerts.ActivationActive
alertConfig.Schedule.QuietHours.Enabled = false
manager.UpdateConfig(alertConfig)
value := 1.0
manager.CheckHost(models.Host{
ID: "startup-warning-host",
Hostname: "startup-warning-host",
Sensors: models.HostSensorSummary{Custom: []models.HostCustomSensorMetric{{
ID: "apt-upgradable",
Name: "Apt Upgradable",
Value: &value,
Status: "warning",
ObservedAt: time.Now().UTC(),
}}},
})
select {
case alert := <-delivered:
if alert.Type != "custom-sensor" || alert.Level != alerts.AlertLevelWarning {
t.Fatalf("startup alert = %#v, want warning custom-sensor", alert)
}
case <-time.After(time.Second):
t.Fatal("custom sensor warning did not reach external delivery before Monitor.Start")
}
}
func TestHostZFSPoolsFromAgentDisksPreservesDatasetFacts(t *testing.T) {
got := hostZFSPoolsFromAgentDisks([]agentshost.Disk{
{Device: "/", Type: "ext4"},