mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-14 00:00:30 +00:00
Deliver alerts from first startup reports
Wire outbound alert callbacks before a monitor is published so immediately reconnecting agents cannot create active warnings in the constructor-to-Start gap. Add a pre-Start custom-sensor regression and bind the ordering in monitoring and agent lifecycle contracts. Change-source: pulse-maintainer
This commit is contained in:
@@ -1857,6 +1857,13 @@ func New(cfg *config.Config) (*Monitor, error) {
|
||||
Version: "2.0.0-go",
|
||||
}
|
||||
|
||||
// Agent endpoints can receive reports as soon as New returns, before the
|
||||
// monitoring loop goroutine reaches Start. Wire external alert delivery now
|
||||
// so a first startup observation cannot create an active alert without also
|
||||
// reaching notifications. Start rewires the escalation callback with its
|
||||
// WebSocket hub and installs lifecycle projection handling.
|
||||
m.wireExternalAlertCallbacks(nil)
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
@@ -1991,25 +1998,7 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
|
||||
|
||||
// Set up alert callbacks
|
||||
m.alertManager.SubscribeLifecycleCallback(m.handleAlertLifecycleEvent)
|
||||
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
|
||||
m.handleAlertFired(alert)
|
||||
})
|
||||
// Set up AI analysis callback - this bypasses activation state and other notification suppression
|
||||
// so AI can analyze alerts even during pending_review setup phase
|
||||
m.alertManager.SetAlertForAICallback(func(alert *alerts.Alert) {
|
||||
log.Debug().Str("alertID", alert.ID).Msg("AI alert callback invoked (bypassing notification suppression)")
|
||||
if m.alertTriggeredAICallback != nil {
|
||||
m.alertTriggeredAICallback(alert)
|
||||
}
|
||||
})
|
||||
m.alertManager.SetResolvedCallback(func(alertID string) {
|
||||
m.handleAlertResolved(alertID)
|
||||
// Don't broadcast full state here - it causes a cascade with many guests.
|
||||
// The frontend will get the updated alerts through the regular broadcast ticker.
|
||||
})
|
||||
m.alertManager.SetEscalateCallback(func(alert *alerts.Alert, level int) {
|
||||
m.handleAlertEscalated(wsHub, alert, level)
|
||||
})
|
||||
m.wireExternalAlertCallbacks(wsHub)
|
||||
m.replayAlertLifecycleProjections()
|
||||
m.reconcileActiveAlertTimelines()
|
||||
m.markDeadManMonitoringProgress(time.Now().UTC())
|
||||
@@ -2142,6 +2131,37 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
|
||||
}
|
||||
}
|
||||
|
||||
// wireExternalAlertCallbacks installs callbacks whose effects leave the alert
|
||||
// manager. It is safe to call again: these are single callback slots, so Start
|
||||
// replaces the constructor-time escalation closure with one carrying its hub.
|
||||
func (m *Monitor) wireExternalAlertCallbacks(wsHub *websocket.Hub) {
|
||||
if m == nil || m.alertManager == nil {
|
||||
return
|
||||
}
|
||||
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
|
||||
m.handleAlertFired(alert)
|
||||
})
|
||||
// AI analysis bypasses activation and notification suppression so findings
|
||||
// can be prepared while alert delivery is still pending review.
|
||||
m.alertManager.SetAlertForAICallback(func(alert *alerts.Alert) {
|
||||
log.Debug().Str("alertID", alert.ID).Msg("AI alert callback invoked (bypassing notification suppression)")
|
||||
m.mu.RLock()
|
||||
callback := m.alertTriggeredAICallback
|
||||
m.mu.RUnlock()
|
||||
if callback != nil {
|
||||
callback(alert)
|
||||
}
|
||||
})
|
||||
m.alertManager.SetResolvedCallback(func(alertID string) {
|
||||
m.handleAlertResolved(alertID)
|
||||
// Don't broadcast full state here - it causes a cascade with many guests.
|
||||
// The frontend gets updated alerts through the regular broadcast ticker.
|
||||
})
|
||||
m.alertManager.SetEscalateCallback(func(alert *alerts.Alert, level int) {
|
||||
m.handleAlertEscalated(wsHub, alert, level)
|
||||
})
|
||||
}
|
||||
|
||||
// poll fetches data from all configured instances
|
||||
func (m *Monitor) poll(_ context.Context, wsHub *websocket.Hub) {
|
||||
defer recoverFromPanic("poll")
|
||||
|
||||
@@ -24,6 +24,48 @@ import (
|
||||
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
|
||||
)
|
||||
|
||||
func TestNewMonitorRoutesStartupCustomSensorWarningBeforeStart(t *testing.T) {
|
||||
monitor, err := New(&config.Config{DataPath: t.TempDir()})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
t.Cleanup(monitor.Stop)
|
||||
|
||||
delivered := make(chan *alerts.Alert, 1)
|
||||
monitor.SetAlertPushCallback(func(alert *alerts.Alert) {
|
||||
delivered <- alert
|
||||
})
|
||||
|
||||
manager := monitor.GetAlertManager()
|
||||
alertConfig := manager.GetConfig()
|
||||
alertConfig.Enabled = true
|
||||
alertConfig.ActivationState = alerts.ActivationActive
|
||||
alertConfig.Schedule.QuietHours.Enabled = false
|
||||
manager.UpdateConfig(alertConfig)
|
||||
|
||||
value := 1.0
|
||||
manager.CheckHost(models.Host{
|
||||
ID: "startup-warning-host",
|
||||
Hostname: "startup-warning-host",
|
||||
Sensors: models.HostSensorSummary{Custom: []models.HostCustomSensorMetric{{
|
||||
ID: "apt-upgradable",
|
||||
Name: "Apt Upgradable",
|
||||
Value: &value,
|
||||
Status: "warning",
|
||||
ObservedAt: time.Now().UTC(),
|
||||
}}},
|
||||
})
|
||||
|
||||
select {
|
||||
case alert := <-delivered:
|
||||
if alert.Type != "custom-sensor" || alert.Level != alerts.AlertLevelWarning {
|
||||
t.Fatalf("startup alert = %#v, want warning custom-sensor", alert)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("custom sensor warning did not reach external delivery before Monitor.Start")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostZFSPoolsFromAgentDisksPreservesDatasetFacts(t *testing.T) {
|
||||
got := hostZFSPoolsFromAgentDisks([]agentshost.Disk{
|
||||
{Device: "/", Type: "ext4"},
|
||||
|
||||
Reference in New Issue
Block a user