Serve fresh alert snapshots from monitor state reads (#1365)

This commit is contained in:
rcourtman
2026-03-27 10:47:22 +00:00
parent 3736574391
commit 963670f01c
2 changed files with 69 additions and 11 deletions
+33 -11
View File
@@ -5480,6 +5480,26 @@ func (m *Monitor) syncAlertsToState() {
}
}
modelAlerts := m.currentModelAlerts()
m.state.UpdateActiveAlerts(modelAlerts)
recentlyResolved := m.currentRecentlyResolvedAlerts()
if len(recentlyResolved) > 0 {
log.Info().Int("count", len(recentlyResolved)).Msg("Syncing recently resolved alerts")
}
m.state.UpdateRecentlyResolved(recentlyResolved)
}
// SyncAlertState is the exported wrapper used by APIs that mutate alerts outside the poll loop.
func (m *Monitor) SyncAlertState() {
m.syncAlertsToState()
}
func (m *Monitor) currentModelAlerts() []models.Alert {
if m.alertManager == nil {
return nil
}
activeAlerts := m.alertManager.GetActiveAlerts()
modelAlerts := make([]models.Alert, 0, len(activeAlerts))
for _, alert := range activeAlerts {
@@ -5504,18 +5524,14 @@ func (m *Monitor) syncAlertsToState() {
log.Debug().Str("alertID", alert.ID).Interface("ackTime", alert.AckTime).Msg("Syncing acknowledged alert")
}
}
m.state.UpdateActiveAlerts(modelAlerts)
recentlyResolved := m.alertManager.GetRecentlyResolved()
if len(recentlyResolved) > 0 {
log.Info().Int("count", len(recentlyResolved)).Msg("Syncing recently resolved alerts")
}
m.state.UpdateRecentlyResolved(recentlyResolved)
return modelAlerts
}
// SyncAlertState is the exported wrapper used by APIs that mutate alerts outside the poll loop.
func (m *Monitor) SyncAlertState() {
m.syncAlertsToState()
func (m *Monitor) currentRecentlyResolvedAlerts() []models.ResolvedAlert {
if m.alertManager == nil {
return nil
}
return m.alertManager.GetRecentlyResolved()
}
// pruneStaleDockerAlerts removes docker alerts that reference hosts no longer present in state.
@@ -9599,7 +9615,13 @@ func (m *Monitor) GetState() models.StateSnapshot {
}
return state
}
return m.state.GetSnapshot()
state := m.state.GetSnapshot()
// Keep externally-served state aligned with the alert manager even between
// explicit sync points, so APIs do not expose stale alert counts/attachments.
state.ActiveAlerts = m.currentModelAlerts()
state.RecentlyResolved = m.currentRecentlyResolvedAlerts()
return state
}
// SetOrgID sets the organization ID for this monitor instance.
@@ -40,6 +40,42 @@ func TestMonitor_GetConnectionStatuses_MockMode_Extra(t *testing.T) {
}
}
func TestMonitor_GetStateRefreshesAlertSnapshots(t *testing.T) {
m := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
}
defer m.alertManager.Stop()
// Simulate stale alert data lingering in state after alerts were cleared.
m.state.UpdateActiveAlerts([]models.Alert{{
ID: "stale-alert",
ResourceID: "vm-1",
Type: "cpu",
Level: "warning",
Message: "stale",
StartTime: time.Now(),
}})
m.alertManager.ClearActiveAlerts()
state := m.GetState()
if len(state.ActiveAlerts) != 0 {
t.Fatalf("expected GetState to drop stale state alerts, got %d", len(state.ActiveAlerts))
}
// Also prove that GetState reflects current alert-manager alerts even before
// an explicit SyncAlertState call updates the cached state.
host := models.DockerHost{ID: "docker-host-1", DisplayName: "docker-host-1"}
m.alertManager.HandleDockerHostOffline(host)
m.alertManager.HandleDockerHostOffline(host)
m.alertManager.HandleDockerHostOffline(host)
state = m.GetState()
if len(state.ActiveAlerts) == 0 {
t.Fatal("expected GetState to include current alert-manager alerts")
}
}
func TestMonitor_Stop_Extra(t *testing.T) {
m := &Monitor{}
m.Stop()