mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-22 03:04:03 +00:00
fix: send notifications for critical alerts restored from disk after restart
addresses #471 when pulse restarts (service restart, container restart, etc), active alerts are loaded from disk but notifications were never sent for these restored alerts. this caused users to miss critical ongoing alerts that existed before the restart. the issue was particularly noticeable with memory alerts on VMs - if a VM's memory was genuinely high and an alert was created, then pulse restarted, the alert would show in 'Active Alerts' but no webhook notification would be sent. however, manually creating a 'fake' alert by lowering thresholds would work because those are new alerts. fix: now sends notifications for restored critical alerts that started within the last 2 hours. adds a 10-second delay after restart to allow the system to stabilize before sending notifications. warning-level alerts are not re-notified to avoid spam on restart.
This commit is contained in:
@@ -2268,6 +2268,23 @@ func (m *Manager) LoadActiveAlerts() error {
|
||||
|
||||
m.activeAlerts[alert.ID] = alert
|
||||
restoredCount++
|
||||
|
||||
// For critical alerts that are still active after restart, send notifications
|
||||
// This ensures users are notified about ongoing critical issues even after service restarts
|
||||
// Only notify for alerts that started recently (within last 2 hours) to avoid spam
|
||||
if alert.Level == AlertLevelCritical && now.Sub(alert.StartTime) < 2*time.Hour {
|
||||
// Use a goroutine and add a small delay to avoid notification spam on startup
|
||||
if m.onAlert != nil {
|
||||
go func(a *Alert) {
|
||||
time.Sleep(10 * time.Second) // Wait for system to stabilize after restart
|
||||
log.Info().
|
||||
Str("alertID", a.ID).
|
||||
Str("resource", a.ResourceName).
|
||||
Msg("Sending notification for restored critical alert")
|
||||
m.onAlert(a)
|
||||
}(alert)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Info().
|
||||
|
||||
Reference in New Issue
Block a user