Files
pulse/internal/monitoring/monitor_alert_sync.go
T
2026-08-27 20:41:08 +01:00

220 lines
6.5 KiB
Go

package monitoring
import (
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
// syncAlertsToState copies the latest alert manager data into the shared state snapshot.
// This keeps WebSocket broadcasts aligned with in-memory acknowledgement updates.
func (m *Monitor) syncAlertsToState() {
if m.pruneStaleDockerAlerts() {
if logging.IsLevelEnabled(zerolog.DebugLevel) {
log.Debug().Msg("pruned stale docker alerts during sync")
}
}
modelAlerts := m.activeAlertsSnapshot()
for _, alert := range modelAlerts {
if alert.Acknowledged && logging.IsLevelEnabled(zerolog.DebugLevel) {
log.Debug().Str("alertID", alert.ID).Interface("ackTime", alert.AckTime).Msg("syncing acknowledged alert")
}
}
m.state.UpdateActiveAlerts(modelAlerts)
recentlyResolved := m.alertManager.GetRecentlyResolved()
if len(recentlyResolved) > 0 && logging.IsLevelEnabled(zerolog.DebugLevel) {
log.Debug().Int("count", len(recentlyResolved)).Msg("syncing recently resolved alerts")
}
m.state.UpdateRecentlyResolved(recentlyResolved)
}
// SyncAlertState is the exported wrapper used by APIs that mutate alerts outside the poll loop.
func (m *Monitor) SyncAlertState() {
m.syncAlertsToState()
}
func (m *Monitor) activeAlertsSnapshot() []models.Alert {
if m == nil {
return nil
}
if m.alertManager == nil {
if m.state == nil {
return nil
}
return m.state.GetSnapshot().ActiveAlerts
}
activeAlerts := m.alertManager.GetActiveAlerts()
modelAlerts := make([]models.Alert, 0, len(activeAlerts))
for _, alert := range activeAlerts {
modelAlerts = append(modelAlerts, models.Alert{
ID: alert.ID,
Type: alert.Type,
Level: string(alert.Level),
ResourceID: alert.ResourceID,
ResourceName: alert.ResourceName,
Node: alert.Node,
NodeDisplayName: alert.NodeDisplayName,
Instance: alert.Instance,
Message: alert.Message,
Value: alert.Value,
Threshold: alert.Threshold,
StartTime: alert.StartTime,
Acknowledged: alert.Acknowledged,
AckTime: alert.AckTime,
AckUser: alert.AckUser,
// GetActiveAlerts returns deep clones, so the map is already private.
Metadata: alert.Metadata,
})
}
return modelAlerts
}
func (m *Monitor) recentlyResolvedAlertsSnapshot() []models.ResolvedAlert {
if m == nil {
return nil
}
if m.alertManager == nil {
if m.state == nil {
return nil
}
return m.state.GetSnapshot().RecentlyResolved
}
return m.alertManager.GetRecentlyResolved()
}
func (m *Monitor) syncUnifiedResourceAlertsToState(resources []unifiedresources.Resource) {
if m == nil || m.alertManager == nil {
return
}
config := m.alertManager.GetConfig()
migrationPlan := alerts.PlanAlertIdentityMigration(config, resources)
if migrationPlan.UnsupportedVersion {
log.Warn().
Int("configVersion", migrationPlan.FromVersion).
Int("supportedVersion", alerts.CurrentAlertIdentitySchemaVersion).
Msg("alert identity config uses a newer schema; leaving persisted identities untouched")
} else if alerts.ApplyAlertIdentityMigration(&config, migrationPlan) {
if m.configPersist == nil {
log.Warn().
Int("fromVersion", migrationPlan.FromVersion).
Int("toVersion", migrationPlan.ToVersion).
Int("removedOverrides", len(migrationPlan.RemovedOverrideKeys)).
Int("addedOverrides", len(migrationPlan.AddedOverrideKeys)).
Int("deferredOverrides", len(migrationPlan.Deferred)).
Msg("cannot persist planned alert identity migration without config persistence")
} else if err := m.configPersist.SaveAlertConfig(config); err != nil {
log.Error().
Err(err).
Int("fromVersion", migrationPlan.FromVersion).
Int("toVersion", migrationPlan.ToVersion).
Msg("failed to persist planned alert identity migration")
} else {
m.alertManager.UpdateConfig(config)
log.Info().
Int("fromVersion", migrationPlan.FromVersion).
Int("toVersion", migrationPlan.ToVersion).
Int("removedOverrides", len(migrationPlan.RemovedOverrideKeys)).
Int("addedOverrides", len(migrationPlan.AddedOverrideKeys)).
Int("deferredOverrides", len(migrationPlan.Deferred)).
Msg("persisted alert identity migration plan")
}
}
m.migrateAvailabilityLinksToCanonicalIDs(resources)
m.alertManager.CheckUnifiedResourceMetricsWithCapacityTrends(resources, m.unifiedStorageCapacityTrends(resources, time.Now()))
m.alertManager.SyncUnifiedResourceIncidents(resources)
m.syncAlertsToState()
}
// pruneStaleDockerAlerts removes docker alerts that reference hosts no longer present in state.
func (m *Monitor) pruneStaleDockerAlerts() bool {
if m.alertManager == nil {
return false
}
readState := m.GetUnifiedReadStateOrSnapshot()
if readState == nil {
return false
}
hosts := readState.DockerHosts()
knownHosts := make(map[string]struct{}, len(hosts)*2)
for _, host := range hosts {
hostID := strings.TrimSpace(host.ID())
if hostID != "" {
knownHosts[hostID] = struct{}{}
}
if sourceID := strings.TrimSpace(host.HostSourceID()); sourceID != "" {
knownHosts[sourceID] = struct{}{}
}
}
if len(knownHosts) == 0 {
// Still allow stale entries to be cleared if no hosts remain.
}
active := m.alertManager.GetActiveAlerts()
processed := make(map[string]struct{})
cleared := false
for _, alert := range active {
var hostID string
switch {
case alert.Type == "docker-host-offline":
hostID = strings.TrimPrefix(strings.TrimSpace(alert.ResourceID), "docker:")
case strings.HasPrefix(alert.ResourceID, "docker:"):
resource := strings.TrimPrefix(alert.ResourceID, "docker:")
if idx := strings.Index(resource, "/"); idx >= 0 {
hostID = resource[:idx]
} else {
hostID = resource
}
default:
continue
}
hostID = strings.TrimSpace(hostID)
if hostID == "" {
continue
}
if _, known := knownHosts[hostID]; known {
continue
}
if _, alreadyCleared := processed[hostID]; alreadyCleared {
continue
}
host := models.DockerHost{
ID: hostID,
DisplayName: alert.ResourceName,
Hostname: alert.Node,
}
if host.DisplayName == "" {
host.DisplayName = hostID
}
if host.Hostname == "" {
host.Hostname = hostID
}
m.alertManager.HandleDockerHostRemoved(host)
processed[hostID] = struct{}{}
cleared = true
}
return cleared
}