Files
pulse/internal/monitoring/monitor_alert_sync.go
T
rcourtman 62c3761ee6 Preserve alert lifecycle truth and safe pinned updates
Rejected active-alert writes still reached history, firing events and
notification dispatch. Make admission explicit across detector and restore
paths so repeated suppressed observations cannot create those side effects.
Reconcile restored alerts when persisted resource policy attaches at startup,
reconcile again after native aliases become canonical, and refresh shared
state before those alerts reach clients.

Preserve occurrence identity when retained alerts re-fire, including durable
history reconstruction and delayed lifecycle replay.

Keep retained table and card components mounted when scrolling shifts the
visible window, so policy editors do not lose unsaved state. Show actual
future maintenance times and the paused-attention effect.

Keep synthetic host machine identities stable across restarts so saved
operator policy remains attached during verification.

Qualify server update and rollback instructions by helper ownership, and
stop signed installer recipes before execution if a download or signature
check fails.

Refs #2237
2026-09-24 17:43:50 +01:00

224 lines
6.7 KiB
Go

package monitoring
import (
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
// syncAlertsToState copies the latest alert manager data into the shared state snapshot.
// This keeps WebSocket broadcasts aligned with in-memory acknowledgement updates.
func (m *Monitor) syncAlertsToState() {
if m.pruneStaleDockerAlerts() {
if logging.IsLevelEnabled(zerolog.DebugLevel) {
log.Debug().Msg("pruned stale docker alerts during sync")
}
}
modelAlerts := m.activeAlertsSnapshot()
for _, alert := range modelAlerts {
if alert.Acknowledged && logging.IsLevelEnabled(zerolog.DebugLevel) {
log.Debug().Str("alertID", alert.ID).Interface("ackTime", alert.AckTime).Msg("syncing acknowledged alert")
}
}
m.state.UpdateActiveAlerts(modelAlerts)
recentlyResolved := m.alertManager.GetRecentlyResolved()
if len(recentlyResolved) > 0 && logging.IsLevelEnabled(zerolog.DebugLevel) {
log.Debug().Int("count", len(recentlyResolved)).Msg("syncing recently resolved alerts")
}
m.state.UpdateRecentlyResolved(recentlyResolved)
}
// SyncAlertState is the exported wrapper used by APIs that mutate alerts outside the poll loop.
func (m *Monitor) SyncAlertState() {
m.syncAlertsToState()
}
func (m *Monitor) activeAlertsSnapshot() []models.Alert {
if m == nil {
return nil
}
if m.alertManager == nil {
if m.state == nil {
return nil
}
return m.state.GetSnapshot().ActiveAlerts
}
activeAlerts := m.alertManager.GetActiveAlerts()
modelAlerts := make([]models.Alert, 0, len(activeAlerts))
for _, alert := range activeAlerts {
modelAlerts = append(modelAlerts, models.Alert{
ID: alert.ID,
Type: alert.Type,
Level: string(alert.Level),
ResourceID: alert.ResourceID,
ResourceName: alert.ResourceName,
Node: alert.Node,
NodeDisplayName: alert.NodeDisplayName,
Instance: alert.Instance,
Message: alert.Message,
Value: alert.Value,
Threshold: alert.Threshold,
StartTime: alert.StartTime,
Acknowledged: alert.Acknowledged,
AckTime: alert.AckTime,
AckUser: alert.AckUser,
// GetActiveAlerts returns deep clones, so the map is already private.
Metadata: alert.Metadata,
})
}
return modelAlerts
}
func (m *Monitor) recentlyResolvedAlertsSnapshot() []models.ResolvedAlert {
if m == nil {
return nil
}
if m.alertManager == nil {
if m.state == nil {
return nil
}
return m.state.GetSnapshot().RecentlyResolved
}
return m.alertManager.GetRecentlyResolved()
}
func (m *Monitor) syncUnifiedResourceAlertsToState(resources []unifiedresources.Resource) {
if m == nil || m.alertManager == nil {
return
}
// Resource publication can make persisted canonical policy resolvable for
// restored native-ID alerts that preceded the first observation at startup.
m.alertManager.ReconcileOperatorIntentState()
config := m.alertManager.GetConfig()
migrationPlan := alerts.PlanAlertIdentityMigration(config, resources)
if migrationPlan.UnsupportedVersion {
log.Warn().
Int("configVersion", migrationPlan.FromVersion).
Int("supportedVersion", alerts.CurrentAlertIdentitySchemaVersion).
Msg("alert identity config uses a newer schema; leaving persisted identities untouched")
} else if alerts.ApplyAlertIdentityMigration(&config, migrationPlan) {
if m.configPersist == nil {
log.Warn().
Int("fromVersion", migrationPlan.FromVersion).
Int("toVersion", migrationPlan.ToVersion).
Int("removedOverrides", len(migrationPlan.RemovedOverrideKeys)).
Int("addedOverrides", len(migrationPlan.AddedOverrideKeys)).
Int("deferredOverrides", len(migrationPlan.Deferred)).
Msg("cannot persist planned alert identity migration without config persistence")
} else if err := m.configPersist.SaveAlertConfig(config); err != nil {
log.Error().
Err(err).
Int("fromVersion", migrationPlan.FromVersion).
Int("toVersion", migrationPlan.ToVersion).
Msg("failed to persist planned alert identity migration")
} else {
m.alertManager.UpdateConfig(config)
log.Info().
Int("fromVersion", migrationPlan.FromVersion).
Int("toVersion", migrationPlan.ToVersion).
Int("removedOverrides", len(migrationPlan.RemovedOverrideKeys)).
Int("addedOverrides", len(migrationPlan.AddedOverrideKeys)).
Int("deferredOverrides", len(migrationPlan.Deferred)).
Msg("persisted alert identity migration plan")
}
}
m.migrateAvailabilityLinksToCanonicalIDs(resources)
m.alertManager.CheckUnifiedResourceMetricsWithCapacityTrends(resources, m.unifiedStorageCapacityTrends(resources, time.Now()))
m.alertManager.SyncUnifiedResourceIncidents(resources)
m.syncAlertsToState()
}
// pruneStaleDockerAlerts removes docker alerts that reference hosts no longer present in state.
func (m *Monitor) pruneStaleDockerAlerts() bool {
if m.alertManager == nil {
return false
}
readState := m.GetUnifiedReadStateOrSnapshot()
if readState == nil {
return false
}
hosts := readState.DockerHosts()
knownHosts := make(map[string]struct{}, len(hosts)*2)
for _, host := range hosts {
hostID := strings.TrimSpace(host.ID())
if hostID != "" {
knownHosts[hostID] = struct{}{}
}
if sourceID := strings.TrimSpace(host.HostSourceID()); sourceID != "" {
knownHosts[sourceID] = struct{}{}
}
}
if len(knownHosts) == 0 {
// Still allow stale entries to be cleared if no hosts remain.
}
active := m.alertManager.GetActiveAlerts()
processed := make(map[string]struct{})
cleared := false
for _, alert := range active {
var hostID string
switch {
case alert.Type == "docker-host-offline":
hostID = strings.TrimPrefix(strings.TrimSpace(alert.ResourceID), "docker:")
case strings.HasPrefix(alert.ResourceID, "docker:"):
resource := strings.TrimPrefix(alert.ResourceID, "docker:")
if idx := strings.Index(resource, "/"); idx >= 0 {
hostID = resource[:idx]
} else {
hostID = resource
}
default:
continue
}
hostID = strings.TrimSpace(hostID)
if hostID == "" {
continue
}
if _, known := knownHosts[hostID]; known {
continue
}
if _, alreadyCleared := processed[hostID]; alreadyCleared {
continue
}
host := models.DockerHost{
ID: hostID,
DisplayName: alert.ResourceName,
Hostname: alert.Node,
}
if host.DisplayName == "" {
host.DisplayName = hostID
}
if host.Hostname == "" {
host.Hostname = hostID
}
m.alertManager.HandleDockerHostRemoved(host)
processed[hostID] = struct{}{}
cleared = true
}
return cleared
}