mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-24 12:42:49 +00:00
62c3761ee6
Rejected active-alert writes still reached history, firing events and notification dispatch. Make admission explicit across detector and restore paths so repeated suppressed observations cannot create those side effects. Reconcile restored alerts when persisted resource policy attaches at startup, reconcile again after native aliases become canonical, and refresh shared state before those alerts reach clients. Preserve occurrence identity when retained alerts re-fire, including durable history reconstruction and delayed lifecycle replay. Keep retained table and card components mounted when scrolling shifts the visible window, so policy editors do not lose unsaved state. Show actual future maintenance times and the paused-attention effect. Keep synthetic host machine identities stable across restarts so saved operator policy remains attached during verification. Qualify server update and rollback instructions by helper ownership, and stop signed installer recipes before execution if a download or signature check fails. Refs #2237
224 lines
6.7 KiB
Go
224 lines
6.7 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
"github.com/rs/zerolog"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
// syncAlertsToState copies the latest alert manager data into the shared state snapshot.
|
|
// This keeps WebSocket broadcasts aligned with in-memory acknowledgement updates.
|
|
func (m *Monitor) syncAlertsToState() {
|
|
if m.pruneStaleDockerAlerts() {
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Msg("pruned stale docker alerts during sync")
|
|
}
|
|
}
|
|
|
|
modelAlerts := m.activeAlertsSnapshot()
|
|
for _, alert := range modelAlerts {
|
|
if alert.Acknowledged && logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Str("alertID", alert.ID).Interface("ackTime", alert.AckTime).Msg("syncing acknowledged alert")
|
|
}
|
|
}
|
|
m.state.UpdateActiveAlerts(modelAlerts)
|
|
|
|
recentlyResolved := m.alertManager.GetRecentlyResolved()
|
|
if len(recentlyResolved) > 0 && logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Int("count", len(recentlyResolved)).Msg("syncing recently resolved alerts")
|
|
}
|
|
m.state.UpdateRecentlyResolved(recentlyResolved)
|
|
}
|
|
|
|
// SyncAlertState is the exported wrapper used by APIs that mutate alerts outside the poll loop.
|
|
func (m *Monitor) SyncAlertState() {
|
|
m.syncAlertsToState()
|
|
}
|
|
|
|
func (m *Monitor) activeAlertsSnapshot() []models.Alert {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
if m.alertManager == nil {
|
|
if m.state == nil {
|
|
return nil
|
|
}
|
|
return m.state.GetSnapshot().ActiveAlerts
|
|
}
|
|
|
|
activeAlerts := m.alertManager.GetActiveAlerts()
|
|
modelAlerts := make([]models.Alert, 0, len(activeAlerts))
|
|
for _, alert := range activeAlerts {
|
|
modelAlerts = append(modelAlerts, models.Alert{
|
|
ID: alert.ID,
|
|
Type: alert.Type,
|
|
Level: string(alert.Level),
|
|
ResourceID: alert.ResourceID,
|
|
ResourceName: alert.ResourceName,
|
|
Node: alert.Node,
|
|
NodeDisplayName: alert.NodeDisplayName,
|
|
Instance: alert.Instance,
|
|
Message: alert.Message,
|
|
Value: alert.Value,
|
|
Threshold: alert.Threshold,
|
|
StartTime: alert.StartTime,
|
|
Acknowledged: alert.Acknowledged,
|
|
AckTime: alert.AckTime,
|
|
AckUser: alert.AckUser,
|
|
// GetActiveAlerts returns deep clones, so the map is already private.
|
|
Metadata: alert.Metadata,
|
|
})
|
|
}
|
|
return modelAlerts
|
|
}
|
|
|
|
func (m *Monitor) recentlyResolvedAlertsSnapshot() []models.ResolvedAlert {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
if m.alertManager == nil {
|
|
if m.state == nil {
|
|
return nil
|
|
}
|
|
return m.state.GetSnapshot().RecentlyResolved
|
|
}
|
|
|
|
return m.alertManager.GetRecentlyResolved()
|
|
}
|
|
|
|
func (m *Monitor) syncUnifiedResourceAlertsToState(resources []unifiedresources.Resource) {
|
|
if m == nil || m.alertManager == nil {
|
|
return
|
|
}
|
|
|
|
// Resource publication can make persisted canonical policy resolvable for
|
|
// restored native-ID alerts that preceded the first observation at startup.
|
|
m.alertManager.ReconcileOperatorIntentState()
|
|
|
|
config := m.alertManager.GetConfig()
|
|
migrationPlan := alerts.PlanAlertIdentityMigration(config, resources)
|
|
if migrationPlan.UnsupportedVersion {
|
|
log.Warn().
|
|
Int("configVersion", migrationPlan.FromVersion).
|
|
Int("supportedVersion", alerts.CurrentAlertIdentitySchemaVersion).
|
|
Msg("alert identity config uses a newer schema; leaving persisted identities untouched")
|
|
} else if alerts.ApplyAlertIdentityMigration(&config, migrationPlan) {
|
|
if m.configPersist == nil {
|
|
log.Warn().
|
|
Int("fromVersion", migrationPlan.FromVersion).
|
|
Int("toVersion", migrationPlan.ToVersion).
|
|
Int("removedOverrides", len(migrationPlan.RemovedOverrideKeys)).
|
|
Int("addedOverrides", len(migrationPlan.AddedOverrideKeys)).
|
|
Int("deferredOverrides", len(migrationPlan.Deferred)).
|
|
Msg("cannot persist planned alert identity migration without config persistence")
|
|
} else if err := m.configPersist.SaveAlertConfig(config); err != nil {
|
|
log.Error().
|
|
Err(err).
|
|
Int("fromVersion", migrationPlan.FromVersion).
|
|
Int("toVersion", migrationPlan.ToVersion).
|
|
Msg("failed to persist planned alert identity migration")
|
|
} else {
|
|
m.alertManager.UpdateConfig(config)
|
|
log.Info().
|
|
Int("fromVersion", migrationPlan.FromVersion).
|
|
Int("toVersion", migrationPlan.ToVersion).
|
|
Int("removedOverrides", len(migrationPlan.RemovedOverrideKeys)).
|
|
Int("addedOverrides", len(migrationPlan.AddedOverrideKeys)).
|
|
Int("deferredOverrides", len(migrationPlan.Deferred)).
|
|
Msg("persisted alert identity migration plan")
|
|
}
|
|
}
|
|
|
|
m.migrateAvailabilityLinksToCanonicalIDs(resources)
|
|
|
|
m.alertManager.CheckUnifiedResourceMetricsWithCapacityTrends(resources, m.unifiedStorageCapacityTrends(resources, time.Now()))
|
|
m.alertManager.SyncUnifiedResourceIncidents(resources)
|
|
m.syncAlertsToState()
|
|
}
|
|
|
|
// pruneStaleDockerAlerts removes docker alerts that reference hosts no longer present in state.
|
|
func (m *Monitor) pruneStaleDockerAlerts() bool {
|
|
if m.alertManager == nil {
|
|
return false
|
|
}
|
|
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return false
|
|
}
|
|
|
|
hosts := readState.DockerHosts()
|
|
knownHosts := make(map[string]struct{}, len(hosts)*2)
|
|
for _, host := range hosts {
|
|
hostID := strings.TrimSpace(host.ID())
|
|
if hostID != "" {
|
|
knownHosts[hostID] = struct{}{}
|
|
}
|
|
if sourceID := strings.TrimSpace(host.HostSourceID()); sourceID != "" {
|
|
knownHosts[sourceID] = struct{}{}
|
|
}
|
|
}
|
|
|
|
if len(knownHosts) == 0 {
|
|
// Still allow stale entries to be cleared if no hosts remain.
|
|
}
|
|
|
|
active := m.alertManager.GetActiveAlerts()
|
|
processed := make(map[string]struct{})
|
|
cleared := false
|
|
|
|
for _, alert := range active {
|
|
var hostID string
|
|
|
|
switch {
|
|
case alert.Type == "docker-host-offline":
|
|
hostID = strings.TrimPrefix(strings.TrimSpace(alert.ResourceID), "docker:")
|
|
case strings.HasPrefix(alert.ResourceID, "docker:"):
|
|
resource := strings.TrimPrefix(alert.ResourceID, "docker:")
|
|
if idx := strings.Index(resource, "/"); idx >= 0 {
|
|
hostID = resource[:idx]
|
|
} else {
|
|
hostID = resource
|
|
}
|
|
default:
|
|
continue
|
|
}
|
|
|
|
hostID = strings.TrimSpace(hostID)
|
|
if hostID == "" {
|
|
continue
|
|
}
|
|
|
|
if _, known := knownHosts[hostID]; known {
|
|
continue
|
|
}
|
|
if _, alreadyCleared := processed[hostID]; alreadyCleared {
|
|
continue
|
|
}
|
|
|
|
host := models.DockerHost{
|
|
ID: hostID,
|
|
DisplayName: alert.ResourceName,
|
|
Hostname: alert.Node,
|
|
}
|
|
if host.DisplayName == "" {
|
|
host.DisplayName = hostID
|
|
}
|
|
if host.Hostname == "" {
|
|
host.Hostname = hostID
|
|
}
|
|
|
|
m.alertManager.HandleDockerHostRemoved(host)
|
|
processed[hostID] = struct{}{}
|
|
cleared = true
|
|
}
|
|
|
|
return cleared
|
|
}
|