Files
pulse/internal/monitoring/monitor_alert_sync.go
T
rcourtman 69b5ec22fd fix(alerts): key docker container overrides by name so toggles survive updates
The #1601 follow-on: per-container alert overrides were keyed by Docker
container ID (docker:{host}/{containerID}), which changes on every
recreate, so each image update silently re-armed alerts the user had
switched off and left a dead entry behind in alerts.json — the unbounded
growth that pushed the reporter's config past the old 64KB body cap
(raised in 38434a513). The v6 thresholds UI additionally wrote keys from
the unified hash id (docker:{host}/app-container-{16hex}), which the
evaluator never read at all.

Overrides now key on stable identity, docker:{host}/{containerName}:

- The evaluator resolves the name key first and falls back to the legacy
  container-ID key so pre-migration entries keep working
  (evaluateDockerContainer, checkDockerContainerState, the
  container-update resolver, and reevaluateActiveAlertsLocked).
- MigrateDockerContainerOverrideKeys runs in the monitor sync next to
  MigrateCanonicalOverrideKeys, driven by the unified resource snapshot:
  it re-homes live legacy-ID and unified-hash keys onto the name key and
  prunes orphaned ID-shaped entries, ending the per-update orphan
  accumulation. Name-keyed entries for absent containers are kept so a
  recreate under the same name still honours them.
- The UI candidate chain (single implementation in alertOverridesModel)
  now leads with docker:{host}/{name} and trails the container-ID, short
  ID, unified-hash and slash-tail forms, so rows bind pre-existing
  overrides of every historical shape and the next save re-homes them.
  Rows carry overrideStorageId/overrideIdCandidates so toggle,
  connectivity, offline-state, edit and remove all write the stable key.
- The ignored-containers card copy now documents the wildcard forms
  (runner-*, *-dev, *staging*) shipped in b5fa6a9af, under the title
  "Ignored container patterns".

Contract deltas: alerts, frontend-primitives, monitoring, and
unified-resources now pin the name-keyed override identity, the single
frontend candidate-chain owner, the sync-cadence migration, and the
resource-facet-backed table identity respectively.

Verified live against a mock instance: a UI toggle persists
docker:{host}/loki and binds back after reload, and seeded
legacy/hash/orphan keys converge to name keys on disk within two sync
ticks. go test ./internal/alerts/... ./internal/monitoring/... green;
recreate survival pinned in
TestDockerContainerOverrideSurvivesContainerRecreate.

Refs #1601
2026-08-06 22:04:04 +01:00

200 lines
5.4 KiB
Go

package monitoring
import (
"strings"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
// syncAlertsToState copies the latest alert manager data into the shared state snapshot.
// This keeps WebSocket broadcasts aligned with in-memory acknowledgement updates.
func (m *Monitor) syncAlertsToState() {
if m.pruneStaleDockerAlerts() {
if logging.IsLevelEnabled(zerolog.DebugLevel) {
log.Debug().Msg("pruned stale docker alerts during sync")
}
}
modelAlerts := m.activeAlertsSnapshot()
for _, alert := range modelAlerts {
if alert.Acknowledged && logging.IsLevelEnabled(zerolog.DebugLevel) {
log.Debug().Str("alertID", alert.ID).Interface("ackTime", alert.AckTime).Msg("syncing acknowledged alert")
}
}
m.state.UpdateActiveAlerts(modelAlerts)
recentlyResolved := m.alertManager.GetRecentlyResolved()
if len(recentlyResolved) > 0 && logging.IsLevelEnabled(zerolog.DebugLevel) {
log.Debug().Int("count", len(recentlyResolved)).Msg("syncing recently resolved alerts")
}
m.state.UpdateRecentlyResolved(recentlyResolved)
}
// SyncAlertState is the exported wrapper used by APIs that mutate alerts outside the poll loop.
func (m *Monitor) SyncAlertState() {
m.syncAlertsToState()
}
func (m *Monitor) activeAlertsSnapshot() []models.Alert {
if m == nil {
return nil
}
if m.alertManager == nil {
if m.state == nil {
return nil
}
return m.state.GetSnapshot().ActiveAlerts
}
activeAlerts := m.alertManager.GetActiveAlerts()
modelAlerts := make([]models.Alert, 0, len(activeAlerts))
for _, alert := range activeAlerts {
modelAlerts = append(modelAlerts, models.Alert{
ID: alert.ID,
Type: alert.Type,
Level: string(alert.Level),
ResourceID: alert.ResourceID,
ResourceName: alert.ResourceName,
Node: alert.Node,
NodeDisplayName: alert.NodeDisplayName,
Instance: alert.Instance,
Message: alert.Message,
Value: alert.Value,
Threshold: alert.Threshold,
StartTime: alert.StartTime,
Acknowledged: alert.Acknowledged,
AckTime: alert.AckTime,
AckUser: alert.AckUser,
// GetActiveAlerts returns deep clones, so the map is already private.
Metadata: alert.Metadata,
})
}
return modelAlerts
}
func (m *Monitor) recentlyResolvedAlertsSnapshot() []models.ResolvedAlert {
if m == nil {
return nil
}
if m.alertManager == nil {
if m.state == nil {
return nil
}
return m.state.GetSnapshot().RecentlyResolved
}
return m.alertManager.GetRecentlyResolved()
}
func (m *Monitor) syncUnifiedResourceAlertsToState(resources []unifiedresources.Resource) {
if m == nil || m.alertManager == nil {
return
}
config := m.alertManager.GetConfig()
migrated := alerts.MigrateCanonicalOverrideKeys(&config, resources)
if alerts.MigrateDockerContainerOverrideKeys(&config, resources) {
migrated = true
}
if migrated {
if m.configPersist == nil {
log.Warn().Msg("cannot persist canonical alert override migration without config persistence")
} else if err := m.configPersist.SaveAlertConfig(config); err != nil {
log.Error().Err(err).Msg("failed to persist canonical alert override migration")
} else {
m.alertManager.UpdateConfig(config)
}
}
m.migrateAvailabilityLinksToCanonicalIDs(resources)
m.alertManager.CheckUnifiedResourceMetrics(resources)
m.alertManager.SyncUnifiedResourceIncidents(resources)
m.syncAlertsToState()
}
// pruneStaleDockerAlerts removes docker alerts that reference hosts no longer present in state.
func (m *Monitor) pruneStaleDockerAlerts() bool {
if m.alertManager == nil {
return false
}
readState := m.GetUnifiedReadStateOrSnapshot()
if readState == nil {
return false
}
hosts := readState.DockerHosts()
knownHosts := make(map[string]struct{}, len(hosts)*2)
for _, host := range hosts {
hostID := strings.TrimSpace(host.ID())
if hostID != "" {
knownHosts[hostID] = struct{}{}
}
if sourceID := strings.TrimSpace(host.HostSourceID()); sourceID != "" {
knownHosts[sourceID] = struct{}{}
}
}
if len(knownHosts) == 0 {
// Still allow stale entries to be cleared if no hosts remain.
}
active := m.alertManager.GetActiveAlerts()
processed := make(map[string]struct{})
cleared := false
for _, alert := range active {
var hostID string
switch {
case alert.Type == "docker-host-offline":
hostID = strings.TrimPrefix(strings.TrimSpace(alert.ResourceID), "docker:")
case strings.HasPrefix(alert.ResourceID, "docker:"):
resource := strings.TrimPrefix(alert.ResourceID, "docker:")
if idx := strings.Index(resource, "/"); idx >= 0 {
hostID = resource[:idx]
} else {
hostID = resource
}
default:
continue
}
hostID = strings.TrimSpace(hostID)
if hostID == "" {
continue
}
if _, known := knownHosts[hostID]; known {
continue
}
if _, alreadyCleared := processed[hostID]; alreadyCleared {
continue
}
host := models.DockerHost{
ID: hostID,
DisplayName: alert.ResourceName,
Hostname: alert.Node,
}
if host.DisplayName == "" {
host.DisplayName = hostID
}
if host.Hostname == "" {
host.Hostname = hostID
}
m.alertManager.HandleDockerHostRemoved(host)
processed[hostID] = struct{}{}
cleared = true
}
return cleared
}