mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-11 14:00:29 +00:00
69b5ec22fd
The #1601 follow-on: per-container alert overrides were keyed by Docker container ID (docker:{host}/{containerID}), which changes on every recreate, so each image update silently re-armed alerts the user had switched off and left a dead entry behind in alerts.json — the unbounded growth that pushed the reporter's config past the old 64KB body cap (raised in38434a513). The v6 thresholds UI additionally wrote keys from the unified hash id (docker:{host}/app-container-{16hex}), which the evaluator never read at all. Overrides now key on stable identity, docker:{host}/{containerName}: - The evaluator resolves the name key first and falls back to the legacy container-ID key so pre-migration entries keep working (evaluateDockerContainer, checkDockerContainerState, the container-update resolver, and reevaluateActiveAlertsLocked). - MigrateDockerContainerOverrideKeys runs in the monitor sync next to MigrateCanonicalOverrideKeys, driven by the unified resource snapshot: it re-homes live legacy-ID and unified-hash keys onto the name key and prunes orphaned ID-shaped entries, ending the per-update orphan accumulation. Name-keyed entries for absent containers are kept so a recreate under the same name still honours them. - The UI candidate chain (single implementation in alertOverridesModel) now leads with docker:{host}/{name} and trails the container-ID, short ID, unified-hash and slash-tail forms, so rows bind pre-existing overrides of every historical shape and the next save re-homes them. Rows carry overrideStorageId/overrideIdCandidates so toggle, connectivity, offline-state, edit and remove all write the stable key. - The ignored-containers card copy now documents the wildcard forms (runner-*, *-dev, *staging*) shipped inb5fa6a9af, under the title "Ignored container patterns". Contract deltas: alerts, frontend-primitives, monitoring, and unified-resources now pin the name-keyed override identity, the single frontend candidate-chain owner, the sync-cadence migration, and the resource-facet-backed table identity respectively. Verified live against a mock instance: a UI toggle persists docker:{host}/loki and binds back after reload, and seeded legacy/hash/orphan keys converge to name keys on disk within two sync ticks. go test ./internal/alerts/... ./internal/monitoring/... green; recreate survival pinned in TestDockerContainerOverrideSurvivesContainerRecreate. Refs #1601
200 lines
5.4 KiB
Go
200 lines
5.4 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"strings"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
"github.com/rs/zerolog"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
// syncAlertsToState copies the latest alert manager data into the shared state snapshot.
|
|
// This keeps WebSocket broadcasts aligned with in-memory acknowledgement updates.
|
|
func (m *Monitor) syncAlertsToState() {
|
|
if m.pruneStaleDockerAlerts() {
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Msg("pruned stale docker alerts during sync")
|
|
}
|
|
}
|
|
|
|
modelAlerts := m.activeAlertsSnapshot()
|
|
for _, alert := range modelAlerts {
|
|
if alert.Acknowledged && logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Str("alertID", alert.ID).Interface("ackTime", alert.AckTime).Msg("syncing acknowledged alert")
|
|
}
|
|
}
|
|
m.state.UpdateActiveAlerts(modelAlerts)
|
|
|
|
recentlyResolved := m.alertManager.GetRecentlyResolved()
|
|
if len(recentlyResolved) > 0 && logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Int("count", len(recentlyResolved)).Msg("syncing recently resolved alerts")
|
|
}
|
|
m.state.UpdateRecentlyResolved(recentlyResolved)
|
|
}
|
|
|
|
// SyncAlertState is the exported wrapper used by APIs that mutate alerts outside the poll loop.
|
|
func (m *Monitor) SyncAlertState() {
|
|
m.syncAlertsToState()
|
|
}
|
|
|
|
func (m *Monitor) activeAlertsSnapshot() []models.Alert {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
if m.alertManager == nil {
|
|
if m.state == nil {
|
|
return nil
|
|
}
|
|
return m.state.GetSnapshot().ActiveAlerts
|
|
}
|
|
|
|
activeAlerts := m.alertManager.GetActiveAlerts()
|
|
modelAlerts := make([]models.Alert, 0, len(activeAlerts))
|
|
for _, alert := range activeAlerts {
|
|
modelAlerts = append(modelAlerts, models.Alert{
|
|
ID: alert.ID,
|
|
Type: alert.Type,
|
|
Level: string(alert.Level),
|
|
ResourceID: alert.ResourceID,
|
|
ResourceName: alert.ResourceName,
|
|
Node: alert.Node,
|
|
NodeDisplayName: alert.NodeDisplayName,
|
|
Instance: alert.Instance,
|
|
Message: alert.Message,
|
|
Value: alert.Value,
|
|
Threshold: alert.Threshold,
|
|
StartTime: alert.StartTime,
|
|
Acknowledged: alert.Acknowledged,
|
|
AckTime: alert.AckTime,
|
|
AckUser: alert.AckUser,
|
|
// GetActiveAlerts returns deep clones, so the map is already private.
|
|
Metadata: alert.Metadata,
|
|
})
|
|
}
|
|
return modelAlerts
|
|
}
|
|
|
|
func (m *Monitor) recentlyResolvedAlertsSnapshot() []models.ResolvedAlert {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
if m.alertManager == nil {
|
|
if m.state == nil {
|
|
return nil
|
|
}
|
|
return m.state.GetSnapshot().RecentlyResolved
|
|
}
|
|
|
|
return m.alertManager.GetRecentlyResolved()
|
|
}
|
|
|
|
func (m *Monitor) syncUnifiedResourceAlertsToState(resources []unifiedresources.Resource) {
|
|
if m == nil || m.alertManager == nil {
|
|
return
|
|
}
|
|
|
|
config := m.alertManager.GetConfig()
|
|
migrated := alerts.MigrateCanonicalOverrideKeys(&config, resources)
|
|
if alerts.MigrateDockerContainerOverrideKeys(&config, resources) {
|
|
migrated = true
|
|
}
|
|
if migrated {
|
|
if m.configPersist == nil {
|
|
log.Warn().Msg("cannot persist canonical alert override migration without config persistence")
|
|
} else if err := m.configPersist.SaveAlertConfig(config); err != nil {
|
|
log.Error().Err(err).Msg("failed to persist canonical alert override migration")
|
|
} else {
|
|
m.alertManager.UpdateConfig(config)
|
|
}
|
|
}
|
|
|
|
m.migrateAvailabilityLinksToCanonicalIDs(resources)
|
|
|
|
m.alertManager.CheckUnifiedResourceMetrics(resources)
|
|
m.alertManager.SyncUnifiedResourceIncidents(resources)
|
|
m.syncAlertsToState()
|
|
}
|
|
|
|
// pruneStaleDockerAlerts removes docker alerts that reference hosts no longer present in state.
|
|
func (m *Monitor) pruneStaleDockerAlerts() bool {
|
|
if m.alertManager == nil {
|
|
return false
|
|
}
|
|
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return false
|
|
}
|
|
|
|
hosts := readState.DockerHosts()
|
|
knownHosts := make(map[string]struct{}, len(hosts)*2)
|
|
for _, host := range hosts {
|
|
hostID := strings.TrimSpace(host.ID())
|
|
if hostID != "" {
|
|
knownHosts[hostID] = struct{}{}
|
|
}
|
|
if sourceID := strings.TrimSpace(host.HostSourceID()); sourceID != "" {
|
|
knownHosts[sourceID] = struct{}{}
|
|
}
|
|
}
|
|
|
|
if len(knownHosts) == 0 {
|
|
// Still allow stale entries to be cleared if no hosts remain.
|
|
}
|
|
|
|
active := m.alertManager.GetActiveAlerts()
|
|
processed := make(map[string]struct{})
|
|
cleared := false
|
|
|
|
for _, alert := range active {
|
|
var hostID string
|
|
|
|
switch {
|
|
case alert.Type == "docker-host-offline":
|
|
hostID = strings.TrimPrefix(strings.TrimSpace(alert.ResourceID), "docker:")
|
|
case strings.HasPrefix(alert.ResourceID, "docker:"):
|
|
resource := strings.TrimPrefix(alert.ResourceID, "docker:")
|
|
if idx := strings.Index(resource, "/"); idx >= 0 {
|
|
hostID = resource[:idx]
|
|
} else {
|
|
hostID = resource
|
|
}
|
|
default:
|
|
continue
|
|
}
|
|
|
|
hostID = strings.TrimSpace(hostID)
|
|
if hostID == "" {
|
|
continue
|
|
}
|
|
|
|
if _, known := knownHosts[hostID]; known {
|
|
continue
|
|
}
|
|
if _, alreadyCleared := processed[hostID]; alreadyCleared {
|
|
continue
|
|
}
|
|
|
|
host := models.DockerHost{
|
|
ID: hostID,
|
|
DisplayName: alert.ResourceName,
|
|
Hostname: alert.Node,
|
|
}
|
|
if host.DisplayName == "" {
|
|
host.DisplayName = hostID
|
|
}
|
|
if host.Hostname == "" {
|
|
host.Hostname = hostID
|
|
}
|
|
|
|
m.alertManager.HandleDockerHostRemoved(host)
|
|
processed[hostID] = struct{}{}
|
|
cleared = true
|
|
}
|
|
|
|
return cleared
|
|
}
|