mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 18:45:53 +00:00
rcourtman
8827defb63
Phase 3 of docs/ALERT_ENGINE_EVOLUTION.md: the effective alert policy for a resource — type default thresholds, the type's DisableAll switches, custom rules, the per-resource override — is now answered by one ordered fold (effectiveAlertPolicyNoLock in alert_policy.go), translated from the persisted AlertConfig. The config keeps its shape; the engine stops reading it piecemeal. Before this, every check path read its own DisableAll* boolean and picked its own override lookup, and the scattered reads drifted (#1738 was an override lookup that existed on some paths and not others; connection.go hand-rolled its own type-to-switch mapping). Now: - resolveResourceThresholds and getGuestThresholds delegate to the fold. - All 40+ DisableAll* reads across the check paths, the config-change reconciliation, and the connection detector go through alertPolicyTypeSwitchesNoLock — the single place those booleans are read on behalf of evaluation. Characterization first: alert_policy_test.go pins the fold against the legacy resolution paths (per-type defaults, overrides, storage aliases, guest custom rules by priority, every type's switch pair) before any call site moved.
1536 lines
51 KiB
Go
1536 lines
51 KiB
Go
package alerts
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
|
|
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
type dockerRestartRecord struct {
|
|
count int
|
|
lastCount int
|
|
times []time.Time // Track restart times for loop detection
|
|
lastChecked time.Time
|
|
}
|
|
|
|
// dockerInstanceName returns the logical instance name used for Docker alerts.
|
|
func dockerInstanceName(host models.DockerHost) string {
|
|
name := strings.TrimSpace(host.DisplayName)
|
|
if name == "" {
|
|
name = strings.TrimSpace(host.Hostname)
|
|
}
|
|
if name == "" {
|
|
return "Docker"
|
|
}
|
|
return fmt.Sprintf("Docker:%s", name)
|
|
}
|
|
|
|
// dockerContainerDisplayName normalizes the container name for alert readability.
|
|
func dockerContainerDisplayName(container models.DockerContainer) string {
|
|
name := strings.TrimSpace(container.Name)
|
|
if strings.HasPrefix(name, "/") {
|
|
name = strings.TrimLeft(name, "/")
|
|
}
|
|
if name == "" {
|
|
containerID := strings.TrimSpace(container.ID)
|
|
if len(containerID) > 12 {
|
|
containerID = containerID[:12]
|
|
}
|
|
return containerID
|
|
}
|
|
return name
|
|
}
|
|
|
|
// dockerContainerOverrideKey builds the durable override key for a container:
|
|
// host + container *name*. Container IDs change on every recreate (image
|
|
// update, compose up), so ID-keyed overrides orphan the moment a container is
|
|
// updated; the name is the identity that survives (#1601). Returns "" when
|
|
// either part is missing so callers fall back to the legacy ID key.
|
|
func dockerContainerOverrideKey(hostID, containerName string) string {
|
|
hostID = strings.TrimSpace(hostID)
|
|
name := strings.TrimLeft(strings.TrimSpace(containerName), "/")
|
|
if hostID == "" || name == "" {
|
|
return ""
|
|
}
|
|
return fmt.Sprintf("docker:%s/%s", hostID, name)
|
|
}
|
|
|
|
// lookupDockerContainerOverrideNoLock resolves a container override,
|
|
// preferring the stable host+name key and falling back to the legacy
|
|
// container-ID resource ID so pre-migration entries keep working.
|
|
// Callers must hold m.mu.
|
|
func (m *Manager) lookupDockerContainerOverrideNoLock(hostID, containerName, legacyResourceID string) (ThresholdConfig, bool) {
|
|
if stableKey := dockerContainerOverrideKey(hostID, containerName); stableKey != "" {
|
|
if override, ok := m.config.Overrides[stableKey]; ok {
|
|
return override, true
|
|
}
|
|
}
|
|
if legacyResourceID != "" {
|
|
if override, ok := m.config.Overrides[legacyResourceID]; ok {
|
|
return override, true
|
|
}
|
|
}
|
|
return ThresholdConfig{}, false
|
|
}
|
|
|
|
// DockerResourceID builds a stable identifier for Docker container alerts.
|
|
// Patrol scope resolution registers these forms as known aliases so an
|
|
// alert's resource ID resolves to the collected container or host.
|
|
func DockerResourceID(hostID, containerID string) string {
|
|
hostID = strings.TrimSpace(hostID)
|
|
containerID = strings.TrimSpace(containerID)
|
|
if containerID == "" {
|
|
if hostID == "" {
|
|
return "docker:unknown"
|
|
}
|
|
return fmt.Sprintf("docker:%s", hostID)
|
|
}
|
|
if hostID == "" {
|
|
return fmt.Sprintf("docker:container/%s", containerID)
|
|
}
|
|
return fmt.Sprintf("docker:%s/%s", hostID, containerID)
|
|
}
|
|
|
|
func normalizeDockerUpdateTrackingPart(part string) string {
|
|
return strings.ToLower(strings.TrimSpace(part))
|
|
}
|
|
|
|
// dockerUpdateTrackingHostKey builds a stable host identity for Docker update timing.
|
|
func dockerUpdateTrackingHostKey(host models.DockerHost) string {
|
|
switch {
|
|
case normalizeDockerUpdateTrackingPart(host.AgentID) != "":
|
|
return "agent:" + normalizeDockerUpdateTrackingPart(host.AgentID)
|
|
case normalizeDockerUpdateTrackingPart(host.TokenID) != "":
|
|
return "token:" + normalizeDockerUpdateTrackingPart(host.TokenID)
|
|
case normalizeDockerUpdateTrackingPart(host.MachineID) != "":
|
|
return "machine:" + normalizeDockerUpdateTrackingPart(host.MachineID)
|
|
case normalizeDockerUpdateTrackingPart(host.Hostname) != "":
|
|
return "hostname:" + normalizeDockerUpdateTrackingPart(host.Hostname)
|
|
case normalizeDockerUpdateTrackingPart(host.ID) != "":
|
|
return "id:" + normalizeDockerUpdateTrackingPart(host.ID)
|
|
case normalizeDockerUpdateTrackingPart(host.DisplayName) != "":
|
|
return "name:" + normalizeDockerUpdateTrackingPart(host.DisplayName)
|
|
default:
|
|
return "unknown-host"
|
|
}
|
|
}
|
|
|
|
func dockerUpdateTrackingContainerKey(container models.DockerContainer) string {
|
|
if containerID := normalizeDockerUpdateTrackingPart(container.ID); containerID != "" {
|
|
return "id:" + containerID
|
|
}
|
|
|
|
name := normalizeDockerUpdateTrackingPart(container.Name)
|
|
name = strings.TrimPrefix(name, "/")
|
|
if name != "" {
|
|
return "name:" + name
|
|
}
|
|
|
|
if image := normalizeDockerUpdateTrackingPart(container.Image); image != "" {
|
|
return "image:" + image
|
|
}
|
|
|
|
return "unknown-container"
|
|
}
|
|
|
|
func dockerUpdateTrackingKey(host models.DockerHost, container models.DockerContainer) string {
|
|
return fmt.Sprintf("docker-update:%s/%s", dockerUpdateTrackingHostKey(host), dockerUpdateTrackingContainerKey(container))
|
|
}
|
|
|
|
func dockerUpdateTrackingHostPrefix(host models.DockerHost) string {
|
|
return fmt.Sprintf("docker-update:%s/", dockerUpdateTrackingHostKey(host))
|
|
}
|
|
|
|
// dockerServiceDisplayName normalizes the service name for alert readability.
|
|
func dockerServiceDisplayName(service models.DockerService) string {
|
|
name := strings.TrimSpace(service.Name)
|
|
if name != "" {
|
|
return name
|
|
}
|
|
serviceID := strings.TrimSpace(service.ID)
|
|
if len(serviceID) > 12 {
|
|
serviceID = serviceID[:12]
|
|
}
|
|
if serviceID == "" {
|
|
return "service"
|
|
}
|
|
return serviceID
|
|
}
|
|
|
|
// DockerServiceResourceID builds a stable identifier for Docker Swarm service
|
|
// alerts. Patrol scope resolution registers this form as a known alias on the
|
|
// owning Docker host record so a service alert's resource ID resolves there.
|
|
func DockerServiceResourceID(hostID, serviceID, serviceName string) string {
|
|
hostID = strings.TrimSpace(hostID)
|
|
normalizedServiceID := strings.TrimSpace(serviceID)
|
|
if normalizedServiceID == "" {
|
|
name := strings.TrimSpace(serviceName)
|
|
if name == "" {
|
|
name = "service"
|
|
}
|
|
builder := strings.Builder{}
|
|
for _, r := range strings.ToLower(name) {
|
|
switch {
|
|
case r >= 'a' && r <= 'z':
|
|
builder.WriteRune(r)
|
|
case r >= '0' && r <= '9':
|
|
builder.WriteRune(r)
|
|
case r == '-', r == '_':
|
|
builder.WriteRune(r)
|
|
case r == ' ' || r == '/' || r == '\\' || r == ':' || r == '.':
|
|
builder.WriteRune('-')
|
|
}
|
|
}
|
|
normalizedServiceID = strings.Trim(builder.String(), "-_")
|
|
if normalizedServiceID == "" {
|
|
normalizedServiceID = "service"
|
|
}
|
|
if len(normalizedServiceID) > 32 {
|
|
normalizedServiceID = normalizedServiceID[:32]
|
|
}
|
|
}
|
|
if hostID == "" {
|
|
return fmt.Sprintf("docker-service:%s", normalizedServiceID)
|
|
}
|
|
return fmt.Sprintf("docker:%s/service/%s", hostID, normalizedServiceID)
|
|
}
|
|
|
|
// matchesDockerIgnoredPrefix reports whether a container name or ID matches
|
|
// any entry in the ignore list. A bare entry keeps its historical
|
|
// prefix-match semantics. Entries may also use the wildcard forms shared
|
|
// with the other ignore lists: "*token" matches a suffix, "*token*" a
|
|
// substring, "token*" an explicit prefix.
|
|
func matchesDockerIgnoredPrefix(name, id string, prefixes []string) bool {
|
|
if len(prefixes) == 0 {
|
|
return false
|
|
}
|
|
|
|
name = strings.ToLower(strings.TrimSpace(name))
|
|
id = strings.ToLower(strings.TrimSpace(id))
|
|
|
|
for _, raw := range prefixes {
|
|
pattern := strings.ToLower(strings.TrimSpace(raw))
|
|
// A pattern that is nothing but wildcards would ignore every
|
|
// container, which is what DisableAllDockerContainers is for.
|
|
if pattern == "" || strings.Trim(pattern, "*") == "" {
|
|
continue
|
|
}
|
|
if matchesIgnoredContainerPattern(name, pattern) || matchesIgnoredContainerPattern(id, pattern) {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func matchesIgnoredContainerPattern(value, pattern string) bool {
|
|
if value == "" {
|
|
return false
|
|
}
|
|
leading := strings.HasPrefix(pattern, "*")
|
|
trailing := strings.HasSuffix(pattern, "*")
|
|
switch {
|
|
case leading && trailing:
|
|
return strings.Contains(value, pattern[1:len(pattern)-1])
|
|
case leading:
|
|
return strings.HasSuffix(value, pattern[1:])
|
|
case trailing:
|
|
return strings.HasPrefix(value, pattern[:len(pattern)-1])
|
|
default:
|
|
return strings.HasPrefix(value, pattern)
|
|
}
|
|
}
|
|
|
|
// CheckDockerHost evaluates Docker host telemetry and container metrics for alerts.
|
|
func (m *Manager) CheckDockerHost(host models.DockerHost) {
|
|
if host.ID == "" {
|
|
return
|
|
}
|
|
|
|
// Fresh telemetry marks the host as online and clears any offline alert.
|
|
m.HandleDockerHostOnline(host)
|
|
|
|
m.mu.RLock()
|
|
alertsEnabled := m.config.Enabled
|
|
disableAllDockerHosts, _ := m.alertPolicyTypeSwitchesNoLock("docker-host")
|
|
ignoredPrefixes := append([]string(nil), m.config.DockerIgnoredContainerPrefixes...)
|
|
m.mu.RUnlock()
|
|
if !alertsEnabled {
|
|
return
|
|
}
|
|
if disableAllDockerHosts {
|
|
return
|
|
}
|
|
|
|
seen := make(map[string]struct{}, len(host.Containers)+len(host.Services))
|
|
seenUpdateTracking := make(map[string]struct{}, len(host.Containers))
|
|
for _, container := range host.Containers {
|
|
containerName := dockerContainerDisplayName(container)
|
|
resourceID := DockerResourceID(host.ID, container.ID)
|
|
updateTrackingKey := dockerUpdateTrackingKey(host, container)
|
|
|
|
if matchesDockerIgnoredPrefix(containerName, container.ID, ignoredPrefixes) {
|
|
log.Debug().
|
|
Str("container", containerName).
|
|
Str("host", host.DisplayName).
|
|
Msg("Skipping Docker container alert evaluation due to ignored prefix")
|
|
m.clearDockerContainerStateAlert(resourceID)
|
|
m.clearDockerContainerHealthAlert(resourceID)
|
|
m.clearDockerContainerMetricAlerts(resourceID)
|
|
m.clearAlert(fmt.Sprintf("docker-container-restart-loop-%s", resourceID))
|
|
m.clearAlert(fmt.Sprintf("docker-container-oom-%s", resourceID))
|
|
m.clearAlert(fmt.Sprintf("docker-container-memory-limit-%s", resourceID))
|
|
m.mu.Lock()
|
|
delete(m.dockerRestartTracking, resourceID)
|
|
m.mu.Unlock()
|
|
m.clearDockerContainerUpdateTracking(resourceID, updateTrackingKey)
|
|
continue
|
|
}
|
|
|
|
seen[resourceID] = struct{}{}
|
|
seenUpdateTracking[updateTrackingKey] = struct{}{}
|
|
m.evaluateDockerContainer(host, container, resourceID)
|
|
}
|
|
|
|
for _, service := range host.Services {
|
|
resourceID := DockerServiceResourceID(host.ID, service.ID, service.Name)
|
|
seen[resourceID] = struct{}{}
|
|
m.evaluateDockerService(host, service, resourceID)
|
|
}
|
|
|
|
m.cleanupDockerContainerAlertsWithTracking(host, seen, seenUpdateTracking)
|
|
}
|
|
|
|
func (m *Manager) evaluateDockerContainer(host models.DockerHost, container models.DockerContainer, resourceID string) {
|
|
m.mu.RLock()
|
|
disableAllContainers, _ := m.alertPolicyTypeSwitchesNoLock("app-container")
|
|
m.mu.RUnlock()
|
|
if disableAllContainers {
|
|
return
|
|
}
|
|
|
|
containerName := dockerContainerDisplayName(container)
|
|
nodeName := strings.TrimSpace(host.Hostname)
|
|
instanceName := dockerInstanceName(host)
|
|
resourceType := "app-container"
|
|
containerTags := dockerLabelTags(container.Labels)
|
|
|
|
m.mu.RLock()
|
|
overrideConfig, hasOverride := m.lookupDockerContainerOverrideNoLock(host.ID, container.Name, resourceID)
|
|
m.mu.RUnlock()
|
|
if hasOverride && overrideConfig.Disabled {
|
|
// Alerts disabled via override; clear any existing alerts and skip evaluation.
|
|
m.clearDockerContainerStateAlert(resourceID)
|
|
m.clearDockerContainerHealthAlert(resourceID)
|
|
m.clearDockerContainerMetricAlerts(resourceID)
|
|
m.clearAlert(fmt.Sprintf("docker-container-update-%s", resourceID))
|
|
m.clearAlert(buildCanonicalStateID(resourceID, resourceID+"-image-update"))
|
|
m.clearDockerContainerUpdateTracking(resourceID, dockerUpdateTrackingKey(host, container))
|
|
return
|
|
}
|
|
|
|
state := strings.ToLower(strings.TrimSpace(container.State))
|
|
if state == "" {
|
|
state = strings.ToLower(strings.TrimSpace(container.Status))
|
|
}
|
|
|
|
if state != "running" {
|
|
m.checkDockerContainerState(host, container, resourceID, containerName, instanceName, nodeName)
|
|
m.clearDockerContainerMetricAlerts(resourceID, "cpu", "memory", "disk")
|
|
} else {
|
|
m.clearDockerContainerStateAlert(resourceID)
|
|
|
|
// Use Docker-specific defaults for containers
|
|
thresholds := ThresholdConfig{
|
|
CPU: &m.config.DockerDefaults.CPU,
|
|
Memory: &m.config.DockerDefaults.Memory,
|
|
Disk: &m.config.DockerDefaults.Disk,
|
|
}
|
|
if hasOverride {
|
|
thresholds = m.applyThresholdOverride(thresholds, overrideConfig)
|
|
}
|
|
|
|
if thresholds.CPU != nil {
|
|
cpuCapacityPercent := models.DockerContainerCPUCapacityPercent(container, host.CPUs)
|
|
cpuMetadata := map[string]interface{}{
|
|
"resourceType": resourceType,
|
|
"hostId": host.ID,
|
|
"hostName": host.DisplayName,
|
|
"hostHostname": host.Hostname,
|
|
"containerId": container.ID,
|
|
"containerName": containerName,
|
|
"image": container.Image,
|
|
"state": container.State,
|
|
"status": container.Status,
|
|
"restartCount": container.RestartCount,
|
|
"metric": "cpu",
|
|
"cpuPercent": cpuCapacityPercent,
|
|
"cpuCapacityPercent": cpuCapacityPercent,
|
|
"cpuRawPercent": container.CPUPercent,
|
|
"cpuCapacityCores": host.CPUs,
|
|
"tags": containerTags,
|
|
}
|
|
spec, err := buildCanonicalMetricSpec(resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "cpu", thresholds.CPU)
|
|
if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("resourceID", resourceID).
|
|
Str("container", containerName).
|
|
Msg("Skipping invalid canonical docker container CPU metric spec")
|
|
} else {
|
|
m.checkMetricWithCanonicalSpec(spec, containerName, nodeName, instanceName, resourceType, cpuCapacityPercent, thresholds.CPU, &metricOptions{Metadata: cpuMetadata})
|
|
}
|
|
}
|
|
|
|
if thresholds.Memory != nil {
|
|
memMetadata := map[string]interface{}{
|
|
"resourceType": resourceType,
|
|
"hostId": host.ID,
|
|
"hostName": host.DisplayName,
|
|
"hostHostname": host.Hostname,
|
|
"containerId": container.ID,
|
|
"containerName": containerName,
|
|
"image": container.Image,
|
|
"state": container.State,
|
|
"status": container.Status,
|
|
"restartCount": container.RestartCount,
|
|
"metric": "memory",
|
|
"memoryPercent": container.MemoryPercent,
|
|
"memoryUsageBytes": container.MemoryUsage,
|
|
"tags": containerTags,
|
|
}
|
|
if container.MemoryLimit > 0 {
|
|
memMetadata["memoryLimitBytes"] = container.MemoryLimit
|
|
}
|
|
spec, err := buildCanonicalMetricSpec(resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "memory", thresholds.Memory)
|
|
if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("resourceID", resourceID).
|
|
Str("container", containerName).
|
|
Msg("Skipping invalid canonical docker container memory metric spec")
|
|
} else {
|
|
m.checkMetricWithCanonicalSpec(spec, containerName, nodeName, instanceName, resourceType, container.MemoryPercent, thresholds.Memory, &metricOptions{Metadata: memMetadata})
|
|
}
|
|
}
|
|
|
|
if thresholds.Disk != nil {
|
|
totalBytes := container.RootFilesystemBytes
|
|
usedBytes := container.WritableLayerBytes
|
|
if totalBytes > 0 && usedBytes >= 0 {
|
|
diskPercent := (float64(usedBytes) / float64(totalBytes)) * 100
|
|
diskMetadata := map[string]interface{}{
|
|
"resourceType": resourceType,
|
|
"hostId": host.ID,
|
|
"hostName": host.DisplayName,
|
|
"hostHostname": host.Hostname,
|
|
"containerId": container.ID,
|
|
"containerName": containerName,
|
|
"image": container.Image,
|
|
"state": container.State,
|
|
"status": container.Status,
|
|
"restartCount": container.RestartCount,
|
|
"metric": "disk",
|
|
"diskPercent": diskPercent,
|
|
"writableLayerBytes": usedBytes,
|
|
"rootFilesystemBytes": totalBytes,
|
|
"mountCount": len(container.Mounts),
|
|
"tags": containerTags,
|
|
}
|
|
if container.BlockIO != nil {
|
|
diskMetadata["blockIoReadBytes"] = container.BlockIO.ReadBytes
|
|
diskMetadata["blockIoWriteBytes"] = container.BlockIO.WriteBytes
|
|
}
|
|
spec, err := buildCanonicalMetricSpec(resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "disk", thresholds.Disk)
|
|
if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("resourceID", resourceID).
|
|
Str("container", containerName).
|
|
Msg("Skipping invalid canonical docker container disk metric spec")
|
|
} else {
|
|
m.checkMetricWithCanonicalSpec(spec, containerName, nodeName, instanceName, resourceType, diskPercent, thresholds.Disk, &metricOptions{Metadata: diskMetadata})
|
|
}
|
|
} else {
|
|
m.clearDockerContainerMetricAlerts(resourceID, "disk")
|
|
}
|
|
}
|
|
}
|
|
|
|
m.checkDockerContainerHealth(host, container, resourceID, containerName, instanceName, nodeName)
|
|
|
|
// Docker-specific checks
|
|
m.checkDockerContainerRestartLoop(host, container, resourceID, containerName, instanceName, nodeName)
|
|
m.checkDockerContainerOOMKill(host, container, resourceID, containerName, instanceName, nodeName)
|
|
m.checkDockerContainerMemoryLimit(host, container, resourceID, containerName, instanceName, nodeName)
|
|
m.checkDockerContainerImageUpdate(host, container, resourceID, containerName, instanceName, nodeName)
|
|
}
|
|
|
|
func (m *Manager) evaluateDockerService(host models.DockerHost, service models.DockerService, resourceID string) {
|
|
m.mu.RLock()
|
|
disableAllServices, _ := m.alertPolicyTypeSwitchesNoLock("docker-service")
|
|
warnPct := m.config.DockerDefaults.ServiceWarnGapPct
|
|
critPct := m.config.DockerDefaults.ServiceCritGapPct
|
|
overrideConfig, hasOverride := m.config.Overrides[resourceID]
|
|
m.mu.RUnlock()
|
|
|
|
if disableAllServices {
|
|
m.clearDockerServiceAlert(resourceID)
|
|
return
|
|
}
|
|
if hasOverride && overrideConfig.Disabled {
|
|
m.clearDockerServiceAlert(resourceID)
|
|
return
|
|
}
|
|
|
|
desired := service.DesiredTasks
|
|
running := service.RunningTasks
|
|
if desired <= 0 {
|
|
m.clearDockerServiceAlert(resourceID)
|
|
return
|
|
}
|
|
|
|
missing := desired - running
|
|
if missing < 0 {
|
|
missing = 0
|
|
}
|
|
|
|
percentMissing := 0.0
|
|
if desired > 0 {
|
|
percentMissing = (float64(missing) / float64(desired)) * 100.0
|
|
}
|
|
|
|
thresholdValue := 0.0
|
|
serviceName := dockerServiceDisplayName(service)
|
|
instanceName := dockerInstanceName(host)
|
|
nodeName := strings.TrimSpace(host.Hostname)
|
|
|
|
metadata := map[string]interface{}{
|
|
"resourceType": "docker-service",
|
|
"hostId": host.ID,
|
|
"hostName": host.DisplayName,
|
|
"hostHostname": host.Hostname,
|
|
"serviceId": service.ID,
|
|
"serviceName": service.Name,
|
|
"stack": service.Stack,
|
|
"mode": service.Mode,
|
|
"desiredTasks": service.DesiredTasks,
|
|
"runningTasks": service.RunningTasks,
|
|
"completedTasks": service.CompletedTasks,
|
|
"missingTasks": missing,
|
|
"percentMissing": percentMissing,
|
|
"tags": dockerLabelTags(service.Labels),
|
|
}
|
|
alertID := fmt.Sprintf("docker-service-health-%s", resourceID)
|
|
|
|
if critPct > 0 && percentMissing >= float64(critPct) {
|
|
thresholdValue = float64(critPct)
|
|
} else if warnPct > 0 && percentMissing >= float64(warnPct) {
|
|
thresholdValue = float64(warnPct)
|
|
}
|
|
|
|
updateState := ""
|
|
updateMessage := ""
|
|
updateSeverity := AlertLevel("")
|
|
if service.UpdateStatus != nil {
|
|
updateState = strings.ToLower(strings.TrimSpace(service.UpdateStatus.State))
|
|
updateMessage = strings.TrimSpace(service.UpdateStatus.Message)
|
|
switch updateState {
|
|
case "paused", "rollback_started", "rollback_paused":
|
|
updateSeverity = AlertLevelWarning
|
|
case "rollback_failed":
|
|
updateSeverity = AlertLevelCritical
|
|
}
|
|
if service.UpdateStatus.CompletedAt != nil && !service.UpdateStatus.CompletedAt.IsZero() {
|
|
metadata["updateCompletedAt"] = service.UpdateStatus.CompletedAt.UTC()
|
|
}
|
|
if updateState != "" {
|
|
metadata["updateState"] = service.UpdateStatus.State
|
|
}
|
|
if updateMessage != "" {
|
|
metadata["updateMessage"] = updateMessage
|
|
}
|
|
}
|
|
|
|
if thresholdValue == 0 && updateSeverity != "" {
|
|
spec, err := buildCanonicalDiscreteStateSpec(resourceID, serviceName, unifiedresources.ResourceTypeDockerService, updateSeverity, 1, false, "update-state",
|
|
[]string{"paused", "rollback_started", "rollback_paused", "rollback_failed"})
|
|
if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("service", serviceName).
|
|
Str("resourceID", resourceID).
|
|
Msg("Skipping invalid canonical docker service update-state spec")
|
|
return
|
|
}
|
|
|
|
message := fmt.Sprintf("Docker service '%s' update state: %s", serviceName, service.UpdateStatus.State)
|
|
if updateMessage != "" {
|
|
message = fmt.Sprintf("%s (%s)", message, updateMessage)
|
|
}
|
|
|
|
_, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
|
|
Spec: spec,
|
|
Evidence: alertspecs.AlertEvidence{ObservedAt: time.Now(), DiscreteState: &alertspecs.DiscreteStateEvidence{StateKey: "update-state", Observed: updateState}},
|
|
AlertID: alertID,
|
|
AlertType: "docker-service-health",
|
|
ResourceID: resourceID,
|
|
ResourceName: serviceName,
|
|
Node: nodeName,
|
|
Instance: instanceName,
|
|
Message: message,
|
|
Value: percentMissing,
|
|
Threshold: 0,
|
|
Metadata: metadata,
|
|
AddToRecent: true,
|
|
AddToHistory: true,
|
|
RateLimit: true,
|
|
NotifyOnSeverityChange: true,
|
|
AddToHistoryOnSeverityChange: true,
|
|
DispatchAsync: true,
|
|
})
|
|
return
|
|
}
|
|
|
|
if thresholdValue == 0 {
|
|
m.clearDockerServiceAlert(resourceID)
|
|
return
|
|
}
|
|
|
|
spec, err := buildCanonicalServiceGapSpec(resourceID, serviceName, unifiedresources.ResourceTypeDockerService, serviceName, float64(warnPct), float64(critPct), false)
|
|
if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("service", serviceName).
|
|
Str("resourceID", resourceID).
|
|
Msg("Skipping invalid canonical docker service gap spec")
|
|
m.clearDockerServiceAlert(resourceID)
|
|
return
|
|
}
|
|
|
|
message := fmt.Sprintf("Docker service '%s' is running %d of %d desired tasks", serviceName, service.RunningTasks, service.DesiredTasks)
|
|
_, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
|
|
Spec: spec,
|
|
Evidence: alertspecs.AlertEvidence{
|
|
ObservedAt: time.Now(),
|
|
ServiceGap: &alertspecs.ServiceGapEvidence{
|
|
Service: serviceName,
|
|
Desired: desired,
|
|
Running: running,
|
|
},
|
|
},
|
|
AlertID: alertID,
|
|
AlertType: "docker-service-health",
|
|
ResourceID: resourceID,
|
|
ResourceName: serviceName,
|
|
Node: nodeName,
|
|
Instance: instanceName,
|
|
Message: message,
|
|
Value: percentMissing,
|
|
Threshold: thresholdValue,
|
|
Metadata: metadata,
|
|
AddToRecent: true,
|
|
AddToHistory: true,
|
|
RateLimit: true,
|
|
NotifyOnSeverityChange: true,
|
|
AddToHistoryOnSeverityChange: true,
|
|
DispatchAsync: true,
|
|
})
|
|
}
|
|
|
|
func (m *Manager) clearDockerServiceAlert(resourceID string) {
|
|
m.clearAlert(canonicalServiceGapStateID(resourceID))
|
|
m.clearAlert(canonicalDiscreteStateStateID(resourceID, "update-state"))
|
|
}
|
|
|
|
// HandleDockerHostOnline clears offline tracking and alerts for a Docker host.
|
|
func (m *Manager) HandleDockerHostOnline(host models.DockerHost) {
|
|
if host.ID == "" {
|
|
return
|
|
}
|
|
|
|
resourceID := fmt.Sprintf("docker:%s", strings.TrimSpace(host.ID))
|
|
alertID := canonicalConnectivityStateID(resourceID)
|
|
|
|
m.mu.Lock()
|
|
exists := m.hasActiveAlertNoLock(alertID)
|
|
// The host was observed online: a healthy observation ends any
|
|
// in-flight offline confirmation run in the core.
|
|
m.core.ApplyDiscrete(reducer.DiscreteSignal{ResourceID: resourceID, Key: canonicalConnectivitySpecID(resourceID), Matched: false, ObservedAt: m.policyNow()}, reducer.DiscreteRule{})
|
|
m.mu.Unlock()
|
|
|
|
if exists {
|
|
m.clearAlert(alertID)
|
|
}
|
|
}
|
|
|
|
// HandleDockerHostRemoved clears all alerts and tracking when a Docker host is deleted.
|
|
func (m *Manager) HandleDockerHostRemoved(host models.DockerHost) {
|
|
if host.ID == "" {
|
|
return
|
|
}
|
|
|
|
// Reuse the online handler to clear offline alerts and tracking.
|
|
m.HandleDockerHostOnline(host)
|
|
// Drop any container alerts and host-scoped tracking entries.
|
|
m.clearDockerHostContainerAlerts(host)
|
|
}
|
|
|
|
// HandleDockerHostOffline raises an alert when a Docker host stops reporting.
|
|
func (m *Manager) HandleDockerHostOffline(host models.DockerHost) {
|
|
if host.ID == "" {
|
|
return
|
|
}
|
|
|
|
m.mu.RLock()
|
|
if !m.config.Enabled {
|
|
m.mu.RUnlock()
|
|
return
|
|
}
|
|
_, disableDockerHostsOffline := m.alertPolicyTypeSwitchesNoLock("docker-host")
|
|
m.mu.RUnlock()
|
|
|
|
resourceID := fmt.Sprintf("docker:%s", strings.TrimSpace(host.ID))
|
|
alertID := canonicalConnectivityStateID(resourceID)
|
|
instanceName := dockerInstanceName(host)
|
|
nodeName := strings.TrimSpace(host.Hostname)
|
|
|
|
if disableDockerHostsOffline {
|
|
m.mu.Lock()
|
|
m.mu.Unlock()
|
|
m.clearAlert(alertID)
|
|
return
|
|
}
|
|
|
|
var disableConnectivity bool
|
|
m.mu.RLock()
|
|
if override, exists := m.config.Overrides[host.ID]; exists {
|
|
disableConnectivity = override.DisableConnectivity
|
|
}
|
|
m.mu.RUnlock()
|
|
|
|
if disableConnectivity {
|
|
m.clearAlert(alertID)
|
|
m.mu.Lock()
|
|
m.mu.Unlock()
|
|
return
|
|
}
|
|
|
|
spec, err := buildCanonicalConnectivitySpec(resourceID, host.DisplayName, unifiedresources.ResourceType("docker-host"), AlertLevelCritical, 3, false)
|
|
if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("dockerHost", host.DisplayName).
|
|
Str("hostID", host.ID).
|
|
Msg("Skipping invalid canonical docker host connectivity spec")
|
|
return
|
|
}
|
|
|
|
result, ok := m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
|
|
Spec: spec,
|
|
Evidence: alertspecs.AlertEvidence{ObservedAt: time.Now(), Connectivity: &alertspecs.ConnectivityEvidence{Signal: "status", Connected: false}},
|
|
AlertID: alertID,
|
|
AlertType: "docker-host-offline",
|
|
ResourceID: resourceID,
|
|
ResourceName: host.DisplayName,
|
|
Node: nodeName,
|
|
Instance: instanceName,
|
|
Message: fmt.Sprintf("Docker host '%s' is offline", host.DisplayName),
|
|
Metadata: map[string]interface{}{
|
|
"resourceType": "docker-host",
|
|
"hostId": host.ID,
|
|
"hostname": host.Hostname,
|
|
"agentId": host.AgentID,
|
|
"displayName": host.DisplayName,
|
|
},
|
|
AddToRecent: true,
|
|
AddToHistory: true,
|
|
RateLimit: true,
|
|
DispatchAsync: false,
|
|
})
|
|
if !ok || result.Transition == nil || result.Transition.Kind != alertspecs.EvaluationTransitionActivated {
|
|
return
|
|
}
|
|
|
|
m.mu.RLock()
|
|
alert, _ := m.getActiveAlertNoLock(alertID)
|
|
m.mu.RUnlock()
|
|
if alert != nil {
|
|
if callbacks := m.getAlertForAICallbacks(); len(callbacks) > 0 {
|
|
alertCopy := cloneAlertForOutput(alert)
|
|
go func(a *Alert, fns []func(*Alert)) {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
log.Error().Interface("panic", r).Str("alertID", a.ID).Msg("panic in AI alert callback")
|
|
}
|
|
}()
|
|
for _, callback := range fns {
|
|
callback(a)
|
|
}
|
|
}(alertCopy, callbacks)
|
|
}
|
|
}
|
|
|
|
m.clearDockerHostContainerAlerts(host)
|
|
}
|
|
|
|
func (m *Manager) checkDockerContainerState(host models.DockerHost, container models.DockerContainer, resourceID, containerName, instanceName, nodeName string) {
|
|
alertID := fmt.Sprintf("docker-container-state-%s", resourceID)
|
|
|
|
m.mu.RLock()
|
|
override, hasOverride := m.lookupDockerContainerOverrideNoLock(host.ID, container.Name, resourceID)
|
|
defaultDisable := m.config.DockerDefaults.StateDisableConnectivity
|
|
defaultSeverity := NormalizePoweredOffSeverity(m.config.DockerDefaults.StatePoweredOffSeverity)
|
|
m.mu.RUnlock()
|
|
|
|
disableConnectivity := defaultDisable
|
|
severity := defaultSeverity
|
|
if hasOverride {
|
|
if defaultDisable && !override.DisableConnectivity {
|
|
disableConnectivity = false
|
|
} else if override.DisableConnectivity {
|
|
disableConnectivity = true
|
|
}
|
|
|
|
if override.PoweredOffSeverity != "" {
|
|
severity = NormalizePoweredOffSeverity(override.PoweredOffSeverity)
|
|
}
|
|
}
|
|
|
|
if disableConnectivity {
|
|
m.clearDockerContainerStateAlert(resourceID)
|
|
return
|
|
}
|
|
|
|
observedState := strings.ToLower(strings.TrimSpace(container.State))
|
|
if observedState == "" {
|
|
observedState = "unknown"
|
|
}
|
|
|
|
spec, err := buildCanonicalDiscreteStateSpec(resourceID, containerName, unifiedresources.ResourceTypeAppContainer, severity, 2, false, "runtime-state",
|
|
[]string{"created", "restarting", "removing", "paused", "exited", "dead", "unknown"})
|
|
if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("resourceID", resourceID).
|
|
Str("container", containerName).
|
|
Msg("Skipping invalid canonical docker container state spec")
|
|
return
|
|
}
|
|
|
|
_, _ = m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
|
|
Spec: spec,
|
|
Evidence: alertspecs.AlertEvidence{
|
|
ObservedAt: time.Now(),
|
|
DiscreteState: &alertspecs.DiscreteStateEvidence{
|
|
StateKey: "runtime-state",
|
|
Observed: observedState,
|
|
},
|
|
},
|
|
AlertID: alertID,
|
|
AlertType: "docker-container-state",
|
|
ResourceID: resourceID,
|
|
ResourceName: containerName,
|
|
Node: nodeName,
|
|
Instance: instanceName,
|
|
Message: fmt.Sprintf("Docker container '%s' is %s", containerName, strings.TrimSpace(container.Status)),
|
|
Metadata: dockerContainerAlertMetadata(host, container, containerName),
|
|
AddToRecent: true,
|
|
AddToHistory: true,
|
|
DispatchAsync: true,
|
|
})
|
|
}
|
|
|
|
func (m *Manager) clearDockerContainerStateAlert(resourceID string) {
|
|
m.mu.Lock()
|
|
m.mu.Unlock()
|
|
m.clearAlert(canonicalDiscreteStateStateID(resourceID, "runtime-state"))
|
|
}
|
|
|
|
func dockerContainerAlertMetadata(host models.DockerHost, container models.DockerContainer, containerName string) map[string]interface{} {
|
|
return map[string]interface{}{
|
|
"resourceType": "app-container",
|
|
"hostId": host.ID,
|
|
"hostName": host.DisplayName,
|
|
"hostHostname": host.Hostname,
|
|
"containerId": container.ID,
|
|
"containerName": containerName,
|
|
"image": container.Image,
|
|
"state": container.State,
|
|
"status": container.Status,
|
|
"tags": dockerLabelTags(container.Labels),
|
|
}
|
|
}
|
|
|
|
func dockerLabelTags(labels map[string]string) []string {
|
|
if len(labels) == 0 {
|
|
return nil
|
|
}
|
|
|
|
tags := make([]string, 0, len(labels))
|
|
for rawKey, rawValue := range labels {
|
|
key := strings.TrimSpace(rawKey)
|
|
if key == "" {
|
|
continue
|
|
}
|
|
value := strings.TrimSpace(rawValue)
|
|
if value == "" {
|
|
tags = append(tags, key)
|
|
continue
|
|
}
|
|
tags = append(tags, key+":"+value)
|
|
}
|
|
sort.Strings(tags)
|
|
return tags
|
|
}
|
|
|
|
func (m *Manager) checkDockerContainerHealth(host models.DockerHost, container models.DockerContainer, resourceID, containerName, instanceName, nodeName string) {
|
|
// Docker retains the last health-check result after a container stops (and
|
|
// while health checks are suspended in other non-running states). That
|
|
// persisted value is no longer live health evidence; the runtime-state alert
|
|
// owns the non-running condition.
|
|
state := strings.ToLower(strings.TrimSpace(container.State))
|
|
if state != "" && state != "running" {
|
|
m.clearDockerContainerHealthAlert(resourceID)
|
|
return
|
|
}
|
|
|
|
health := strings.ToLower(strings.TrimSpace(container.Health))
|
|
if health == "" || health == "none" || health == "healthy" || health == "starting" {
|
|
m.clearDockerContainerHealthAlert(resourceID)
|
|
return
|
|
}
|
|
|
|
alertID := fmt.Sprintf("docker-container-health-%s", resourceID)
|
|
spec, err := buildCanonicalHealthAssessmentSpec(resourceID+"-health", resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "docker-container-health", nil, false)
|
|
if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("resourceID", resourceID).
|
|
Str("container", containerName).
|
|
Msg("Skipping invalid canonical docker container health spec")
|
|
return
|
|
}
|
|
|
|
severity := alertspecs.AlertSeverityWarning
|
|
if health == "unhealthy" {
|
|
severity = alertspecs.AlertSeverityCritical
|
|
}
|
|
|
|
metadata := dockerContainerAlertMetadata(host, container, containerName)
|
|
metadata["health"] = container.Health
|
|
|
|
_, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
|
|
Spec: spec,
|
|
Evidence: alertspecs.AlertEvidence{
|
|
ObservedAt: time.Now(),
|
|
HealthAssessment: &alertspecs.HealthAssessmentEvidence{
|
|
Signal: "docker-container-health",
|
|
Severity: severity,
|
|
Codes: []string{health},
|
|
},
|
|
},
|
|
AlertID: alertID,
|
|
AlertType: "docker-container-health",
|
|
ResourceID: resourceID,
|
|
ResourceName: containerName,
|
|
Node: nodeName,
|
|
Instance: instanceName,
|
|
Message: fmt.Sprintf("Docker container '%s' health is %s", containerName, container.Health),
|
|
Metadata: metadata,
|
|
AddToRecent: true,
|
|
AddToHistory: true,
|
|
DispatchAsync: false,
|
|
NotifyOnSeverityChange: true,
|
|
AddToHistoryOnSeverityChange: true,
|
|
})
|
|
|
|
log.Warn().
|
|
Str("container", containerName).
|
|
Str("host", host.DisplayName).
|
|
Str("health", container.Health).
|
|
Msg("Docker container health alert raised")
|
|
}
|
|
|
|
func (m *Manager) clearDockerContainerHealthAlert(resourceID string) {
|
|
m.clearAlert(buildCanonicalStateID(resourceID, resourceID+"-health"))
|
|
}
|
|
|
|
// checkDockerContainerRestartLoop detects containers stuck in a restart loop
|
|
func (m *Manager) checkDockerContainerRestartLoop(host models.DockerHost, container models.DockerContainer, resourceID, containerName, instanceName, nodeName string) {
|
|
alertID := fmt.Sprintf("docker-container-restart-loop-%s", resourceID)
|
|
now := time.Now()
|
|
|
|
// Get config values with defaults
|
|
restartThreshold := m.config.DockerDefaults.RestartCount
|
|
if restartThreshold == 0 {
|
|
restartThreshold = 3 // Default: 3 restarts
|
|
}
|
|
timeWindow := m.config.DockerDefaults.RestartWindow
|
|
if timeWindow == 0 {
|
|
timeWindow = 300 // Default: 5 minutes (300 seconds)
|
|
}
|
|
|
|
m.mu.Lock()
|
|
|
|
record, exists := m.dockerRestartTracking[resourceID]
|
|
if !exists {
|
|
record = &dockerRestartRecord{
|
|
count: container.RestartCount,
|
|
lastCount: container.RestartCount,
|
|
times: []time.Time{},
|
|
lastChecked: now,
|
|
}
|
|
m.dockerRestartTracking[resourceID] = record
|
|
m.mu.Unlock()
|
|
return
|
|
}
|
|
|
|
// If restart count increased, track it
|
|
if container.RestartCount > record.lastCount {
|
|
newRestarts := container.RestartCount - record.lastCount
|
|
for i := 0; i < newRestarts; i++ {
|
|
record.times = append(record.times, now)
|
|
}
|
|
record.lastCount = container.RestartCount
|
|
}
|
|
|
|
// Clean up old restart times outside the window
|
|
cutoff := now.Add(-time.Duration(timeWindow) * time.Second)
|
|
var recentRestarts []time.Time
|
|
for _, t := range record.times {
|
|
if t.After(cutoff) {
|
|
recentRestarts = append(recentRestarts, t)
|
|
}
|
|
}
|
|
record.times = recentRestarts
|
|
record.lastChecked = now
|
|
|
|
recentCount := len(record.times)
|
|
m.mu.Unlock()
|
|
|
|
// Check if we have a restart loop
|
|
if recentCount > restartThreshold {
|
|
spec, err := buildCanonicalSeverityThresholdSpec(resourceID+"-restart-loop", resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "restart-count-window", 0, float64(restartThreshold+1), false)
|
|
if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("resourceID", resourceID).
|
|
Str("container", containerName).
|
|
Msg("Skipping invalid canonical docker container restart loop spec")
|
|
return
|
|
}
|
|
|
|
metadata := dockerContainerAlertMetadata(host, container, containerName)
|
|
metadata["restartCount"] = container.RestartCount
|
|
metadata["recentRestarts"] = recentCount
|
|
|
|
_, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
|
|
Spec: spec,
|
|
Evidence: alertspecs.AlertEvidence{
|
|
ObservedAt: now,
|
|
SeverityThreshold: &alertspecs.SeverityThresholdEvidence{
|
|
Metric: "restart-count-window",
|
|
Direction: alertspecs.ThresholdDirectionAbove,
|
|
Observed: float64(recentCount),
|
|
},
|
|
},
|
|
AlertID: alertID,
|
|
AlertType: "docker-container-restart-loop",
|
|
ResourceID: resourceID,
|
|
ResourceName: containerName,
|
|
Node: nodeName,
|
|
Instance: instanceName,
|
|
Message: fmt.Sprintf("Docker container '%s' has restarted %d times in the last %d minutes (restart loop detected)", containerName, recentCount, timeWindow/60),
|
|
Metadata: metadata,
|
|
AddToRecent: true,
|
|
AddToHistory: true,
|
|
})
|
|
|
|
log.Warn().
|
|
Str("container", containerName).
|
|
Str("host", host.DisplayName).
|
|
Int("restarts", recentCount).
|
|
Msg("Docker container restart loop detected")
|
|
} else {
|
|
// Clear alert if restart loop has stopped
|
|
m.clearAlert(buildCanonicalStateID(resourceID, resourceID+"-restart-loop"))
|
|
}
|
|
}
|
|
|
|
// checkDockerContainerOOMKill detects when the runtime explicitly reports that
|
|
// a container was killed due to out of memory.
|
|
func (m *Manager) checkDockerContainerOOMKill(host models.DockerHost, container models.DockerContainer, resourceID, containerName, instanceName, nodeName string) {
|
|
alertID := fmt.Sprintf("docker-container-oom-%s", resourceID)
|
|
|
|
state := strings.ToLower(strings.TrimSpace(container.State))
|
|
if (state == "exited" || state == "dead") && container.OOMKilled != nil && *container.OOMKilled {
|
|
spec, err := buildCanonicalHealthAssessmentSpec(resourceID+"-oom-kill", resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "docker-container-exit", []string{"oom-kill"}, false)
|
|
if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("resourceID", resourceID).
|
|
Str("container", containerName).
|
|
Msg("Skipping invalid canonical docker container OOM spec")
|
|
return
|
|
}
|
|
|
|
metadata := dockerContainerAlertMetadata(host, container, containerName)
|
|
metadata["exitCode"] = container.ExitCode
|
|
metadata["oomKilled"] = true
|
|
metadata["memoryUsageBytes"] = container.MemoryUsage
|
|
metadata["memoryLimitBytes"] = container.MemoryLimit
|
|
|
|
_, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
|
|
Spec: spec,
|
|
Evidence: alertspecs.AlertEvidence{
|
|
ObservedAt: time.Now(),
|
|
HealthAssessment: &alertspecs.HealthAssessmentEvidence{
|
|
Signal: "docker-container-exit",
|
|
Severity: alertspecs.AlertSeverityCritical,
|
|
Codes: []string{"oom-kill"},
|
|
},
|
|
},
|
|
AlertID: alertID,
|
|
AlertType: "docker-container-oom-kill",
|
|
ResourceID: resourceID,
|
|
ResourceName: containerName,
|
|
Node: nodeName,
|
|
Instance: instanceName,
|
|
Message: fmt.Sprintf("Docker container '%s' was killed due to out of memory (OOM)", containerName),
|
|
Metadata: metadata,
|
|
AddToRecent: true,
|
|
AddToHistory: true,
|
|
})
|
|
|
|
log.Error().
|
|
Str("container", containerName).
|
|
Str("host", host.DisplayName).
|
|
Int64("memoryUsage", container.MemoryUsage).
|
|
Int64("memoryLimit", container.MemoryLimit).
|
|
Msg("Docker container OOM killed")
|
|
} else {
|
|
// Unknown reports from older agents and explicit false reports both fail
|
|
// closed. Exit code 137 proves SIGKILL only; it is not OOM evidence.
|
|
m.clearAlert(buildCanonicalStateID(resourceID, resourceID+"-oom-kill"))
|
|
}
|
|
}
|
|
|
|
// checkDockerContainerMemoryLimit alerts when container approaches its memory limit
|
|
func (m *Manager) checkDockerContainerMemoryLimit(host models.DockerHost, container models.DockerContainer, resourceID, containerName, instanceName, nodeName string) {
|
|
// Only check if container is running and has a memory limit
|
|
state := strings.ToLower(strings.TrimSpace(container.State))
|
|
if state != "running" || container.MemoryLimit <= 0 {
|
|
return
|
|
}
|
|
|
|
alertID := fmt.Sprintf("docker-container-memory-limit-%s", resourceID)
|
|
|
|
// Get config values with defaults
|
|
warnThreshold := float64(m.config.DockerDefaults.MemoryWarnPct)
|
|
if warnThreshold == 0 {
|
|
warnThreshold = 90.0 // Default: 90%
|
|
}
|
|
criticalThreshold := float64(m.config.DockerDefaults.MemoryCriticalPct)
|
|
if criticalThreshold == 0 {
|
|
criticalThreshold = 95.0 // Default: 95%
|
|
}
|
|
|
|
// Calculate percentage of limit used
|
|
limitPercent := (float64(container.MemoryUsage) / float64(container.MemoryLimit)) * 100
|
|
|
|
clearThreshold := warnThreshold - 5
|
|
recovery := clearThreshold
|
|
spec, err := buildCanonicalSeverityThresholdSpecWithRecovery(resourceID+"-memory-limit", resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "memory-limit-percent", warnThreshold, criticalThreshold, &recovery, false)
|
|
if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("resourceID", resourceID).
|
|
Str("container", containerName).
|
|
Msg("Skipping invalid canonical docker container memory limit spec")
|
|
return
|
|
}
|
|
|
|
metadata := dockerContainerAlertMetadata(host, container, containerName)
|
|
metadata["memoryUsageBytes"] = container.MemoryUsage
|
|
metadata["memoryLimitBytes"] = container.MemoryLimit
|
|
metadata["limitPercent"] = limitPercent
|
|
|
|
_, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
|
|
Spec: spec,
|
|
Evidence: alertspecs.AlertEvidence{
|
|
ObservedAt: time.Now(),
|
|
SeverityThreshold: &alertspecs.SeverityThresholdEvidence{
|
|
Metric: "memory-limit-percent",
|
|
Direction: alertspecs.ThresholdDirectionAbove,
|
|
Observed: limitPercent,
|
|
},
|
|
},
|
|
AlertID: alertID,
|
|
AlertType: "docker-container-memory-limit",
|
|
ResourceID: resourceID,
|
|
ResourceName: containerName,
|
|
Node: nodeName,
|
|
Instance: instanceName,
|
|
Message: fmt.Sprintf("Docker container '%s' is using %.1f%% of its memory limit (%d MB / %d MB)", containerName, limitPercent, container.MemoryUsage/(1024*1024), container.MemoryLimit/(1024*1024)),
|
|
Metadata: metadata,
|
|
AddToRecent: true,
|
|
AddToHistory: true,
|
|
})
|
|
|
|
if limitPercent >= warnThreshold {
|
|
log.Warn().
|
|
Str("container", containerName).
|
|
Str("host", host.DisplayName).
|
|
Float64("limitPercent", limitPercent).
|
|
Msg("Docker container approaching memory limit")
|
|
}
|
|
}
|
|
|
|
func (m *Manager) clearDockerContainerMetricAlerts(resourceID string, metrics ...string) {
|
|
if len(metrics) == 0 {
|
|
metrics = []string{"cpu", "memory", "disk"}
|
|
}
|
|
for _, metric := range metrics {
|
|
m.clearAlert(canonicalMetricStateID(resourceID, metric))
|
|
}
|
|
}
|
|
|
|
func (m *Manager) clearDockerContainerUpdateTracking(resourceID, trackingKey string) {
|
|
m.mu.Lock()
|
|
delete(m.dockerUpdateFirstSeen, resourceID)
|
|
if trackingKey != "" {
|
|
delete(m.dockerUpdateFirstSeenByIdentity, trackingKey)
|
|
}
|
|
m.mu.Unlock()
|
|
}
|
|
|
|
func dockerUpdateTrackingKeyFromAlert(alert *Alert) string {
|
|
if alert == nil || alert.Metadata == nil {
|
|
return ""
|
|
}
|
|
|
|
metadataString := func(key string) string {
|
|
value, ok := alert.Metadata[key]
|
|
if !ok || value == nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(fmt.Sprint(value))
|
|
}
|
|
|
|
host := models.DockerHost{
|
|
ID: metadataString("hostId"),
|
|
DisplayName: metadataString("hostName"),
|
|
Hostname: metadataString("hostHostname"),
|
|
}
|
|
container := models.DockerContainer{
|
|
ID: metadataString("containerId"),
|
|
Name: metadataString("containerName"),
|
|
Image: metadataString("image"),
|
|
}
|
|
|
|
if host.ID == "" && host.DisplayName == "" && host.Hostname == "" &&
|
|
container.ID == "" && container.Name == "" && container.Image == "" {
|
|
return ""
|
|
}
|
|
|
|
return dockerUpdateTrackingKey(host, container)
|
|
}
|
|
|
|
func (m *Manager) clearDockerContainerUpdateStateLocked(alert *Alert) {
|
|
if alert == nil {
|
|
return
|
|
}
|
|
|
|
if alert.ResourceID != "" {
|
|
delete(m.dockerUpdateFirstSeen, alert.ResourceID)
|
|
}
|
|
if trackingKey := dockerUpdateTrackingKeyFromAlert(alert); trackingKey != "" {
|
|
delete(m.dockerUpdateFirstSeenByIdentity, trackingKey)
|
|
}
|
|
}
|
|
|
|
func (m *Manager) clearDockerContainerUpdateAlertsLocked() {
|
|
toClear := make([]string, 0)
|
|
for storageKey, alert := range m.activeAlerts {
|
|
if alert == nil {
|
|
continue
|
|
}
|
|
alertID := effectiveAlertID(alert, storageKey)
|
|
if alert.Type != "docker-container-update" && !strings.HasPrefix(alertID, "docker-container-update-") {
|
|
continue
|
|
}
|
|
m.clearDockerContainerUpdateStateLocked(alert)
|
|
toClear = append(toClear, alertID)
|
|
}
|
|
for _, alertID := range toClear {
|
|
m.clearAlertNoLock(alertID)
|
|
}
|
|
}
|
|
|
|
func (m *Manager) shouldResolveDockerContainerUpdateAlertLocked(alert *Alert) bool {
|
|
if alert == nil {
|
|
return false
|
|
}
|
|
|
|
if allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("app-container"); allDisabled || m.config.DockerDefaults.UpdateAlertDelayHours < 0 {
|
|
m.clearDockerContainerUpdateStateLocked(alert)
|
|
return true
|
|
}
|
|
|
|
containerName := strings.TrimSpace(alert.ResourceName)
|
|
containerID := ""
|
|
hostID := ""
|
|
if alert.Metadata != nil {
|
|
if value, ok := alert.Metadata["containerName"].(string); ok && containerName == "" {
|
|
containerName = value
|
|
}
|
|
if value, ok := alert.Metadata["containerId"].(string); ok {
|
|
containerID = value
|
|
}
|
|
if value, ok := alert.Metadata["hostId"].(string); ok {
|
|
hostID = value
|
|
}
|
|
}
|
|
|
|
if override, exists := m.lookupDockerContainerOverrideNoLock(hostID, containerName, alert.ResourceID); exists && override.Disabled {
|
|
m.clearDockerContainerUpdateStateLocked(alert)
|
|
return true
|
|
}
|
|
if matchesDockerIgnoredPrefix(containerName, containerID, m.config.DockerIgnoredContainerPrefixes) {
|
|
m.clearDockerContainerUpdateStateLocked(alert)
|
|
return true
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func (m *Manager) touchDockerContainerUpdateAlert(alertID string) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
if alert, exists := m.getActiveAlertNoLock(alertID); exists && alert != nil {
|
|
alert.LastSeen = time.Now()
|
|
}
|
|
}
|
|
|
|
// checkDockerContainerImageUpdate checks if an image update has been pending for too long
|
|
func (m *Manager) checkDockerContainerImageUpdate(host models.DockerHost, container models.DockerContainer, resourceID, containerName, instanceName, nodeName string) {
|
|
alertID := fmt.Sprintf("docker-container-update-%s", resourceID)
|
|
canonicalAlertID := buildCanonicalStateID(resourceID, resourceID+"-image-update")
|
|
updateTrackingKey := dockerUpdateTrackingKey(host, container)
|
|
|
|
// Check if update detection is enabled
|
|
m.mu.RLock()
|
|
delayHours := m.config.DockerDefaults.UpdateAlertDelayHours
|
|
m.mu.RUnlock()
|
|
|
|
// Negative value means disabled
|
|
if delayHours < 0 {
|
|
m.clearAlert(canonicalAlertID)
|
|
m.clearDockerContainerUpdateTracking(resourceID, updateTrackingKey)
|
|
return
|
|
}
|
|
|
|
// Check if this container has an update status reported
|
|
if container.UpdateStatus == nil {
|
|
// Missing update status means the condition is unknown, not resolved.
|
|
// Preserve any active alert and first-seen tracking until we see an affirmative clear.
|
|
m.touchDockerContainerUpdateAlert(canonicalAlertID)
|
|
return
|
|
}
|
|
|
|
// Check for errors in update detection (don't alert on errors)
|
|
if container.UpdateStatus.Error != "" {
|
|
// A failed update check cannot confirm the pending update has been resolved.
|
|
m.touchDockerContainerUpdateAlert(canonicalAlertID)
|
|
return
|
|
}
|
|
|
|
// Check if an update is available
|
|
if !container.UpdateStatus.UpdateAvailable {
|
|
// No update available - clear tracking and alert
|
|
m.clearAlert(canonicalAlertID)
|
|
m.clearDockerContainerUpdateTracking(resourceID, updateTrackingKey)
|
|
return
|
|
}
|
|
|
|
// Update is available - track when we first saw it
|
|
m.mu.Lock()
|
|
firstSeen, exists := m.dockerUpdateFirstSeenByIdentity[updateTrackingKey]
|
|
if !exists {
|
|
firstSeen, exists = m.dockerUpdateFirstSeen[resourceID]
|
|
}
|
|
if !exists {
|
|
firstSeen = time.Now()
|
|
}
|
|
m.dockerUpdateFirstSeen[resourceID] = firstSeen
|
|
m.dockerUpdateFirstSeenByIdentity[updateTrackingKey] = firstSeen
|
|
m.mu.Unlock()
|
|
|
|
// Check if we've exceeded the delay threshold
|
|
pendingDuration := time.Since(firstSeen)
|
|
threshold := time.Duration(delayHours) * time.Hour
|
|
if pendingDuration < threshold {
|
|
// Not yet time to alert
|
|
log.Debug().
|
|
Str("container", containerName).
|
|
Str("host", host.DisplayName).
|
|
Str("image", container.Image).
|
|
Dur("pending", pendingDuration).
|
|
Dur("threshold", threshold).
|
|
Msg("Container update pending but below alert threshold")
|
|
return
|
|
}
|
|
|
|
// Create or update the alert
|
|
pendingHours := int(pendingDuration.Hours())
|
|
spec, err := buildCanonicalSeverityThresholdSpec(resourceID+"-image-update", resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "image-update-hours", float64(delayHours), 0, false)
|
|
if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("resourceID", resourceID).
|
|
Str("container", containerName).
|
|
Msg("Skipping invalid canonical docker container update spec")
|
|
return
|
|
}
|
|
|
|
metadata := dockerContainerAlertMetadata(host, container, containerName)
|
|
metadata["currentDigest"] = container.UpdateStatus.CurrentDigest
|
|
metadata["latestDigest"] = container.UpdateStatus.LatestDigest
|
|
metadata["lastChecked"] = container.UpdateStatus.LastChecked
|
|
metadata["firstSeen"] = firstSeen
|
|
metadata["pendingHours"] = pendingHours
|
|
metadata["thresholdHours"] = delayHours
|
|
|
|
_, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
|
|
Spec: spec,
|
|
Evidence: alertspecs.AlertEvidence{
|
|
ObservedAt: time.Now(),
|
|
SeverityThreshold: &alertspecs.SeverityThresholdEvidence{
|
|
Metric: "image-update-hours",
|
|
Direction: alertspecs.ThresholdDirectionAbove,
|
|
Observed: pendingDuration.Hours(),
|
|
},
|
|
},
|
|
AlertID: alertID,
|
|
AlertType: "docker-container-update",
|
|
ResourceID: resourceID,
|
|
ResourceName: containerName,
|
|
Node: nodeName,
|
|
Instance: instanceName,
|
|
Message: fmt.Sprintf("Docker container '%s' has an image update available for %d hours", containerName, pendingHours),
|
|
StartTimeOverride: firstSeen,
|
|
Metadata: metadata,
|
|
AddToRecent: true,
|
|
AddToHistory: true,
|
|
})
|
|
|
|
log.Warn().
|
|
Str("container", containerName).
|
|
Str("host", host.DisplayName).
|
|
Str("image", container.Image).
|
|
Int("pendingHours", pendingHours).
|
|
Msg("Docker container has pending image update")
|
|
}
|
|
|
|
func (m *Manager) cleanupDockerContainerAlerts(host models.DockerHost, seen map[string]struct{}) {
|
|
m.cleanupDockerContainerAlertsWithTracking(host, seen, nil)
|
|
}
|
|
|
|
func (m *Manager) cleanupDockerContainerAlertsWithTracking(host models.DockerHost, seen map[string]struct{}, seenUpdateTracking map[string]struct{}) {
|
|
prefix := fmt.Sprintf("docker:%s/", strings.TrimSpace(host.ID))
|
|
updateTrackingPrefix := dockerUpdateTrackingHostPrefix(host)
|
|
|
|
m.mu.Lock()
|
|
toClear := make([]string, 0)
|
|
for storageKey, alert := range m.activeAlerts {
|
|
alertID := effectiveAlertID(alert, storageKey)
|
|
if !strings.HasPrefix(alert.ResourceID, prefix) {
|
|
continue
|
|
}
|
|
if _, exists := seen[alert.ResourceID]; exists {
|
|
continue
|
|
}
|
|
toClear = append(toClear, alertID)
|
|
}
|
|
// Removed containers stop being observed, so their in-flight
|
|
// confirmation runs in the core would never see a closing observation.
|
|
for _, resourceID := range m.core.PendingResourceIDs() {
|
|
if strings.HasPrefix(resourceID, prefix) {
|
|
if _, exists := seen[resourceID]; !exists {
|
|
m.core.DropPendingForResource(resourceID)
|
|
}
|
|
}
|
|
}
|
|
// Cleanup update tracking for removed containers
|
|
for resourceID := range m.dockerUpdateFirstSeen {
|
|
if strings.HasPrefix(resourceID, prefix) {
|
|
if _, exists := seen[resourceID]; !exists {
|
|
delete(m.dockerUpdateFirstSeen, resourceID)
|
|
}
|
|
}
|
|
}
|
|
if seenUpdateTracking != nil {
|
|
for trackingKey := range m.dockerUpdateFirstSeenByIdentity {
|
|
if !strings.HasPrefix(trackingKey, updateTrackingPrefix) {
|
|
continue
|
|
}
|
|
if _, exists := seenUpdateTracking[trackingKey]; !exists {
|
|
delete(m.dockerUpdateFirstSeenByIdentity, trackingKey)
|
|
}
|
|
}
|
|
}
|
|
m.mu.Unlock()
|
|
|
|
for _, alertID := range toClear {
|
|
m.clearAlert(alertID)
|
|
}
|
|
}
|
|
|
|
func (m *Manager) clearDockerHostContainerAlerts(host models.DockerHost) {
|
|
prefix := fmt.Sprintf("docker:%s/", strings.TrimSpace(host.ID))
|
|
updateTrackingPrefix := dockerUpdateTrackingHostPrefix(host)
|
|
|
|
m.mu.Lock()
|
|
toClear := make([]string, 0)
|
|
for storageKey, alert := range m.activeAlerts {
|
|
alertID := effectiveAlertID(alert, storageKey)
|
|
if strings.HasPrefix(alert.ResourceID, prefix) {
|
|
toClear = append(toClear, alertID)
|
|
}
|
|
}
|
|
for _, resourceID := range m.core.PendingResourceIDs() {
|
|
if strings.HasPrefix(resourceID, prefix) {
|
|
m.core.DropPendingForResource(resourceID)
|
|
}
|
|
}
|
|
for resourceID := range m.dockerRestartTracking {
|
|
if strings.HasPrefix(resourceID, prefix) {
|
|
delete(m.dockerRestartTracking, resourceID)
|
|
}
|
|
}
|
|
for resourceID := range m.dockerUpdateFirstSeen {
|
|
if strings.HasPrefix(resourceID, prefix) {
|
|
delete(m.dockerUpdateFirstSeen, resourceID)
|
|
}
|
|
}
|
|
for trackingKey := range m.dockerUpdateFirstSeenByIdentity {
|
|
if strings.HasPrefix(trackingKey, updateTrackingPrefix) {
|
|
delete(m.dockerUpdateFirstSeenByIdentity, trackingKey)
|
|
}
|
|
}
|
|
m.mu.Unlock()
|
|
|
|
for _, alertID := range toClear {
|
|
m.clearAlert(alertID)
|
|
}
|
|
}
|