pulse/internal/alerts/docker.go

package alerts

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
	alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rs/zerolog/log"
)

type dockerRestartRecord struct {
	count       int
	lastCount   int
	times       []time.Time // Track restart times for loop detection
	lastChecked time.Time
}

// dockerInstanceName returns the logical instance name used for Docker alerts.
func dockerInstanceName(host models.DockerHost) string {
	name := strings.TrimSpace(host.DisplayName)
	if name == "" {
		name = strings.TrimSpace(host.Hostname)
	}
	if name == "" {
		return "Docker"
	}
	return fmt.Sprintf("Docker:%s", name)
}

// dockerContainerDisplayName normalizes the container name for alert readability.
func dockerContainerDisplayName(container models.DockerContainer) string {
	name := strings.TrimSpace(container.Name)
	if strings.HasPrefix(name, "/") {
		name = strings.TrimLeft(name, "/")
	}
	if name == "" {
		containerID := strings.TrimSpace(container.ID)
		if len(containerID) > 12 {
			containerID = containerID[:12]
		}
		return containerID
	}
	return name
}

// dockerContainerOverrideKey builds the durable override key for a container:
// host + container *name*. Container IDs change on every recreate (image
// update, compose up), so ID-keyed overrides orphan the moment a container is
// updated; the name is the identity that survives (#1601). Returns "" when
// either part is missing so callers fall back to the legacy ID key.
func dockerContainerOverrideKey(hostID, containerName string) string {
	hostID = strings.TrimSpace(hostID)
	name := strings.TrimLeft(strings.TrimSpace(containerName), "/")
	if hostID == "" || name == "" {
		return ""
	}
	return fmt.Sprintf("docker:%s/%s", hostID, name)
}

// lookupDockerContainerOverrideNoLock resolves a container override,
// preferring the stable host+name key and falling back to the legacy
// container-ID resource ID so pre-migration entries keep working.
// Callers must hold m.mu.
func (m *Manager) lookupDockerContainerOverrideNoLock(hostID, containerName, legacyResourceID string) (ThresholdConfig, bool) {
	if stableKey := dockerContainerOverrideKey(hostID, containerName); stableKey != "" {
		if override, ok := m.config.Overrides[stableKey]; ok {
			return override, true
		}
	}
	if legacyResourceID != "" {
		if override, ok := m.config.Overrides[legacyResourceID]; ok {
			return override, true
		}
	}
	return ThresholdConfig{}, false
}

// dockerNameReferencePrefix marks the name of a Docker container or Swarm
// service that has no ID in the slot of an alert reference that otherwise
// holds the ID. Docker and Swarm IDs never contain a colon, so a name
// reference can never equal an ID reference or the host's own reference.
const dockerNameReferencePrefix = "name:"

// DockerHostResourceID builds the reference of a Docker host's own alerts
// ("docker:<host ID>"). Only the host's connectivity alert uses it. Returns ""
// without a host ID.
func DockerHostResourceID(hostID string) string {
	hostID = strings.TrimSpace(hostID)
	if hostID == "" {
		return ""
	}
	return "docker:" + hostID
}

// DockerContainerResourceID builds the reference of a Docker container's
// alerts: "docker:<host ID>/<container ID>". A container reported without an
// ID is referenced by its name, "docker:<host ID>/name:<name>", so it never
// shares its host's reference or another container's. Returns "" for a
// container with neither, which has no identity to alert under. Patrol scope
// resolution registers the same forms as aliases of the collected container.
func DockerContainerResourceID(hostID, containerID, containerName string) string {
	hostID = strings.TrimSpace(hostID)
	ref := strings.TrimSpace(containerID)
	if ref == "" {
		name := strings.TrimLeft(strings.TrimSpace(containerName), "/")
		if hostID == "" || name == "" {
			return ""
		}
		ref = dockerNameReferencePrefix + name
	}
	if hostID == "" {
		return fmt.Sprintf("docker:container/%s", ref)
	}
	return fmt.Sprintf("docker:%s/%s", hostID, ref)
}

func normalizeDockerUpdateTrackingPart(part string) string {
	return strings.ToLower(strings.TrimSpace(part))
}

// dockerUpdateTrackingHostKey builds a stable host identity for Docker update timing.
func dockerUpdateTrackingHostKey(host models.DockerHost) string {
	switch {
	case normalizeDockerUpdateTrackingPart(host.AgentID) != "":
		return "agent:" + normalizeDockerUpdateTrackingPart(host.AgentID)
	case normalizeDockerUpdateTrackingPart(host.TokenID) != "":
		return "token:" + normalizeDockerUpdateTrackingPart(host.TokenID)
	case normalizeDockerUpdateTrackingPart(host.MachineID) != "":
		return "machine:" + normalizeDockerUpdateTrackingPart(host.MachineID)
	case normalizeDockerUpdateTrackingPart(host.Hostname) != "":
		return "hostname:" + normalizeDockerUpdateTrackingPart(host.Hostname)
	case normalizeDockerUpdateTrackingPart(host.ID) != "":
		return "id:" + normalizeDockerUpdateTrackingPart(host.ID)
	case normalizeDockerUpdateTrackingPart(host.DisplayName) != "":
		return "name:" + normalizeDockerUpdateTrackingPart(host.DisplayName)
	default:
		return "unknown-host"
	}
}

func dockerUpdateTrackingContainerKey(container models.DockerContainer) string {
	if containerID := normalizeDockerUpdateTrackingPart(container.ID); containerID != "" {
		return "id:" + containerID
	}

	name := normalizeDockerUpdateTrackingPart(container.Name)
	name = strings.TrimPrefix(name, "/")
	if name != "" {
		return "name:" + name
	}

	if image := normalizeDockerUpdateTrackingPart(container.Image); image != "" {
		return "image:" + image
	}

	return "unknown-container"
}

func dockerUpdateTrackingKey(host models.DockerHost, container models.DockerContainer) string {
	return fmt.Sprintf("docker-update:%s/%s", dockerUpdateTrackingHostKey(host), dockerUpdateTrackingContainerKey(container))
}

func dockerUpdateTrackingHostPrefix(host models.DockerHost) string {
	return fmt.Sprintf("docker-update:%s/", dockerUpdateTrackingHostKey(host))
}

// dockerServiceDisplayName normalizes the service name for alert readability.
func dockerServiceDisplayName(service models.DockerService) string {
	name := strings.TrimSpace(service.Name)
	if name != "" {
		return name
	}
	serviceID := strings.TrimSpace(service.ID)
	if len(serviceID) > 12 {
		serviceID = serviceID[:12]
	}
	if serviceID == "" {
		return "service"
	}
	return serviceID
}

// DockerServiceResourceID builds the reference of a Docker Swarm service's
// alerts: "docker:<host ID>/service/<service ID>". A service reported without
// an ID is referenced by its normalized name, "docker:<host ID>/service/name:<name>",
// so it never shares another service's ID reference. Returns "" for a service
// with neither, which has no identity to alert under. Patrol scope resolution
// registers the same forms as aliases on the owning Docker host record so a
// service alert's resource ID resolves there.
func DockerServiceResourceID(hostID, serviceID, serviceName string) string {
	hostID = strings.TrimSpace(hostID)
	normalizedServiceID := strings.TrimSpace(serviceID)
	if normalizedServiceID == "" {
		name := strings.TrimSpace(serviceName)
		builder := strings.Builder{}
		for _, r := range strings.ToLower(name) {
			switch {
			case r >= 'a' && r <= 'z':
				builder.WriteRune(r)
			case r >= '0' && r <= '9':
				builder.WriteRune(r)
			case r == '-', r == '_':
				builder.WriteRune(r)
			case r == ' ' || r == '/' || r == '\\' || r == ':' || r == '.':
				builder.WriteRune('-')
			}
		}
		normalizedServiceID = strings.Trim(builder.String(), "-_")
		if normalizedServiceID == "" {
			return ""
		}
		if len(normalizedServiceID) > 32 {
			normalizedServiceID = normalizedServiceID[:32]
		}
		normalizedServiceID = dockerNameReferencePrefix + normalizedServiceID
	}
	if hostID == "" {
		return fmt.Sprintf("docker-service:%s", normalizedServiceID)
	}
	return fmt.Sprintf("docker:%s/service/%s", hostID, normalizedServiceID)
}

// matchesDockerIgnoredPrefix reports whether a container name or ID matches
// any entry in the ignore list. A bare entry keeps its historical
// prefix-match semantics. Entries may also use the wildcard forms shared
// with the other ignore lists: "*token" matches a suffix, "*token*" a
// substring, "token*" an explicit prefix.
func matchesDockerIgnoredPrefix(name, id string, prefixes []string) bool {
	if len(prefixes) == 0 {
		return false
	}

	name = strings.ToLower(strings.TrimSpace(name))
	id = strings.ToLower(strings.TrimSpace(id))

	for _, raw := range prefixes {
		pattern := strings.ToLower(strings.TrimSpace(raw))
		// A pattern that is nothing but wildcards would ignore every
		// container, which is what DisableAllDockerContainers is for.
		if pattern == "" || strings.Trim(pattern, "*") == "" {
			continue
		}
		if matchesIgnoredContainerPattern(name, pattern) || matchesIgnoredContainerPattern(id, pattern) {
			return true
		}
	}

	return false
}

func matchesIgnoredContainerPattern(value, pattern string) bool {
	if value == "" {
		return false
	}
	leading := strings.HasPrefix(pattern, "*")
	trailing := strings.HasSuffix(pattern, "*")
	switch {
	case leading && trailing:
		return strings.Contains(value, pattern[1:len(pattern)-1])
	case leading:
		return strings.HasSuffix(value, pattern[1:])
	case trailing:
		return strings.HasPrefix(value, pattern[:len(pattern)-1])
	default:
		return strings.HasPrefix(value, pattern)
	}
}

// CheckDockerHost evaluates Docker host telemetry and container metrics for alerts.
func (m *Manager) CheckDockerHost(host models.DockerHost) {
	if host.ID == "" {
		return
	}

	// Fresh telemetry marks the host as online and clears any offline alert.
	m.HandleDockerHostOnline(host)

	m.mu.RLock()
	alertsEnabled := m.config.Enabled
	disableAllDockerHosts, _ := m.alertPolicyTypeSwitchesNoLock("docker-host")
	ignoredPrefixes := append([]string(nil), m.config.DockerIgnoredContainerPrefixes...)
	m.mu.RUnlock()
	if !alertsEnabled {
		return
	}
	if disableAllDockerHosts {
		return
	}

	seen := make(map[string]struct{}, len(host.Containers)+len(host.Services))
	seenUpdateTracking := make(map[string]struct{}, len(host.Containers))
	for _, container := range host.Containers {
		resourceID := DockerContainerResourceID(host.ID, container.ID, container.Name)
		if resourceID == "" {
			// Neither an ID nor a name: nothing identifies its alerts.
			continue
		}
		containerName := dockerContainerDisplayName(container)
		updateTrackingKey := dockerUpdateTrackingKey(host, container)

		if matchesDockerIgnoredPrefix(containerName, container.ID, ignoredPrefixes) {
			log.Debug().
				Str("container", containerName).
				Str("host", host.DisplayName).
				Msg("Skipping Docker container alert evaluation due to ignored prefix")
			m.clearDockerContainerStateAlert(resourceID)
			m.clearDockerContainerHealthAlert(resourceID)
			m.clearDockerContainerMetricAlerts(resourceID)
			m.clearAlert(fmt.Sprintf("docker-container-restart-loop-%s", resourceID))
			m.clearAlert(fmt.Sprintf("docker-container-oom-%s", resourceID))
			m.clearAlert(fmt.Sprintf("docker-container-memory-limit-%s", resourceID))
			m.mu.Lock()
			delete(m.dockerRestartTracking, resourceID)
			m.mu.Unlock()
			m.clearDockerContainerUpdateTracking(resourceID, updateTrackingKey)
			continue
		}

		seen[resourceID] = struct{}{}
		seenUpdateTracking[updateTrackingKey] = struct{}{}
		m.evaluateDockerContainer(host, container, resourceID)
	}

	for _, service := range host.Services {
		resourceID := DockerServiceResourceID(host.ID, service.ID, service.Name)
		if resourceID == "" {
			continue
		}
		seen[resourceID] = struct{}{}
		m.evaluateDockerService(host, service, resourceID)
	}

	m.cleanupDockerContainerAlertsWithTracking(host, seen, seenUpdateTracking)
}

func (m *Manager) evaluateDockerContainer(host models.DockerHost, container models.DockerContainer, resourceID string) {
	m.mu.RLock()
	disableAllContainers, _ := m.alertPolicyTypeSwitchesNoLock("app-container")
	m.mu.RUnlock()
	if disableAllContainers {
		return
	}

	containerName := dockerContainerDisplayName(container)
	nodeName := strings.TrimSpace(host.Hostname)
	instanceName := dockerInstanceName(host)
	resourceType := "app-container"
	containerTags := dockerLabelTags(container.Labels)

	// Resolve the metric thresholds while the lock is held: the Docker defaults
	// are fields of m.config, which UpdateConfig replaces, and the override
	// still points into it. applyThresholdOverride clones what it keeps.
	m.mu.RLock()
	overrideConfig, hasOverride := m.lookupDockerContainerOverrideNoLock(host.ID, container.Name, resourceID)
	thresholds := ThresholdConfig{
		CPU:    cloneThreshold(&m.config.DockerDefaults.CPU),
		Memory: cloneThreshold(&m.config.DockerDefaults.Memory),
		Disk:   cloneThreshold(&m.config.DockerDefaults.Disk),
	}
	if hasOverride {
		thresholds = m.applyThresholdOverride(thresholds, overrideConfig)
	}
	m.mu.RUnlock()
	if hasOverride && overrideConfig.Disabled {
		// Alerts disabled via override; clear any existing alerts and skip evaluation.
		m.clearDockerContainerStateAlert(resourceID)
		m.clearDockerContainerHealthAlert(resourceID)
		m.clearDockerContainerMetricAlerts(resourceID)
		m.clearAlert(fmt.Sprintf("docker-container-update-%s", resourceID))
		m.clearAlert(buildCanonicalStateID(resourceID, resourceID+"-image-update"))
		m.clearDockerContainerUpdateTracking(resourceID, dockerUpdateTrackingKey(host, container))
		return
	}

	state := strings.ToLower(strings.TrimSpace(container.State))
	if state == "" {
		state = strings.ToLower(strings.TrimSpace(container.Status))
	}

	if state != "running" {
		m.checkDockerContainerState(host, container, resourceID, containerName, instanceName, nodeName)
		m.clearDockerContainerMetricAlerts(resourceID, "cpu", "memory", "disk")
	} else {
		m.clearDockerContainerStateAlert(resourceID)

		if thresholds.CPU != nil {
			cpuCapacityPercent := models.DockerContainerCPUCapacityPercent(container, host.CPUs)
			cpuMetadata := map[string]interface{}{
				"resourceType":       resourceType,
				"hostId":             host.ID,
				"hostName":           host.DisplayName,
				"hostHostname":       host.Hostname,
				"containerId":        container.ID,
				"containerName":      containerName,
				"image":              container.Image,
				"state":              container.State,
				"status":             container.Status,
				"restartCount":       container.RestartCount,
				"metric":             "cpu",
				"cpuPercent":         cpuCapacityPercent,
				"cpuCapacityPercent": cpuCapacityPercent,
				"cpuRawPercent":      container.CPUPercent,
				"cpuCapacityCores":   host.CPUs,
				"tags":               containerTags,
			}
			spec, err := buildCanonicalMetricSpec(resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "cpu", thresholds.CPU)
			if err != nil {
				log.Warn().
					Err(err).
					Str("resourceID", resourceID).
					Str("container", containerName).
					Msg("Skipping invalid canonical docker container CPU metric spec")
			} else {
				m.checkMetricWithCanonicalSpec(spec, containerName, nodeName, instanceName, resourceType, cpuCapacityPercent, thresholds.CPU, &metricOptions{Metadata: cpuMetadata})
			}
		}

		if thresholds.Memory != nil {
			memMetadata := map[string]interface{}{
				"resourceType":     resourceType,
				"hostId":           host.ID,
				"hostName":         host.DisplayName,
				"hostHostname":     host.Hostname,
				"containerId":      container.ID,
				"containerName":    containerName,
				"image":            container.Image,
				"state":            container.State,
				"status":           container.Status,
				"restartCount":     container.RestartCount,
				"metric":           "memory",
				"memoryPercent":    container.MemoryPercent,
				"memoryUsageBytes": container.MemoryUsage,
				"tags":             containerTags,
			}
			if container.MemoryLimit > 0 {
				memMetadata["memoryLimitBytes"] = container.MemoryLimit
			}
			spec, err := buildCanonicalMetricSpec(resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "memory", thresholds.Memory)
			if err != nil {
				log.Warn().
					Err(err).
					Str("resourceID", resourceID).
					Str("container", containerName).
					Msg("Skipping invalid canonical docker container memory metric spec")
			} else {
				m.checkMetricWithCanonicalSpec(spec, containerName, nodeName, instanceName, resourceType, container.MemoryPercent, thresholds.Memory, &metricOptions{Metadata: memMetadata})
			}
		}

		if thresholds.Disk != nil {
			totalBytes := container.RootFilesystemBytes
			usedBytes := container.WritableLayerBytes
			if totalBytes > 0 && usedBytes >= 0 {
				diskPercent := (float64(usedBytes) / float64(totalBytes)) * 100
				diskMetadata := map[string]interface{}{
					"resourceType":        resourceType,
					"hostId":              host.ID,
					"hostName":            host.DisplayName,
					"hostHostname":        host.Hostname,
					"containerId":         container.ID,
					"containerName":       containerName,
					"image":               container.Image,
					"state":               container.State,
					"status":              container.Status,
					"restartCount":        container.RestartCount,
					"metric":              "disk",
					"diskPercent":         diskPercent,
					"writableLayerBytes":  usedBytes,
					"rootFilesystemBytes": totalBytes,
					"mountCount":          len(container.Mounts),
					"tags":                containerTags,
				}
				if container.BlockIO != nil {
					diskMetadata["blockIoReadBytes"] = container.BlockIO.ReadBytes
					diskMetadata["blockIoWriteBytes"] = container.BlockIO.WriteBytes
				}
				spec, err := buildCanonicalMetricSpec(resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "disk", thresholds.Disk)
				if err != nil {
					log.Warn().
						Err(err).
						Str("resourceID", resourceID).
						Str("container", containerName).
						Msg("Skipping invalid canonical docker container disk metric spec")
				} else {
					m.checkMetricWithCanonicalSpec(spec, containerName, nodeName, instanceName, resourceType, diskPercent, thresholds.Disk, &metricOptions{Metadata: diskMetadata})
				}
			} else {
				m.clearDockerContainerMetricAlerts(resourceID, "disk")
			}
		}
	}

	m.checkDockerContainerHealth(host, container, resourceID, containerName, instanceName, nodeName)

	// Docker-specific checks
	m.checkDockerContainerRestartLoop(host, container, resourceID, containerName, instanceName, nodeName)
	m.checkDockerContainerOOMKill(host, container, resourceID, containerName, instanceName, nodeName)
	m.checkDockerContainerMemoryLimit(host, container, resourceID, containerName, instanceName, nodeName)
	m.checkDockerContainerImageUpdate(host, container, resourceID, containerName, instanceName, nodeName)
}

func (m *Manager) evaluateDockerService(host models.DockerHost, service models.DockerService, resourceID string) {
	m.mu.RLock()
	disableAllServices, _ := m.alertPolicyTypeSwitchesNoLock("docker-service")
	warnPct := m.config.DockerDefaults.ServiceWarnGapPct
	critPct := m.config.DockerDefaults.ServiceCritGapPct
	overrideConfig, hasOverride := m.config.Overrides[resourceID]
	m.mu.RUnlock()

	if disableAllServices {
		m.clearDockerServiceAlert(resourceID)
		return
	}
	if hasOverride && overrideConfig.Disabled {
		m.clearDockerServiceAlert(resourceID)
		return
	}

	desired := service.DesiredTasks
	running := service.RunningTasks
	if desired <= 0 {
		m.clearDockerServiceAlert(resourceID)
		return
	}

	missing := desired - running
	if missing < 0 {
		missing = 0
	}

	percentMissing := 0.0
	if desired > 0 {
		percentMissing = (float64(missing) / float64(desired)) * 100.0
	}

	thresholdValue := 0.0
	serviceName := dockerServiceDisplayName(service)
	instanceName := dockerInstanceName(host)
	nodeName := strings.TrimSpace(host.Hostname)

	metadata := map[string]interface{}{
		"resourceType":   "docker-service",
		"hostId":         host.ID,
		"hostName":       host.DisplayName,
		"hostHostname":   host.Hostname,
		"serviceId":      service.ID,
		"serviceName":    service.Name,
		"stack":          service.Stack,
		"mode":           service.Mode,
		"desiredTasks":   service.DesiredTasks,
		"runningTasks":   service.RunningTasks,
		"completedTasks": service.CompletedTasks,
		"missingTasks":   missing,
		"percentMissing": percentMissing,
		"tags":           dockerLabelTags(service.Labels),
	}
	alertID := fmt.Sprintf("docker-service-health-%s", resourceID)

	if critPct > 0 && percentMissing >= float64(critPct) {
		thresholdValue = float64(critPct)
	} else if warnPct > 0 && percentMissing >= float64(warnPct) {
		thresholdValue = float64(warnPct)
	}

	updateState := ""
	updateMessage := ""
	updateSeverity := AlertLevel("")
	if service.UpdateStatus != nil {
		updateState = strings.ToLower(strings.TrimSpace(service.UpdateStatus.State))
		updateMessage = strings.TrimSpace(service.UpdateStatus.Message)
		switch updateState {
		case "paused", "rollback_started", "rollback_paused":
			updateSeverity = AlertLevelWarning
		case "rollback_failed":
			updateSeverity = AlertLevelCritical
		}
		if service.UpdateStatus.CompletedAt != nil && !service.UpdateStatus.CompletedAt.IsZero() {
			metadata["updateCompletedAt"] = service.UpdateStatus.CompletedAt.UTC()
		}
		if updateState != "" {
			metadata["updateState"] = service.UpdateStatus.State
		}
		if updateMessage != "" {
			metadata["updateMessage"] = updateMessage
		}
	}

	if thresholdValue == 0 && updateSeverity != "" {
		spec, err := buildCanonicalDiscreteStateSpec(resourceID, serviceName, unifiedresources.ResourceTypeDockerService, updateSeverity, 1, false, "update-state",
			[]string{"paused", "rollback_started", "rollback_paused", "rollback_failed"})
		if err != nil {
			log.Warn().
				Err(err).
				Str("service", serviceName).
				Str("resourceID", resourceID).
				Msg("Skipping invalid canonical docker service update-state spec")
			return
		}

		message := fmt.Sprintf("Docker service '%s' update state: %s", serviceName, service.UpdateStatus.State)
		if updateMessage != "" {
			message = fmt.Sprintf("%s (%s)", message, updateMessage)
		}

		_, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
			Spec:                         spec,
			Evidence:                     alertspecs.AlertEvidence{ObservedAt: time.Now(), DiscreteState: &alertspecs.DiscreteStateEvidence{StateKey: "update-state", Observed: updateState}},
			AlertID:                      alertID,
			AlertType:                    "docker-service-health",
			ResourceID:                   resourceID,
			ResourceName:                 serviceName,
			Node:                         nodeName,
			Instance:                     instanceName,
			Message:                      message,
			Value:                        percentMissing,
			Threshold:                    0,
			Metadata:                     metadata,
			AddToRecent:                  true,
			AddToHistory:                 true,
			RateLimit:                    true,
			NotifyOnSeverityChange:       true,
			AddToHistoryOnSeverityChange: true,
			DispatchAsync:                true,
		})
		return
	}

	if thresholdValue == 0 {
		m.clearDockerServiceAlert(resourceID)
		return
	}

	spec, err := buildCanonicalServiceGapSpec(resourceID, serviceName, unifiedresources.ResourceTypeDockerService, serviceName, float64(warnPct), float64(critPct), false)
	if err != nil {
		log.Warn().
			Err(err).
			Str("service", serviceName).
			Str("resourceID", resourceID).
			Msg("Skipping invalid canonical docker service gap spec")
		m.clearDockerServiceAlert(resourceID)
		return
	}

	message := fmt.Sprintf("Docker service '%s' is running %d of %d desired tasks", serviceName, service.RunningTasks, service.DesiredTasks)
	_, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
		Spec: spec,
		Evidence: alertspecs.AlertEvidence{
			ObservedAt: time.Now(),
			ServiceGap: &alertspecs.ServiceGapEvidence{
				Service: serviceName,
				Desired: desired,
				Running: running,
			},
		},
		AlertID:                      alertID,
		AlertType:                    "docker-service-health",
		ResourceID:                   resourceID,
		ResourceName:                 serviceName,
		Node:                         nodeName,
		Instance:                     instanceName,
		Message:                      message,
		Value:                        percentMissing,
		Threshold:                    thresholdValue,
		Metadata:                     metadata,
		AddToRecent:                  true,
		AddToHistory:                 true,
		RateLimit:                    true,
		NotifyOnSeverityChange:       true,
		AddToHistoryOnSeverityChange: true,
		DispatchAsync:                true,
	})
}

func (m *Manager) clearDockerServiceAlert(resourceID string) {
	m.clearAlert(canonicalServiceGapStateID(resourceID))
	m.clearAlert(canonicalDiscreteStateStateID(resourceID, "update-state"))
}

// HandleDockerHostOnline clears offline tracking and alerts for a Docker host.
func (m *Manager) HandleDockerHostOnline(host models.DockerHost) {
	if host.ID == "" {
		return
	}

	resourceID := DockerHostResourceID(host.ID)
	alertID := canonicalConnectivityStateID(resourceID)

	m.mu.Lock()
	exists := m.hasActiveAlertNoLock(alertID)
	// The host was observed online: a healthy observation ends any
	// in-flight offline confirmation run in the core.
	m.core.ApplyDiscrete(reducer.DiscreteSignal{ResourceID: resourceID, Key: canonicalConnectivitySpecID(resourceID), Matched: false, ObservedAt: m.policyNow()}, reducer.DiscreteRule{})
	m.mu.Unlock()

	if exists {
		m.clearAlert(alertID)
	}
}

// HandleDockerHostRemoved clears all alerts and tracking when a Docker host is deleted.
func (m *Manager) HandleDockerHostRemoved(host models.DockerHost) {
	if host.ID == "" {
		return
	}

	// Reuse the online handler to clear offline alerts and tracking.
	m.HandleDockerHostOnline(host)
	// Drop any container alerts and host-scoped tracking entries.
	m.clearDockerHostContainerAlerts(host)
}

// HandleDockerHostOffline raises an alert when a Docker host stops reporting.
func (m *Manager) HandleDockerHostOffline(host models.DockerHost) {
	if host.ID == "" {
		return
	}

	m.mu.RLock()
	if !m.config.Enabled {
		m.mu.RUnlock()
		return
	}
	_, disableDockerHostsOffline := m.alertPolicyTypeSwitchesNoLock("docker-host")
	m.mu.RUnlock()

	resourceID := DockerHostResourceID(host.ID)
	alertID := canonicalConnectivityStateID(resourceID)
	instanceName := dockerInstanceName(host)
	nodeName := strings.TrimSpace(host.Hostname)

	if disableDockerHostsOffline {
		m.mu.Lock()
		m.mu.Unlock()
		m.clearAlert(alertID)
		return
	}

	var disableConnectivity bool
	m.mu.RLock()
	if override, exists := m.config.Overrides[host.ID]; exists {
		disableConnectivity = override.DisableConnectivity
	}
	m.mu.RUnlock()

	if disableConnectivity {
		m.clearAlert(alertID)
		m.mu.Lock()
		m.mu.Unlock()
		return
	}

	spec, err := buildCanonicalConnectivitySpec(resourceID, host.DisplayName, unifiedresources.ResourceType("docker-host"), AlertLevelCritical, 3, false)
	if err != nil {
		log.Warn().
			Err(err).
			Str("dockerHost", host.DisplayName).
			Str("hostID", host.ID).
			Msg("Skipping invalid canonical docker host connectivity spec")
		return
	}

	result, ok := m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
		Spec:         spec,
		Evidence:     alertspecs.AlertEvidence{ObservedAt: time.Now(), Connectivity: &alertspecs.ConnectivityEvidence{Signal: "status", Connected: false}},
		AlertID:      alertID,
		AlertType:    "docker-host-offline",
		ResourceID:   resourceID,
		ResourceName: host.DisplayName,
		Node:         nodeName,
		Instance:     instanceName,
		Message:      fmt.Sprintf("Docker host '%s' is offline", host.DisplayName),
		Metadata: map[string]interface{}{
			"resourceType": "docker-host",
			"hostId":       host.ID,
			"hostname":     host.Hostname,
			"agentId":      host.AgentID,
			"displayName":  host.DisplayName,
		},
		AddToRecent:   true,
		AddToHistory:  true,
		RateLimit:     true,
		DispatchAsync: false,
	})
	if !ok || result.Transition == nil || result.Transition.Kind != alertspecs.EvaluationTransitionActivated {
		return
	}

	m.mu.RLock()
	alert, _ := m.getActiveAlertNoLock(alertID)
	m.mu.RUnlock()
	if alert != nil {
		if callbacks := m.getAlertForAICallbacks(); len(callbacks) > 0 {
			alertCopy := cloneAlertForOutput(alert)
			go func(a *Alert, fns []func(*Alert)) {
				defer func() {
					if r := recover(); r != nil {
						log.Error().Interface("panic", r).Str("alertID", a.ID).Msg("panic in AI alert callback")
					}
				}()
				for _, callback := range fns {
					callback(a)
				}
			}(alertCopy, callbacks)
		}
	}

	m.clearDockerHostContainerAlerts(host)
}

func (m *Manager) checkDockerContainerState(host models.DockerHost, container models.DockerContainer, resourceID, containerName, instanceName, nodeName string) {
	alertID := fmt.Sprintf("docker-container-state-%s", resourceID)

	m.mu.RLock()
	override, hasOverride := m.lookupDockerContainerOverrideNoLock(host.ID, container.Name, resourceID)
	defaultDisable := m.config.DockerDefaults.StateDisableConnectivity
	defaultSeverity := NormalizePoweredOffSeverity(m.config.DockerDefaults.StatePoweredOffSeverity)
	m.mu.RUnlock()

	disableConnectivity := defaultDisable
	severity := defaultSeverity
	if hasOverride {
		// Same rule as applyThresholdOverride: an override turns state
		// alerts back on over a disabled default only when it carries the
		// row's own severity. DisableConnectivity is a plain bool, so an
		// override saved for a CPU or memory threshold alone reads false
		// here, and used to re-enable state alerts the operator had turned
		// off for every container.
		if override.DisableConnectivity {
			disableConnectivity = true
		} else if override.PoweredOffSeverity != "" {
			disableConnectivity = false
		}

		if override.PoweredOffSeverity != "" {
			severity = NormalizePoweredOffSeverity(override.PoweredOffSeverity)
		}
	}

	if disableConnectivity {
		m.clearDockerContainerStateAlert(resourceID)
		return
	}

	observedState := strings.ToLower(strings.TrimSpace(container.State))
	if observedState == "" {
		observedState = "unknown"
	}

	spec, err := buildCanonicalDiscreteStateSpec(resourceID, containerName, unifiedresources.ResourceTypeAppContainer, severity, 2, false, "runtime-state",
		[]string{"created", "restarting", "removing", "paused", "exited", "dead", "unknown"})
	if err != nil {
		log.Warn().
			Err(err).
			Str("resourceID", resourceID).
			Str("container", containerName).
			Msg("Skipping invalid canonical docker container state spec")
		return
	}

	_, _ = m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
		Spec: spec,
		Evidence: alertspecs.AlertEvidence{
			ObservedAt: time.Now(),
			DiscreteState: &alertspecs.DiscreteStateEvidence{
				StateKey: "runtime-state",
				Observed: observedState,
			},
		},
		AlertID:       alertID,
		AlertType:     "docker-container-state",
		ResourceID:    resourceID,
		ResourceName:  containerName,
		Node:          nodeName,
		Instance:      instanceName,
		Message:       fmt.Sprintf("Docker container '%s' is %s", containerName, strings.TrimSpace(container.Status)),
		Metadata:      dockerContainerAlertMetadata(host, container, containerName),
		AddToRecent:   true,
		AddToHistory:  true,
		DispatchAsync: true,
	})
}

func (m *Manager) clearDockerContainerStateAlert(resourceID string) {
	m.mu.Lock()
	m.mu.Unlock()
	m.clearAlert(canonicalDiscreteStateStateID(resourceID, "runtime-state"))
}

func dockerContainerAlertMetadata(host models.DockerHost, container models.DockerContainer, containerName string) map[string]interface{} {
	return map[string]interface{}{
		"resourceType":  "app-container",
		"hostId":        host.ID,
		"hostName":      host.DisplayName,
		"hostHostname":  host.Hostname,
		"containerId":   container.ID,
		"containerName": containerName,
		"image":         container.Image,
		"state":         container.State,
		"status":        container.Status,
		"tags":          dockerLabelTags(container.Labels),
	}
}

func dockerLabelTags(labels map[string]string) []string {
	if len(labels) == 0 {
		return nil
	}

	tags := make([]string, 0, len(labels))
	for rawKey, rawValue := range labels {
		key := strings.TrimSpace(rawKey)
		if key == "" {
			continue
		}
		value := strings.TrimSpace(rawValue)
		if value == "" {
			tags = append(tags, key)
			continue
		}
		tags = append(tags, key+":"+value)
	}
	sort.Strings(tags)
	return tags
}

func (m *Manager) checkDockerContainerHealth(host models.DockerHost, container models.DockerContainer, resourceID, containerName, instanceName, nodeName string) {
	// Docker retains the last health-check result after a container stops (and
	// while health checks are suspended in other non-running states). That
	// persisted value is no longer live health evidence; the runtime-state alert
	// owns the non-running condition.
	state := strings.ToLower(strings.TrimSpace(container.State))
	if state != "" && state != "running" {
		m.clearDockerContainerHealthAlert(resourceID)
		return
	}

	health := strings.ToLower(strings.TrimSpace(container.Health))
	if health == "" || health == "none" || health == "healthy" || health == "starting" {
		m.clearDockerContainerHealthAlert(resourceID)
		return
	}

	alertID := fmt.Sprintf("docker-container-health-%s", resourceID)
	spec, err := buildCanonicalHealthAssessmentSpec(resourceID+"-health", resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "docker-container-health", nil, false)
	if err != nil {
		log.Warn().
			Err(err).
			Str("resourceID", resourceID).
			Str("container", containerName).
			Msg("Skipping invalid canonical docker container health spec")
		return
	}

	severity := alertspecs.AlertSeverityWarning
	if health == "unhealthy" {
		severity = alertspecs.AlertSeverityCritical
	}

	metadata := dockerContainerAlertMetadata(host, container, containerName)
	metadata["health"] = container.Health

	_, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
		Spec: spec,
		Evidence: alertspecs.AlertEvidence{
			ObservedAt: time.Now(),
			HealthAssessment: &alertspecs.HealthAssessmentEvidence{
				Signal:   "docker-container-health",
				Severity: severity,
				Codes:    []string{health},
			},
		},
		AlertID:                      alertID,
		AlertType:                    "docker-container-health",
		ResourceID:                   resourceID,
		ResourceName:                 containerName,
		Node:                         nodeName,
		Instance:                     instanceName,
		Message:                      fmt.Sprintf("Docker container '%s' health is %s", containerName, container.Health),
		Metadata:                     metadata,
		AddToRecent:                  true,
		AddToHistory:                 true,
		DispatchAsync:                false,
		NotifyOnSeverityChange:       true,
		AddToHistoryOnSeverityChange: true,
	})

	log.Warn().
		Str("container", containerName).
		Str("host", host.DisplayName).
		Str("health", container.Health).
		Msg("Docker container health alert raised")
}

func (m *Manager) clearDockerContainerHealthAlert(resourceID string) {
	m.clearAlert(buildCanonicalStateID(resourceID, resourceID+"-health"))
}

// checkDockerContainerRestartLoop detects containers stuck in a restart loop
func (m *Manager) checkDockerContainerRestartLoop(host models.DockerHost, container models.DockerContainer, resourceID, containerName, instanceName, nodeName string) {
	alertID := fmt.Sprintf("docker-container-restart-loop-%s", resourceID)
	now := time.Now()

	m.mu.Lock()

	// NormalizeDockerDefaults owns the factory values; 0 here is the
	// operator's Off, so either one set to 0 turns restart-loop alerts off.
	restartThreshold := m.config.DockerDefaults.RestartCount
	timeWindow := m.config.DockerDefaults.RestartWindow
	if restartThreshold <= 0 || timeWindow <= 0 {
		delete(m.dockerRestartTracking, resourceID)
		m.mu.Unlock()
		m.clearAlert(buildCanonicalStateID(resourceID, resourceID+"-restart-loop"))
		return
	}

	record, exists := m.dockerRestartTracking[resourceID]
	if !exists {
		record = &dockerRestartRecord{
			count:       container.RestartCount,
			lastCount:   container.RestartCount,
			times:       []time.Time{},
			lastChecked: now,
		}
		m.dockerRestartTracking[resourceID] = record
		m.mu.Unlock()
		return
	}

	// If restart count increased, track it
	if container.RestartCount > record.lastCount {
		newRestarts := container.RestartCount - record.lastCount
		for i := 0; i < newRestarts; i++ {
			record.times = append(record.times, now)
		}
		record.lastCount = container.RestartCount
	}

	// Clean up old restart times outside the window
	cutoff := now.Add(-time.Duration(timeWindow) * time.Second)
	var recentRestarts []time.Time
	for _, t := range record.times {
		if t.After(cutoff) {
			recentRestarts = append(recentRestarts, t)
		}
	}
	record.times = recentRestarts
	record.lastChecked = now

	recentCount := len(record.times)
	m.mu.Unlock()

	// Check if we have a restart loop
	if recentCount > restartThreshold {
		spec, err := buildCanonicalSeverityThresholdSpec(resourceID+"-restart-loop", resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "restart-count-window", 0, float64(restartThreshold+1), false)
		if err != nil {
			log.Warn().
				Err(err).
				Str("resourceID", resourceID).
				Str("container", containerName).
				Msg("Skipping invalid canonical docker container restart loop spec")
			return
		}

		metadata := dockerContainerAlertMetadata(host, container, containerName)
		metadata["restartCount"] = container.RestartCount
		metadata["recentRestarts"] = recentCount

		_, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
			Spec: spec,
			Evidence: alertspecs.AlertEvidence{
				ObservedAt: now,
				SeverityThreshold: &alertspecs.SeverityThresholdEvidence{
					Metric:    "restart-count-window",
					Direction: alertspecs.ThresholdDirectionAbove,
					Observed:  float64(recentCount),
				},
			},
			AlertID:      alertID,
			AlertType:    "docker-container-restart-loop",
			ResourceID:   resourceID,
			ResourceName: containerName,
			Node:         nodeName,
			Instance:     instanceName,
			Message:      fmt.Sprintf("Docker container '%s' has restarted %d times in the last %d minutes (restart loop detected)", containerName, recentCount, timeWindow/60),
			Metadata:     metadata,
			AddToRecent:  true,
			AddToHistory: true,
		})

		log.Warn().
			Str("container", containerName).
			Str("host", host.DisplayName).
			Int("restarts", recentCount).
			Msg("Docker container restart loop detected")
	} else {
		// Clear alert if restart loop has stopped
		m.clearAlert(buildCanonicalStateID(resourceID, resourceID+"-restart-loop"))
	}
}

// checkDockerContainerOOMKill detects when the runtime explicitly reports that
// a container was killed due to out of memory.
func (m *Manager) checkDockerContainerOOMKill(host models.DockerHost, container models.DockerContainer, resourceID, containerName, instanceName, nodeName string) {
	alertID := fmt.Sprintf("docker-container-oom-%s", resourceID)

	state := strings.ToLower(strings.TrimSpace(container.State))
	if (state == "exited" || state == "dead") && container.OOMKilled != nil && *container.OOMKilled {
		spec, err := buildCanonicalHealthAssessmentSpec(resourceID+"-oom-kill", resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "docker-container-exit", []string{"oom-kill"}, false)
		if err != nil {
			log.Warn().
				Err(err).
				Str("resourceID", resourceID).
				Str("container", containerName).
				Msg("Skipping invalid canonical docker container OOM spec")
			return
		}

		metadata := dockerContainerAlertMetadata(host, container, containerName)
		metadata["exitCode"] = container.ExitCode
		metadata["oomKilled"] = true
		metadata["memoryUsageBytes"] = container.MemoryUsage
		metadata["memoryLimitBytes"] = container.MemoryLimit

		_, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
			Spec: spec,
			Evidence: alertspecs.AlertEvidence{
				ObservedAt: time.Now(),
				HealthAssessment: &alertspecs.HealthAssessmentEvidence{
					Signal:   "docker-container-exit",
					Severity: alertspecs.AlertSeverityCritical,
					Codes:    []string{"oom-kill"},
				},
			},
			AlertID:      alertID,
			AlertType:    "docker-container-oom-kill",
			ResourceID:   resourceID,
			ResourceName: containerName,
			Node:         nodeName,
			Instance:     instanceName,
			Message:      fmt.Sprintf("Docker container '%s' was killed due to out of memory (OOM)", containerName),
			Metadata:     metadata,
			AddToRecent:  true,
			AddToHistory: true,
		})

		log.Error().
			Str("container", containerName).
			Str("host", host.DisplayName).
			Int64("memoryUsage", container.MemoryUsage).
			Int64("memoryLimit", container.MemoryLimit).
			Msg("Docker container OOM killed")
	} else {
		// Unknown reports from older agents and explicit false reports both fail
		// closed. Exit code 137 proves SIGKILL only; it is not OOM evidence.
		m.clearAlert(buildCanonicalStateID(resourceID, resourceID+"-oom-kill"))
	}
}

// checkDockerContainerMemoryLimit alerts when container approaches its memory limit
func (m *Manager) checkDockerContainerMemoryLimit(host models.DockerHost, container models.DockerContainer, resourceID, containerName, instanceName, nodeName string) {
	// Only check if container is running and has a memory limit
	state := strings.ToLower(strings.TrimSpace(container.State))
	if state != "running" || container.MemoryLimit <= 0 {
		return
	}

	alertID := fmt.Sprintf("docker-container-memory-limit-%s", resourceID)

	// Get config values with defaults
	m.mu.RLock()
	warnThreshold := float64(m.config.DockerDefaults.MemoryWarnPct)
	criticalThreshold := float64(m.config.DockerDefaults.MemoryCriticalPct)
	m.mu.RUnlock()
	// NormalizeDockerDefaults owns the factory values; 0 here is the
	// operator's Off for that level.
	if warnThreshold <= 0 && criticalThreshold <= 0 {
		m.clearAlert(buildCanonicalStateID(resourceID, resourceID+"-memory-limit"))
		return
	}

	// Calculate percentage of limit used
	limitPercent := (float64(container.MemoryUsage) / float64(container.MemoryLimit)) * 100

	activation := warnThreshold
	if activation <= 0 {
		activation = criticalThreshold
	}
	recovery := activation - 5
	spec, err := buildCanonicalSeverityThresholdSpecWithRecovery(resourceID+"-memory-limit", resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "memory-limit-percent", warnThreshold, criticalThreshold, &recovery, false)
	if err != nil {
		log.Warn().
			Err(err).
			Str("resourceID", resourceID).
			Str("container", containerName).
			Msg("Skipping invalid canonical docker container memory limit spec")
		return
	}

	metadata := dockerContainerAlertMetadata(host, container, containerName)
	metadata["memoryUsageBytes"] = container.MemoryUsage
	metadata["memoryLimitBytes"] = container.MemoryLimit
	metadata["limitPercent"] = limitPercent

	_, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
		Spec: spec,
		Evidence: alertspecs.AlertEvidence{
			ObservedAt: time.Now(),
			SeverityThreshold: &alertspecs.SeverityThresholdEvidence{
				Metric:    "memory-limit-percent",
				Direction: alertspecs.ThresholdDirectionAbove,
				Observed:  limitPercent,
			},
		},
		AlertID:      alertID,
		AlertType:    "docker-container-memory-limit",
		ResourceID:   resourceID,
		ResourceName: containerName,
		Node:         nodeName,
		Instance:     instanceName,
		Message:      fmt.Sprintf("Docker container '%s' is using %.1f%% of its memory limit (%d MB / %d MB)", containerName, limitPercent, container.MemoryUsage/(1024*1024), container.MemoryLimit/(1024*1024)),
		Metadata:     metadata,
		AddToRecent:  true,
		AddToHistory: true,
	})

	if limitPercent >= warnThreshold {
		log.Warn().
			Str("container", containerName).
			Str("host", host.DisplayName).
			Float64("limitPercent", limitPercent).
			Msg("Docker container approaching memory limit")
	}
}

func (m *Manager) clearDockerContainerMetricAlerts(resourceID string, metrics ...string) {
	if len(metrics) == 0 {
		metrics = []string{"cpu", "memory", "disk"}
	}
	for _, metric := range metrics {
		m.clearAlert(canonicalMetricStateID(resourceID, metric))
	}
}

func (m *Manager) clearDockerContainerUpdateTracking(resourceID, trackingKey string) {
	m.mu.Lock()
	delete(m.dockerUpdateFirstSeen, resourceID)
	delete(m.dockerUpdateLastObserved, resourceID)
	if trackingKey != "" {
		delete(m.dockerUpdateFirstSeenByIdentity, trackingKey)
		delete(m.dockerUpdateLastObserved, trackingKey)
	}
	m.mu.Unlock()
}

func dockerUpdateTrackingKeyFromAlert(alert *Alert) string {
	if alert == nil || alert.Metadata == nil {
		return ""
	}

	metadataString := func(key string) string {
		value, ok := alert.Metadata[key]
		if !ok || value == nil {
			return ""
		}
		return strings.TrimSpace(fmt.Sprint(value))
	}

	host := models.DockerHost{
		ID:          metadataString("hostId"),
		DisplayName: metadataString("hostName"),
		Hostname:    metadataString("hostHostname"),
	}
	container := models.DockerContainer{
		ID:    metadataString("containerId"),
		Name:  metadataString("containerName"),
		Image: metadataString("image"),
	}

	if host.ID == "" && host.DisplayName == "" && host.Hostname == "" &&
		container.ID == "" && container.Name == "" && container.Image == "" {
		return ""
	}

	return dockerUpdateTrackingKey(host, container)
}

func (m *Manager) clearDockerContainerUpdateStateLocked(alert *Alert) {
	if alert == nil {
		return
	}

	if alert.ResourceID != "" {
		delete(m.dockerUpdateFirstSeen, alert.ResourceID)
		delete(m.dockerUpdateLastObserved, alert.ResourceID)
	}
	if trackingKey := dockerUpdateTrackingKeyFromAlert(alert); trackingKey != "" {
		delete(m.dockerUpdateFirstSeenByIdentity, trackingKey)
		delete(m.dockerUpdateLastObserved, trackingKey)
	}
}

func (m *Manager) clearDockerContainerUpdateAlertsLocked() {
	toClear := make([]string, 0)
	for storageKey, alert := range m.activeAlerts {
		if alert == nil {
			continue
		}
		alertID := effectiveAlertID(alert, storageKey)
		if alert.Type != "docker-container-update" && !strings.HasPrefix(alertID, "docker-container-update-") {
			continue
		}
		m.clearDockerContainerUpdateStateLocked(alert)
		toClear = append(toClear, alertID)
	}
	for _, alertID := range toClear {
		m.clearAlertNoLock(alertID)
	}
}

func (m *Manager) shouldResolveDockerContainerUpdateAlertLocked(alert *Alert) bool {
	if alert == nil {
		return false
	}

	if allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("app-container"); allDisabled || m.config.DockerDefaults.UpdateAlertDelayHours < 0 {
		m.clearDockerContainerUpdateStateLocked(alert)
		return true
	}

	containerName := strings.TrimSpace(alert.ResourceName)
	containerID := ""
	hostID := ""
	if alert.Metadata != nil {
		if value, ok := alert.Metadata["containerName"].(string); ok && containerName == "" {
			containerName = value
		}
		if value, ok := alert.Metadata["containerId"].(string); ok {
			containerID = value
		}
		if value, ok := alert.Metadata["hostId"].(string); ok {
			hostID = value
		}
	}

	if override, exists := m.lookupDockerContainerOverrideNoLock(hostID, containerName, alert.ResourceID); exists && override.Disabled {
		m.clearDockerContainerUpdateStateLocked(alert)
		return true
	}
	if matchesDockerIgnoredPrefix(containerName, containerID, m.config.DockerIgnoredContainerPrefixes) {
		m.clearDockerContainerUpdateStateLocked(alert)
		return true
	}

	return false
}

func (m *Manager) touchDockerContainerUpdateAlert(alertID string) {
	m.mu.Lock()
	defer m.mu.Unlock()

	if alert, exists := m.getActiveAlertNoLock(alertID); exists && alert != nil {
		alert.LastSeen = time.Now()
	}
}

// checkDockerContainerImageUpdate checks if an image update has been pending for too long
func (m *Manager) checkDockerContainerImageUpdate(host models.DockerHost, container models.DockerContainer, resourceID, containerName, instanceName, nodeName string) {
	alertID := fmt.Sprintf("docker-container-update-%s", resourceID)
	canonicalAlertID := buildCanonicalStateID(resourceID, resourceID+"-image-update")
	updateTrackingKey := dockerUpdateTrackingKey(host, container)

	// Check if update detection is enabled
	m.mu.RLock()
	delayHours := m.config.DockerDefaults.UpdateAlertDelayHours
	m.mu.RUnlock()

	// Negative value means disabled
	if delayHours < 0 {
		m.clearAlert(canonicalAlertID)
		m.clearDockerContainerUpdateTracking(resourceID, updateTrackingKey)
		return
	}

	// Reports refresh tracking activity even when the registry result is cached
	// or unavailable. Do not create a pending condition from unknown evidence.
	m.mu.Lock()
	if _, exists := m.dockerUpdateFirstSeen[resourceID]; exists {
		m.dockerUpdateLastObserved[resourceID] = time.Now()
	}
	if _, exists := m.dockerUpdateFirstSeenByIdentity[updateTrackingKey]; exists {
		m.dockerUpdateLastObserved[updateTrackingKey] = time.Now()
	}
	m.mu.Unlock()

	// Check if this container has an update status reported
	if container.UpdateStatus == nil {
		// Missing update status means the condition is unknown, not resolved.
		// Preserve any active alert and first-seen tracking until we see an affirmative clear.
		m.touchDockerContainerUpdateAlert(canonicalAlertID)
		return
	}

	// Check for errors in update detection (don't alert on errors)
	if container.UpdateStatus.Error != "" {
		// A failed update check cannot confirm the pending update has been resolved.
		m.touchDockerContainerUpdateAlert(canonicalAlertID)
		return
	}

	// "No update" is a comparison result only when the agent knew the local
	// digest. Agents before #2353 compared an empty digest after a failed image
	// inspect and reported no update, which resolved a pending alert and
	// restarted its delay. Without a current digest the result is unknown.
	if !container.UpdateStatus.UpdateAvailable && strings.TrimSpace(container.UpdateStatus.CurrentDigest) == "" {
		m.touchDockerContainerUpdateAlert(canonicalAlertID)
		return
	}

	// Check if an update is available
	if !container.UpdateStatus.UpdateAvailable {
		// No update available - clear tracking and alert
		m.clearAlert(canonicalAlertID)
		m.clearDockerContainerUpdateTracking(resourceID, updateTrackingKey)
		return
	}

	// Update is available - track when we first saw it
	m.mu.Lock()
	firstSeen, exists := m.dockerUpdateFirstSeenByIdentity[updateTrackingKey]
	if !exists {
		firstSeen, exists = m.dockerUpdateFirstSeen[resourceID]
	}
	// Tracking maps are process-local, but an active occurrence survives a
	// restart. Its start is the original pending age, not a new delay window.
	// Otherwise positive reports return below-threshold without refreshing
	// LastSeen, and hourly cleanup can resolve a still-asserted update.
	if active, ok := m.getActiveAlertNoLock(canonicalAlertID); ok && active != nil &&
		active.Type == "docker-container-update" && active.ResourceID == resourceID &&
		!active.StartTime.IsZero() {
		firstSeen, exists = active.StartTime, true
	} else if strings.TrimSpace(container.ID) == "" {
		// A container without an ID used to record its pending update under
		// its host's reference. Keep that age; the host's report cleanup then
		// clears the old occurrence.
		legacyRef := DockerHostResourceID(host.ID)
		if legacy, ok := m.getActiveAlertNoLock(buildCanonicalStateID(legacyRef, legacyRef+"-image-update")); ok &&
			isLegacyDockerHostContainerAlert(legacy, host) && legacy.Type == "docker-container-update" &&
			alertMetadataString(legacy, "containerName") == containerName && !legacy.StartTime.IsZero() {
			firstSeen, exists = legacy.StartTime, true
		}
	}
	if !exists {
		firstSeen = time.Now()
	}
	m.dockerUpdateFirstSeen[resourceID] = firstSeen
	m.dockerUpdateFirstSeenByIdentity[updateTrackingKey] = firstSeen
	m.dockerUpdateLastObserved[resourceID] = time.Now()
	m.dockerUpdateLastObserved[updateTrackingKey] = time.Now()
	m.mu.Unlock()

	// Check if we've exceeded the delay threshold
	pendingDuration := time.Since(firstSeen)
	threshold := time.Duration(delayHours) * time.Hour
	if pendingDuration < threshold {
		// Not yet time to alert
		log.Debug().
			Str("container", containerName).
			Str("host", host.DisplayName).
			Str("image", container.Image).
			Dur("pending", pendingDuration).
			Dur("threshold", threshold).
			Msg("Container update pending but below alert threshold")
		return
	}

	// Create or update the alert
	pendingHours := int(pendingDuration.Hours())
	spec, err := buildCanonicalSeverityThresholdSpec(resourceID+"-image-update", resourceID, containerName, unifiedresources.ResourceTypeAppContainer, "image-update-hours", float64(delayHours), 0, false)
	if err != nil {
		log.Warn().
			Err(err).
			Str("resourceID", resourceID).
			Str("container", containerName).
			Msg("Skipping invalid canonical docker container update spec")
		return
	}

	metadata := dockerContainerAlertMetadata(host, container, containerName)
	metadata["currentDigest"] = container.UpdateStatus.CurrentDigest
	metadata["latestDigest"] = container.UpdateStatus.LatestDigest
	metadata["lastChecked"] = container.UpdateStatus.LastChecked
	metadata["firstSeen"] = firstSeen
	metadata["pendingHours"] = pendingHours
	metadata["thresholdHours"] = delayHours

	_, _ = m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
		Spec: spec,
		Evidence: alertspecs.AlertEvidence{
			ObservedAt: time.Now(),
			SeverityThreshold: &alertspecs.SeverityThresholdEvidence{
				Metric:    "image-update-hours",
				Direction: alertspecs.ThresholdDirectionAbove,
				Observed:  pendingDuration.Hours(),
			},
		},
		AlertID:           alertID,
		AlertType:         "docker-container-update",
		ResourceID:        resourceID,
		ResourceName:      containerName,
		Node:              nodeName,
		Instance:          instanceName,
		Message:           fmt.Sprintf("Docker container '%s' has an image update available for %d hours", containerName, pendingHours),
		StartTimeOverride: firstSeen,
		Metadata:          metadata,
		AddToRecent:       true,
		AddToHistory:      true,
	})

	log.Warn().
		Str("container", containerName).
		Str("host", host.DisplayName).
		Str("image", container.Image).
		Int("pendingHours", pendingHours).
		Msg("Docker container has pending image update")
}

func (m *Manager) cleanupDockerContainerAlerts(host models.DockerHost, seen map[string]struct{}) {
	m.cleanupDockerContainerAlertsWithTracking(host, seen, nil)
}

// isLegacyDockerHostContainerAlert reports an alert that a container without
// an ID on this host raised under the host's own reference, before such
// containers had a reference of their own. Only the host's connectivity alert
// belongs there, so no producer refreshes these. The recorded host and empty
// container ID must match too: an alert restored with a resource ID cut short
// at a "::" in another host's ID ("<base>::<suffix>") can carry this host's
// reference.
func isLegacyDockerHostContainerAlert(alert *Alert, host models.DockerHost) bool {
	hostRef := DockerHostResourceID(host.ID)
	return alert != nil && hostRef != "" && alert.ResourceID == hostRef && alert.Type != "docker-host-offline" &&
		alertMetadataString(alert, "hostId") == strings.TrimSpace(host.ID) && alertMetadataString(alert, "containerId") == ""
}

func (m *Manager) cleanupDockerContainerAlertsWithTracking(host models.DockerHost, seen map[string]struct{}, seenUpdateTracking map[string]struct{}) {
	prefix := fmt.Sprintf("docker:%s/", strings.TrimSpace(host.ID))
	updateTrackingPrefix := dockerUpdateTrackingHostPrefix(host)

	m.mu.Lock()
	toClear := make([]string, 0)
	for storageKey, alert := range m.activeAlerts {
		alertID := effectiveAlertID(alert, storageKey)
		if isLegacyDockerHostContainerAlert(alert, host) {
			toClear = append(toClear, alertID)
			continue
		}
		if !strings.HasPrefix(alert.ResourceID, prefix) {
			continue
		}
		if _, exists := seen[alert.ResourceID]; exists {
			continue
		}
		toClear = append(toClear, alertID)
	}
	// Removed containers stop being observed, so their in-flight
	// confirmation runs in the core would never see a closing observation.
	for _, resourceID := range m.core.PendingResourceIDs() {
		if strings.HasPrefix(resourceID, prefix) {
			if _, exists := seen[resourceID]; !exists {
				m.core.DropPendingForResource(resourceID)
			}
		}
	}
	// Cleanup update tracking for removed containers
	for resourceID := range m.dockerUpdateFirstSeen {
		if strings.HasPrefix(resourceID, prefix) {
			if _, exists := seen[resourceID]; !exists {
				delete(m.dockerUpdateFirstSeen, resourceID)
				delete(m.dockerUpdateLastObserved, resourceID)
			}
		}
	}
	if seenUpdateTracking != nil {
		for trackingKey := range m.dockerUpdateFirstSeenByIdentity {
			if !strings.HasPrefix(trackingKey, updateTrackingPrefix) {
				continue
			}
			if _, exists := seenUpdateTracking[trackingKey]; !exists {
				delete(m.dockerUpdateFirstSeenByIdentity, trackingKey)
				delete(m.dockerUpdateLastObserved, trackingKey)
			}
		}
	}
	m.mu.Unlock()

	for _, alertID := range toClear {
		m.clearAlert(alertID)
	}
}

func (m *Manager) clearDockerHostContainerAlerts(host models.DockerHost) {
	prefix := fmt.Sprintf("docker:%s/", strings.TrimSpace(host.ID))
	updateTrackingPrefix := dockerUpdateTrackingHostPrefix(host)

	m.mu.Lock()
	toClear := make([]string, 0)
	for storageKey, alert := range m.activeAlerts {
		alertID := effectiveAlertID(alert, storageKey)
		if strings.HasPrefix(alert.ResourceID, prefix) || isLegacyDockerHostContainerAlert(alert, host) {
			toClear = append(toClear, alertID)
		}
	}
	for _, resourceID := range m.core.PendingResourceIDs() {
		if strings.HasPrefix(resourceID, prefix) {
			m.core.DropPendingForResource(resourceID)
		}
	}
	for resourceID := range m.dockerRestartTracking {
		if strings.HasPrefix(resourceID, prefix) {
			delete(m.dockerRestartTracking, resourceID)
		}
	}
	for resourceID := range m.dockerUpdateFirstSeen {
		if strings.HasPrefix(resourceID, prefix) {
			delete(m.dockerUpdateFirstSeen, resourceID)
			delete(m.dockerUpdateLastObserved, resourceID)
		}
	}
	for trackingKey := range m.dockerUpdateFirstSeenByIdentity {
		if strings.HasPrefix(trackingKey, updateTrackingPrefix) {
			delete(m.dockerUpdateFirstSeenByIdentity, trackingKey)
			delete(m.dockerUpdateLastObserved, trackingKey)
		}
	}
	m.mu.Unlock()

	for _, alertID := range toClear {
		m.clearAlert(alertID)
	}
}