pulse/internal/alerts/node.go

package alerts

import (
	"fmt"
	"strings"
	"time"

	alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rs/zerolog/log"
)

// CheckNode checks a node against thresholds
func (m *Manager) CheckNode(node models.Node) {
	// Cache display name so all alerts (including guest alerts on this node) can resolve it.
	m.UpdateNodeDisplayName(node.Instance, node.Name, node.DisplayName)
	for _, alias := range node.NativeNameAliases {
		m.UpdateNodeDisplayName(node.Instance, alias, node.DisplayName)
	}

	m.mu.RLock()
	if !m.config.Enabled {
		m.mu.RUnlock()
		return
	}
	if allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("node"); allDisabled {
		m.mu.RUnlock()
		// Clear any existing node alerts when all node alerts are disabled
		m.mu.Lock()
		// Clear offline tracking
		// Clear all possible node alert types
		alertTypes := []string{"cpu", "memory", "disk", "temperature"}
		for _, alertType := range alertTypes {
			alertID := canonicalMetricStateID(node.ID, alertType)
			if m.hasActiveAlertNoLock(alertID) {
				m.clearAlertNoLock(alertID)
				log.Info().
					Str("alertID", alertID).
					Str("node", node.Name).
					Msg("Cleared node alert - all node alerts disabled")
			}
		}
		// Clear offline alert
		offlineAlertID := canonicalConnectivityStateID(node.ID)
		if m.hasActiveAlertNoLock(offlineAlertID) {
			m.clearAlertNoLock(offlineAlertID)
			log.Info().
				Str("alertID", offlineAlertID).
				Str("node", node.Name).
				Msg("Cleared offline alert - all node alerts disabled")
		}
		m.mu.Unlock()
		return
	}
	_, disableNodesOffline := m.alertPolicyTypeSwitchesNoLock("node")
	thresholds := m.resolveResourceThresholds("node", node.ID)
	m.mu.RUnlock()

	if thresholds.Disabled {
		m.mu.Lock()
		m.mu.Unlock()
		for _, alertID := range []string{
			canonicalMetricStateID(node.ID, "cpu"),
			canonicalMetricStateID(node.ID, "memory"),
			canonicalMetricStateID(node.ID, "disk"),
			canonicalMetricStateID(node.ID, "temperature"),
			canonicalConnectivityStateID(node.ID),
		} {
			m.clearAlert(alertID)
		}
		return
	}

	if disableNodesOffline || thresholds.DisableConnectivity {
		// Clear tracking and any existing offline alerts when globally disabled
		m.mu.Lock()
		m.mu.Unlock()
		m.clearAlert(canonicalConnectivityStateID(node.ID))
	} else {
		// CRITICAL: Check if node is offline first
		if node.Status == "offline" || node.ConnectionHealth == "error" || node.ConnectionHealth == "failed" {
			m.checkNodeOffline(node)

			// Clear resource alerts if node is offline/unreachable.
			// This prevents stale alerts from persisting when we can't get new data.
			metrics := []string{"cpu", "memory", "disk", "temperature"}
			for _, metric := range metrics {
				m.clearAlert(canonicalMetricStateID(node.ID, metric))
			}
		} else {
			// Clear any existing offline alert if node is back online
			m.clearNodeOfflineAlert(node)

			// Check each metric (only if node is online and reachable)
			// Check for host agent deduplication: a usage metric that a linked host
			// agent evaluates belongs to the agent resource (CheckHost raises it), so
			// the node releases its own copy instead of alerting twice. A metric the
			// agent does not evaluate (agent alerts disabled, threshold off, usage
			// unknown) stays with the node, so the machine is never left unmonitored.
			owners := m.linkedHostAgentsForNode(node.ID)
			input := &UnifiedResourceInput{
				ID:       node.ID,
				Type:     "node",
				Name:     node.Name,
				Node:     node.Name,
				Instance: node.Instance,
			}
			released := make([]string, 0, 3)
			if owners.cpu.cpu {
				released = append(released, "cpu")
			} else {
				input.CPU = &UnifiedResourceMetric{Percent: node.CPU * 100}
			}
			if owners.memory.memory {
				released = append(released, "memory")
			} else if node.Memory.HasKnownUsage() {
				input.Memory = &UnifiedResourceMetric{Percent: node.Memory.Usage}
			}
			if owners.disk.disk {
				released = append(released, "disk")
			} else if percent, observed := unifiedresources.DiskUsagePercent(node.Disk); observed {
				input.Disk = &UnifiedResourceMetric{Percent: percent}
			} else if !liveHysteresisThreshold(thresholds.Disk) {
				// Explicit policy still releases the alert, even without a
				// reading. Otherwise leave it unknown: missing capacity or
				// contradictory used space is not measured zero/recovery.
				input.Disk = &UnifiedResourceMetric{}
			}
			m.releaseNodeMetricAlerts(node, owners, released...)
			m.evaluateUnifiedMetrics(input, thresholds, nil, "temperature") // checkNodeTemperature owns its observation gaps.

			// CPU temperature stays with the node even when a host agent runs on it:
			// CheckHost has no CPU temperature metric, and the node poll already
			// merges the agent's sensor readings into node.Temperature.
			m.checkNodeTemperature(node, thresholds.Temperature)
		}
	}
}

// checkNodeTemperature evaluates the node CPU temperature alert.
// A disabled threshold (nil or trigger 0) is always passed through so any
// existing alert is cleared. Otherwise a missing CPU reading is not evidence:
// feeding 0°C would resolve an open alert without proving the node cooled down,
// so the open alert is kept and only its timing runs are interrupted.
func (m *Manager) checkNodeTemperature(node models.Node, threshold *HysteresisThreshold) {
	// The alert judges the same reading the node's table, drawer and history
	// show, so an alert never names a value the operator cannot find.
	var temp float64
	if primary := models.PrimaryCPUTemperatureCelsius(node.Temperature); primary != nil {
		temp = *primary
	}
	spec, err := buildCanonicalMetricSpec(node.ID, node.Name, unifiedresources.ResourceType("node"), "temperature", threshold)
	if err != nil {
		log.Warn().
			Err(err).
			Str("resourceID", node.ID).
			Str("node", node.Name).
			Msg("Skipping invalid canonical node temperature metric spec")
		return
	}
	if temp <= 0 && !spec.Disabled {
		m.interruptMetricRun(spec)
		return
	}
	m.checkMetricWithCanonicalSpec(spec, node.Name, node.Name, node.Instance, "node", temp, threshold, nil)
}

// releaseNodeMetricAlerts stops node-side evaluation of the given metrics the
// same way a disabled threshold does: any pending run is dropped and any open
// node alert is resolved. Without this, a node alert that was open when a host
// agent registered would stay frozen until the agent went offline. Each close
// names an agent that actually covers that metric, not an arbitrary sibling.
func (m *Manager) releaseNodeMetricAlerts(node models.Node, owners hostAgentUsageOwners, metrics ...string) {
	for _, metric := range metrics {
		agent := owners.forMetric(metric)
		resolution := &AlertResolution{
			Reason:              AlertResolutionMovedToAgent,
			SuccessorResourceID: hostResourceID(agent.agentID),
			SuccessorName:       agent.agentName,
		}
		spec, err := buildCanonicalMetricSpec(node.ID, node.Name, unifiedresources.ResourceType("node"), metric, nil)
		if err != nil {
			log.Warn().
				Err(err).
				Str("resourceID", node.ID).
				Str("node", node.Name).
				Str("metric", metric).
				Msg("Skipping invalid canonical node metric spec")
			continue
		}
		m.releaseCanonicalMetricAlert(spec, node.Name, node.Name, node.Instance, "node", 0, resolution)
	}
}

// UpdateNodeDisplayName caches the display name for a node/host so alerts
// can resolve it without needing the full model object.
func nodeDisplayNameCacheKey(instance, name string) string {
	return strings.TrimSpace(instance) + "\x00" + strings.TrimSpace(name)
}

func (m *Manager) UpdateNodeDisplayName(parts ...string) {
	var instance, name, displayName string
	switch len(parts) {
	case 2:
		name, displayName = parts[0], parts[1]
	case 3:
		instance, name, displayName = parts[0], parts[1], parts[2]
	default:
		return
	}

	instance = strings.TrimSpace(instance)
	name = strings.TrimSpace(name)
	if name == "" {
		return
	}
	displayName = strings.TrimSpace(displayName)
	m.mu.Lock()
	if instance != "" {
		key := nodeDisplayNameCacheKey(instance, name)
		if displayName != "" && displayName != name {
			m.instanceNodeDisplayNames[key] = displayName
		} else {
			delete(m.instanceNodeDisplayNames, key)
		}
	} else {
		if displayName != "" && displayName != name {
			m.nodeDisplayNames[name] = displayName
		} else {
			delete(m.nodeDisplayNames, name)
		}
	}
	m.mu.Unlock()
}

// resolveNodeDisplayName returns the cached display name for a node, or empty
// string if none is set. Caller must hold m.mu (read or write).
func (m *Manager) resolveNodeDisplayName(instance, node string) string {
	if instance = strings.TrimSpace(instance); instance != "" {
		if displayName, ok := m.instanceNodeDisplayNames[nodeDisplayNameCacheKey(instance, node)]; ok {
			return displayName
		}
	}
	return m.nodeDisplayNames[node]
}

// checkNodeOffline creates an alert for offline nodes after confirmation
func (m *Manager) checkNodeOffline(node models.Node) {
	alertID := fmt.Sprintf("node-offline-%s", node.ID)
	correlation := NewSharedSystemAlertCorrelation(
		"pve:"+strings.TrimSpace(node.Instance),
		AlertCorrelationRoleSupporting,
		"proxmox-node-membership",
	)

	m.mu.Lock()
	m.mu.Unlock()

	thresholds := m.resolveResourceThresholds("node", node.ID)
	spec, err := buildCanonicalConnectivitySpec(node.ID, node.Name, unifiedresources.ResourceType("node"), AlertLevelCritical, 3, thresholds.Disabled || thresholds.DisableConnectivity)
	if err != nil {
		log.Warn().
			Err(err).
			Str("node", node.Name).
			Str("nodeID", node.ID).
			Msg("Skipping invalid canonical node connectivity spec")
		return
	}

	_, _ = m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
		Spec:         spec,
		Evidence:     alertspecs.AlertEvidence{ObservedAt: time.Now(), Connectivity: &alertspecs.ConnectivityEvidence{Signal: "status", Connected: false}},
		AlertID:      alertID,
		AlertType:    "connectivity",
		ResourceID:   node.ID,
		ResourceName: node.Name,
		Node:         node.Name,
		Instance:     node.Instance,
		Message:      fmt.Sprintf("Node '%s' is offline", node.Name),
		Correlation:  correlation,
		Metadata: map[string]interface{}{
			"resourceType":     "node",
			"status":           node.Status,
			"connectionHealth": node.ConnectionHealth,
		},
		AddToRecent:   true,
		AddToHistory:  true,
		RateLimit:     true,
		DispatchAsync: false,
	})
}

// clearNodeOfflineAlert removes offline alert when node comes back online
func (m *Manager) clearNodeOfflineAlert(node models.Node) {
	alertID := canonicalConnectivityStateID(node.ID)

	m.mu.Lock()
	defer m.mu.Unlock()
	defer m.shadowObserveRecoveryNoLock(node.ID, canonicalConnectivitySpecID(node.ID), alertID, offlineRecoveryConfirmationsDefault)

	// Reset the legacy offline counter; the reducer core owns the
	// confirmation run itself.

	m.resolveDiscreteRecoveryNoLock(node.ID, canonicalConnectivitySpecID(node.ID), alertID, offlineRecoveryConfirmationsDefault, "Node", node.Name, node.Instance)
}