pulse/internal/monitoring/monitor_polling_node_helpers.go

package monitoring

import (
	"context"
	stderrors "errors"
	"fmt"
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
	"github.com/rs/zerolog/log"
)

func resolveNodeConnectionInfo(instanceCfg *config.PVEInstance, discoveryCfg config.DiscoveryConfig, nodeName string) (string, string) {
	connectionHost := instanceCfg.Host
	guestURL := instanceCfg.GuestURL
	if instanceCfg.IsCluster && len(instanceCfg.ClusterEndpoints) > 0 {
		hasFingerprint := instanceCfg.Fingerprint != ""
		for _, ep := range instanceCfg.ClusterEndpoints {
			if strings.EqualFold(ep.NodeName, nodeName) {
				if effective := clusterEndpointRuntimeURL(ep, instanceCfg.VerifySSL, hasFingerprint, discoveryCfg); effective != "" {
					connectionHost = effective
				}
				if ep.GuestURL != "" {
					guestURL = ep.GuestURL
				}
				break
			}
		}
	}

	return connectionHost, guestURL
}

func (m *Monitor) determineNodeIDAndStatus(instanceName string, instanceCfg *config.PVEInstance, node proxmox.Node) (string, string) {
	// Apply grace period for node status to prevent flapping
	// The configured instance is the canonical provider identity. Normally a
	// cluster name remains the stable scope used by earlier Pulse releases,
	// but two unrelated clusters are allowed to share that display name. In
	// that ambiguous case the instance name prevents cross-cluster resource
	// collisions without churning IDs for established unambiguous clusters.
	nodeID := m.pveNodeID(instanceName, instanceCfg, node.Node)
	effectiveStatus := node.Status
	now := time.Now()
	gracePeriod := m.pveNodeOfflineGracePeriod()

	m.mu.Lock()
	if strings.ToLower(node.Status) == "online" {
		// Node is online - update last-online timestamp
		m.nodeLastOnline[nodeID] = now
	} else {
		// Node is reported as offline - check grace period
		lastOnline, exists := m.nodeLastOnline[nodeID]
		if exists && now.Sub(lastOnline) < gracePeriod {
			// Still within grace period - preserve online status
			effectiveStatus = "online"
			log.Debug().
				Str("instance", instanceName).
				Str("node", node.Node).
				Dur("timeSinceOnline", now.Sub(lastOnline)).
				Dur("gracePeriod", gracePeriod).
				Msg("Node offline but within grace period - preserving online status")
		} else {
			// Grace period expired or never seen online - mark as offline
			if exists {
				log.Info().
					Str("instance", instanceName).
					Str("node", node.Node).
					Dur("timeSinceOnline", now.Sub(lastOnline)).
					Msg("Node offline and grace period expired - marking as offline")
			}
		}
	}
	m.mu.Unlock()

	return nodeID, effectiveStatus
}

func (m *Monitor) pveNodeID(instanceName string, instanceCfg *config.PVEInstance, nativeName string) string {
	if instanceCfg != nil && instanceCfg.IsCluster {
		if identityID := config.PVEClusterNodeIdentityForName(instanceCfg, nativeName); identityID != "" {
			return identityID
		}
	}
	return m.pveNodeIdentityScope(instanceName, instanceCfg) + "-" + nativeName
}

// pveNodeTLSFingerprint resolves the TOFU-captured TLS certificate
// fingerprint recorded in configuration for one node's API endpoint. The
// fingerprint is per-machine identity evidence for node state aggregation, so
// a cluster node only inherits the fingerprint of its own named endpoint -
// never the instance-level fingerprint, which pins whichever member the
// connection URL happens to reach. Unknown stays empty and asserts nothing.
func pveNodeTLSFingerprint(instanceCfg *config.PVEInstance, nodeName string) string {
	if instanceCfg == nil {
		return ""
	}
	if !instanceCfg.IsCluster {
		return strings.TrimSpace(instanceCfg.Fingerprint)
	}
	nodeName = strings.TrimSpace(nodeName)
	match := ""
	for _, endpoint := range instanceCfg.ClusterEndpoints {
		if !strings.EqualFold(strings.TrimSpace(endpoint.NodeName), nodeName) {
			continue
		}
		fingerprint := strings.TrimSpace(endpoint.Fingerprint)
		if fingerprint == "" {
			continue
		}
		if match != "" && !strings.EqualFold(match, fingerprint) {
			// Ambiguous duplicate endpoint records fail unknown.
			return ""
		}
		match = fingerprint
	}
	return match
}

func (m *Monitor) pveNodeUsesProviderScopedIdentity(instanceName string, instanceCfg *config.PVEInstance) bool {
	if instanceCfg == nil || !instanceCfg.IsCluster || strings.TrimSpace(instanceCfg.ClusterName) == "" {
		return false
	}
	return !strings.EqualFold(
		strings.TrimSpace(m.pveNodeIdentityScope(instanceName, instanceCfg)),
		strings.TrimSpace(instanceCfg.ClusterName),
	)
}

// nodeTemperatureCarryFloor is the shortest time a node may keep showing its
// last temperature after collection stops returning one, sized to ride out the
// SSH collector's early failure backoff steps (30s, 1m, 2m).
const nodeTemperatureCarryFloor = 5 * time.Minute

// nodeTemperatureCarryWindow bounds how old a carried-over node temperature may
// be. Beyond it the reading no longer describes the node, so it stops being
// presented as available.
func (m *Monitor) nodeTemperatureCarryWindow() time.Duration {
	return resourceStaleThresholdForPollInterval(effectivePVEPollingIntervalForConfig(m.config), nodeTemperatureCarryFloor)
}

func (m *Monitor) collectNodeTemperatureData(
	ctx context.Context,
	instanceName string,
	instanceCfg *config.PVEInstance,
	node proxmox.Node,
	modelNode *models.Node,
	prevInstanceNodes []models.Node,
	effectiveStatus string,
) {
	if modelNode == nil {
		return
	}

	// Collect temperature data via SSH (non-blocking, best effort)
	// Only attempt for online nodes when temperature monitoring is enabled
	// Check per-node setting first, fall back to global setting
	tempMonitoringEnabled := m.config.TemperatureMonitoringEnabled
	if instanceCfg.TemperatureMonitoringEnabled != nil {
		tempMonitoringEnabled = *instanceCfg.TemperatureMonitoringEnabled
	}
	if effectiveStatus == "online" && tempMonitoringEnabled {
		// First, check if there's a matching host agent with temperature data.
		// Host agent temperatures are preferred because they don't require SSH access.
		// Match by the polled node's identity, not its name, so two connections'
		// same-named nodes (e.g., "px1" at two sites) never share an agent.
		hostAgentTemp := m.getHostAgentTemperatureForNode(*modelNode)
		if hostAgentTemp != nil {
			log.Debug().
				Str("node", node.Node).
				Float64("cpuPackage", hostAgentTemp.CPUPackage).
				Float64("cpuMax", hostAgentTemp.CPUMax).
				Int("nvmeCount", len(hostAgentTemp.NVMe)).
				Msg("Using temperature data from host agent")
		}

		// If no host agent temp or we need additional data (SMART), try SSH/proxy collection
		var sshTemp *models.Temperature
		var err error
		if m.tempCollector != nil {
			// Temperature collection is best-effort - use a short timeout to avoid blocking node polling
			// Use context.Background() so the timeout is truly independent of the parent polling context
			// If SSH is slow or unresponsive, we'll preserve previous temperature data
			tempCtx, tempCancel := context.WithTimeout(context.Background(), 10*time.Second)
			defer tempCancel()

			// Determine SSH hostname to use (most robust approach):
			// Prefer the resolved host for this node, with cluster overrides when available.
			sshHost := modelNode.Host
			foundNodeEndpoint := false

			if modelNode.IsClusterMember && instanceCfg.IsCluster {
				// Try to find specific endpoint configuration for this node
				if len(instanceCfg.ClusterEndpoints) > 0 {
					hasFingerprint := instanceCfg.Fingerprint != ""
					for _, ep := range instanceCfg.ClusterEndpoints {
						if strings.EqualFold(ep.NodeName, node.Node) {
							if effective := clusterEndpointEffectiveURL(ep, instanceCfg.VerifySSL, hasFingerprint); effective != "" {
								sshHost = effective
								foundNodeEndpoint = true
							}
							break
						}
					}
				}

				// If no specific endpoint found, fall back to node name
				if !foundNodeEndpoint {
					sshHost = node.Node
					log.Debug().
						Str("node", node.Node).
						Str("instance", instanceCfg.Name).
						Msg("Node endpoint not found in cluster metadata - falling back to node name for temperature collection")
				}
			}

			if strings.TrimSpace(sshHost) == "" {
				sshHost = node.Node
			}

			// A recent host agent payload is authoritative for this node.
			// Legacy SSH remains only as a fallback when no usable local
			// temperature reading has arrived from the linked agent.
			skipSSHCollection := shouldSkipTemperatureSSHCollection(hostAgentTemp)

			if !skipSSHCollection {
				sshTemp, err = m.tempCollector.CollectTemperature(tempCtx, sshHost, node.Node)
				if err != nil && hostAgentTemp == nil {
					log.Debug().
						Str("node", node.Node).
						Str("sshHost", sshHost).
						Bool("isCluster", modelNode.IsClusterMember).
						Int("endpointCount", len(instanceCfg.ClusterEndpoints)).
						Msg("Temperature collection failed - check SSH access")
				}
			}

			// Debug: log SSH temp details before merge
			if sshTemp != nil {
				log.Debug().
					Str("node", node.Node).
					Bool("sshTempAvailable", sshTemp.Available).
					Bool("sshHasSMART", sshTemp.HasSMART).
					Int("sshSMARTCount", len(sshTemp.SMART)).
					Bool("sshHasNVMe", sshTemp.HasNVMe).
					Int("sshNVMeCount", len(sshTemp.NVMe)).
					Msg("SSH temperature data before merge")
			} else {
				log.Debug().
					Str("node", node.Node).
					Msg("SSH temperature data is nil")
			}
		}

		// Merge host agent and SSH temperatures
		temp := mergeTemperatureData(hostAgentTemp, sshTemp)

		if temp != nil && temp.Available {
			// Get the current CPU temperature (prefer package, fall back to max)
			currentTemp := temp.CPUPackage
			if currentTemp == 0 && temp.CPUMax > 0 {
				currentTemp = temp.CPUMax
			}

			// Find previous temperature data for this node to preserve min/max
			var prevTemp *models.Temperature
			for _, prevNode := range prevInstanceNodes {
				if prevNode.ID == modelNode.ID && prevNode.Temperature != nil {
					prevTemp = prevNode.Temperature
					break
				}
			}

			// Initialize or update min/max tracking
			if prevTemp != nil && prevTemp.CPUMin > 0 {
				// Preserve existing min/max and update if necessary
				temp.CPUMin = prevTemp.CPUMin
				temp.CPUMaxRecord = prevTemp.CPUMaxRecord
				temp.MinRecorded = prevTemp.MinRecorded
				temp.MaxRecorded = prevTemp.MaxRecorded

				// Update min if current is lower
				if currentTemp > 0 && currentTemp < temp.CPUMin {
					temp.CPUMin = currentTemp
					temp.MinRecorded = time.Now()
				}

				// Update max if current is higher
				if currentTemp > temp.CPUMaxRecord {
					temp.CPUMaxRecord = currentTemp
					temp.MaxRecorded = time.Now()
				}
			} else if currentTemp > 0 {
				// First reading - initialize min/max to current value
				temp.CPUMin = currentTemp
				temp.CPUMaxRecord = currentTemp
				temp.MinRecorded = time.Now()
				temp.MaxRecorded = time.Now()
			}

			modelNode.Temperature = temp

			// Determine source for logging
			tempSource := "ssh"
			if hostAgentTemp != nil && sshTemp == nil {
				tempSource = "agent"
			} else if hostAgentTemp != nil && sshTemp != nil {
				tempSource = "agent+ssh"
			}

			log.Debug().
				Str("node", node.Node).
				Str("source", tempSource).
				Float64("cpuPackage", temp.CPUPackage).
				Float64("cpuMax", temp.CPUMax).
				Float64("cpuMin", temp.CPUMin).
				Float64("cpuMaxRecord", temp.CPUMaxRecord).
				Int("nvmeCount", len(temp.NVMe)).
				Msg("Collected temperature data")
		} else {
			// Temperature data returned but not available (temp != nil && !temp.Available)
			// OR no temperature data from any source - preserve previous temperature if available
			// This prevents the temperature column from flickering when collection temporarily fails
			var prevTemp *models.Temperature
			for _, prevNode := range prevInstanceNodes {
				if prevNode.ID == modelNode.ID && prevNode.Temperature != nil && prevNode.Temperature.Available {
					prevTemp = prevNode.Temperature
					break
				}
			}

			if m.carriedTemperatureOutlivesAgentLease(*modelNode, prevTemp, time.Now()) {
				// The reading came from a host agent that has stopped reporting.
				// Its reporting lease already rode out transient gaps, so
				// carrying it would outlive that lease.
				log.Debug().
					Str("node", node.Node).
					Time("lastUpdate", prevTemp.LastUpdate).
					Msg("Dropped temperature data (host agent stopped reporting)")
			} else if prevTemp != nil && time.Since(prevTemp.LastUpdate) > m.nodeTemperatureCarryWindow() {
				// Collection has stayed broken past the carry window. Re-presenting
				// the old reading would let alerts, history and the UI treat it as
				// a live measurement forever, so the node has no reading now.
				log.Debug().
					Str("node", node.Node).
					Bool("isCluster", modelNode.IsClusterMember).
					Time("lastUpdate", prevTemp.LastUpdate).
					Msg("Dropped stale temperature data (collection has not returned a reading within the carry window)")
			} else if preserved := carriedNodeTemperature(prevTemp); preserved != nil {
				// The carried copy keeps its original LastUpdate to indicate staleness.
				modelNode.Temperature = preserved
				log.Debug().
					Str("node", node.Node).
					Bool("isCluster", modelNode.IsClusterMember).
					Float64("cpuPackage", preserved.CPUPackage).
					Time("lastUpdate", preserved.LastUpdate).
					Msg("Preserved previous temperature data (current collection failed or unavailable)")
			} else {
				log.Debug().
					Str("node", node.Node).
					Bool("isCluster", modelNode.IsClusterMember).
					Msg("No temperature data available (collection failed, no previous data to preserve)")
			}
		}
	}
}

// carriedNodeTemperature returns the part of an earlier reading a node may
// keep presenting while collection fails: its CPU and GPU temperatures, with
// the original LastUpdate. Disk temperatures (SMART, NVMe) are left out. They
// also stamp the node's physical disks, which have their own freshness and
// standby state that a carried copy cannot follow: a disk that spun down would
// be stamped, and recorded, with its pre-sleep reading. Returns nil when the
// earlier reading held nothing else.
func carriedNodeTemperature(prev *models.Temperature) *models.Temperature {
	if prev == nil || !prev.Available {
		return nil
	}
	carried := *prev
	carried.Cores = append([]models.CoreTemp(nil), prev.Cores...)
	carried.GPU = append([]models.GPUTemp(nil), prev.GPU...)
	carried.SMART = []models.DiskTemp{}
	carried.HasSMART = false
	carried.NVMe = []models.NVMeTemp{}
	carried.HasNVMe = false
	if !hasUsableTemperatureReading(&carried) {
		return nil
	}
	return &carried
}

func (m *Monitor) applyNodePendingUpdates(ctx context.Context, instanceName string, client PVEClientInterface, node proxmox.Node, nodeID string, effectiveStatus string, modelNode *models.Node) {
	if modelNode == nil {
		return
	}
	modelNode.PendingUpdatesStatus = "not_checked"
	modelNode.PendingUpdatesReason = ""
	if effectiveStatus != "online" {
		modelNode.PendingUpdatesReason = "node_offline"
		return
	}

	// Poll pending apt updates (less frequently - every 30 minutes)
	// Only for online nodes to avoid wasting API calls on offline nodes
	if effectiveStatus == "online" {
		now := time.Now()
		m.mu.RLock()
		if m.nodePendingUpdatesCache == nil {
			m.mu.RUnlock()
			m.mu.Lock()
			if m.nodePendingUpdatesCache == nil {
				m.nodePendingUpdatesCache = make(map[string]pendingUpdatesCache)
			}
			m.mu.Unlock()
			m.mu.RLock()
		}
		cached, hasCached := m.nodePendingUpdatesCache[nodeID]
		m.mu.RUnlock()

		if !hasCached || now.Sub(cached.checkedAt) >= pendingUpdatesCacheTTL {
			// Time to check for updates
			pendingPkgs, err := client.GetNodePendingUpdates(ctx, node.Node)
			if err != nil {
				// Preserve the last successful value, but mark it stale rather than
				// presenting a permission or reachability failure as a confirmed zero.
				log.Debug().
					Err(err).
					Str("node", node.Node).
					Str("instance", instanceName).
					Msg("Could not check pending apt updates with configured Proxmox credentials")
				modelNode.PendingUpdatesReason = pendingUpdatesFailureReason(err)
				if hasCached {
					modelNode.PendingUpdates = cached.count
					modelNode.PendingUpdatesCheckedAt = cached.checkedAt
					modelNode.PendingUpdatesStatus = "stale"
				} else {
					modelNode.PendingUpdatesStatus = "unavailable"
				}
			} else {
				updateCount := len(pendingPkgs)
				modelNode.PendingUpdates = updateCount
				modelNode.PendingUpdatesCheckedAt = now
				modelNode.PendingUpdatesStatus = "checked"

				// Cache the result
				m.mu.Lock()
				m.nodePendingUpdatesCache[nodeID] = pendingUpdatesCache{
					count:     updateCount,
					checkedAt: now,
				}
				m.mu.Unlock()

				log.Debug().
					Str("node", node.Node).
					Str("instance", instanceName).
					Int("pendingUpdates", updateCount).
					Msg("Checked pending apt updates")
			}
		} else {
			// Use cached value
			modelNode.PendingUpdates = cached.count
			modelNode.PendingUpdatesCheckedAt = cached.checkedAt
			modelNode.PendingUpdatesStatus = "checked"
		}
	}
}

func pendingUpdatesFailureReason(err error) string {
	if err == nil {
		return ""
	}
	message := strings.ToLower(err.Error())
	switch {
	case strings.Contains(message, "403"), strings.Contains(message, "permission"):
		return "permission_denied"
	case strings.Contains(message, "no healthy nodes"), strings.Contains(message, "595"),
		strings.Contains(message, "offline"), strings.Contains(message, "unavailable"),
		strings.Contains(message, "connection"), strings.Contains(message, "timeout"):
		return "source_unavailable"
	default:
		return "check_failed"
	}
}

func (m *Monitor) recordNodePollMetrics(instanceName string, node proxmox.Node, modelNode *models.Node, nodeStart time.Time) {
	if m.pollMetrics == nil || modelNode == nil {
		return
	}

	nodeNameLabel := strings.TrimSpace(node.Node)
	if nodeNameLabel == "" {
		nodeNameLabel = strings.TrimSpace(modelNode.DisplayName)
	}
	if nodeNameLabel == "" {
		nodeNameLabel = "unknown-node"
	}

	success := true
	nodeErrReason := ""
	health := strings.ToLower(strings.TrimSpace(modelNode.ConnectionHealth))
	if health != "" && health != "healthy" {
		success = false
		nodeErrReason = fmt.Sprintf("connection health %s", health)
	}

	status := strings.ToLower(strings.TrimSpace(modelNode.Status))
	if success && status != "" && status != "online" {
		success = false
		nodeErrReason = fmt.Sprintf("status %s", status)
	}

	var nodeErr error
	if !success {
		if nodeErrReason == "" {
			nodeErrReason = "unknown node error"
		}
		nodeErr = stderrors.New(nodeErrReason)
	}

	m.pollMetrics.RecordNodeResult(NodePollResult{
		InstanceName: instanceName,
		InstanceType: "pve",
		NodeName:     nodeNameLabel,
		Success:      success,
		Error:        nodeErr,
		StartTime:    nodeStart,
		EndTime:      time.Now(),
	})
}