pulse/pkg/proxmox/cluster_client.go

package proxmox

import (
	"context"
	"errors"
	"fmt"
	"math/rand"
	"regexp"
	"slices"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/securityutil"
	"github.com/rcourtman/pulse-go-rewrite/pkg/tlsutil"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
)

// ClusterClient wraps multiple Proxmox clients for cluster-aware operations
type ClusterClient struct {
	mu                              sync.RWMutex
	clientInitMu                    sync.Mutex // Serialize lazy construction while a trust probe releases mu.
	name                            string
	clients                         map[string]*Client   // Key is node name
	endpoints                       []string             // All available endpoints
	endpointFingerprints            map[string]string    // Per-endpoint TLS fingerprints (TOFU)
	conflictingEndpointFingerprints map[string]bool      // Exact duplicate saved rows refused by the producer.
	nodeHealth                      map[string]bool      // Track node health
	lastHealthCheck                 map[string]time.Time // Track last health check time
	lastError                       map[string]string    // Track last error per endpoint
	config                          ClientConfig         // Base config (auth info)
	rateLimitUntil                  map[string]time.Time // Cooldown window for rate-limited endpoints
	recoveryInProgress              bool                 // Prevent overlapping background recovery sweeps
}

const (
	rateLimitBaseDelay        = 150 * time.Millisecond
	rateLimitMaxJitter        = 200 * time.Millisecond
	rateLimitRetryBudget      = 2
	backgroundRecoveryTimeout = 6 * time.Second
	unhealthyRecoveryInterval = 10 * time.Second
)

var statusCodePattern = regexp.MustCompile(`(?i)(?:api error|status)\s+(\d{3})`)

var transientRateLimitStatusCodes = map[int]struct{}{
	408: {},
	425: {}, // Too Early
	429: {},
	502: {},
	503: {},
	504: {},
}

var errConflictingEndpointFingerprints = errors.New("TLS trust conflict - equivalent endpoint URLs have different saved fingerprints; independently verify and reconcile saved trust")

// isEndpointConnectivityError reports whether an error indicates the endpoint
// itself is unreachable (TCP, DNS, TLS failures). Application-level errors
// (HTTP responses, parsing errors) mean the endpoint IS reachable.
func isEndpointConnectivityError(err error) bool {
	if err == nil {
		return false
	}
	errStr := strings.ToLower(err.Error())

	// If we received an HTTP response from Proxmox (any status code),
	// the endpoint is reachable.
	if strings.Contains(errStr, "api error") {
		return false
	}

	// TCP/DNS connectivity failures
	if strings.Contains(errStr, "connection refused") ||
		strings.Contains(errStr, "connection reset") ||
		strings.Contains(errStr, "no such host") ||
		strings.Contains(errStr, "network is unreachable") ||
		strings.Contains(errStr, "no route to host") ||
		strings.Contains(errStr, "i/o timeout") ||
		strings.Contains(errStr, "dial tcp") ||
		strings.Contains(errStr, "dial:") {
		return true
	}

	// TLS failures
	if strings.Contains(errStr, "tls handshake") ||
		strings.Contains(errStr, "tls:") ||
		strings.Contains(errStr, "certificate") ||
		strings.Contains(errStr, "fingerprint mismatch") {
		return true
	}

	return false
}

// sanitizeEndpointError transforms raw Go errors into user-friendly messages
// for display in the UI. The original error is preserved in logs.
func sanitizeEndpointError(errMsg string) string {
	if errMsg == "" {
		return errMsg
	}
	if strings.Contains(errMsg, errConflictingEndpointFingerprints.Error()) {
		return errConflictingEndpointFingerprints.Error()
	}

	lower := strings.ToLower(errMsg)

	// Context deadline exceeded - usually means slow API response
	if strings.Contains(lower, "context deadline exceeded") {
		// Check for specific causes
		if strings.Contains(lower, "/storage") {
			return "Request timed out - storage API slow (check for unreachable PBS/NFS/Ceph backends)"
		}
		if strings.Contains(lower, "pbs-") || strings.Contains(lower, ":8007") {
			return "Request timed out - PBS storage backend unreachable"
		}
		return "Request timed out - Proxmox API may be slow or waiting on unreachable backend services"
	}

	// Client timeout - similar to context deadline
	if strings.Contains(lower, "client.timeout exceeded") {
		return "Connection timed out - Proxmox API not responding in time"
	}

	// Connection refused
	if strings.Contains(lower, "connection refused") {
		return "Connection refused - Proxmox API not running or firewall blocking"
	}

	// No route to host
	if strings.Contains(lower, "no route to host") {
		return "Network unreachable - check network connectivity to Proxmox host"
	}

	// TLS/certificate errors
	if strings.Contains(lower, "fingerprint mismatch") {
		return "TLS fingerprint changed - independently verify the certificate before updating the saved fingerprint"
	}
	if strings.Contains(lower, "certificate") || strings.Contains(lower, "x509") {
		return "TLS certificate error - check SSL settings or add fingerprint"
	}

	// Auth errors - keep these specific
	if strings.Contains(lower, "authentication") || strings.Contains(lower, "401") || strings.Contains(lower, "403") {
		return "Authentication failed - check API token or credentials"
	}

	// PBS-specific errors
	if strings.Contains(lower, "can't connect to") && strings.Contains(lower, ":8007") {
		return "PBS storage unreachable - check Proxmox Backup Server connectivity"
	}

	// Return original if no transformation applies
	return errMsg
}

// NewClusterClient creates a new cluster-aware client.
// endpointFingerprints is an optional map of endpoint URL -> TLS fingerprint for per-node certificate verification.
// This enables TOFU (Trust On First Use) for clusters with unique self-signed certs per node.
// Optional conflictingEndpoints retain contradictory exact duplicate saved rows
// which a map cannot represent. They refuse only those endpoint trust origins.
func NewClusterClient(name string, config ClientConfig, endpoints []string, endpointFingerprints map[string]string, conflictingEndpoints ...string) *ClusterClient {
	cc, _ := NewClusterClientWithContext(context.Background(), name, config, endpoints, endpointFingerprints, conflictingEndpoints...)
	return cc
}

// NewClusterClientWithContext keeps synchronous startup checks and password
// session construction within the caller's lifetime. Endpoint failures still
// produce a recoverable cluster; cancellation does not publish a partial one.
func NewClusterClientWithContext(ctx context.Context, name string, config ClientConfig, endpoints []string, endpointFingerprints map[string]string, conflictingEndpoints ...string) (*ClusterClient, error) {
	if err := ctx.Err(); err != nil {
		return nil, err
	}
	// The caller may reuse configuration storage after construction. Keep live
	// routing and first-use trust on the same membership snapshot registered for
	// guest-command coordination, including during initial health checks.
	endpoints = slices.Clone(endpoints)
	registerGuestAgentEndpoints(config.Host, endpoints)
	// The client owns subsequent first-use captures. Do not mutate the caller's
	// saved trust map or let another client overwrite it without our mutex.
	ownedFingerprints := make(map[string]string, len(endpointFingerprints))
	for endpoint, fingerprint := range endpointFingerprints {
		ownedFingerprints[endpoint] = fingerprint
	}
	ownedConflicts := make(map[string]bool, len(conflictingEndpoints))
	for _, endpoint := range conflictingEndpoints {
		ownedConflicts[endpoint] = true
	}
	cc := &ClusterClient{
		name:                            name,
		clients:                         make(map[string]*Client),
		endpoints:                       endpoints,
		endpointFingerprints:            ownedFingerprints,
		conflictingEndpointFingerprints: ownedConflicts,
		nodeHealth:                      make(map[string]bool),
		lastHealthCheck:                 make(map[string]time.Time),
		lastError:                       make(map[string]string),
		config:                          config,
		rateLimitUntil:                  make(map[string]time.Time),
	}

	// Initialize all endpoints as unknown (will be tested on first use)
	// Start optimistically - assume healthy until proven otherwise
	// This allows operations to be attempted even if initial health check fails
	for _, endpoint := range endpoints {
		cc.nodeHealth[endpoint] = true // Start optimistic, will be marked unhealthy if operations fail
	}

	// Do a quick parallel health check on initialization (synchronous to avoid race)
	// This will mark unhealthy nodes but won't prevent trying them later
	cc.initialHealthCheck(ctx)
	if err := ctx.Err(); err != nil {
		for _, client := range cc.clients {
			client.httpClient.CloseIdleConnections()
		}
		return nil, err
	}

	return cc, nil
}

// getEndpointFingerprint returns the TLS fingerprint to use for a specific endpoint.
// It prefers the per-endpoint fingerprint (TOFU) over the base config fingerprint.
func (cc *ClusterClient) getEndpointFingerprint(endpoint string) (string, error) {
	cc.mu.RLock()
	defer cc.mu.RUnlock()
	return cc.getEndpointFingerprintLocked(endpoint)
}

func (cc *ClusterClient) getEndpointFingerprintLocked(endpoint string) (string, error) {
	if fp, known, err := cc.knownEndpointFingerprintLocked(endpoint); known || err != nil {
		return fp, err
	}
	return cc.config.Fingerprint, nil
}

func clusterEndpointKey(endpoint string) (string, bool) {
	u, err := securityutil.NormalizeHTTPBaseURL(endpoint, "https")
	if err != nil {
		return "", false
	}
	// Keep the existing TLS origin/base-path identity, without joining cluster
	// aliases or guest admission's IPv4-mapped equivalence. Serializing guest
	// work is not permission to share saved pins across request authorities.
	return proxmoxEndpointKey(u.String(), false)
}

func (cc *ClusterClient) knownEndpointFingerprintLocked(endpoint string) (string, bool, error) {
	key, valid := clusterEndpointKey(endpoint)
	for conflict := range cc.conflictingEndpointFingerprints {
		conflictKey, conflictValid := clusterEndpointKey(conflict)
		if conflict == endpoint || (valid && conflictValid && conflictKey == key) {
			return "", true, errConflictingEndpointFingerprints
		}
	}
	var fingerprint string
	known := false
	for storedEndpoint, fp := range cc.endpointFingerprints {
		if fp == "" {
			continue
		}
		storedKey, storedValid := clusterEndpointKey(storedEndpoint)
		if storedEndpoint != endpoint && (!valid || !storedValid || storedKey != key) {
			continue
		}
		// Compare the same colon/case normalization used by the TLS verifier.
		// Exact URL spelling must not choose a winner over another saved pin
		// for this authority, nor may Go map order decide which leaf is trusted.
		normalized := strings.ToLower(strings.ReplaceAll(fp, ":", ""))
		if known && strings.ToLower(strings.ReplaceAll(fingerprint, ":", "")) != normalized {
			return "", true, errConflictingEndpointFingerprints
		}
		// Keep a nonempty, malformed saved pin nonempty: returning its empty
		// normalized form would accidentally select CA or explicit opt-out mode.
		// Agreeing representations use a deterministic original spelling.
		if !known || fp < fingerprint {
			fingerprint = fp
		}
		known = true
	}
	return fingerprint, known, nil
}

// CheckEndpointFingerprintConflicts checks saved trust without constructing a
// client, probing TLS or changing any pin. Discovery must not discard ambiguity
// by collapsing saved rows or replacing their addresses during a refresh.
// This checks agreement only; malformed nonempty pins remain the TLS verifier's
// responsibility and are never treated as absent trust here.
func CheckEndpointFingerprintConflicts(fingerprints map[string]string, conflictingEndpoints ...string) error {
	if len(conflictingEndpoints) > 0 {
		return errConflictingEndpointFingerprints
	}
	cc := ClusterClient{endpointFingerprints: fingerprints}
	for endpoint := range fingerprints {
		if _, _, err := cc.knownEndpointFingerprintLocked(endpoint); err != nil {
			return err
		}
	}
	return nil
}

// Only a declared, previously untrusted member may use the primary pin as a
// first-use bootstrap. A known member or the configured primary must keep its
// established pin; recovery is never permission to replace it.
func (cc *ClusterClient) canTrustMemberOnFirstUseLocked(endpoint string) bool {
	key, valid := clusterEndpointKey(endpoint)
	primary, primaryValid := clusterEndpointKey(cc.config.Host)
	if !valid || !primaryValid || key == primary || cc.config.Fingerprint == "" {
		return false
	}
	if _, known, err := cc.knownEndpointFingerprintLocked(endpoint); known || err != nil {
		return false
	}
	for _, member := range cc.endpoints {
		if memberKey, ok := clusterEndpointKey(member); ok && memberKey == key {
			return true
		}
	}
	return false
}

func isFingerprintMismatchError(err error) bool {
	var mismatch *tlsutil.FingerprintMismatchError
	return errors.As(err, &mismatch)
}

// refreshTOFUFingerprintAndRetry captures a new member's first-use pin, never
// refreshes an established pin. Returns whether a pinned retry was attempted.
func (cc *ClusterClient) refreshTOFUFingerprintAndRetry(ctx context.Context, endpoint string, timeout time.Duration, lastErr error) (*Client, error, bool) {
	if ctx.Err() != nil || !isFingerprintMismatchError(lastErr) {
		return nil, lastErr, false
	}

	cc.mu.RLock()
	firstUse := cc.canTrustMemberOnFirstUseLocked(endpoint)
	cc.mu.RUnlock()
	if !firstUse {
		return nil, lastErr, false
	}

	newFingerprint, err := tlsutil.FetchFingerprintContext(ctx, endpoint)
	if err != nil {
		log.Warn().
			Str("cluster", cc.name).
			Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(endpoint)).
			Err(err).
			Msg("Failed to refresh TOFU fingerprint after mismatch")
		return nil, lastErr, false
	}
	if ctx.Err() != nil {
		return nil, lastErr, false
	}

	cc.mu.Lock()
	// Cancellation may have arrived while waiting for the trust mutex too.
	// Ending that caller cannot establish a new runtime pin.
	if ctx.Err() != nil {
		cc.mu.Unlock()
		return nil, lastErr, false
	}
	captured := false
	// A concurrent first-use capture may have established trust while the TLS
	// probe was in flight. Use that winner's pin, not our later certificate.
	if existing, known, trustErr := cc.knownEndpointFingerprintLocked(endpoint); trustErr != nil {
		cc.mu.Unlock()
		return nil, trustErr, false
	} else if known {
		newFingerprint = existing
	} else {
		if !cc.canTrustMemberOnFirstUseLocked(endpoint) {
			cc.mu.Unlock()
			return nil, lastErr, false
		}
		cc.endpointFingerprints[endpoint] = newFingerprint
		captured = true
	}
	cc.mu.Unlock()
	if captured {
		log.Warn().
			Str("cluster", cc.name).
			Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(endpoint)).
			Msg("Trusting cluster member certificate on first use; the pinned primary fingerprint cannot match per-node certificates")
	}

	cfg := cc.config
	cfg.Host = endpoint
	cfg.Fingerprint = newFingerprint
	cfg.Timeout = timeout

	retryClient, err := NewClientWithContext(ctx, cfg)
	if err != nil {
		return nil, err, true
	}

	retryCtx, cancel := context.WithTimeout(ctx, timeout)
	_, err = retryClient.GetNodes(retryCtx)
	cancel()

	return retryClient, err, true
}

// probeEndpoint applies the same first-use boundary whether TLS verification
// fails during password construction or the subsequent token-authenticated
// nodes read. Capture is still credential-free and only a declared, unknown
// member can establish a pin; a saved/primary pin is never replaced.
// The returned client is a disposable probe, not an accepted polling session.
func (cc *ClusterClient) probeEndpoint(ctx context.Context, cfg ClientConfig) (*Client, []Node, error) {
	fingerprint, err := cc.getEndpointFingerprint(cfg.Host)
	if err != nil {
		return nil, nil, err
	}
	cfg.Fingerprint = fingerprint
	client, err := NewClientWithContext(ctx, cfg)
	var nodes []Node
	if err == nil {
		probeCtx, cancel := context.WithTimeout(ctx, cfg.Timeout)
		nodes, err = client.GetNodes(probeCtx)
		cancel()
	}
	if retryClient, retryErr, retried := cc.refreshTOFUFingerprintAndRetry(ctx, cfg.Host, cfg.Timeout, err); retried {
		if client != nil {
			client.httpClient.CloseIdleConnections()
		}
		// The pinned retry already performed its nodes read. Do not repeat it
		// just to fill a diagnostic count or after a refused password login.
		return retryClient, nil, retryErr
	}
	return client, nodes, err
}

// Resolve saved trust again before constructing the full polling session.
// Never turn a resolution failure into an empty pin (CA or opt-out mode).
func (cc *ClusterClient) newEndpointClient(ctx context.Context, cfg ClientConfig) (*Client, error) {
	fingerprint, err := cc.getEndpointFingerprint(cfg.Host)
	if err != nil {
		return nil, err
	}
	cfg.Fingerprint = fingerprint
	return NewClientWithContext(ctx, cfg)
}

// initialHealthCheck performs a quick parallel health check on all endpoints
func (cc *ClusterClient) initialHealthCheck(ctx context.Context) {
	// Skip initial health check if there's only one endpoint
	// For single-endpoint clusters (using main host for routing), assume healthy
	if len(cc.endpoints) == 1 {
		log.Info().
			Str("cluster", cc.name).
			Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(cc.endpoints[0])).
			Msg("Single endpoint cluster - skipping initial health check")
		return
	}

	// For multi-node clusters, do a very quick check but don't mark unhealthy immediately
	// This prevents nodes from being marked unhealthy due to temporary startup conditions

	var wg sync.WaitGroup
	for _, endpoint := range cc.endpoints {
		wg.Add(1)
		go func(ep string) {
			defer wg.Done()

			// Try a quick connection test with slightly longer timeout for initial check
			cfg := cc.config
			cfg.Host = ep
			cfg.Timeout = 5 * time.Second

			testClient, _, err := cc.probeEndpoint(ctx, cfg)
			if testClient != nil {
				defer testClient.httpClient.CloseIdleConnections()
			}

			// The probe reads /nodes, not a guest endpoint. Provider text about
			// QGA cannot turn a failed nodes/login request into usable polling.
			if err == nil {
				// Node is healthy - create a proper client with full timeout for actual use
				fullCfg := cc.config
				fullCfg.Host = ep
				fullClient, clientErr := cc.newEndpointClient(ctx, fullCfg)

				cc.mu.Lock()
				if clientErr != nil {
					cc.nodeHealth[ep] = false
					cc.lastError[ep] = sanitizeEndpointError(clientErr.Error())
					cc.lastHealthCheck[ep] = time.Now()
					cc.mu.Unlock()
					log.Warn().
						Str("cluster", cc.name).
						Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).
						Err(clientErr).
						Msg("Failed to create full client after successful health check")
				} else {
					cc.nodeHealth[ep] = true
					delete(cc.lastError, ep)
					cc.lastHealthCheck[ep] = time.Now()
					cc.clients[ep] = fullClient // Store the full client, not test client
					cc.mu.Unlock()
					log.Info().
						Str("cluster", cc.name).
						Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).
						Msg("Cluster endpoint passed initial health check")
				}
			} else {
				// Real connectivity issue
				cc.mu.Lock()
				cc.nodeHealth[ep] = false
				cc.lastError[ep] = sanitizeEndpointError(err.Error())
				cc.lastHealthCheck[ep] = time.Now()
				cc.mu.Unlock()
				log.Info().
					Str("cluster", cc.name).
					Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).
					Err(err).
					Msg("Cluster endpoint failed initial health check")
			}
		}(endpoint)
	}

	// Wait for all checks to complete
	wg.Wait()

	log.Info().
		Str("cluster", cc.name).
		Int("total", len(cc.endpoints)).
		Msg("Initial cluster health check completed")
}

func (cc *ClusterClient) availableEndpointsLocked(now time.Time) (healthyEndpoints, coolingEndpoints []string) {
	for _, endpoint := range cc.endpoints {
		if !cc.nodeHealth[endpoint] {
			continue
		}
		if cooldown, exists := cc.rateLimitUntil[endpoint]; exists {
			if now.Before(cooldown) {
				coolingEndpoints = append(coolingEndpoints, endpoint)
				continue
			}
			delete(cc.rateLimitUntil, endpoint)
		}
		healthyEndpoints = append(healthyEndpoints, endpoint)
	}
	return healthyEndpoints, coolingEndpoints
}

func (cc *ClusterClient) startBackgroundRecoveryLocked() {
	if cc.recoveryInProgress {
		return
	}
	cc.recoveryInProgress = true

	go func() {
		defer func() {
			cc.mu.Lock()
			cc.recoveryInProgress = false
			cc.mu.Unlock()
		}()

		recoveryCtx, cancel := context.WithTimeout(context.Background(), backgroundRecoveryTimeout)
		defer cancel()
		cc.recoverUnhealthyNodes(recoveryCtx)
	}()
}

func (cc *ClusterClient) hasRecoverableUnhealthyEndpointLocked(now time.Time) bool {
	for _, endpoint := range cc.endpoints {
		if cc.nodeHealth[endpoint] {
			continue
		}
		lastCheck, checked := cc.lastHealthCheck[endpoint]
		if !checked || now.Sub(lastCheck) >= unhealthyRecoveryInterval {
			return true
		}
	}
	return false
}

type endpointHealthDiagnostic struct {
	Endpoint string `json:"endpoint"`
	Healthy  bool   `json:"healthy"`
}

// endpointHealthDiagnosticsLocked is logging-only. A slice preserves multiple
// invalid endpoints with the same safe label without exposing map keys. cc.mu
// is held by the caller; routing and health storage retain their own identities.
func (cc *ClusterClient) endpointHealthDiagnosticsLocked() []endpointHealthDiagnostic {
	statuses := make([]endpointHealthDiagnostic, 0, len(cc.nodeHealth))
	for endpoint, healthy := range cc.nodeHealth {
		statuses = append(statuses, endpointHealthDiagnostic{securityutil.InfrastructureEndpointDiagnostic(endpoint), healthy})
	}
	return statuses
}

// getHealthyClient returns the first available endpoint in configured priority
// order. The first endpoint is the operator-configured cluster URL; discovered
// member addresses are failover candidates rather than a random load-balancing
// pool.
func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error) {
	if err := ctx.Err(); err != nil {
		return nil, err
	}
	cc.mu.Lock()
	defer cc.mu.Unlock()
	if err := ctx.Err(); err != nil {
		return nil, err
	}

	now := time.Now()
	healthyEndpoints, coolingEndpoints := cc.availableEndpointsLocked(now)

	if len(healthyEndpoints) == 0 && len(coolingEndpoints) > 0 {
		// Nothing is immediately available, fall back to endpoints that are in cooldown
		healthyEndpoints = append(healthyEndpoints, coolingEndpoints...)
	}

	// Count unhealthy endpoints for logging and recovery decisions
	unhealthyCount := 0
	for _, healthy := range cc.nodeHealth {
		if !healthy {
			unhealthyCount++
		}
	}

	// Log at warn level if no healthy endpoints to aid troubleshooting
	if len(healthyEndpoints) == 0 && len(coolingEndpoints) == 0 {
		log.Warn().
			Str("cluster", cc.name).
			Int("healthy", len(healthyEndpoints)).
			Int("total", len(cc.nodeHealth)).
			Func(func(event *zerolog.Event) {
				event.Interface("nodeHealth", cc.endpointHealthDiagnosticsLocked())
			}).
			Msg("No healthy endpoints available - attempting recovery")
	} else {
		log.Debug().
			Str("cluster", cc.name).
			Int("healthy", len(healthyEndpoints)).
			Int("cooling", len(coolingEndpoints)).
			Int("total", len(cc.nodeHealth)).
			Func(func(event *zerolog.Event) {
				event.Interface("nodeHealth", cc.endpointHealthDiagnosticsLocked())
			}).
			Msg("Checking for healthy endpoints")
	}

	// Keep failover evidence fresh, but do not make a healthy API request wait
	// for member addresses that may only be reachable on the cluster network.
	if unhealthyCount > 0 {
		if len(healthyEndpoints) > 0 {
			if cc.hasRecoverableUnhealthyEndpointLocked(now) {
				cc.startBackgroundRecoveryLocked()
			}
		} else {
			// With no usable authority, recovery remains synchronous so this
			// request can fail over to a member that has just become reachable.
			func() {
				cc.mu.Unlock()
				defer cc.mu.Lock()
				cc.recoverUnhealthyNodes(ctx)
			}()
			if err := ctx.Err(); err != nil {
				return nil, err
			}

			now = time.Now()
			healthyEndpoints, coolingEndpoints = cc.availableEndpointsLocked(now)
			if len(healthyEndpoints) == 0 && len(coolingEndpoints) > 0 {
				healthyEndpoints = append(healthyEndpoints, coolingEndpoints...)
			}
		}
	}

	if len(healthyEndpoints) == 0 {
		// If still no healthy endpoints and we only have one endpoint,
		// try to use it anyway (could be temporarily unreachable)
		if len(cc.endpoints) == 1 {
			log.Warn().
				Str("cluster", cc.name).
				Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(cc.endpoints[0])).
				Msg("Single endpoint appears unhealthy but attempting to use it anyway")
			healthyEndpoints = cc.endpoints
			// Selection is not recovery evidence. Keep the failure until a
			// successful ordinary request clears it, even with a cached client.
		} else {
			// Provide detailed error with endpoint status. Surfacing the
			// per-endpoint reason (already sanitized) preserves the original
			// failure category so callers — and the connections aggregator's
			// auth-error regex in particular — can distinguish "authentication
			// failed" from "network unreachable" instead of seeing every
			// cluster-wide failure as "unreachable".
			unhealthyList := make([]string, 0, len(cc.endpoints))
			endpointReasons := make([]string, 0, len(cc.endpoints))
			for _, ep := range cc.endpoints {
				if !cc.nodeHealth[ep] {
					unhealthyList = append(unhealthyList, ep)
					if reason := strings.TrimSpace(cc.lastError[ep]); reason != "" {
						endpointReasons = append(endpointReasons, fmt.Sprintf("%s: %s", securityutil.InfrastructureEndpointDiagnostic(ep), reason))
					} else {
						endpointReasons = append(endpointReasons, fmt.Sprintf("%s: no recorded reason", securityutil.InfrastructureEndpointDiagnostic(ep)))
					}
				}
			}
			log.Error().
				Str("cluster", cc.name).
				Strs("unhealthyEndpoints", securityutil.InfrastructureEndpointsDiagnostic(unhealthyList)).
				Int("totalEndpoints", len(cc.endpoints)).
				Msg("All cluster endpoints are unhealthy - verify network connectivity and API accessibility from Pulse server")
			return nil, fmt.Errorf("no healthy nodes available in cluster %s (%d endpoints failed: %s)", cc.name, len(cc.endpoints), strings.Join(endpointReasons, "; "))
		}
	}

	selectedEndpoint := healthyEndpoints[0]

	// Get or create client for this endpoint
	client, exists := cc.clients[selectedEndpoint]
	if !exists {
		// Selection previously serialized construction under mu. Keep that
		// property without holding the trust/health mutex during capture:
		// concurrent first readers must not create a password-login storm.
		cc.mu.Unlock()
		cc.clientInitMu.Lock()
		cc.mu.Lock()
		defer cc.clientInitMu.Unlock()
		if err := ctx.Err(); err != nil {
			return nil, err
		}
		if accepted := cc.clients[selectedEndpoint]; accepted != nil {
			return accepted, nil
		}
		// Create new client with shorter timeout for initial test
		cfg := cc.config
		cfg.Host = selectedEndpoint

		// First try with a short timeout to quickly detect offline nodes
		testCfg := cfg
		testCfg.Timeout = 3 * time.Second

		// First-use capture owns cc.mu internally. Release selection's lock
		// while probing, just as synchronous unhealthy recovery does above.
		// Other readers can continue using an already accepted sibling.
		var testClient *Client
		var testNodes []Node
		var testErr error
		func() {
			cc.mu.Unlock()
			defer cc.mu.Lock()
			testClient, testNodes, testErr = cc.probeEndpoint(ctx, testCfg)
		}()
		if testClient != nil {
			defer testClient.httpClient.CloseIdleConnections()
		}
		if err := ctx.Err(); err != nil {
			return nil, err
		}

		if testErr != nil {
			// Check if this is a transient rate limit error that shouldn't mark the node unhealthy
			if isRateLimited, _ := isTransientRateLimitError(testErr); isRateLimited {
				log.Debug().
					Str("cluster", cc.name).
					Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint)).
					Err(testErr).
					Msg("Ignoring transient rate limit error during connectivity test")
				// Continue with client creation since the node is accessible, just rate limited
			} else {
				cc.nodeHealth[selectedEndpoint] = false
				cc.lastError[selectedEndpoint] = sanitizeEndpointError(testErr.Error())
				cc.lastHealthCheck[selectedEndpoint] = time.Now()
				log.Warn().
					Str("cluster", cc.name).
					Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint)).
					Err(testErr).
					Msg("Failed to read Proxmox nodes; endpoint removed from rotation until next refresh")
				return nil, fmt.Errorf("endpoint %s failed connectivity test: %w", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint), testErr)
			}
		}

		if testErr == nil {
			log.Debug().
				Str("cluster", cc.name).
				Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint)).
				Func(func(event *zerolog.Event) {
					if testNodes != nil {
						event.Int("nodes", len(testNodes))
					}
				}).
				Msg("Cluster endpoint passed connectivity test")
		}

		// Use the first-use winner's pin for the full polling session too.
		fingerprint, trustErr := cc.getEndpointFingerprintLocked(selectedEndpoint)
		if trustErr != nil {
			cc.nodeHealth[selectedEndpoint] = false
			cc.lastError[selectedEndpoint] = sanitizeEndpointError(trustErr.Error())
			cc.lastHealthCheck[selectedEndpoint] = time.Now()
			return nil, trustErr
		}
		cfg.Fingerprint = fingerprint
		// Create the actual client with full timeout
		newClient, err := NewClientWithContext(ctx, cfg)
		if err != nil {
			if ctx.Err() != nil {
				return nil, ctx.Err()
			}
			// This shouldn't happen since we just tested it
			cc.nodeHealth[selectedEndpoint] = false
			cc.lastError[selectedEndpoint] = sanitizeEndpointError(err.Error())
			cc.lastHealthCheck[selectedEndpoint] = time.Now()
			return nil, fmt.Errorf("failed to create client for %s: %w", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint), err)
		}

		// A successful nodes probe and accepted polling session establish
		// recovery. The existing rate-limit exception only admits an ordinary
		// request; it cannot clear a recorded failure before that request succeeds.
		if testErr == nil {
			cc.nodeHealth[selectedEndpoint] = true
			cc.lastHealthCheck[selectedEndpoint] = time.Now()
			delete(cc.lastError, selectedEndpoint)
		}
		cc.clients[selectedEndpoint] = newClient
		client = newClient
	}

	return client, nil
}

// markUnhealthyWithError marks an endpoint as unhealthy and captures the error
func (cc *ClusterClient) markUnhealthyWithError(endpoint string, errMsg string) {
	cc.mu.Lock()
	defer cc.mu.Unlock()

	if cc.nodeHealth[endpoint] {
		log.Warn().
			Str("cluster", cc.name).
			Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(endpoint)).
			Str("error", errMsg).
			Msg("Marking cluster node as unhealthy")
		cc.nodeHealth[endpoint] = false
	}
	if errMsg != "" {
		cc.lastError[endpoint] = sanitizeEndpointError(errMsg)
	}
	cc.lastHealthCheck[endpoint] = time.Now()
}

// clearEndpointError removes any cached error for an endpoint after successful operations
// and marks the endpoint as healthy since the operation succeeded
func (cc *ClusterClient) clearEndpointError(endpoint string) {
	cc.mu.Lock()
	defer cc.mu.Unlock()
	delete(cc.lastError, endpoint)
	delete(cc.rateLimitUntil, endpoint)
	// Mark endpoint healthy since operation succeeded - this ensures degraded
	// clusters recover once endpoints start responding again
	cc.nodeHealth[endpoint] = true
}

// recoverUnhealthyNodes attempts to recover unhealthy nodes
func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {
	if ctx.Err() != nil {
		return
	}
	cc.mu.RLock()
	unhealthyEndpoints := make([]string, 0)
	throttledEndpoints := make([]string, 0)
	now := time.Now()
	for endpoint, healthy := range cc.nodeHealth {
		if !healthy {
			// Skip if we checked this endpoint recently.
			// Balance between recovery speed and avoiding excessive checks
			if lastCheck, exists := cc.lastHealthCheck[endpoint]; exists {
				if now.Sub(lastCheck) < unhealthyRecoveryInterval {
					throttledEndpoints = append(throttledEndpoints, endpoint)
					continue
				}
			}
			unhealthyEndpoints = append(unhealthyEndpoints, endpoint)
		}
	}
	cc.mu.RUnlock()

	if len(unhealthyEndpoints) == 0 {
		if len(throttledEndpoints) > 0 {
			log.Debug().
				Str("cluster", cc.name).
				Strs("throttledEndpoints", securityutil.InfrastructureEndpointsDiagnostic(throttledEndpoints)).
				Msg("Skipping recovery check - endpoints checked recently")
		}
		return
	}

	log.Info().
		Str("cluster", cc.name).
		Strs("unhealthyEndpoints", securityutil.InfrastructureEndpointsDiagnostic(unhealthyEndpoints)).
		Int("count", len(unhealthyEndpoints)).
		Msg("Attempting to recover unhealthy cluster endpoints")

	// Test all unhealthy endpoints concurrently with a short timeout
	var wg sync.WaitGroup
	recoveredEndpoints := make(chan string, len(unhealthyEndpoints))

	var failureMu sync.Mutex
	failureReasons := make(map[string]string, len(unhealthyEndpoints))
	recordFailure := func(ep string, err error) {
		if ctx.Err() != nil {
			return
		}
		sanitized := sanitizeEndpointError(err.Error())
		failureMu.Lock()
		failureReasons[ep] = sanitized
		failureMu.Unlock()
		cc.mu.Lock()
		cc.lastError[ep] = sanitized
		cc.mu.Unlock()
	}

	for _, endpoint := range unhealthyEndpoints {
		wg.Add(1)
		go func(ep string) {
			defer wg.Done()
			if ctx.Err() != nil {
				return
			}

			// Update last check time
			cc.mu.Lock()
			cc.lastHealthCheck[ep] = now
			cc.mu.Unlock()

			// Try to create a client and test connection
			// Note: 5-second timeout needed because TLS handshake to Proxmox API
			// typically takes ~3 seconds on local networks
			cfg := cc.config
			cfg.Host = ep
			cfg.Timeout = 5 * time.Second

			testClient, _, err := cc.probeEndpoint(ctx, cfg)
			if testClient != nil {
				defer testClient.httpClient.CloseIdleConnections()
			}
			if ctx.Err() != nil {
				return
			}

			// Recovery requires the actual nodes probe to succeed; a guest-like
			// error body is not evidence about this endpoint's nodes inventory.
			if err == nil {
				// Store the client with original timeout
				cfg.Timeout = cc.config.Timeout
				fullClient, clientErr := cc.newEndpointClient(ctx, cfg)
				if clientErr != nil {
					// A short probe is not a usable polling session. Password
					// authentication may fail when constructing the actual client;
					// retain the existing client and failure instead of publishing
					// a healthy endpoint with a nil client.
					recordFailure(ep, clientErr)
					log.Debug().
						Str("cluster", cc.name).
						Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).
						Err(clientErr).
						Msg("Failed to create full client after successful recovery probe")
					return
				}

				cc.mu.Lock()
				if ctx.Err() != nil {
					cc.mu.Unlock()
					fullClient.httpClient.CloseIdleConnections()
					return
				}
				cc.nodeHealth[ep] = true
				delete(cc.lastError, ep)
				cc.lastHealthCheck[ep] = time.Now()
				cc.clients[ep] = fullClient
				cc.mu.Unlock()
				recoveredEndpoints <- ep

				log.Info().
					Str("cluster", cc.name).
					Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).
					Msg("Recovered unhealthy cluster node")
			} else {
				recordFailure(ep, err)
				log.Debug().
					Str("cluster", cc.name).
					Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).
					Err(err).
					Msg("Recovery attempt failed - endpoint still unhealthy")
			}
		}(endpoint)
	}

	// Wait for all recovery attempts to complete
	go func() {
		wg.Wait()
		close(recoveredEndpoints)
	}()

	// Count recovered endpoints
	recoveredCount := 0
	for range recoveredEndpoints {
		recoveredCount++
	}

	// Log recovery summary
	if recoveredCount > 0 {
		log.Info().
			Str("cluster", cc.name).
			Int("recovered", recoveredCount).
			Int("attempted", len(unhealthyEndpoints)).
			Msg("Cluster endpoint recovery completed")
	} else if len(unhealthyEndpoints) > 0 {
		// Include the per-endpoint failure reasons: without them this summary
		// is the only non-debug signal and leaves operators guessing whether
		// the cause is DNS, TLS, auth, or routing. Refs: #1665
		failureMu.Lock()
		reasons := make([]string, 0, len(failureReasons))
		for _, ep := range unhealthyEndpoints {
			if reason, ok := failureReasons[ep]; ok {
				reasons = append(reasons, securityutil.InfrastructureEndpointDiagnostic(ep)+": "+reason)
			}
		}
		failureMu.Unlock()
		log.Warn().
			Str("cluster", cc.name).
			Int("attempted", len(unhealthyEndpoints)).
			Strs("failedEndpoints", securityutil.InfrastructureEndpointsDiagnostic(unhealthyEndpoints)).
			Strs("failureReasons", reasons).
			Msg("No endpoints recovered - cluster may be unreachable from Pulse server")
	}
}

// executeWithFailover executes a function with automatic failover
func (cc *ClusterClient) executeWithFailover(ctx context.Context, fn func(*Client) error) error {
	baseRetries := len(cc.endpoints)
	maxRetries := baseRetries + rateLimitRetryBudget
	var lastErr error

	log.Debug().
		Str("cluster", cc.name).
		Int("maxRetries", maxRetries).
		Msg("Starting executeWithFailover")

	for i := 0; i < maxRetries; i++ {
		client, err := cc.getHealthyClient(ctx)
		if err != nil {
			log.Debug().
				Str("cluster", cc.name).
				Err(err).
				Int("attempt", i+1).
				Msg("Failed to get healthy client")
			return err
		}

		// Get the endpoint for this client
		var clientEndpoint string
		cc.mu.RLock()
		for endpoint, c := range cc.clients {
			if c == client {
				clientEndpoint = endpoint
				break
			}
		}
		cc.mu.RUnlock()

		// Execute the function
		err = fn(client)
		if err == nil {
			// Clear any stale error for this endpoint on success
			cc.clearEndpointError(clientEndpoint)
			return nil
		}
		lastErr = err

		// Rate limit - retry with backoff (must check first)
		if isRateLimited, statusCode := isTransientRateLimitError(err); isRateLimited {
			backoff := calculateRateLimitBackoff(i)
			cc.applyRateLimitCooldown(clientEndpoint, backoff)

			event := log.Warn().
				Str("cluster", cc.name).
				Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(clientEndpoint)).
				Err(err).
				Dur("backoff", backoff).
				Int("attempt", i+1)
			if statusCode != 0 {
				event = event.Int("status", statusCode)
			}
			event.Msg("Rate limited by cluster node, retrying with backoff")

			timer := time.NewTimer(backoff)
			select {
			case <-ctx.Done():
				if !timer.Stop() {
					<-timer.C
				}
				return fmt.Errorf("context canceled while backing off after rate limit: %w", ctx.Err())
			case <-timer.C:
			}

			continue
		}

		// Auth errors - return immediately without retry
		if isAuthError(err) {
			return err
		}

		// Only mark endpoint unhealthy for actual connectivity failures
		// (connection refused, TLS errors, DNS failures, etc.).
		// Any HTTP response from Proxmox - even a 500 - means the endpoint
		// is reachable and the request just failed at the application level.
		if isEndpointConnectivityError(err) {
			cc.markUnhealthyWithError(clientEndpoint, err.Error())
			log.Warn().
				Str("cluster", cc.name).
				Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(clientEndpoint)).
				Err(err).
				Int("attempt", i+1).
				Msg("Connectivity error, trying next cluster endpoint")
			continue
		}

		// Endpoint is reachable but the specific request failed
		// (HTTP 4xx/5xx, parsing error, VM-specific error, etc.)
		log.Debug().
			Str("cluster", cc.name).
			Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(clientEndpoint)).
			Err(err).
			Msg("Request-level error, endpoint reachable - not marking unhealthy")
		return err
	}

	if lastErr != nil {
		return fmt.Errorf("all cluster nodes failed for %s: %w", cc.name, lastErr)
	}

	return fmt.Errorf("all cluster nodes failed for %s", cc.name)
}

func (cc *ClusterClient) applyRateLimitCooldown(endpoint string, backoff time.Duration) {
	if endpoint == "" {
		return
	}

	cc.mu.Lock()
	defer cc.mu.Unlock()
	if cc.rateLimitUntil == nil {
		cc.rateLimitUntil = make(map[string]time.Time)
	}
	cc.rateLimitUntil[endpoint] = time.Now().Add(backoff)
}

func calculateRateLimitBackoff(attempt int) time.Duration {
	// Linear backoff with jitter keeps retries gentle while avoiding thundering herd
	base := rateLimitBaseDelay * time.Duration(attempt+1)
	if rateLimitMaxJitter <= 0 {
		return base
	}

	jitter := time.Duration(rand.Int63n(rateLimitMaxJitter.Nanoseconds()+1)) * time.Nanosecond
	return base + jitter
}

func isTransientRateLimitError(err error) (bool, int) {
	if err == nil {
		return false, 0
	}

	errStr := err.Error()
	statusCode := extractStatusCode(errStr)
	if statusCode != 0 {
		if _, ok := transientRateLimitStatusCodes[statusCode]; ok {
			return true, statusCode
		}
	}

	lowerErr := strings.ToLower(errStr)
	if strings.Contains(lowerErr, "rate limit") || strings.Contains(lowerErr, "too many requests") {
		if statusCode == 0 {
			statusCode = 429
		}
		return true, statusCode
	}

	return false, statusCode
}

func extractStatusCode(errStr string) int {
	matches := statusCodePattern.FindStringSubmatch(errStr)
	if len(matches) != 2 {
		return 0
	}

	code, err := strconv.Atoi(matches[1])
	if err != nil {
		return 0
	}

	return code
}

func isNotImplementedError(errStr string) bool {
	lower := strings.ToLower(errStr)
	if !strings.Contains(lower, "not implemented") {
		return false
	}

	// Common formatting: "status 501", "error 501", "api error 501"
	if strings.Contains(lower, " 501") || strings.Contains(lower, "status 501") || strings.Contains(lower, "error 501") {
		return true
	}

	// Fallback to explicit HTTP status detection
	if extractStatusCode(errStr) == 501 {
		return true
	}

	return false
}

// GetHealthStatus returns the health status of all nodes
func (cc *ClusterClient) GetHealthStatus() map[string]bool {
	cc.mu.RLock()
	defer cc.mu.RUnlock()

	status := make(map[string]bool)
	for endpoint, healthy := range cc.nodeHealth {
		status[endpoint] = healthy
	}
	return status
}

// EndpointHealth contains health information for a single endpoint
type EndpointHealth struct {
	Healthy   bool
	LastCheck time.Time
	LastError string
}

// GetHealthStatusWithErrors returns detailed health status including error messages
func (cc *ClusterClient) GetHealthStatusWithErrors() map[string]EndpointHealth {
	cc.mu.RLock()
	defer cc.mu.RUnlock()

	status := make(map[string]EndpointHealth)
	for endpoint, healthy := range cc.nodeHealth {
		status[endpoint] = EndpointHealth{
			Healthy:   healthy,
			LastCheck: cc.lastHealthCheck[endpoint],
			LastError: cc.lastError[endpoint],
		}
	}
	return status
}

// Implement all the Client methods with failover

func (cc *ClusterClient) GetNodes(ctx context.Context) ([]Node, error) {
	log.Debug().
		Str("cluster", cc.name).
		Msg("ClusterClient.GetNodes called")

	var result []Node
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		nodes, err := client.GetNodes(ctx)
		if err != nil {
			return err
		}
		result = nodes
		return nil
	})

	if err != nil {
		log.Warn().
			Str("cluster", cc.name).
			Err(err).
			Msg("ClusterClient.GetNodes failed")
	} else {
		log.Info().
			Str("cluster", cc.name).
			Int("count", len(result)).
			Msg("ClusterClient.GetNodes succeeded")
	}

	return result, err
}

func (cc *ClusterClient) GetNodeStatus(ctx context.Context, node string) (*NodeStatus, error) {
	var result *NodeStatus
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		status, err := client.GetNodeStatus(ctx, node)
		if err != nil {
			return err
		}
		result = status
		return nil
	})
	return result, err
}

// GetNodeNetworkInterfaces returns the configured interfaces for a node using
// the same cluster failover boundary as the rest of the node inventory.
func (cc *ClusterClient) GetNodeNetworkInterfaces(ctx context.Context, node string) ([]NodeNetworkInterface, error) {
	var result []NodeNetworkInterface
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		interfaces, err := client.GetNodeNetworkInterfaces(ctx, node)
		if err != nil {
			return err
		}
		result = interfaces
		return nil
	})
	return result, err
}

func (cc *ClusterClient) GetNodeRRDData(ctx context.Context, node, timeframe, cf string, ds []string) ([]NodeRRDPoint, error) {
	var result []NodeRRDPoint
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		points, err := client.GetNodeRRDData(ctx, node, timeframe, cf, ds)
		if err != nil {
			return err
		}
		result = points
		return nil
	})
	return result, err
}

func (cc *ClusterClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]GuestRRDPoint, error) {
	var result []GuestRRDPoint
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		points, err := client.GetLXCRRDData(ctx, node, vmid, timeframe, cf, ds)
		if err != nil {
			return err
		}
		result = points
		return nil
	})
	return result, err
}

func (cc *ClusterClient) GetVMRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]GuestRRDPoint, error) {
	var result []GuestRRDPoint
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		points, err := client.GetVMRRDData(ctx, node, vmid, timeframe, cf, ds)
		if err != nil {
			return err
		}
		result = points
		return nil
	})
	return result, err
}

func (cc *ClusterClient) GetVMs(ctx context.Context, node string) ([]VM, error) {
	var result []VM
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		vms, err := client.GetVMs(ctx, node)
		if err != nil {
			return err
		}
		result = vms
		return nil
	})

	// Don't return error for transient connectivity issues - preserve UI state
	if err != nil && strings.Contains(err.Error(), "no healthy nodes available") {
		log.Debug().
			Str("cluster", cc.name).
			Str("node", node).
			Err(err).
			Msg("No healthy nodes for GetVMs - returning empty list to preserve UI state")
		return []VM{}, nil
	}

	return result, err
}

func (cc *ClusterClient) GetContainers(ctx context.Context, node string) ([]Container, error) {
	var result []Container
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		containers, err := client.GetContainers(ctx, node)
		if err != nil {
			return err
		}
		result = containers
		return nil
	})

	// Don't return error for transient connectivity issues - preserve UI state
	if err != nil && strings.Contains(err.Error(), "no healthy nodes available") {
		log.Debug().
			Str("cluster", cc.name).
			Str("node", node).
			Err(err).
			Msg("No healthy nodes for GetContainers - returning empty list to preserve UI state")
		return []Container{}, nil
	}

	return result, err
}

func (cc *ClusterClient) GetStorage(ctx context.Context, node string) ([]Storage, error) {
	var result []Storage
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		storage, err := client.GetStorage(ctx, node)
		if err != nil {
			return err
		}
		result = storage
		return nil
	})
	return result, err
}

func (cc *ClusterClient) GetAllStorage(ctx context.Context) ([]Storage, error) {
	var result []Storage
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		storage, err := client.GetAllStorage(ctx)
		if err != nil {
			return err
		}
		result = storage
		return nil
	})
	return result, err
}

func (cc *ClusterClient) GetBackupTasks(ctx context.Context) ([]Task, error) {
	var result []Task
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		tasks, err := client.GetBackupTasks(ctx)
		if err != nil {
			return err
		}
		result = tasks
		return nil
	})
	return result, err
}

func (cc *ClusterClient) GetTaskLog(ctx context.Context, node, upid string) ([]TaskLogLine, error) {
	var result []TaskLogLine
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		lines, err := client.GetTaskLog(ctx, node, upid)
		if err != nil {
			return err
		}
		result = lines
		return nil
	})
	return result, err
}

func (cc *ClusterClient) GetReplicationStatus(ctx context.Context) ([]ReplicationJob, error) {
	var result []ReplicationJob
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		jobs, err := client.GetReplicationStatus(ctx)
		if err != nil {
			return err
		}
		result = jobs
		return nil
	})
	return result, err
}

func (cc *ClusterClient) GetStorageContent(ctx context.Context, node, storage string) ([]StorageContent, error) {
	var result []StorageContent
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		content, err := client.GetStorageContent(ctx, node, storage)
		if err != nil {
			return err
		}
		result = content
		return nil
	})
	return result, err
}

// GetCephStatus returns Ceph cluster status information with failover support.
func (cc *ClusterClient) GetCephStatus(ctx context.Context) (*CephStatus, error) {
	var result *CephStatus
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		status, err := client.GetCephStatus(ctx)
		if err != nil {
			return err
		}
		result = status
		return nil
	})
	return result, err
}

// GetCephDF returns Ceph capacity information with failover support.
func (cc *ClusterClient) GetCephDF(ctx context.Context) (*CephDF, error) {
	var result *CephDF
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		df, err := client.GetCephDF(ctx)
		if err != nil {
			return err
		}
		result = df
		return nil
	})
	return result, err
}

func (cc *ClusterClient) GetVMSnapshots(ctx context.Context, node string, vmid int) ([]Snapshot, error) {
	var result []Snapshot
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		snapshots, err := client.GetVMSnapshots(ctx, node, vmid)
		if err != nil {
			return err
		}
		result = snapshots
		return nil
	})
	return result, err
}

func (cc *ClusterClient) GetContainerSnapshots(ctx context.Context, node string, vmid int) ([]Snapshot, error) {
	var result []Snapshot
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		snapshots, err := client.GetContainerSnapshots(ctx, node, vmid)
		if err != nil {
			return err
		}
		result = snapshots
		return nil
	})
	return result, err
}

func (cc *ClusterClient) GetVMStatus(ctx context.Context, node string, vmid int) (*VMStatus, error) {
	var result *VMStatus
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		status, err := client.GetVMStatus(ctx, node, vmid)
		if err != nil {
			return err
		}
		result = status
		return nil
	})
	return result, err
}

func (cc *ClusterClient) GetVMConfig(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
	var result map[string]interface{}
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		config, err := client.GetVMConfig(ctx, node, vmid)
		if err != nil {
			return err
		}
		result = config
		return nil
	})
	return result, err
}

func (cc *ClusterClient) GetVMAgentInfo(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
	var result map[string]interface{}
	err := cc.executeGuestAgent(ctx, func(client *Client) error {
		info, err := client.GetVMAgentInfo(ctx, node, vmid)
		if err != nil {
			return err
		}
		result = info
		return nil
	})
	return result, err
}

// GetVMAgentVersion returns the guest agent version for the VM.
func (cc *ClusterClient) GetVMAgentVersion(ctx context.Context, node string, vmid int) (string, error) {
	var version string
	err := cc.executeGuestAgent(ctx, func(client *Client) error {
		v, err := client.GetVMAgentVersion(ctx, node, vmid)
		if err != nil {
			return err
		}
		version = v
		return nil
	})
	return version, err
}

// GetVMFSInfo returns filesystem information from QEMU guest agent
func (cc *ClusterClient) GetVMFSInfo(ctx context.Context, node string, vmid int) ([]VMFileSystem, error) {
	var result []VMFileSystem
	err := cc.executeGuestAgent(ctx, func(client *Client) error {
		info, err := client.GetVMFSInfo(ctx, node, vmid)
		// Preserve valid peers and the incomplete-inventory error together.
		// Neither a partial response nor its error grants another endpoint read.
		result = info
		return err
	})
	return result, err
}

// GetVMNetworkInterfaces returns guest network interfaces from the QEMU agent
func (cc *ClusterClient) GetVMNetworkInterfaces(ctx context.Context, node string, vmid int) ([]VMNetworkInterface, error) {
	var result []VMNetworkInterface
	err := cc.executeGuestAgent(ctx, func(client *Client) error {
		interfaces, err := client.GetVMNetworkInterfaces(ctx, node, vmid)
		if err != nil {
			return err
		}
		result = interfaces
		return nil
	})
	return result, err
}

// GetVMMemAvailableFromAgent reads /proc/meminfo via the QEMU guest agent to get MemAvailable.
func (cc *ClusterClient) GetVMMemAvailableFromAgent(ctx context.Context, node string, vmid int) (uint64, error) {
	var result uint64
	err := cc.executeGuestAgent(ctx, func(client *Client) error {
		available, err := client.GetVMMemAvailableFromAgent(ctx, node, vmid)
		if err != nil {
			return err
		}
		result = available
		return nil
	})
	return result, err
}

// GetVMMemoryAvailabilityFromAgent reads cache-aware /proc/meminfo evidence
// through the first healthy cluster client.
func (cc *ClusterClient) GetVMMemoryAvailabilityFromAgent(ctx context.Context, node string, vmid int) (LinuxMemoryAvailability, error) {
	var result LinuxMemoryAvailability
	err := cc.executeGuestAgent(ctx, func(client *Client) error {
		availability, err := client.GetVMMemoryAvailabilityFromAgent(ctx, node, vmid)
		if err != nil {
			return err
		}
		result = availability
		return nil
	})
	return result, err
}

// GetClusterResources returns all resources (VMs, containers) across the cluster in a single call
func (cc *ClusterClient) GetClusterResources(ctx context.Context, resourceType string) ([]ClusterResource, error) {
	var result []ClusterResource
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		resources, err := client.GetClusterResources(ctx, resourceType)
		if err != nil {
			return err
		}
		result = resources
		return nil
	})
	return result, err
}

// GetClusterOptions fetches datacenter options via the first healthy node.
func (cc *ClusterClient) GetClusterOptions(ctx context.Context) (*ClusterOptions, error) {
	var result *ClusterOptions
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		opts, err := client.GetClusterOptions(ctx)
		if err != nil {
			return err
		}
		result = opts
		return nil
	})
	return result, err
}

// GetContainerStatus returns the status of a specific container
func (cc *ClusterClient) GetContainerStatus(ctx context.Context, node string, vmid int) (*Container, error) {
	var result *Container
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		status, err := client.GetContainerStatus(ctx, node, vmid)
		if err != nil {
			return err
		}
		result = status
		return nil
	})
	return result, err
}

// GetContainerConfig returns the configuration of a specific container
func (cc *ClusterClient) GetContainerConfig(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
	var result map[string]interface{}
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		config, err := client.GetContainerConfig(ctx, node, vmid)
		if err != nil {
			return err
		}
		result = config
		return nil
	})
	return result, err
}

// GetContainerInterfaces returns interface details for a container
func (cc *ClusterClient) GetContainerInterfaces(ctx context.Context, node string, vmid int) ([]ContainerInterface, error) {
	var result []ContainerInterface
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		interfaces, err := client.GetContainerInterfaces(ctx, node, vmid)
		if err != nil {
			return err
		}
		result = interfaces
		return nil
	})
	return result, err
}

// IsClusterMember checks if this node is part of a cluster
func (cc *ClusterClient) IsClusterMember(ctx context.Context) (bool, error) {
	var result bool
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		isMember, err := client.IsClusterMember(ctx)
		if err != nil {
			return err
		}
		result = isMember
		return nil
	})
	return result, err
}

// GetZFSPoolStatus returns ZFS pool status for a node
func (cc *ClusterClient) GetZFSPoolStatus(ctx context.Context, node string) ([]ZFSPoolStatus, error) {
	var result []ZFSPoolStatus
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		pools, err := client.GetZFSPoolStatus(ctx, node)
		if err != nil {
			return err
		}
		result = pools
		return nil
	})
	return result, err
}

// GetZFSPoolsWithDetails returns ZFS pools with full details for a node
func (cc *ClusterClient) GetZFSPoolsWithDetails(ctx context.Context, node string) ([]ZFSPoolInfo, error) {
	var result []ZFSPoolInfo
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		pools, err := client.GetZFSPoolsWithDetails(ctx, node)
		if err != nil {
			return err
		}
		result = pools
		return nil
	})
	return result, err
}

// Helper to check if error is auth-related
func (cc *ClusterClient) GetDisks(ctx context.Context, node string) ([]Disk, error) {
	var result []Disk
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		disks, err := client.GetDisks(ctx, node)
		if err != nil {
			return err
		}
		result = disks
		return nil
	})

	return result, err
}

// GetNodePendingUpdates returns pending apt updates for a node with failover support
func (cc *ClusterClient) GetNodePendingUpdates(ctx context.Context, node string) ([]AptPackage, error) {
	var result []AptPackage
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		pkgs, err := client.GetNodePendingUpdates(ctx, node)
		if err != nil {
			return err
		}
		result = pkgs
		return nil
	})

	return result, err
}

// GetClusterStatus returns the cluster status including all nodes with failover support.
func (cc *ClusterClient) GetClusterStatus(ctx context.Context) ([]ClusterStatus, error) {
	var result []ClusterStatus
	err := cc.executeWithFailover(ctx, func(client *Client) error {
		status, err := client.GetClusterStatus(ctx)
		if err != nil {
			return err
		}
		result = status
		return nil
	})

	return result, err
}

// IsQuorate checks if the cluster has quorum by querying the Proxmox cluster status.
// Returns true if the cluster is quorate (has enough votes for consensus), false otherwise.
// This is the authoritative check for cluster health - a cluster with quorum is healthy
// even if some nodes are intentionally offline (e.g., backup nodes not running).
func (cc *ClusterClient) IsQuorate(ctx context.Context) (bool, error) {
	status, err := cc.GetClusterStatus(ctx)
	if err != nil {
		return false, err
	}

	// Look for the cluster entry which has the quorate field
	for _, s := range status {
		if s.Type == "cluster" {
			return s.Quorate == 1, nil
		}
	}

	// If no cluster entry found, this might be a standalone node - consider it healthy
	return true, nil
}

// isAuthError checks if an error is an authentication error
func isAuthError(err error) bool {
	if err == nil {
		return false
	}
	errStr := err.Error()
	return strings.Contains(errStr, "authentication") ||
		strings.Contains(errStr, "401") ||
		strings.Contains(errStr, "403")
}