pulse/internal/monitoring/monitor_client_reconnect.go

package monitoring

import (
	"context"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	monitorerrors "github.com/rcourtman/pulse-go-rewrite/internal/monitoring/errors"
	"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
	"github.com/rs/zerolog/log"
)

var connRetryDelays = []time.Duration{
	5 * time.Second,
	10 * time.Second,
	20 * time.Second,
	40 * time.Second,
	60 * time.Second,
}

// retryFailedConnections attempts to recreate clients that failed during initialization
// This handles cases where Proxmox/network isn't ready when Pulse starts. Keep
// the capped retry cadence until every active connection has a client or the
// monitor stops: providers list clients, so an expired startup retry would leave
// a missing connection outside ordinary polling permanently.
func (m *Monitor) retryFailedConnections(ctx context.Context) {
	defer recoverFromPanic("retryFailedConnections")

	// Retry schedule: 5s, 10s, 20s, 40s, 60s, then every 60s.
	retryDelays := connRetryDelays
	retryIndex := 0

	for {
		// Stop when the monitor is cancelled, including while waiting below.
		select {
		case <-ctx.Done():
			return
		default:
		}

		// Calculate next retry delay
		var delay time.Duration
		if retryIndex < len(retryDelays) {
			delay = retryDelays[retryIndex]
			retryIndex++
		} else {
			delay = 60 * time.Second // Continue retrying every 60s
		}

		// Wait before retry
		select {
		case <-time.After(delay):
		case <-ctx.Done():
			return
		}

		// Check for missing clients and try to recreate them
		m.mu.Lock()
		missingPVE := []config.PVEInstance{}
		missingPBS := []config.PBSInstance{}
		connectionTimeout := 0 * time.Second

		if m.config != nil {
			connectionTimeout = m.config.ConnectionTimeout

			// Find PVE instances without clients. Skip paused instances —
			// a missing client for a paused instance is intentional, not a failure.
			for _, pve := range m.config.PVEInstances {
				if pve.Disabled {
					continue
				}
				if _, exists := m.pveClients[pve.Name]; !exists {
					missingPVE = append(missingPVE, pve)
				}
			}

			// Find PBS instances without clients. Skip paused instances —
			// a missing client for a paused instance is intentional, not a failure.
			for _, pbsInst := range m.config.PBSInstances {
				if pbsInst.Disabled {
					continue
				}
				if _, exists := m.pbsClients[pbsInst.Name]; !exists {
					missingPBS = append(missingPBS, pbsInst)
				}
			}
		}
		m.mu.Unlock()

		// If no missing clients, we're done
		if len(missingPVE) == 0 && len(missingPBS) == 0 {
			log.Info().Msg("all client connections established successfully")
			return
		}

		log.Info().
			Int("missingPVE", len(missingPVE)).
			Int("missingPBS", len(missingPBS)).
			Dur("nextRetry", delay).
			Msg("Attempting to reconnect failed clients")

		// Try to recreate PVE clients
		for _, pve := range missingPVE {
			if ctx.Err() != nil {
				return
			}
			if pve.IsCluster && len(pve.ClusterEndpoints) > 0 {
				// Create cluster client
				endpoints, endpointFingerprints, conflictingEndpoints := m.buildClusterEndpointsForReconnect(pve)

				clientConfig := config.CreateProxmoxConfig(&pve)
				clientConfig.Timeout = connectionTimeout
				clusterClient, err := proxmox.NewClusterClientWithContext(ctx, pve.Name, clientConfig, endpoints, endpointFingerprints, conflictingEndpoints...)
				if ctx.Err() != nil {
					return
				}
				if err != nil {
					log.Warn().Err(err).Str("instance", pve.Name).Msg("Failed to reconnect cluster client, will retry")
					continue
				}

				m.replacePVEClient(pve.Name, clusterClient)
				m.setProviderConnectionHealth(InstanceTypePVE, pve.Name, true)

				log.Info().
					Str("instance", pve.Name).
					Str("cluster", pve.ClusterName).
					Msg("Successfully reconnected cluster client")
				continue
			}

			// Create regular client
			clientConfig := config.CreateProxmoxConfig(&pve)
			clientConfig.Timeout = connectionTimeout
			client, err := newProxmoxClientFunc(ctx, clientConfig)
			if ctx.Err() != nil {
				return
			}
			if err != nil {
				log.Warn().
					Err(err).
					Str("instance", pve.Name).
					Msg("Failed to reconnect PVE client, will retry")
				continue
			}

			m.replacePVEClient(pve.Name, client)
			m.setProviderConnectionHealth(InstanceTypePVE, pve.Name, true)

			log.Info().
				Str("instance", pve.Name).
				Msg("Successfully reconnected PVE client")
		}

		// Try to recreate PBS clients
		for _, pbsInst := range missingPBS {
			if ctx.Err() != nil {
				return
			}
			clientConfig := config.CreatePBSConfig(&pbsInst)
			clientConfig.Timeout = 60 * time.Second
			client, err := pbs.NewClientWithContext(ctx, clientConfig)
			if ctx.Err() != nil {
				return
			}
			if err != nil {
				monErr := monitorerrors.WrapConnectionError("recreate_pbs_client", pbsInst.Name, err)
				log.Warn().
					Err(monErr).
					Str("instance", pbsInst.Name).
					Msg("Failed to reconnect PBS client, will retry")
				m.recordTaskResult(InstanceTypePBS, pbsInst.Name, monErr)
				if m.stalenessTracker != nil {
					m.stalenessTracker.UpdateError(InstanceTypePBS, pbsInst.Name)
				}
				m.publishPBSConnectionOutcome(models.PBSInstance{
					ID:       PBSMonitorResourceID(pbsInst.Name),
					Name:     pbsInst.Name,
					Host:     pbsInst.Host,
					GuestURL: pbsInst.GuestURL,
					Version:  "unknown",
					LastSeen: time.Now(),
				}, monErr)
				continue
			}

			m.mu.Lock()
			m.pbsClients[pbsInst.Name] = client
			m.mu.Unlock()

			log.Info().
				Str("instance", pbsInst.Name).
				Msg("PBS client recreated; awaiting connectivity poll")
		}
	}
}