pulse/cmd/pulse-agent/main.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/fs"
	"math"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"reflect"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
	"github.com/rcourtman/pulse-go-rewrite/internal/agenttarget"
	"github.com/rcourtman/pulse-go-rewrite/internal/agentupdate"
	pulseconfig "github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/dockeragent"
	"github.com/rcourtman/pulse-go-rewrite/internal/hostagent"
	"github.com/rcourtman/pulse-go-rewrite/internal/kubernetesagent"
	pulselogging "github.com/rcourtman/pulse-go-rewrite/internal/logging"
	"github.com/rcourtman/pulse-go-rewrite/internal/remoteconfig"
	"github.com/rcourtman/pulse-go-rewrite/internal/utils"
	agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
	"github.com/rcourtman/pulse-go-rewrite/pkg/securityutil"
	"github.com/rs/zerolog"
	gohost "github.com/shirou/gopsutil/v4/host"
	"golang.org/x/sync/errgroup"
)

var (
	Version = "dev"

	// Prometheus metrics
	agentInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Name: "pulse_agent_info",
		Help: "Information about the Pulse agent",
	}, []string{"version", "host_enabled", "docker_enabled", "kubernetes_enabled"})

	agentUp = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "pulse_agent_up",
		Help: "Whether the Pulse agent is running (1 = up, 0 = down)",
	})

	agentModuleEnabled = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Name: "pulse_agent_module_enabled",
		Help: "Whether a Pulse Unified Agent module is configured to run (1 = enabled, 0 = disabled)",
	}, []string{"module"})

	agentModuleReady = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Name: "pulse_agent_module_ready",
		Help: "Whether a configured Pulse Unified Agent module initialized successfully (1 = ready, 0 = not ready)",
	}, []string{"module"})
)

const (
	agentLogMaxSizeMB  = 25
	agentLogMaxAgeDays = 14
)

// Runnable is an interface for agents that can be run
type Runnable interface {
	Run(ctx context.Context) error
}

type RemoteConfigApplier interface {
	ApplyRemoteConfig(settings map[string]interface{}, commandsEnabled *bool)
}

// Runnable closer for the Docker / Podman collection module which needs cleanup.
type RunnableCloser interface {
	Runnable
	Close() error
}

var (
	// For testing - wrappers to return interfaces
	newDockerAgent func(dockeragent.Config) (RunnableCloser, error) = func(c dockeragent.Config) (RunnableCloser, error) {
		return dockeragent.New(c)
	}
	newKubeAgent func(kubernetesagent.Config) (Runnable, error) = func(c kubernetesagent.Config) (Runnable, error) {
		return kubernetesagent.New(c)
	}
	newHostAgent func(hostagent.Config) (Runnable, error) = func(c hostagent.Config) (Runnable, error) {
		return hostagent.New(c)
	}
	newUpdater              func(agentupdate.Config) *agentupdate.Updater = agentupdate.New
	lookPath                                                              = exec.LookPath
	runAsWindowsServiceFunc                                               = runAsWindowsService

	// For testing
	retryInitialDelay           = 5 * time.Second
	retryMaxDelay               = 5 * time.Minute
	remoteConfigRefreshInterval = 1 * time.Minute
)

type multiValue []string

func (m *multiValue) String() string {
	return strings.Join(*m, ",")
}

func (m *multiValue) Set(value string) error {
	*m = append(*m, value)
	return nil
}

func main() {
	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
	defer cancel()

	if err := run(ctx, os.Args[1:], os.Getenv); err != nil {
		if err == flag.ErrHelp {
			os.Exit(0)
		}
		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
		os.Exit(1)
	}
}

func run(ctx context.Context, args []string, getenv func(string) string) error {
	// 1. Parse Configuration
	cfg, err := loadConfig(args, getenv)
	if err != nil {
		if err == flag.ErrHelp {
			return err
		}
		return fmt.Errorf("failed to load unified agent configuration: %w", err)
	}

	// 2. Setup Logging
	logger, closeLogger, err := configureAgentLogger(cfg)
	if err != nil {
		return fmt.Errorf("failed to configure unified agent logging: %w", err)
	}
	defer closeLogger()
	cfg.Logger = &logger

	if cfg.InsecureSkipVerify && cfg.ServerFingerprint == "" {
		logger.Warn().
			Str("component", "startup").
			Str("action", "tls_skip_verify_enabled").
			Msg("TLS verification disabled for agent connections (self-signed cert mode)")
	} else if cfg.ServerFingerprint != "" {
		logger.Info().
			Str("component", "startup").
			Str("action", "tls_server_fingerprint_enabled").
			Msg("Using pinned server certificate fingerprint for agent connections")
	}

	// 2a. Handle Self-Test
	if cfg.SelfTest {
		logger.Info().Msg("Self-test passed: config loaded and logger initialized")
		return nil
	}

	if err := secureAgentStateDir(cfg.StateDir); err != nil {
		logger.Warn().
			Err(err).
			Str("path", cfg.StateDir).
			Msg("Failed to enforce owner-only agent state directory permissions")
	}

	// 2b. Compute Agent ID if missing (needed for remote config)
	// We replicate the logic from hostagent.New to ensure we get the same ID
	lookupHostname := strings.TrimSpace(cfg.HostnameOverride)
	if cfg.AgentID == "" && cfg.AgentIDFile != "" {
		if persisted, err := readAgentIDFile(cfg.AgentIDFile); err == nil && persisted != "" {
			cfg.AgentID = persisted
			logger.Info().
				Str("path", cfg.AgentIDFile).
				Str("agentID", cfg.AgentID).
				Msg("Loaded persisted agent ID")
		} else if err != nil && !errors.Is(err, fs.ErrNotExist) {
			logger.Warn().
				Err(err).
				Str("path", cfg.AgentIDFile).
				Msg("Failed to read agent-id-file; will fall back to machine-id and rewrite the file")
		}
	}
	if cfg.AgentID == "" {
		// Use a short timeout for host info
		hCtx, hCancel := context.WithTimeout(ctx, 5*time.Second)
		info, err := gohost.InfoWithContext(hCtx)
		hCancel()
		if err == nil {
			if lookupHostname == "" {
				lookupHostname = strings.TrimSpace(info.Hostname)
			}
			collector := hostagent.NewDefaultCollector()
			machineID := hostagent.GetReliableMachineID(collector, info.HostID, logger)
			cfg.AgentID = machineID
			if cfg.AgentID == "" {
				// Fallback to hostname
				cfg.AgentID = lookupHostname
			}
		} else {
			logger.Warn().
				Err(err).
				Str("component", "startup").
				Str("action", "agent_id_host_info_failed").
				Msg("Failed to fetch host info for Agent ID generation")
		}
	}
	if cfg.AgentID != "" && cfg.AgentIDFile != "" {
		if err := writeAgentIDFile(cfg.AgentIDFile, cfg.AgentID); err != nil {
			logger.Warn().
				Err(err).
				Str("path", cfg.AgentIDFile).
				Msg("Failed to persist agent ID; ID will be re-derived on next start")
		}
	}
	if lookupHostname == "" {
		lookupHostname = strings.TrimSpace(cfg.HostnameOverride)
		if lookupHostname == "" {
			if name, err := os.Hostname(); err == nil {
				lookupHostname = strings.TrimSpace(name)
			}
		}
	}

	var remoteConfigClient *remoteconfig.Client
	var remoteConfigAppliers []RemoteConfigApplier

	// 2c. Fetch Remote Config
	// Only if we have enough info to contact server
	if cfg.PulseURL != "" && cfg.APIToken != "" && cfg.AgentID != "" {
		logger.Debug().Msg("Fetching remote configuration...")
		remoteConfigClient = remoteconfig.New(remoteconfig.Config{
			PulseURL:           cfg.PulseURL,
			APIToken:           cfg.APIToken,
			AgentID:            cfg.AgentID,
			Hostname:           lookupHostname,
			InsecureSkipVerify: cfg.InsecureSkipVerify,
			CACertPath:         cfg.CACertPath,
			ServerFingerprint:  cfg.ServerFingerprint,
			Logger:             logger,
		})
		defer remoteConfigClient.Close()

		// Use a short timeout for config fetch so we don't block startup too long
		rcCtx, rcCancel := context.WithTimeout(ctx, 10*time.Second)
		settings, commandsEnabled, err := remoteConfigClient.Fetch(rcCtx)
		rcCancel()

		if err != nil {
			// Just log warning and proceed with local config
			logger.Warn().
				Err(err).
				Str("component", "remote_config").
				Str("action", "fetch_failed").
				Msg("Failed to fetch remote config - using local (or previously cached) defaults")
		} else {
			logger.Info().
				Str("component", "remote_config").
				Str("action", "fetch_succeeded").
				Msg("Successfully fetched remote configuration")
			if commandsEnabled != nil {
				cfg.EnableCommands = *commandsEnabled
				logger.Info().
					Str("component", "remote_config").
					Str("action", "apply_enable_commands").
					Bool("enabled", cfg.EnableCommands).
					Msg("Applied remote command execution setting")
			}
			if len(settings) > 0 {
				applyRemoteSettings(&cfg, settings, &logger)
			}
			if remoteconfig.HasAppliedDesiredConfig(commandsEnabled, settings) {
				metadata, metadataErr := remoteconfig.BuildDesiredConfigMetadata(commandsEnabled, settings)
				if metadataErr != nil {
					logger.Warn().Err(metadataErr).Msg("Failed to derive applied managed configuration fingerprint")
				} else {
					cfg.AppliedConfig = &agentshost.ConfigFingerprint{Version: metadata.Version, Hash: metadata.Hash}
				}
			}
		}
	}

	// 3. Check if running as Windows service
	ranAsService, err := runAsWindowsServiceFunc(cfg, logger)
	if err != nil {
		return fmt.Errorf("Windows service failed: %w", err)
	}
	if ranAsService {
		return nil
	}

	g, ctx := errgroup.WithContext(ctx)

	// Resolve automatic runtime selection before publishing startup and
	// readiness state so the process never claims a module set it did not
	// actually attempt to start.
	if !cfg.EnableDocker && !cfg.DockerConfigured {
		if _, err := lookPath("docker"); err == nil {
			logger.Info().Msg("Auto-detected Docker binary, enabling Docker monitoring")
			cfg.EnableDocker = true
		} else if _, err := lookPath("podman"); err == nil {
			logger.Info().Msg("Auto-detected Podman binary, enabling Docker monitoring")
			cfg.EnableDocker = true
		} else {
			logger.Debug().Msg("Docker/Podman not found, skipping Docker monitoring")
		}
	}

	logger.Info().
		Str("version", Version).
		Str("pulse_url", cfg.PulseURL).
		Int("observer_destinations", len(cfg.Observers)).
		Bool("host_enabled", cfg.EnableHost).
		Bool("docker_enabled", cfg.EnableDocker).
		Bool("kubernetes_enabled", cfg.EnableKubernetes).
		Bool("proxmox_mode", cfg.EnableProxmox).
		Bool("auto_update", !cfg.DisableAutoUpdate).
		Msg("Starting Pulse Unified Agent")

	if cfg.AllowPlaintextHTTP {
		logger.Warn().
			Str("pulse_url", cfg.PulseURL).
			Msg("--allow-plaintext-http is set: the agent API token travels in cleartext to any non-loopback Pulse URL; only use this on a network you fully control")
	}
	for _, observer := range cfg.Observers {
		if observer.AllowPlaintextHTTP {
			logger.Warn().
				Str("destination", observer.Name).
				Msg("Observer destination explicitly permits plaintext HTTP; its API token may travel in cleartext")
		}
	}

	// 5. Set prometheus info metric
	agentInfo.WithLabelValues(
		Version,
		fmt.Sprintf("%t", cfg.EnableHost),
		fmt.Sprintf("%t", cfg.EnableDocker),
		fmt.Sprintf("%t", cfg.EnableKubernetes),
	).Set(1)
	agentUp.Set(1)

	// 6. Start Health/Metrics Server
	var ready atomic.Bool
	runtimeStatus := newRuntimeHealth(&ready, map[string]bool{
		"host":       cfg.EnableHost,
		"docker":     cfg.EnableDocker,
		"kubernetes": cfg.EnableKubernetes,
	})
	if cfg.HealthAddr != "" {
		startHealthServer(ctx, cfg.HealthAddr, &ready, &logger, runtimeStatus)
	}

	// 7. Start Auto-Updater
	updater := newUpdater(agentupdate.Config{
		PulseURL:           cfg.PulseURL,
		APIToken:           cfg.APIToken,
		AgentName:          "pulse-agent",
		CurrentVersion:     Version,
		StateDir:           cfg.StateDir,
		CheckInterval:      1 * time.Hour,
		InsecureSkipVerify: cfg.InsecureSkipVerify,
		CACertPath:         cfg.CACertPath,
		ServerFingerprint:  cfg.ServerFingerprint,
		Logger:             &logger,
		Disabled:           cfg.DisableAutoUpdate,
	})

	g.Go(func() error {
		updater.RunLoop(ctx)
		return nil
	})

	// The host module starts before the Docker module (which may only come up
	// after daemon retries), so the typed container-update bridge late-binds.
	dockerUpdaterBridge := &lateBoundDockerUpdater{}

	// 8. Start Host Agent (if enabled)
	if cfg.EnableHost {
		hostCfg := hostagent.Config{
			PulseURL:            cfg.PulseURL,
			APIToken:            cfg.APIToken,
			Interval:            cfg.Interval,
			HostnameOverride:    cfg.HostnameOverride,
			AgentID:             cfg.AgentID,
			AgentType:           "unified",
			AgentVersion:        Version,
			Tags:                cfg.Tags,
			InsecureSkipVerify:  cfg.InsecureSkipVerify,
			CACertPath:          cfg.CACertPath,
			ServerFingerprint:   cfg.ServerFingerprint,
			DeploySSHUser:       cfg.DeploySSHUser,
			LogLevel:            cfg.LogLevel,
			Logger:              &logger,
			EnableProxmox:       cfg.EnableProxmox,
			ProxmoxType:         cfg.ProxmoxType,
			EnableCommands:      cfg.EnableCommands,
			Enroll:              cfg.Enroll,
			DiskExclude:         cfg.DiskExclude,
			StateDir:            cfg.StateDir,
			ReportIP:            cfg.ReportIP,
			DisableCeph:         cfg.DisableCeph,
			AvailabilityTargets: cfg.AvailabilityTargets,
			AppliedConfig:       cfg.AppliedConfig,
			UpdateStatus:        updater.Snapshot,
			ModuleStatus:        runtimeStatus.moduleStatuses,
			Observers:           hostObserverTargets(cfg.Observers),

			DockerContainerUpdater: dockerUpdaterBridge,
		}

		agent, err := newHostAgent(hostCfg)
		if err != nil {
			return fmt.Errorf("failed to initialize host agent: %w", err)
		}
		if applier, ok := agent.(RemoteConfigApplier); ok {
			remoteConfigAppliers = append(remoteConfigAppliers, applier)
		}
		runtimeStatus.setState("host", moduleStateRunning, nil)

		g.Go(func() error {
			logger.Info().Msg("Host agent module started")
			return agent.Run(ctx)
		})
	}

	// 9. Start Docker / Podman module (if enabled)
	var dockerAgent RunnableCloser
	if cfg.EnableDocker {
		dockerCfg := dockeragent.Config{
			PulseURL:            cfg.PulseURL,
			APIToken:            cfg.APIToken,
			Interval:            cfg.Interval,
			HostnameOverride:    cfg.HostnameOverride,
			AgentID:             cfg.AgentID,
			AgentType:           "unified",
			AgentVersion:        Version,
			InsecureSkipVerify:  cfg.InsecureSkipVerify,
			CACertPath:          cfg.CACertPath,
			ServerFingerprint:   cfg.ServerFingerprint,
			DisableAutoUpdate:   cfg.DisableAutoUpdate,
			DisableUpdateChecks: cfg.DisableDockerUpdateChecks,
			Runtime:             cfg.DockerRuntime,
			LogLevel:            cfg.LogLevel,
			Logger:              &logger,
			SwarmScope:          "node",
			IncludeContainers:   true,
			IncludeServices:     true,
			IncludeTasks:        true,
			CollectDiskMetrics:  false,
			DiskExclude:         cfg.DiskExclude,
			Targets:             dockerReportTargets(cfg),
		}

		dockerAgent, err = newDockerAgent(dockerCfg)
		if err == nil {
			dockerUpdaterBridge.set(dockerAgent)
		}
		if err != nil {
			runtimeStatus.setState("docker", moduleStateRetrying, err)
			// Docker isn't available yet - start retry loop in background
			logger.Warn().
				Err(err).
				Str("component", "docker_agent").
				Str("action", "initialization_failed_retry_scheduled").
				Msg("Docker not available, will retry with exponential backoff")

			g.Go(func() error {
				agent := initDockerWithRetry(ctx, dockerCfg, &logger)
				if agent != nil {
					dockerAgent = agent
					dockerUpdaterBridge.set(agent)
					runtimeStatus.setState("docker", moduleStateRunning, nil)
					logger.Info().Msg("Docker / Podman module started (after retry)")
					return agent.Run(ctx)
				}
				// Docker never became available, continue without it
				return nil
			})
		} else {
			runtimeStatus.setState("docker", moduleStateRunning, nil)
			g.Go(func() error {
				logger.Info().Msg("Docker / Podman module started")
				return dockerAgent.Run(ctx)
			})
		}
	}

	// 10. Start Kubernetes Agent (if enabled)
	if cfg.EnableKubernetes {
		kubeCfg := kubernetesagent.Config{
			PulseURL:              cfg.PulseURL,
			APIToken:              cfg.APIToken,
			Interval:              cfg.Interval,
			AgentID:               cfg.AgentID,
			AgentType:             "unified",
			AgentVersion:          Version,
			InsecureSkipVerify:    cfg.InsecureSkipVerify,
			CACertPath:            cfg.CACertPath,
			ServerFingerprint:     cfg.ServerFingerprint,
			LogLevel:              cfg.LogLevel,
			Logger:                &logger,
			KubeconfigPath:        cfg.KubeconfigPath,
			KubeContext:           cfg.KubeContext,
			IncludeNamespaces:     cfg.KubeIncludeNamespaces,
			ExcludeNamespaces:     cfg.KubeExcludeNamespaces,
			IncludeAllPods:        cfg.KubeIncludeAllPods,
			IncludeAllDeployments: cfg.KubeIncludeAllDeployments,
			MaxPods:               cfg.KubeMaxPods,
			Targets:               kubernetesReportTargets(cfg),
		}

		agent, err := newKubeAgent(kubeCfg)
		if err != nil {
			runtimeStatus.setState("kubernetes", moduleStateRetrying, err)
			logger.Warn().
				Err(err).
				Str("component", "kubernetes_agent").
				Str("action", "initialization_failed_retry_scheduled").
				Msg("Kubernetes not available, will retry with exponential backoff")

			g.Go(func() error {
				retried := initKubernetesWithRetry(ctx, kubeCfg, &logger)
				if retried != nil {
					runtimeStatus.setState("kubernetes", moduleStateRunning, nil)
					logger.Info().Msg("Kubernetes agent module started (after retry)")
					return retried.Run(ctx)
				}
				return nil
			})
		} else {
			runtimeStatus.setState("kubernetes", moduleStateRunning, nil)
			g.Go(func() error {
				logger.Info().Msg("Kubernetes agent module started")
				return agent.Run(ctx)
			})
		}
	}

	if remoteConfigClient != nil && len(remoteConfigAppliers) > 0 {
		g.Go(func() error {
			runRemoteConfigLoop(ctx, remoteConfigClient, remoteConfigAppliers, &logger)
			return nil
		})
	}

	// 11. Wait for all agents to exit
	if err := g.Wait(); err != nil && err != context.Canceled {
		logger.Error().Err(err).Msg("Agent terminated with error")
		agentUp.Set(0)
		cleanupDockerAgent(dockerAgent, &logger)
		return fmt.Errorf("unified agent runtime failed: %w", err)
	}

	// 12. Cleanup
	agentUp.Set(0)
	cleanupDockerAgent(dockerAgent, &logger)

	logger.Info().Msg("Pulse Unified Agent stopped")
	return nil
}

func configureAgentLogger(cfg Config) (zerolog.Logger, func(), error) {
	zerolog.SetGlobalLevel(cfg.LogLevel)
	if cfg.LogFile == "" {
		logger := zerolog.New(os.Stdout).With().Timestamp().Logger()
		return logger, func() {}, nil
	}

	logger, closer, err := pulselogging.NewStandaloneLogger(pulselogging.Config{
		Format:     "json",
		Level:      cfg.LogLevel.String(),
		Component:  "pulse-agent",
		FilePath:   cfg.LogFile,
		MaxSizeMB:  agentLogMaxSizeMB,
		MaxAgeDays: agentLogMaxAgeDays,
		Compress:   true,
	}, os.Stdout)
	if err != nil {
		return zerolog.Logger{}, func() {}, fmt.Errorf("initialize log file %q: %w", cfg.LogFile, err)
	}
	return logger, func() {
		if closer != nil {
			_ = closer.Close()
		}
	}, nil
}

func secureAgentStateDir(path string) error {
	path = strings.TrimSpace(path)
	if path == "" {
		return nil
	}
	if err := os.MkdirAll(path, 0o700); err != nil {
		return fmt.Errorf("create state directory: %w", err)
	}
	if err := os.Chmod(path, 0o700); err != nil {
		return fmt.Errorf("chmod state directory: %w", err)
	}
	return nil
}

// readAgentIDFile reads a persisted agent identifier from the given path.
func readAgentIDFile(path string) (string, error) {
	if path == "" {
		return "", nil
	}
	data, err := os.ReadFile(path)
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(data)), nil
}

// writeAgentIDFile persists the agent identifier atomically for future starts.
func writeAgentIDFile(path, id string) error {
	if path == "" || id == "" {
		return nil
	}

	dir := filepath.Dir(path)
	if dir != "" && dir != "." {
		if err := os.MkdirAll(dir, 0o700); err != nil {
			return fmt.Errorf("create agent-id-file directory: %w", err)
		}
	}

	tmp, err := os.CreateTemp(dir, ".agent-id-*")
	if err != nil {
		return fmt.Errorf("create agent-id-file temp: %w", err)
	}
	tmpPath := tmp.Name()
	cleanup := func() { _ = os.Remove(tmpPath) }

	if _, err := tmp.WriteString(id + "\n"); err != nil {
		_ = tmp.Close()
		cleanup()
		return fmt.Errorf("write agent-id-file: %w", err)
	}
	if err := tmp.Chmod(0o600); err != nil {
		_ = tmp.Close()
		cleanup()
		return fmt.Errorf("chmod agent-id-file: %w", err)
	}
	if err := tmp.Close(); err != nil {
		cleanup()
		return fmt.Errorf("close agent-id-file: %w", err)
	}
	if err := os.Rename(tmpPath, path); err != nil {
		cleanup()
		return fmt.Errorf("rename agent-id-file: %w", err)
	}
	return nil
}

func runRemoteConfigLoop(ctx context.Context, client *remoteconfig.Client, appliers []RemoteConfigApplier, logger *zerolog.Logger) {
	if client == nil || len(appliers) == 0 {
		return
	}
	ticker := time.NewTicker(remoteConfigRefreshInterval)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
		}

		fetchCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
		settings, commandsEnabled, err := client.Fetch(fetchCtx)
		cancel()
		if err != nil {
			logger.Warn().
				Err(err).
				Str("component", "remote_config").
				Str("action", "refresh_failed").
				Msg("Failed to refresh remote config")
			continue
		}
		for _, applier := range appliers {
			applier.ApplyRemoteConfig(settings, commandsEnabled)
		}
		logger.Debug().
			Str("component", "remote_config").
			Str("action", "refresh_applied").
			Int("settings_count", len(settings)).
			Bool("commands_enabled_present", commandsEnabled != nil).
			Msg("Applied refreshed remote config")
	}
}

func cleanupDockerAgent(agent RunnableCloser, logger *zerolog.Logger) {
	if agent == nil || reflect.ValueOf(agent).IsNil() {
		return
	}
	if err := agent.Close(); err != nil {
		logger.Warn().
			Err(err).
			Str("component", "docker_agent").
			Str("action", "shutdown_failed").
			Msg("Failed to close Docker / Podman module")
	}
}

func healthHandler(ready *atomic.Bool, runtimes ...*runtimeHealth) http.Handler {
	mux := http.NewServeMux()
	var runtimeStatus *runtimeHealth
	if len(runtimes) > 0 {
		runtimeStatus = runtimes[0]
	}

	// Liveness probe - always returns 200 if server is running
	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("ok"))
	})

	// Readiness probe - returns 200 only when agents are initialized
	mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) {
		if runtimeStatus != nil {
			snapshot := runtimeStatus.snapshot()
			w.Header().Set("Content-Type", "application/json")
			if !snapshot.Ready {
				w.WriteHeader(http.StatusServiceUnavailable)
			}
			_ = json.NewEncoder(w).Encode(snapshot)
			return
		}
		if ready.Load() {
			w.WriteHeader(http.StatusOK)
			w.Write([]byte("ok"))
		} else {
			w.WriteHeader(http.StatusServiceUnavailable)
			w.Write([]byte("not ready"))
		}
	})

	mux.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		if runtimeStatus == nil {
			_ = json.NewEncoder(w).Encode(runtimeHealthSnapshot{Ready: ready.Load()})
			return
		}
		_ = json.NewEncoder(w).Encode(runtimeStatus.snapshot())
	})

	// Prometheus metrics
	mux.Handle("/metrics", promhttp.Handler())
	return mux
}

func startHealthServer(ctx context.Context, addr string, ready *atomic.Bool, logger *zerolog.Logger, runtimes ...*runtimeHealth) {
	srv := &http.Server{
		Addr:         addr,
		Handler:      healthHandler(ready, runtimes...),
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 10 * time.Second,
		IdleTimeout:  30 * time.Second,
	}

	go func() {
		<-ctx.Done()
		shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		if err := srv.Shutdown(shutdownCtx); err != nil && err != http.ErrServerClosed {
			logger.Warn().
				Err(err).
				Str("component", "health_server").
				Str("action", "shutdown_failed").
				Str("addr", addr).
				Msg("Failed to shut down health server")
		}
	}()

	go func() {
		logger.Info().
			Str("component", "health_server").
			Str("action", "listening").
			Str("addr", addr).
			Msg("Health/metrics server listening")
		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			logger.Warn().
				Err(err).
				Str("component", "health_server").
				Str("action", "stopped_unexpectedly").
				Str("addr", addr).
				Msg("Health server stopped unexpectedly")
		}
	}()
}

type Config struct {
	PulseURL           string
	APIToken           string
	Interval           time.Duration
	HostnameOverride   string
	AgentID            string
	AgentIDFile        string
	Tags               []string
	InsecureSkipVerify bool
	AllowPlaintextHTTP bool
	CACertPath         string
	ServerFingerprint  string
	ObserversFile      string
	Observers          []agenttarget.Observer
	DeploySSHUser      string
	LogLevel           zerolog.Level
	LogFile            string
	Logger             *zerolog.Logger

	// Module flags
	EnableHost               bool
	EnableDocker             bool
	DockerConfigured         bool
	DockerExplicitlyDisabled bool
	EnableKubernetes         bool
	EnableProxmox            bool
	ProxmoxType              string // "pve", "pbs", or "" for auto-detect

	// Auto-update
	DisableAutoUpdate         bool
	DisableDockerUpdateChecks bool   // Disable Docker image update detection
	DockerRuntime             string // Force Docker / Podman runtime: docker, podman, or auto

	// Security
	EnableCommands bool // Enable Pulse command execution for Patrol actions and Proxmox LXC Docker inventory (disabled by default)

	// Enrollment
	Enroll bool // Exchange bootstrap token for runtime token on startup

	// State directory
	StateDir string // Persistent state directory for agent-id, proxmox registration, etc.

	// Disk filtering
	DiskExclude []string // Mount points or patterns to exclude from disk monitoring

	// Network configuration
	ReportIP    string // IP address to report (for multi-NIC systems)
	DisableCeph bool   // Disable local Ceph status polling
	SelfTest    bool   // Perform self-test and exit

	// AvailabilityTargets are externally probed availability checks assigned to
	// this agent by the server. Remote config is the only source.
	AvailabilityTargets []pulseconfig.AvailabilityTarget

	// Health/metrics server
	HealthAddr    string
	AppliedConfig *agentshost.ConfigFingerprint

	// Kubernetes
	KubeconfigPath            string
	KubeContext               string
	KubeIncludeNamespaces     []string
	KubeExcludeNamespaces     []string
	KubeIncludeAllPods        bool
	KubeIncludeAllDeployments bool
	KubeMaxPods               int
}

func hostObserverTargets(observers []agenttarget.Observer) []hostagent.ObserverTarget {
	targets := make([]hostagent.ObserverTarget, 0, len(observers))
	for _, observer := range observers {
		targets = append(targets, hostagent.ObserverTarget{
			Name:               observer.Name,
			ID:                 observer.ID,
			PulseURL:           observer.URL,
			APIToken:           observer.Token,
			InsecureSkipVerify: observer.InsecureSkipVerify,
			AllowPlaintextHTTP: observer.AllowPlaintextHTTP,
			CACertPath:         observer.CACertPath,
			ServerFingerprint:  observer.ServerFingerprint,
			ProvisionProxmox:   observer.ProvisionProxmox,
		})
	}
	return targets
}

func dockerReportTargets(cfg Config) []dockeragent.TargetConfig {
	targets := make([]dockeragent.TargetConfig, 0, len(cfg.Observers)+1)
	targets = append(targets, dockeragent.TargetConfig{
		Name:               "primary",
		URL:                cfg.PulseURL,
		Token:              cfg.APIToken,
		InsecureSkipVerify: cfg.InsecureSkipVerify,
		AllowPlaintextHTTP: cfg.AllowPlaintextHTTP,
		CACertPath:         cfg.CACertPath,
		ServerFingerprint:  cfg.ServerFingerprint,
		Authoritative:      true,
	})
	for _, observer := range cfg.Observers {
		targets = append(targets, dockeragent.TargetConfig{
			Name:               observer.Name,
			URL:                observer.URL,
			Token:              observer.Token,
			InsecureSkipVerify: observer.InsecureSkipVerify,
			AllowPlaintextHTTP: observer.AllowPlaintextHTTP,
			CACertPath:         observer.CACertPath,
			ServerFingerprint:  observer.ServerFingerprint,
		})
	}
	return targets
}

func kubernetesReportTargets(cfg Config) []kubernetesagent.TargetConfig {
	targets := make([]kubernetesagent.TargetConfig, 0, len(cfg.Observers)+1)
	targets = append(targets, kubernetesagent.TargetConfig{
		Name:               "primary",
		URL:                cfg.PulseURL,
		Token:              cfg.APIToken,
		InsecureSkipVerify: cfg.InsecureSkipVerify,
		AllowPlaintextHTTP: cfg.AllowPlaintextHTTP,
		CACertPath:         cfg.CACertPath,
		ServerFingerprint:  cfg.ServerFingerprint,
		Authoritative:      true,
	})
	for _, observer := range cfg.Observers {
		targets = append(targets, kubernetesagent.TargetConfig{
			Name:               observer.Name,
			URL:                observer.URL,
			Token:              observer.Token,
			InsecureSkipVerify: observer.InsecureSkipVerify,
			AllowPlaintextHTTP: observer.AllowPlaintextHTTP,
			CACertPath:         observer.CACertPath,
			ServerFingerprint:  observer.ServerFingerprint,
		})
	}
	return targets
}

func loadConfig(args []string, getenv func(string) string) (Config, error) {
	// Environment Variables
	envURL := strings.TrimSpace(getenv("PULSE_URL"))
	envToken := strings.TrimSpace(getenv("PULSE_TOKEN"))
	envInterval := strings.TrimSpace(getenv("PULSE_INTERVAL"))
	envHostname := strings.TrimSpace(getenv("PULSE_HOSTNAME"))
	envAgentID := strings.TrimSpace(getenv("PULSE_AGENT_ID"))
	envAgentIDFile := strings.TrimSpace(getenv("PULSE_AGENT_ID_FILE"))
	envInsecure := strings.TrimSpace(getenv("PULSE_INSECURE_SKIP_VERIFY"))
	envCACertPath := strings.TrimSpace(getenv("PULSE_CACERT"))
	envServerFingerprint := strings.TrimSpace(getenv("PULSE_SERVER_FINGERPRINT"))
	envDeploySSHUser := strings.TrimSpace(getenv("PULSE_DEPLOY_SSH_USER"))
	envTags := strings.TrimSpace(getenv("PULSE_TAGS"))
	envLogLevel := strings.TrimSpace(getenv("LOG_LEVEL"))
	envLogFile := strings.TrimSpace(getenv("PULSE_LOG_FILE"))
	envEnableHost := strings.TrimSpace(getenv("PULSE_ENABLE_HOST"))
	envEnableDocker := strings.TrimSpace(getenv("PULSE_ENABLE_DOCKER"))
	envEnableKubernetes := strings.TrimSpace(getenv("PULSE_ENABLE_KUBERNETES"))
	envEnableProxmox := strings.TrimSpace(getenv("PULSE_ENABLE_PROXMOX"))
	envProxmoxType := strings.TrimSpace(getenv("PULSE_PROXMOX_TYPE"))
	envDisableAutoUpdate := strings.TrimSpace(getenv("PULSE_DISABLE_AUTO_UPDATE"))
	envDisableDockerUpdateChecks := strings.TrimSpace(getenv("PULSE_DISABLE_DOCKER_UPDATE_CHECKS"))
	envDockerRuntime := strings.TrimSpace(getenv("PULSE_DOCKER_RUNTIME"))
	envEnableCommands := strings.TrimSpace(getenv("PULSE_ENABLE_COMMANDS"))
	envDisableCommands := strings.TrimSpace(getenv("PULSE_DISABLE_COMMANDS")) // deprecated
	envHealthAddr := strings.TrimSpace(getenv("PULSE_HEALTH_ADDR"))
	envKubeconfig := strings.TrimSpace(getenv("PULSE_KUBECONFIG"))
	envKubeContext := strings.TrimSpace(getenv("PULSE_KUBE_CONTEXT"))
	envKubeIncludeNamespaces := strings.TrimSpace(getenv("PULSE_KUBE_INCLUDE_NAMESPACES"))
	envKubeExcludeNamespaces := strings.TrimSpace(getenv("PULSE_KUBE_EXCLUDE_NAMESPACES"))
	envKubeIncludeAllPods := strings.TrimSpace(getenv("PULSE_KUBE_INCLUDE_ALL_PODS"))
	if envKubeIncludeAllPods == "" {
		// Backwards compatibility for older env var name.
		envKubeIncludeAllPods = strings.TrimSpace(getenv("PULSE_KUBE_INCLUDE_ALL_POD_FILES"))
	}
	envKubeIncludeAllDeployments := strings.TrimSpace(getenv("PULSE_KUBE_INCLUDE_ALL_DEPLOYMENTS"))
	envKubeMaxPods := strings.TrimSpace(getenv("PULSE_KUBE_MAX_PODS"))
	envStateDir := strings.TrimSpace(getenv("PULSE_STATE_DIR"))
	envDiskExclude := strings.TrimSpace(getenv("PULSE_DISK_EXCLUDE"))
	envReportIP := strings.TrimSpace(getenv("PULSE_REPORT_IP"))
	envDisableCeph := strings.TrimSpace(getenv("PULSE_DISABLE_CEPH"))
	envObserversFile := strings.TrimSpace(getenv("PULSE_OBSERVERS_FILE"))

	// Defaults
	defaultInterval := 30 * time.Second
	if envInterval != "" {
		if parsed, err := time.ParseDuration(envInterval); err == nil {
			defaultInterval = parsed
		}
	}

	defaultEnableHost := true
	if envEnableHost != "" {
		defaultEnableHost = utils.ParseBool(envEnableHost)
	}

	defaultEnableDocker := false
	if envEnableDocker != "" {
		defaultEnableDocker = utils.ParseBool(envEnableDocker)
	}

	defaultEnableKubernetes := false
	if envEnableKubernetes != "" {
		defaultEnableKubernetes = utils.ParseBool(envEnableKubernetes)
	}

	defaultEnableProxmox := false
	if envEnableProxmox != "" {
		defaultEnableProxmox = utils.ParseBool(envEnableProxmox)
	}

	healthAddrDisabledByEnv := false
	defaultHealthAddr := envHealthAddr
	switch strings.ToLower(defaultHealthAddr) {
	case "off", "none", "disabled":
		defaultHealthAddr = ""
		healthAddrDisabledByEnv = true
	}
	if defaultHealthAddr == "" && !healthAddrDisabledByEnv {
		defaultHealthAddr = "127.0.0.1:9191"
	}

	// Flags
	fs := flag.NewFlagSet("pulse-agent", flag.ContinueOnError)
	urlFlag := fs.String("url", envURL, "Pulse server URL")
	tokenFlag := fs.String("token", envToken, "Pulse API token (prefer --token-file for security)")
	tokenFileFlag := fs.String("token-file", "", "Path to file containing Pulse API token (more secure than --token)")
	intervalFlag := fs.Duration("interval", defaultInterval, "Reporting interval")
	hostnameFlag := fs.String("hostname", envHostname, "Override hostname")
	agentIDFlag := fs.String("agent-id", envAgentID, "Override agent identifier")
	agentIDFileFlag := fs.String("agent-id-file", envAgentIDFile, "Path to a file storing the agent identifier (read on start, written on first start). Mount this file as a volume to keep the agent identity stable across container recreation.")
	insecureFlag := fs.Bool("insecure", utils.ParseBool(envInsecure), "Skip TLS verification")
	allowPlaintextHTTPFlag := fs.Bool("allow-plaintext-http", utils.ParseBool(strings.TrimSpace(getenv("PULSE_AGENT_ALLOW_PLAINTEXT_HTTP"))), "Allow plain HTTP to a Pulse server that does not look local (sends the API token in cleartext; only for networks you fully control)")
	caCertFlag := fs.String("cacert", envCACertPath, "Path to custom CA bundle for agent HTTPS transport")
	serverFingerprintFlag := fs.String("server-fingerprint", envServerFingerprint, "Expected Pulse server TLS certificate fingerprint (SHA256)")
	observersFileFlag := fs.String("observers-file", envObserversFile, "Absolute path to a private JSON file defining report-only Pulse observer destinations")
	deploySSHUserFlag := fs.String("deploy-ssh-user", envDeploySSHUser, "SSH user for peer deploy fan-out (default: root; non-root requires passwordless sudo)")
	logLevelFlag := fs.String("log-level", defaultLogLevel(envLogLevel), "Log level")
	logFileFlag := fs.String("log-file", envLogFile, "Write rotating JSON logs to this file")

	enableHostFlag := fs.Bool("enable-host", defaultEnableHost, "Enable Host Agent module")
	enableDockerFlag := fs.Bool("enable-docker", defaultEnableDocker, "Enable Docker / Podman Agent module")
	enableKubernetesFlag := fs.Bool("enable-kubernetes", defaultEnableKubernetes, "Enable Kubernetes Agent module")
	enableProxmoxFlag := fs.Bool("enable-proxmox", defaultEnableProxmox, "Enable Proxmox mode (creates API token, registers node)")
	proxmoxTypeFlag := fs.String("proxmox-type", envProxmoxType, "Proxmox type: pve or pbs (auto-detected if not specified)")
	disableAutoUpdateFlag := fs.Bool("disable-auto-update", utils.ParseBool(envDisableAutoUpdate), "Disable automatic updates")
	disableDockerUpdateChecksFlag := fs.Bool("disable-docker-update-checks", utils.ParseBool(envDisableDockerUpdateChecks), "Disable Docker image update detection (avoids Docker Hub rate limits)")
	dockerRuntimeFlag := fs.String("docker-runtime", envDockerRuntime, "Docker / Podman runtime: auto, docker, or podman (default: auto)")
	enableCommandsFlag := fs.Bool("enable-commands", utils.ParseBool(envEnableCommands), "Enable Pulse command execution for Patrol actions and Proxmox LXC Docker inventory (disabled by default)")
	disableCommandsFlag := fs.Bool("disable-commands", false, "[DEPRECATED] Commands are now disabled by default; use --enable-commands to enable")
	enrollFlag := fs.Bool("enroll", false, "Exchange bootstrap token for runtime token (used by deploy wizard)")
	healthAddrFlag := fs.String("health-addr", defaultHealthAddr, "Health/metrics server address (empty to disable)")
	kubeconfigFlag := fs.String("kubeconfig", envKubeconfig, "Path to kubeconfig (optional; uses in-cluster config if available)")
	kubeContextFlag := fs.String("kube-context", envKubeContext, "Kubeconfig context (optional)")
	kubeIncludeAllPodsFlag := fs.Bool("kube-include-all-pods", utils.ParseBool(envKubeIncludeAllPods), "Include all non-succeeded pods (may be large)")
	kubeIncludeAllDeploymentsFlag := fs.Bool("kube-include-all-deployments", utils.ParseBool(envKubeIncludeAllDeployments), "Include all deployments, not just problem ones")
	kubeMaxPodsFlag := fs.Int("kube-max-pods", defaultInt(envKubeMaxPods, 200), "Max pods included in report")
	stateDirFlag := fs.String("state-dir", envStateDir, "Persistent state directory (default: platform service state directory)")
	reportIPFlag := fs.String("report-ip", envReportIP, "IP address to report (for multi-NIC systems)")
	disableCephFlag := fs.Bool("disable-ceph", utils.ParseBool(envDisableCeph), "Disable local Ceph status polling")
	showVersion := fs.Bool("version", false, "Print the agent version and exit")
	selfTest := fs.Bool("self-test", false, "Perform self-test and exit (used during auto-update)")

	var tagFlags multiValue
	fs.Var(&tagFlags, "tag", "Tag to apply (repeatable)")
	var kubeIncludeNamespaceFlags multiValue
	fs.Var(&kubeIncludeNamespaceFlags, "kube-include-namespace", "Namespace to include (repeatable; default is all)")
	var kubeExcludeNamespaceFlags multiValue
	fs.Var(&kubeExcludeNamespaceFlags, "kube-exclude-namespace", "Namespace to exclude (repeatable)")
	var diskExcludeFlags multiValue
	fs.Var(&diskExcludeFlags, "disk-exclude", "Device name/path or mount point pattern to exclude from disk monitoring (repeatable)")

	if err := fs.Parse(args); err != nil {
		return Config{}, err
	}

	if *showVersion {
		fmt.Println(Version)
		return Config{}, flag.ErrHelp
	}

	// Validation
	pulseURL := strings.TrimSpace(*urlFlag)
	if pulseURL == "" {
		pulseURL = "http://localhost:7655"
	}

	// Record operator plaintext consent before any module validates the Pulse
	// URL; the startup warning is emitted once the logger exists in run().
	securityutil.SetOperatorPlaintextHTTPConsent(*allowPlaintextHTTPFlag)

	// Resolve the state directory before any implicit token or identity path.
	// A custom instance must never borrow the default instance's credentials.
	stateDir := strings.TrimSpace(*stateDirFlag)
	if stateDir == "" {
		stateDir = defaultAgentStateDir()
	}
	agentIDFile := strings.TrimSpace(*agentIDFileFlag)
	if agentIDFile == "" {
		agentIDFile = filepath.Join(stateDir, "agent-id")
	}

	// Resolve token with priority: --token > --token-file > env > state-dir file.
	token := resolveToken(*tokenFlag, *tokenFileFlag, envToken, stateDir)
	observers, err := agenttarget.LoadObservers(strings.TrimSpace(*observersFileFlag), pulseURL)
	if err != nil {
		return Config{}, fmt.Errorf("load observer destinations: %w", err)
	}

	// When --enroll is set and a runtime token already exists from a previous
	// enrollment, use it instead of the bootstrap token embedded in the service
	// config. This ensures the agent survives process and server restarts.
	if *enrollFlag {
		runtimeTokenPath := filepath.Join(stateDir, "runtime.token")
		if content, err := os.ReadFile(runtimeTokenPath); err == nil {
			if t := strings.TrimSpace(string(content)); t != "" {
				token = t
			}
		}
	}

	if token == "" && *enrollFlag && !*selfTest {
		return Config{}, fmt.Errorf("Pulse API token is required for enrollment (use --token, --token-file, PULSE_TOKEN env, or %s)", defaultTokenFilePath())
	}

	logLevel, err := parseLogLevel(*logLevelFlag)
	if err != nil {
		return Config{}, fmt.Errorf("invalid log level %q: %w", strings.TrimSpace(*logLevelFlag), err)
	}
	interval := *intervalFlag
	if interval <= 0 {
		return Config{}, fmt.Errorf("interval must be greater than 0 (got %s)", interval)
	}
	kubeMaxPods := *kubeMaxPodsFlag
	if kubeMaxPods <= 0 {
		return Config{}, fmt.Errorf("kube-max-pods must be greater than 0 (got %d)", kubeMaxPods)
	}
	dockerRuntime, err := normalizeDockerRuntime(*dockerRuntimeFlag)
	if err != nil {
		return Config{}, err
	}
	deploySSHUser, err := hostagent.NormalizeDeploySSHUser(*deploySSHUserFlag)
	if err != nil {
		return Config{}, err
	}

	tags := gatherTags(envTags, tagFlags)
	kubeIncludeNamespaces := gatherCSV(envKubeIncludeNamespaces, kubeIncludeNamespaceFlags)
	kubeExcludeNamespaces := gatherCSV(envKubeExcludeNamespaces, kubeExcludeNamespaceFlags)
	diskExclude := gatherCSV(envDiskExclude, diskExcludeFlags)

	// Check if Docker was explicitly configured via fs or env. An explicit
	// local disable is a privacy boundary and cannot be reversed by remote
	// profile config or auto-detection.
	dockerConfigured := envEnableDocker != ""
	dockerExplicitlyDisabled := envEnableDocker != "" && !utils.ParseBool(envEnableDocker)
	if !dockerConfigured {
		fs.Visit(func(f *flag.Flag) {
			if f.Name == "enable-docker" {
				dockerConfigured = true
				dockerExplicitlyDisabled = !*enableDockerFlag
			}
		})
	} else {
		fs.Visit(func(f *flag.Flag) {
			if f.Name == "enable-docker" {
				dockerExplicitlyDisabled = !*enableDockerFlag
			}
		})
	}

	return Config{
		PulseURL:                  pulseURL,
		APIToken:                  token,
		Interval:                  interval,
		HostnameOverride:          strings.TrimSpace(*hostnameFlag),
		AgentID:                   strings.TrimSpace(*agentIDFlag),
		AgentIDFile:               agentIDFile,
		Tags:                      tags,
		InsecureSkipVerify:        *insecureFlag,
		AllowPlaintextHTTP:        *allowPlaintextHTTPFlag,
		CACertPath:                strings.TrimSpace(*caCertFlag),
		ServerFingerprint:         strings.TrimSpace(*serverFingerprintFlag),
		ObserversFile:             strings.TrimSpace(*observersFileFlag),
		Observers:                 observers,
		DeploySSHUser:             deploySSHUser,
		LogLevel:                  logLevel,
		LogFile:                   strings.TrimSpace(*logFileFlag),
		EnableHost:                *enableHostFlag,
		EnableDocker:              *enableDockerFlag,
		DockerConfigured:          dockerConfigured,
		DockerExplicitlyDisabled:  dockerExplicitlyDisabled,
		EnableKubernetes:          *enableKubernetesFlag,
		EnableProxmox:             *enableProxmoxFlag,
		ProxmoxType:               strings.TrimSpace(*proxmoxTypeFlag),
		DisableAutoUpdate:         *disableAutoUpdateFlag,
		DisableDockerUpdateChecks: *disableDockerUpdateChecksFlag,
		DockerRuntime:             dockerRuntime,
		EnableCommands:            resolveEnableCommands(*enableCommandsFlag, *disableCommandsFlag, envEnableCommands, envDisableCommands),
		Enroll:                    *enrollFlag,
		HealthAddr:                strings.TrimSpace(*healthAddrFlag),
		KubeconfigPath:            strings.TrimSpace(*kubeconfigFlag),
		KubeContext:               strings.TrimSpace(*kubeContextFlag),
		KubeIncludeNamespaces:     kubeIncludeNamespaces,
		KubeExcludeNamespaces:     kubeExcludeNamespaces,
		KubeIncludeAllPods:        *kubeIncludeAllPodsFlag,
		KubeIncludeAllDeployments: *kubeIncludeAllDeploymentsFlag,
		KubeMaxPods:               kubeMaxPods,
		StateDir:                  stateDir,
		DiskExclude:               diskExclude,
		ReportIP:                  strings.TrimSpace(*reportIPFlag),
		DisableCeph:               *disableCephFlag,
		SelfTest:                  *selfTest,
	}, nil
}

func gatherTags(env string, flags []string) []string {
	tags := make([]string, 0)
	if env != "" {
		for _, tag := range strings.Split(env, ",") {
			tag = strings.TrimSpace(tag)
			if tag != "" {
				tags = append(tags, tag)
			}
		}
	}
	for _, tag := range flags {
		tag = strings.TrimSpace(tag)
		if tag != "" {
			tags = append(tags, tag)
		}
	}
	return tags
}

func gatherCSV(env string, flags []string) []string {
	values := make([]string, 0)
	if env != "" {
		for _, value := range strings.Split(env, ",") {
			value = strings.TrimSpace(value)
			if value != "" {
				values = append(values, value)
			}
		}
	}
	for _, value := range flags {
		value = strings.TrimSpace(value)
		if value != "" {
			values = append(values, value)
		}
	}
	return values
}

func defaultInt(value string, fallback int) int {
	value = strings.TrimSpace(value)
	if value == "" {
		return fallback
	}
	parsed, err := strconv.Atoi(value)
	if err != nil {
		return fallback
	}
	return parsed
}

func normalizeDockerRuntime(value string) (string, error) {
	runtime := strings.ToLower(strings.TrimSpace(value))
	switch runtime {
	case "", "auto", "default":
		return "", nil
	case "docker", "podman":
		return runtime, nil
	default:
		return "", fmt.Errorf("invalid docker runtime %q: must be auto, docker, or podman", value)
	}
}

func parseLogLevel(value string) (zerolog.Level, error) {
	normalized := strings.ToLower(strings.TrimSpace(value))
	if normalized == "" {
		return zerolog.InfoLevel, nil
	}
	return zerolog.ParseLevel(normalized)
}

func defaultLogLevel(envValue string) string {
	if strings.TrimSpace(envValue) == "" {
		return "info"
	}
	return envValue
}

// resolveEnableCommands determines whether command execution should be enabled.
// Priority: --enable-commands > --disable-commands (deprecated) > PULSE_ENABLE_COMMANDS > PULSE_DISABLE_COMMANDS (deprecated)
// Default: disabled (false) for security
func resolveEnableCommands(enableFlag, disableFlag bool, envEnable, envDisable string) bool {
	// If --enable-commands is explicitly set, use it
	if enableFlag {
		return true
	}

	// Backwards compat: if --disable-commands was used, log deprecation but respect it
	// (disableFlag being true means commands should be disabled, which is already the default)
	if disableFlag {
		fmt.Fprintln(os.Stderr, "warning: --disable-commands is deprecated and no longer needed (commands are disabled by default). Use --enable-commands to enable.")
		return false
	}

	// Check environment variables
	if envEnable != "" {
		return utils.ParseBool(envEnable)
	}

	// Backwards compat: PULSE_DISABLE_COMMANDS=true means commands disabled (already default)
	// PULSE_DISABLE_COMMANDS=false means commands enabled (backwards compat)
	if envDisable != "" {
		fmt.Fprintln(os.Stderr, "warning: PULSE_DISABLE_COMMANDS is deprecated. Use PULSE_ENABLE_COMMANDS=true to enable commands.")
		// Invert: DISABLE=false means enable
		return !utils.ParseBool(envDisable)
	}

	// Default: commands disabled
	return false
}

// resolveToken resolves the API token with priority:
// 1. --token flag (direct value)
// 2. --token-file flag (read from file)
// 3. PULSE_TOKEN environment variable
// 4. Token file under the resolved state directory
//
// Reading from a file is more secure than CLI args as tokens won't appear in `ps` output.
func resolveToken(tokenFlag, tokenFileFlag, envToken, stateDir string) string {
	return resolveTokenInternal(tokenFlag, tokenFileFlag, envToken, stateDir, os.ReadFile)
}

// defaultAgentStateDir mirrors where each platform's installer keeps agent
// state: %ProgramData%\Pulse on Windows (see scripts/install.ps1), the
// systemd/launchd convention /var/lib/pulse-agent everywhere else.
func defaultAgentStateDir() string {
	if runtime.GOOS == "windows" {
		if pd := strings.TrimSpace(os.Getenv("ProgramData")); pd != "" {
			return filepath.Join(pd, "Pulse")
		}
		return `C:\ProgramData\Pulse`
	}
	return "/var/lib/pulse-agent"
}

func defaultTokenFilePath() string {
	return filepath.Join(defaultAgentStateDir(), "token")
}

func resolveTokenInternal(tokenFlag, tokenFileFlag, envToken, stateDir string, readFile func(string) ([]byte, error)) string {
	// 1. Direct token from --token flag
	if t := strings.TrimSpace(tokenFlag); t != "" {
		return t
	}

	// 2. Token from --token-file flag
	if tokenFileFlag != "" {
		if content, err := readFile(tokenFileFlag); err == nil {
			if t := strings.TrimSpace(string(content)); t != "" {
				return t
			}
		}
	}

	// 3. PULSE_TOKEN environment variable
	if t := strings.TrimSpace(envToken); t != "" {
		return t
	}

	// 4. Token file in the resolved state directory. When stateDir is custom,
	// do not fall through to the default instance's token.
	tokenFile := filepath.Join(strings.TrimSpace(stateDir), "token")
	if strings.TrimSpace(stateDir) == "" {
		tokenFile = defaultTokenFilePath()
	}
	if content, err := readFile(tokenFile); err == nil {
		if t := strings.TrimSpace(string(content)); t != "" {
			return t
		}
	}

	return ""
}

// initModuleWithRetry attempts to initialize an agent module with exponential
// backoff. It returns the module when its backend becomes available, or a zero
// value if the context is cancelled. component and displayName feed the
// structured log fields; unavailableMsg is the per-module retry log line.
// Retry intervals: 5s, 10s, 20s, 40s, 80s, 160s, then cap at 5 minutes.
func initModuleWithRetry[T any](ctx context.Context, logger *zerolog.Logger, component, displayName, unavailableMsg string, connect func() (T, error)) T {
	const multiplier = 2.0

	var zero T
	delay := retryInitialDelay
	attempt := 0

	for {
		if err := ctx.Err(); err != nil {
			logger.Info().Msg(displayName + " retry cancelled, context done")
			return zero
		}

		module, err := connect()
		if err == nil {
			logger.Info().
				Str("component", component).
				Str("action", "retry_connect_succeeded").
				Int("attempts", attempt+1).
				Msg("Successfully connected to " + displayName + " after retry")
			return module
		}

		attempt++
		retryLogEvent(logger, attempt).
			Err(err).
			Str("component", component).
			Str("action", "retry_connect_failed").
			Int("attempt", attempt).
			Str("next_retry", delay.String()).
			Msg(unavailableMsg)

		if !waitForRetryDelay(ctx, delay) {
			logger.Info().
				Str("component", component).
				Str("action", "retry_cancelled").
				Msg(displayName + " retry cancelled, context done")
			return zero
		}

		// Calculate next delay with exponential backoff, capped at retryMaxDelay
		delay = time.Duration(float64(delay) * multiplier)
		if delay > retryMaxDelay {
			delay = retryMaxDelay
		}
	}
}

// initDockerWithRetry attempts to initialize the Docker / Podman collection module with exponential backoff.
// It returns the module when Docker / Podman becomes available, or nil if the context is cancelled.
// lateBoundDockerUpdater satisfies hostagent.DockerContainerUpdater while the
// Docker module comes up (or never does). The host command client holds this
// bridge for the process lifetime; set installs the module's implementation.
type lateBoundDockerUpdater struct {
	mu      sync.RWMutex
	updater hostagent.DockerContainerUpdater
}

func (b *lateBoundDockerUpdater) set(candidate any) {
	updater, ok := candidate.(hostagent.DockerContainerUpdater)
	if !ok {
		fmt.Fprintf(os.Stderr, "docker update bridge: %T does not implement the typed container updater\n", candidate)
		return
	}
	b.mu.Lock()
	b.updater = updater
	b.mu.Unlock()
}

func (b *lateBoundDockerUpdater) TypedContainerUpdate(ctx context.Context, runtime, containerID, expectedImageDigest string, progress func(string)) (agentexec.DockerContainerUpdateOutcome, error) {
	b.mu.RLock()
	updater := b.updater
	b.mu.RUnlock()
	if updater == nil {
		return agentexec.DockerContainerUpdateOutcome{}, fmt.Errorf("docker module is not running on this agent")
	}
	return updater.TypedContainerUpdate(ctx, runtime, containerID, expectedImageDigest, progress)
}

func initDockerWithRetry(ctx context.Context, cfg dockeragent.Config, logger *zerolog.Logger) RunnableCloser {
	return initModuleWithRetry(ctx, logger, "docker_agent", "Docker", "Docker not available, will retry", func() (RunnableCloser, error) {
		return newDockerAgent(cfg)
	})
}

// initKubernetesWithRetry attempts to initialize the Kubernetes agent with exponential backoff.
// It returns the agent when Kubernetes becomes available, or nil if the context is cancelled.
func initKubernetesWithRetry(ctx context.Context, cfg kubernetesagent.Config, logger *zerolog.Logger) Runnable {
	return initModuleWithRetry(ctx, logger, "kubernetes_agent", "Kubernetes", "Kubernetes still not available, will retry", func() (Runnable, error) {
		return newKubeAgent(cfg)
	})
}

func waitForRetryDelay(ctx context.Context, delay time.Duration) bool {
	timer := time.NewTimer(delay)
	defer func() {
		if !timer.Stop() {
			select {
			case <-timer.C:
			default:
			}
		}
	}()

	select {
	case <-ctx.Done():
		return false
	case <-timer.C:
		return true
	}
}

// retryLogEvent returns a zerolog event at a level that decreases with attempt count
// to avoid flooding logs on misconfigured systems with unbounded retries.
//   - Attempts 1-10:  Warn  (initial visibility)
//   - Attempts 11-50: Info  (still visible, less noisy)
//   - Attempts 51+:   Debug (effectively silent unless debug logging enabled)
func retryLogEvent(logger *zerolog.Logger, attempt int) *zerolog.Event {
	switch {
	case attempt <= 10:
		return logger.Warn()
	case attempt <= 50:
		return logger.Info()
	default:
		return logger.Debug()
	}
}

// applyRemoteSettings merges remote settings into the local configuration.
// Supported keys:
// - enable_host (bool)
// - enable_docker (bool)
// - enable_kubernetes (bool)
// - enable_proxmox (bool)
// - proxmox_type (string)
// - docker_runtime (string)
// - disable_auto_update (bool)
// - disable_docker_update_checks (bool)
// - kube_include_all_pods (bool)
// - kube_include_all_deployments (bool)
// - log_level (string)
// - interval (string/duration)
// - report_ip (string)
// - disable_ceph (bool)
// - availabilityTargets (array of assigned availability checks)
func applyRemoteSettings(cfg *Config, settings map[string]interface{}, logger *zerolog.Logger) {
	for k, v := range settings {
		switch k {
		case "enable_host":
			if b, ok := v.(bool); ok {
				cfg.EnableHost = b
				logger.Info().Bool("val", b).Msg("Remote config: enable_host")
			}
		case "enable_docker":
			if b, ok := v.(bool); ok {
				if b && cfg.DockerExplicitlyDisabled {
					cfg.DockerConfigured = true
					logger.Info().Msg("Remote config: enable_docker ignored because Docker / Podman monitoring is locally disabled")
					continue
				}
				cfg.EnableDocker = b
				cfg.DockerConfigured = true
				logger.Info().Bool("val", b).Msg("Remote config: enable_docker")
			}
		case "enable_kubernetes":
			if b, ok := v.(bool); ok {
				cfg.EnableKubernetes = b
				logger.Info().Bool("val", b).Msg("Remote config: enable_kubernetes")
			}
		case "enable_proxmox":
			if b, ok := v.(bool); ok {
				cfg.EnableProxmox = b
				logger.Info().Bool("val", b).Msg("Remote config: enable_proxmox")
			}
		case "proxmox_type":
			if s, ok := v.(string); ok {
				normalized := strings.TrimSpace(strings.ToLower(s))
				if normalized == "auto" {
					normalized = ""
				}
				cfg.ProxmoxType = normalized
				logger.Info().Str("val", s).Msg("Remote config: proxmox_type")
			}
		case "docker_runtime":
			if s, ok := v.(string); ok {
				runtime, err := normalizeDockerRuntime(s)
				if err != nil {
					logger.Warn().Str("val", s).Msg("Remote config: ignoring invalid docker_runtime value")
					continue
				}
				cfg.DockerRuntime = runtime
				logger.Info().Str("val", s).Msg("Remote config: docker_runtime")
			}
		case "log_level":
			if s, ok := v.(string); ok {
				if l, err := zerolog.ParseLevel(s); err == nil {
					cfg.LogLevel = l
					zerolog.SetGlobalLevel(l)
					logger.Info().Str("val", s).Msg("Remote config: log_level")
				}
			}
		case "interval":
			if s, ok := v.(string); ok {
				if d, err := time.ParseDuration(s); err == nil && d > 0 {
					cfg.Interval = d
					logger.Info().Str("val", s).Msg("Remote config: interval")
				} else {
					logger.Warn().Str("val", s).Msg("Remote config: ignoring invalid interval value")
				}
			} else if f, ok := v.(float64); ok {
				if math.IsNaN(f) || math.IsInf(f, 0) || f <= 0 {
					logger.Warn().Float64("val", f).Msg("Remote config: ignoring invalid interval value")
					continue
				}
				// JSON numbers are floats, assume seconds.
				cfg.Interval = time.Duration(f * float64(time.Second))
				logger.Info().Float64("val", f).Msg("Remote config: interval (s)")
			}
		case "disable_auto_update":
			if b, ok := v.(bool); ok {
				cfg.DisableAutoUpdate = b
				logger.Info().Bool("val", b).Msg("Remote config: disable_auto_update")
			}
		case "disable_docker_update_checks":
			if b, ok := v.(bool); ok {
				cfg.DisableDockerUpdateChecks = b
				logger.Info().Bool("val", b).Msg("Remote config: disable_docker_update_checks")
			}
		case "kube_include_all_pods":
			if b, ok := v.(bool); ok {
				cfg.KubeIncludeAllPods = b
				logger.Info().Bool("val", b).Msg("Remote config: kube_include_all_pods")
			}
		case "kube_include_all_deployments":
			if b, ok := v.(bool); ok {
				cfg.KubeIncludeAllDeployments = b
				logger.Info().Bool("val", b).Msg("Remote config: kube_include_all_deployments")
			}
		case "report_ip":
			if s, ok := v.(string); ok {
				cfg.ReportIP = s
				logger.Info().Str("val", s).Msg("Remote config: report_ip")
			}
		case "disable_ceph":
			if b, ok := v.(bool); ok {
				cfg.DisableCeph = b
				logger.Info().Bool("val", b).Msg("Remote config: disable_ceph")
			}
		case "availabilityTargets":
			targets, err := hostagent.AvailabilityTargetsFromSetting(v)
			if err != nil {
				logger.Warn().Err(err).Msg("Remote config: ignoring unreadable availabilityTargets value")
				continue
			}
			cfg.AvailabilityTargets = targets
			logger.Info().Int("count", len(targets)).Msg("Remote config: availabilityTargets")
		}
	}
}

func remoteDurationSetting(settings map[string]interface{}, key string) (time.Duration, bool) {
	value, ok := settings[key]
	if !ok {
		return 0, false
	}

	switch typed := value.(type) {
	case string:
		parsed, err := time.ParseDuration(typed)
		if err != nil {
			return 0, false
		}
		return parsed, true
	case float64:
		return time.Duration(typed * float64(time.Second)), true
	case int:
		return time.Duration(typed) * time.Second, true
	case int64:
		return time.Duration(typed) * time.Second, true
	default:
		return 0, false
	}
}