From d0191d136ffefb920bdcacca22467b53e3361507 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Mon, 5 Jan 2026 23:34:33 +0000 Subject: [PATCH] fix: Add configurable poll timeout and handle external Ceph storage Changes: 1. Add MAX_POLL_TIMEOUT env var for large Proxmox clusters that need more than 3 minutes for polling (default: 3m, minimum: 30s) 2. Handle external Ceph storage gracefully - don't mark nodes unhealthy when Proxmox returns 'binary not installed' (e.g., for Ceph not managed by Proxmox) Related to #965 --- internal/config/config.go | 17 ++++++++++++++++- internal/monitoring/monitor.go | 10 ++++++++-- internal/monitoring/monitor_timeout_test.go | 16 ++++++++++++++++ pkg/proxmox/cluster_client.go | 2 ++ 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 4705b3f07..bbe5e101c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -119,7 +119,8 @@ type Config struct { GuestMetadataRetryBackoff time.Duration `envconfig:"GUEST_METADATA_RETRY_BACKOFF" default:"30s" json:"guestMetadataRetryBackoff"` GuestMetadataMaxConcurrent int `envconfig:"GUEST_METADATA_MAX_CONCURRENT" default:"4" json:"guestMetadataMaxConcurrent"` DNSCacheTimeout time.Duration `envconfig:"DNS_CACHE_TIMEOUT" default:"5m" json:"dnsCacheTimeout"` - SSHPort int `envconfig:"SSH_PORT" default:"22" json:"sshPort"` // Default SSH port for temperature monitoring + SSHPort int `envconfig:"SSH_PORT" default:"22" json:"sshPort"` // Default SSH port for temperature monitoring + MaxPollTimeout time.Duration `envconfig:"MAX_POLL_TIMEOUT" default:"3m" json:"-"` // Maximum poll timeout for large clusters (default 3m) // Metrics retention settings (tiered storage) // These control how long historical metrics are retained at each aggregation level. @@ -594,6 +595,7 @@ func Load() (*Config, error) { DiscoveryEnabled: false, DiscoverySubnet: "auto", TemperatureMonitoringEnabled: true, + MaxPollTimeout: 3 * time.Minute, // Default max poll timeout for large clusters EnableSensorProxy: false, EnvOverrides: make(map[string]bool), AgentConnectURL: "", @@ -1446,6 +1448,19 @@ func Load() (*Config, error) { log.Info().Dur("timeout", d).Msg("Connection timeout overridden by CONNECTION_TIMEOUT env var") } } + if maxPollTimeout := os.Getenv("MAX_POLL_TIMEOUT"); maxPollTimeout != "" { + if d, err := time.ParseDuration(maxPollTimeout); err == nil { + if d >= 30*time.Second { // Minimum 30 seconds + cfg.MaxPollTimeout = d + cfg.EnvOverrides["maxPollTimeout"] = true + log.Info().Dur("timeout", d).Msg("Max poll timeout overridden by MAX_POLL_TIMEOUT env var") + } else { + log.Warn().Dur("value", d).Msg("MAX_POLL_TIMEOUT too low (minimum 30s), using default") + } + } else { + log.Warn().Str("value", maxPollTimeout).Msg("Invalid MAX_POLL_TIMEOUT value, using default") + } + } if allowedOrigins := os.Getenv("ALLOWED_ORIGINS"); allowedOrigins != "" { cfg.AllowedOrigins = allowedOrigins cfg.EnvOverrides["allowedOrigins"] = true diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 67bbf1de6..b7e964f46 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -4529,8 +4529,14 @@ func derivePollTimeout(cfg *config.Config) time.Duration { if timeout < minTaskTimeout { timeout = minTaskTimeout } - if timeout > maxTaskTimeout { - timeout = maxTaskTimeout + // Use configurable max timeout from config (set via MAX_POLL_TIMEOUT env var) + // Falls back to hardcoded maxTaskTimeout if config is nil or MaxPollTimeout not set + maxTimeout := maxTaskTimeout + if cfg != nil && cfg.MaxPollTimeout > 0 { + maxTimeout = cfg.MaxPollTimeout + } + if timeout > maxTimeout { + timeout = maxTimeout } return timeout } diff --git a/internal/monitoring/monitor_timeout_test.go b/internal/monitoring/monitor_timeout_test.go index c516925b3..8433d8d48 100644 --- a/internal/monitoring/monitor_timeout_test.go +++ b/internal/monitoring/monitor_timeout_test.go @@ -39,6 +39,22 @@ func TestDerivePollTimeout(t *testing.T) { }, want: maxTaskTimeout, }, + { + name: "respects custom MaxPollTimeout", + cfg: &config.Config{ + ConnectionTimeout: 3 * time.Minute, + MaxPollTimeout: 10 * time.Minute, + }, + want: 6 * time.Minute, // 2 * ConnectionTimeout, still under MaxPollTimeout + }, + { + name: "custom MaxPollTimeout caps at configured value", + cfg: &config.Config{ + ConnectionTimeout: 10 * time.Minute, + MaxPollTimeout: 5 * time.Minute, + }, + want: 5 * time.Minute, // Capped at MaxPollTimeout + }, } for _, tt := range tests { diff --git a/pkg/proxmox/cluster_client.go b/pkg/proxmox/cluster_client.go index c6a40e042..908e5cd83 100644 --- a/pkg/proxmox/cluster_client.go +++ b/pkg/proxmox/cluster_client.go @@ -724,6 +724,8 @@ func (cc *ClusterClient) executeWithFailover(ctx context.Context, fn func(*Clien // PBS storage errors - Proxmox can't reach PBS, but node is still reachable (strings.Contains(errStr, "500") && strings.Contains(errStr, "pbs-") && strings.Contains(errStr, "error fetching datastores")) || (strings.Contains(errStr, "500") && strings.Contains(errStr, "Can't connect to") && strings.Contains(errStr, ":8007")) || + // External Ceph errors - Ceph not managed by Proxmox, but node is still reachable + (strings.Contains(errStr, "500") && strings.Contains(errStr, "binary not installed")) || // RRD data timeouts - secondary metric fetch failures, node is still working (strings.Contains(errStr, "context deadline exceeded") && strings.Contains(errStr, "/rrddata")) || (strings.Contains(errStr, "context deadline exceeded") && strings.Contains(errStr, "/lxc/") && strings.Contains(errStr, "rrd")) ||