fix: Add configurable poll timeout and handle external Ceph storage

Changes:
1. Add MAX_POLL_TIMEOUT env var for large Proxmox clusters that need
   more than 3 minutes for polling (default: 3m, minimum: 30s)
2. Handle external Ceph storage gracefully - don't mark nodes unhealthy
   when Proxmox returns 'binary not installed' (e.g., for Ceph not
   managed by Proxmox)

Related to #965
This commit is contained in:
rcourtman
2026-01-05 23:34:33 +00:00
parent c6182b2ed3
commit d0191d136f
4 changed files with 42 additions and 3 deletions
+16 -1
View File
@@ -119,7 +119,8 @@ type Config struct {
GuestMetadataRetryBackoff time.Duration `envconfig:"GUEST_METADATA_RETRY_BACKOFF" default:"30s" json:"guestMetadataRetryBackoff"`
GuestMetadataMaxConcurrent int `envconfig:"GUEST_METADATA_MAX_CONCURRENT" default:"4" json:"guestMetadataMaxConcurrent"`
DNSCacheTimeout time.Duration `envconfig:"DNS_CACHE_TIMEOUT" default:"5m" json:"dnsCacheTimeout"`
SSHPort int `envconfig:"SSH_PORT" default:"22" json:"sshPort"` // Default SSH port for temperature monitoring
SSHPort int `envconfig:"SSH_PORT" default:"22" json:"sshPort"` // Default SSH port for temperature monitoring
MaxPollTimeout time.Duration `envconfig:"MAX_POLL_TIMEOUT" default:"3m" json:"-"` // Maximum poll timeout for large clusters (default 3m)
// Metrics retention settings (tiered storage)
// These control how long historical metrics are retained at each aggregation level.
@@ -594,6 +595,7 @@ func Load() (*Config, error) {
DiscoveryEnabled: false,
DiscoverySubnet: "auto",
TemperatureMonitoringEnabled: true,
MaxPollTimeout: 3 * time.Minute, // Default max poll timeout for large clusters
EnableSensorProxy: false,
EnvOverrides: make(map[string]bool),
AgentConnectURL: "",
@@ -1446,6 +1448,19 @@ func Load() (*Config, error) {
log.Info().Dur("timeout", d).Msg("Connection timeout overridden by CONNECTION_TIMEOUT env var")
}
}
if maxPollTimeout := os.Getenv("MAX_POLL_TIMEOUT"); maxPollTimeout != "" {
if d, err := time.ParseDuration(maxPollTimeout); err == nil {
if d >= 30*time.Second { // Minimum 30 seconds
cfg.MaxPollTimeout = d
cfg.EnvOverrides["maxPollTimeout"] = true
log.Info().Dur("timeout", d).Msg("Max poll timeout overridden by MAX_POLL_TIMEOUT env var")
} else {
log.Warn().Dur("value", d).Msg("MAX_POLL_TIMEOUT too low (minimum 30s), using default")
}
} else {
log.Warn().Str("value", maxPollTimeout).Msg("Invalid MAX_POLL_TIMEOUT value, using default")
}
}
if allowedOrigins := os.Getenv("ALLOWED_ORIGINS"); allowedOrigins != "" {
cfg.AllowedOrigins = allowedOrigins
cfg.EnvOverrides["allowedOrigins"] = true
+8 -2
View File
@@ -4529,8 +4529,14 @@ func derivePollTimeout(cfg *config.Config) time.Duration {
if timeout < minTaskTimeout {
timeout = minTaskTimeout
}
if timeout > maxTaskTimeout {
timeout = maxTaskTimeout
// Use configurable max timeout from config (set via MAX_POLL_TIMEOUT env var)
// Falls back to hardcoded maxTaskTimeout if config is nil or MaxPollTimeout not set
maxTimeout := maxTaskTimeout
if cfg != nil && cfg.MaxPollTimeout > 0 {
maxTimeout = cfg.MaxPollTimeout
}
if timeout > maxTimeout {
timeout = maxTimeout
}
return timeout
}
@@ -39,6 +39,22 @@ func TestDerivePollTimeout(t *testing.T) {
},
want: maxTaskTimeout,
},
{
name: "respects custom MaxPollTimeout",
cfg: &config.Config{
ConnectionTimeout: 3 * time.Minute,
MaxPollTimeout: 10 * time.Minute,
},
want: 6 * time.Minute, // 2 * ConnectionTimeout, still under MaxPollTimeout
},
{
name: "custom MaxPollTimeout caps at configured value",
cfg: &config.Config{
ConnectionTimeout: 10 * time.Minute,
MaxPollTimeout: 5 * time.Minute,
},
want: 5 * time.Minute, // Capped at MaxPollTimeout
},
}
for _, tt := range tests {
+2
View File
@@ -724,6 +724,8 @@ func (cc *ClusterClient) executeWithFailover(ctx context.Context, fn func(*Clien
// PBS storage errors - Proxmox can't reach PBS, but node is still reachable
(strings.Contains(errStr, "500") && strings.Contains(errStr, "pbs-") && strings.Contains(errStr, "error fetching datastores")) ||
(strings.Contains(errStr, "500") && strings.Contains(errStr, "Can't connect to") && strings.Contains(errStr, ":8007")) ||
// External Ceph errors - Ceph not managed by Proxmox, but node is still reachable
(strings.Contains(errStr, "500") && strings.Contains(errStr, "binary not installed")) ||
// RRD data timeouts - secondary metric fetch failures, node is still working
(strings.Contains(errStr, "context deadline exceeded") && strings.Contains(errStr, "/rrddata")) ||
(strings.Contains(errStr, "context deadline exceeded") && strings.Contains(errStr, "/lxc/") && strings.Contains(errStr, "rrd")) ||