mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
fix: Add configurable poll timeout and handle external Ceph storage
Changes: 1. Add MAX_POLL_TIMEOUT env var for large Proxmox clusters that need more than 3 minutes for polling (default: 3m, minimum: 30s) 2. Handle external Ceph storage gracefully - don't mark nodes unhealthy when Proxmox returns 'binary not installed' (e.g., for Ceph not managed by Proxmox) Related to #965
This commit is contained in:
@@ -119,7 +119,8 @@ type Config struct {
|
||||
GuestMetadataRetryBackoff time.Duration `envconfig:"GUEST_METADATA_RETRY_BACKOFF" default:"30s" json:"guestMetadataRetryBackoff"`
|
||||
GuestMetadataMaxConcurrent int `envconfig:"GUEST_METADATA_MAX_CONCURRENT" default:"4" json:"guestMetadataMaxConcurrent"`
|
||||
DNSCacheTimeout time.Duration `envconfig:"DNS_CACHE_TIMEOUT" default:"5m" json:"dnsCacheTimeout"`
|
||||
SSHPort int `envconfig:"SSH_PORT" default:"22" json:"sshPort"` // Default SSH port for temperature monitoring
|
||||
SSHPort int `envconfig:"SSH_PORT" default:"22" json:"sshPort"` // Default SSH port for temperature monitoring
|
||||
MaxPollTimeout time.Duration `envconfig:"MAX_POLL_TIMEOUT" default:"3m" json:"-"` // Maximum poll timeout for large clusters (default 3m)
|
||||
|
||||
// Metrics retention settings (tiered storage)
|
||||
// These control how long historical metrics are retained at each aggregation level.
|
||||
@@ -594,6 +595,7 @@ func Load() (*Config, error) {
|
||||
DiscoveryEnabled: false,
|
||||
DiscoverySubnet: "auto",
|
||||
TemperatureMonitoringEnabled: true,
|
||||
MaxPollTimeout: 3 * time.Minute, // Default max poll timeout for large clusters
|
||||
EnableSensorProxy: false,
|
||||
EnvOverrides: make(map[string]bool),
|
||||
AgentConnectURL: "",
|
||||
@@ -1446,6 +1448,19 @@ func Load() (*Config, error) {
|
||||
log.Info().Dur("timeout", d).Msg("Connection timeout overridden by CONNECTION_TIMEOUT env var")
|
||||
}
|
||||
}
|
||||
if maxPollTimeout := os.Getenv("MAX_POLL_TIMEOUT"); maxPollTimeout != "" {
|
||||
if d, err := time.ParseDuration(maxPollTimeout); err == nil {
|
||||
if d >= 30*time.Second { // Minimum 30 seconds
|
||||
cfg.MaxPollTimeout = d
|
||||
cfg.EnvOverrides["maxPollTimeout"] = true
|
||||
log.Info().Dur("timeout", d).Msg("Max poll timeout overridden by MAX_POLL_TIMEOUT env var")
|
||||
} else {
|
||||
log.Warn().Dur("value", d).Msg("MAX_POLL_TIMEOUT too low (minimum 30s), using default")
|
||||
}
|
||||
} else {
|
||||
log.Warn().Str("value", maxPollTimeout).Msg("Invalid MAX_POLL_TIMEOUT value, using default")
|
||||
}
|
||||
}
|
||||
if allowedOrigins := os.Getenv("ALLOWED_ORIGINS"); allowedOrigins != "" {
|
||||
cfg.AllowedOrigins = allowedOrigins
|
||||
cfg.EnvOverrides["allowedOrigins"] = true
|
||||
|
||||
@@ -4529,8 +4529,14 @@ func derivePollTimeout(cfg *config.Config) time.Duration {
|
||||
if timeout < minTaskTimeout {
|
||||
timeout = minTaskTimeout
|
||||
}
|
||||
if timeout > maxTaskTimeout {
|
||||
timeout = maxTaskTimeout
|
||||
// Use configurable max timeout from config (set via MAX_POLL_TIMEOUT env var)
|
||||
// Falls back to hardcoded maxTaskTimeout if config is nil or MaxPollTimeout not set
|
||||
maxTimeout := maxTaskTimeout
|
||||
if cfg != nil && cfg.MaxPollTimeout > 0 {
|
||||
maxTimeout = cfg.MaxPollTimeout
|
||||
}
|
||||
if timeout > maxTimeout {
|
||||
timeout = maxTimeout
|
||||
}
|
||||
return timeout
|
||||
}
|
||||
|
||||
@@ -39,6 +39,22 @@ func TestDerivePollTimeout(t *testing.T) {
|
||||
},
|
||||
want: maxTaskTimeout,
|
||||
},
|
||||
{
|
||||
name: "respects custom MaxPollTimeout",
|
||||
cfg: &config.Config{
|
||||
ConnectionTimeout: 3 * time.Minute,
|
||||
MaxPollTimeout: 10 * time.Minute,
|
||||
},
|
||||
want: 6 * time.Minute, // 2 * ConnectionTimeout, still under MaxPollTimeout
|
||||
},
|
||||
{
|
||||
name: "custom MaxPollTimeout caps at configured value",
|
||||
cfg: &config.Config{
|
||||
ConnectionTimeout: 10 * time.Minute,
|
||||
MaxPollTimeout: 5 * time.Minute,
|
||||
},
|
||||
want: 5 * time.Minute, // Capped at MaxPollTimeout
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -724,6 +724,8 @@ func (cc *ClusterClient) executeWithFailover(ctx context.Context, fn func(*Clien
|
||||
// PBS storage errors - Proxmox can't reach PBS, but node is still reachable
|
||||
(strings.Contains(errStr, "500") && strings.Contains(errStr, "pbs-") && strings.Contains(errStr, "error fetching datastores")) ||
|
||||
(strings.Contains(errStr, "500") && strings.Contains(errStr, "Can't connect to") && strings.Contains(errStr, ":8007")) ||
|
||||
// External Ceph errors - Ceph not managed by Proxmox, but node is still reachable
|
||||
(strings.Contains(errStr, "500") && strings.Contains(errStr, "binary not installed")) ||
|
||||
// RRD data timeouts - secondary metric fetch failures, node is still working
|
||||
(strings.Contains(errStr, "context deadline exceeded") && strings.Contains(errStr, "/rrddata")) ||
|
||||
(strings.Contains(errStr, "context deadline exceeded") && strings.Contains(errStr, "/lxc/") && strings.Contains(errStr, "rrd")) ||
|
||||
|
||||
Reference in New Issue
Block a user