From c5841bfe3db21df735473e2ee507a54b1b8febd4 Mon Sep 17 00:00:00 2001 From: Pulse Monitor Date: Fri, 5 Sep 2025 17:41:30 +0000 Subject: [PATCH] improve: handle unavailable storage more gracefully (addresses #418) - Reduced storage API timeout from 120s to 15s to prevent blocking when storage mounts are unavailable - Added graceful error handling for storage timeouts - continues with partial data instead of failing - Improved error messages to clarify when timeouts are likely due to unavailable storage (e.g., NFS mounts) This prevents Pulse from marking nodes as unhealthy when storage endpoints timeout due to temporarily unavailable network storage. --- internal/monitoring/monitor_optimized.go | 12 +++++++++++- pkg/proxmox/client.go | 21 ++++++++++++--------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/internal/monitoring/monitor_optimized.go b/internal/monitoring/monitor_optimized.go index c96e5c94d..aa4b67c68 100644 --- a/internal/monitoring/monitor_optimized.go +++ b/internal/monitoring/monitor_optimized.go @@ -480,7 +480,17 @@ func (m *Monitor) pollStorageWithNodesOptimized(ctx context.Context, instanceNam // Fetch storage for this node nodeStorage, err := client.GetStorage(ctx, n.Node) if err != nil { - // Log more details about the failure + // Handle timeout gracefully - unavailable storage (e.g., NFS mounts) can cause this + if strings.Contains(err.Error(), "timeout") || strings.Contains(err.Error(), "deadline exceeded") { + log.Warn(). + Str("node", n.Node). + Str("instance", instanceName). + Msg("Storage query timed out - likely due to unavailable storage mounts. Continuing without storage data for this node.") + // Return empty storage list but don't fail the node + resultChan <- nodeResult{node: n.Node, storage: []models.Storage{}} + return + } + // For other errors, log as error log.Error(). Err(err). Str("node", n.Node). diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.go index 8af00fa7f..a159c8f00 100644 --- a/pkg/proxmox/client.go +++ b/pkg/proxmox/client.go @@ -556,11 +556,12 @@ func (c *Client) GetContainers(ctx context.Context, node string) ([]Container, e // GetStorage returns storage information for a specific node func (c *Client) GetStorage(ctx context.Context, node string) ([]Storage, error) { // Storage queries can take longer on large clusters or slow storage backends - // Create a new context with extended timeout if the original doesn't have one + // Create a new context with shorter timeout for storage API calls + // Storage endpoints can hang when NFS/network storage is unavailable storageCtx := ctx - if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 120*time.Second { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second { var cancel context.CancelFunc - storageCtx, cancel = context.WithTimeout(ctx, 120*time.Second) + storageCtx, cancel = context.WithTimeout(ctx, 15*time.Second) defer cancel() } @@ -584,11 +585,12 @@ func (c *Client) GetStorage(ctx context.Context, node string) ([]Storage, error) // GetAllStorage returns storage information across all nodes func (c *Client) GetAllStorage(ctx context.Context) ([]Storage, error) { // Storage queries can take longer on large clusters - // Create a new context with extended timeout if the original doesn't have one + // Create a new context with shorter timeout for storage API calls + // Storage endpoints can hang when NFS/network storage is unavailable storageCtx := ctx - if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 120*time.Second { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second { var cancel context.CancelFunc - storageCtx, cancel = context.WithTimeout(ctx, 120*time.Second) + storageCtx, cancel = context.WithTimeout(ctx, 15*time.Second) defer cancel() } @@ -677,11 +679,12 @@ func (c *Client) GetBackupTasks(ctx context.Context) ([]Task, error) { // GetStorageContent returns the content of a specific storage func (c *Client) GetStorageContent(ctx context.Context, node, storage string) ([]StorageContent, error) { // Storage content queries can take longer on large storages - // Create a new context with extended timeout if the original doesn't have one + // Create a new context with shorter timeout for storage API calls + // Storage endpoints can hang when NFS/network storage is unavailable storageCtx := ctx - if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 120*time.Second { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second { var cancel context.CancelFunc - storageCtx, cancel = context.WithTimeout(ctx, 120*time.Second) + storageCtx, cancel = context.WithTimeout(ctx, 15*time.Second) defer cancel() }