diff --git a/internal/monitoring/monitor_optimized.go b/internal/monitoring/monitor_optimized.go index c96e5c94d..aa4b67c68 100644 --- a/internal/monitoring/monitor_optimized.go +++ b/internal/monitoring/monitor_optimized.go @@ -480,7 +480,17 @@ func (m *Monitor) pollStorageWithNodesOptimized(ctx context.Context, instanceNam // Fetch storage for this node nodeStorage, err := client.GetStorage(ctx, n.Node) if err != nil { - // Log more details about the failure + // Handle timeout gracefully - unavailable storage (e.g., NFS mounts) can cause this + if strings.Contains(err.Error(), "timeout") || strings.Contains(err.Error(), "deadline exceeded") { + log.Warn(). + Str("node", n.Node). + Str("instance", instanceName). + Msg("Storage query timed out - likely due to unavailable storage mounts. Continuing without storage data for this node.") + // Return empty storage list but don't fail the node + resultChan <- nodeResult{node: n.Node, storage: []models.Storage{}} + return + } + // For other errors, log as error log.Error(). Err(err). Str("node", n.Node). diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.go index 8af00fa7f..a159c8f00 100644 --- a/pkg/proxmox/client.go +++ b/pkg/proxmox/client.go @@ -556,11 +556,12 @@ func (c *Client) GetContainers(ctx context.Context, node string) ([]Container, e // GetStorage returns storage information for a specific node func (c *Client) GetStorage(ctx context.Context, node string) ([]Storage, error) { // Storage queries can take longer on large clusters or slow storage backends - // Create a new context with extended timeout if the original doesn't have one + // Create a new context with shorter timeout for storage API calls + // Storage endpoints can hang when NFS/network storage is unavailable storageCtx := ctx - if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 120*time.Second { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second { var cancel context.CancelFunc - storageCtx, cancel = context.WithTimeout(ctx, 120*time.Second) + storageCtx, cancel = context.WithTimeout(ctx, 15*time.Second) defer cancel() } @@ -584,11 +585,12 @@ func (c *Client) GetStorage(ctx context.Context, node string) ([]Storage, error) // GetAllStorage returns storage information across all nodes func (c *Client) GetAllStorage(ctx context.Context) ([]Storage, error) { // Storage queries can take longer on large clusters - // Create a new context with extended timeout if the original doesn't have one + // Create a new context with shorter timeout for storage API calls + // Storage endpoints can hang when NFS/network storage is unavailable storageCtx := ctx - if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 120*time.Second { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second { var cancel context.CancelFunc - storageCtx, cancel = context.WithTimeout(ctx, 120*time.Second) + storageCtx, cancel = context.WithTimeout(ctx, 15*time.Second) defer cancel() } @@ -677,11 +679,12 @@ func (c *Client) GetBackupTasks(ctx context.Context) ([]Task, error) { // GetStorageContent returns the content of a specific storage func (c *Client) GetStorageContent(ctx context.Context, node, storage string) ([]StorageContent, error) { // Storage content queries can take longer on large storages - // Create a new context with extended timeout if the original doesn't have one + // Create a new context with shorter timeout for storage API calls + // Storage endpoints can hang when NFS/network storage is unavailable storageCtx := ctx - if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 120*time.Second { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second { var cancel context.CancelFunc - storageCtx, cancel = context.WithTimeout(ctx, 120*time.Second) + storageCtx, cancel = context.WithTimeout(ctx, 15*time.Second) defer cancel() }