gsadmin/pulse
1
0
Fork 0
mirror of https://github.com/rcourtman/Pulse.git synced 2026-09-22 19:23:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

improve: handle unavailable storage more gracefully (addresses #418)

Browse Source 
- Reduced storage API timeout from 120s to 15s to prevent blocking when storage mounts are unavailable
- Added graceful error handling for storage timeouts - continues with partial data instead of failing
- Improved error messages to clarify when timeouts are likely due to unavailable storage (e.g., NFS mounts)

This prevents Pulse from marking nodes as unhealthy when storage endpoints timeout due to temporarily unavailable network storage.
This commit is contained in:
Pulse Monitor
2025-09-05 17:41:30 +00:00
parent 86843c7a7f
commit c5841bfe3d
2 changed files with 23 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/monitoring/monitor_optimized.go
+11 -1
View File
@@ -480,7 +480,17 @@ func (m *Monitor) pollStorageWithNodesOptimized(ctx context.Context, instanceNam
// Fetch storage for this node
nodeStorage, err := client.GetStorage(ctx, n.Node)
if err != nil {
// Log more details about the failure
// Handle timeout gracefully - unavailable storage (e.g., NFS mounts) can cause this
if strings.Contains(err.Error(), "timeout") || strings.Contains(err.Error(), "deadline exceeded") {
log.Warn().
Str("node", n.Node).
Str("instance", instanceName).
Msg("Storage query timed out - likely due to unavailable storage mounts. Continuing without storage data for this node.")
// Return empty storage list but don't fail the node
resultChan <- nodeResult{node: n.Node, storage: []models.Storage{}}
return
}
// For other errors, log as error
log.Error().
Err(err).
Str("node", n.Node).
pkg/proxmox/client.go
+12 -9
View File
@@ -556,11 +556,12 @@ func (c *Client) GetContainers(ctx context.Context, node string) ([]Container, e
// GetStorage returns storage information for a specific node
func (c *Client) GetStorage(ctx context.Context, node string) ([]Storage, error) {
// Storage queries can take longer on large clusters or slow storage backends
// Create a new context with extended timeout if the original doesn't have one
// Create a new context with shorter timeout for storage API calls
// Storage endpoints can hang when NFS/network storage is unavailable
storageCtx := ctx
if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 120*time.Second {
if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second {
var cancel context.CancelFunc
storageCtx, cancel = context.WithTimeout(ctx, 120*time.Second)
storageCtx, cancel = context.WithTimeout(ctx, 15*time.Second)
defer cancel()
}
@@ -584,11 +585,12 @@ func (c *Client) GetStorage(ctx context.Context, node string) ([]Storage, error)
// GetAllStorage returns storage information across all nodes
func (c *Client) GetAllStorage(ctx context.Context) ([]Storage, error) {
// Storage queries can take longer on large clusters
// Create a new context with extended timeout if the original doesn't have one
// Create a new context with shorter timeout for storage API calls
// Storage endpoints can hang when NFS/network storage is unavailable
storageCtx := ctx
if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 120*time.Second {
if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second {
var cancel context.CancelFunc
storageCtx, cancel = context.WithTimeout(ctx, 120*time.Second)
storageCtx, cancel = context.WithTimeout(ctx, 15*time.Second)
defer cancel()
}
@@ -677,11 +679,12 @@ func (c *Client) GetBackupTasks(ctx context.Context) ([]Task, error) {
// GetStorageContent returns the content of a specific storage
func (c *Client) GetStorageContent(ctx context.Context, node, storage string) ([]StorageContent, error) {
// Storage content queries can take longer on large storages
// Create a new context with extended timeout if the original doesn't have one
// Create a new context with shorter timeout for storage API calls
// Storage endpoints can hang when NFS/network storage is unavailable
storageCtx := ctx
if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 120*time.Second {
if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second {
var cancel context.CancelFunc
storageCtx, cancel = context.WithTimeout(ctx, 120*time.Second)
storageCtx, cancel = context.WithTimeout(ctx, 15*time.Second)
defer cancel()
}