diff --git a/frontend-modern/src/components/Storage/DiskList.tsx b/frontend-modern/src/components/Storage/DiskList.tsx new file mode 100644 index 000000000..6fcf75eae --- /dev/null +++ b/frontend-modern/src/components/Storage/DiskList.tsx @@ -0,0 +1,167 @@ +import { Component, For, Show, createMemo } from 'solid-js'; +import { formatBytes } from '@/utils/format'; +import type { PhysicalDisk } from '@/types/api'; + +interface DiskListProps { + disks: PhysicalDisk[]; + selectedNode: string | null; + searchTerm: string; +} + +export const DiskList: Component = (props) => { + // Filter disks based on selected node and search term + const filteredDisks = createMemo(() => { + let disks = props.disks || []; + + // Filter by node if selected + if (props.selectedNode) { + disks = disks.filter(d => d.node === props.selectedNode); + } + + // Filter by search term + if (props.searchTerm) { + const term = props.searchTerm.toLowerCase(); + disks = disks.filter(d => + d.model.toLowerCase().includes(term) || + d.devPath.toLowerCase().includes(term) || + d.serial.toLowerCase().includes(term) || + d.node.toLowerCase().includes(term) + ); + } + + // Sort by node and devPath + return disks.sort((a, b) => { + if (a.node !== b.node) return a.node.localeCompare(b.node); + return a.devPath.localeCompare(b.devPath); + }); + }); + + // Get health status color and icon + const getHealthStatus = (disk: PhysicalDisk) => { + if (disk.health === 'PASSED') { + // Check wearout for SSDs + if (disk.wearout > 0 && disk.wearout < 10) { + return { color: 'text-yellow-500', icon: '⚠️', text: 'Low Life' }; + } + return { color: 'text-green-500', icon: '✅', text: 'Healthy' }; + } else if (disk.health === 'FAILED') { + return { color: 'text-red-500', icon: '❌', text: 'Failed' }; + } + return { color: 'text-gray-500', icon: '❓', text: 'Unknown' }; + }; + + // Get disk type badge color + const getDiskTypeBadge = (type: string) => { + switch (type.toLowerCase()) { + case 'nvme': + return 'bg-purple-100 text-purple-800'; + case 'sata': + return 'bg-blue-100 text-blue-800'; + case 'sas': + return 'bg-indigo-100 text-indigo-800'; + default: + return 'bg-gray-100 text-gray-800'; + } + }; + + return ( +
+ +
+ No physical disks found + {props.selectedNode && ` for node ${props.selectedNode}`} + {props.searchTerm && ` matching "${props.searchTerm}"`} +
+ + + + {(disk) => { + const health = getHealthStatus(disk); + + return ( +
+
+
+ {/* Header with model and health */} +
+ + {health.icon} + +

+ {disk.model || 'Unknown Model'} +

+ + {disk.type.toUpperCase()} + +
+ + {/* Disk details */} +
+
+ Node: + {disk.node} +
+
+ Path: + {disk.devPath} +
+
+ Size: + {formatBytes(disk.size)} +
+
+ Usage: + {disk.used || 'Unknown'} +
+
+ + {/* Additional metrics for SSDs */} + 0}> +
+
+ SSD Life Remaining: +
+
+
= 50 ? 'bg-green-500' : + disk.wearout >= 20 ? 'bg-yellow-500' : + disk.wearout >= 10 ? 'bg-orange-500' : + 'bg-red-500' + }`} + style={`width: ${disk.wearout}%`} + /> +
+
+ {disk.wearout}% +
+
+ + + {/* Temperature if available */} + 0}> +
+ Temperature: + 70 ? 'text-red-500' : + disk.temperature > 60 ? 'text-yellow-500' : + 'text-green-500' + }`}> + {disk.temperature}°C + +
+ + + {/* Serial number (smaller, muted) */} +
+ Serial: {disk.serial} +
+
+
+
+ ); + }} + +
+ ); +}; \ No newline at end of file diff --git a/frontend-modern/src/components/Storage/Storage.tsx b/frontend-modern/src/components/Storage/Storage.tsx index 0e5f2f259..d5b1a2350 100644 --- a/frontend-modern/src/components/Storage/Storage.tsx +++ b/frontend-modern/src/components/Storage/Storage.tsx @@ -7,11 +7,13 @@ import type { Storage as StorageType } from '@/types/api'; import { ComponentErrorBoundary } from '@/components/ErrorBoundary'; import { UnifiedNodeSelector } from '@/components/shared/UnifiedNodeSelector'; import { StorageFilter } from './StorageFilter'; +import { DiskList } from './DiskList'; const Storage: Component = () => { const { state, connected, activeAlerts, initialDataReceived } = useWebSocket(); const [viewMode, setViewMode] = createSignal<'node' | 'storage'>('node'); + const [tabView, setTabView] = createSignal<'pools' | 'disks'>('pools'); const [searchTerm, setSearchTerm] = createSignal(''); const [selectedNode, setSelectedNode] = createSignal(null); // TODO: Implement sorting in sortedStorage function @@ -220,16 +222,62 @@ const Storage: Component = () => { searchTerm={searchTerm()} /> - {/* Storage Filter */} - {}} - setSortDirection={() => {}} - searchInputRef={(el) => searchInputRef = el} - /> + {/* Tab Toggle */} +
+ +
+ + {/* Show Storage Filter only for pools */} + + {}} + setSortDirection={() => {}} + searchInputRef={(el) => searchInputRef = el} + /> + + + {/* Show simple search for disks */} + +
+ setSearchTerm(e.currentTarget.value)} + class="w-full px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500" + /> +
+ {/* Loading State */} @@ -267,21 +315,23 @@ const Storage: Component = () => {
- {/* No results found message */} - -
-
- - - -

No storage found

-

No storage matches your search "{searchTerm()}"

+ {/* Conditional rendering based on tab */} + + {/* No results found message for storage pools */} + +
+
+ + + +

No storage found

+

No storage matches your search "{searchTerm()}"

+
-
- - - {/* Storage Table - shows for both PVE and PBS storage */} - 0}> + + + {/* Storage Table - shows for both PVE and PBS storage */} + 0}>
@@ -356,6 +406,22 @@ const Storage: Component = () => { {storage.name} + {/* ZFS Health Badge */} + + + {storage.zfsPool.state} + + + {/* ZFS Error Badge */} + 0 || storage.zfsPool.writeErrors > 0 || storage.zfsPool.checksumErrors > 0)}> + + ERRORS + + @@ -475,10 +541,18 @@ const Storage: Component = () => {
+ + + + {/* Physical Disks Tab */} + + - {/* Tooltip System */} -
); }; diff --git a/frontend-modern/src/types/api.ts b/frontend-modern/src/types/api.ts index f8f4a8c93..5b5c57d01 100644 --- a/frontend-modern/src/types/api.ts +++ b/frontend-modern/src/types/api.ts @@ -5,6 +5,7 @@ export interface State { vms: VM[]; containers: Container[]; storage: Storage[]; + physicalDisks: PhysicalDisk[]; pbs: PBSInstance[]; pbsBackups: PBSBackup[]; metrics: Metric[]; @@ -248,6 +249,23 @@ export interface Disk { usage: number; } +export interface PhysicalDisk { + id: string; + node: string; + instance: string; + devPath: string; + model: string; + serial: string; + type: 'nvme' | 'sata' | 'sas' | string; + size: number; + health: 'PASSED' | 'FAILED' | 'UNKNOWN' | string; + wearout: number; // 0-100, 100 is best (percentage life remaining) + temperature: number; + rpm: number; + used: string; + lastChecked: string; +} + export interface CPUInfo { model: string; cores: number; diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 03fa8e432..d3dd2b711 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -11,6 +11,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/utils" + "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" "github.com/rs/zerolog/log" ) @@ -2242,3 +2243,132 @@ func (m *Manager) periodicSaveAlerts() { } } } + +// CheckDiskHealth checks disk health and creates alerts if needed +func (m *Manager) CheckDiskHealth(instance, node string, disk proxmox.Disk) { + // Create unique alert ID for this disk + alertID := fmt.Sprintf("disk-health-%s-%s-%s", instance, node, disk.DevPath) + + m.mu.Lock() + defer m.mu.Unlock() + + // Check if disk health is not PASSED + if disk.Health != "PASSED" && disk.Health != "" { + // Check if alert already exists + if _, exists := m.activeAlerts[alertID]; !exists { + // Create new health alert + alert := &Alert{ + ID: alertID, + Type: "disk-health", + Level: AlertLevelCritical, + ResourceID: fmt.Sprintf("%s-%s", node, disk.DevPath), + ResourceName: fmt.Sprintf("%s (%s)", disk.Model, disk.DevPath), + Node: node, + Instance: instance, + Message: fmt.Sprintf("Disk health check failed: %s", disk.Health), + Value: 0, // Not applicable for health status + Threshold: 0, + StartTime: time.Now(), + LastSeen: time.Now(), + Metadata: map[string]interface{}{ + "disk_path": disk.DevPath, + "disk_model": disk.Model, + "disk_serial": disk.Serial, + "disk_type": disk.Type, + "disk_health": disk.Health, + "disk_size": disk.Size, + }, + } + + m.activeAlerts[alertID] = alert + m.recentAlerts[alertID] = alert + m.historyManager.AddAlert(*alert) + + if m.onAlert != nil { + m.onAlert(alert) + } + + log.Error(). + Str("node", node). + Str("disk", disk.DevPath). + Str("model", disk.Model). + Str("health", disk.Health). + Msg("Disk health alert created") + } + } else { + // Disk is healthy, clear alert if it exists + m.clearAlertNoLock(alertID) + } + + // Check for low wearout (SSD life remaining) + if disk.Wearout > 0 && disk.Wearout < 10 { + wearoutAlertID := fmt.Sprintf("disk-wearout-%s-%s-%s", instance, node, disk.DevPath) + + if _, exists := m.activeAlerts[wearoutAlertID]; !exists { + // Create wearout alert + alert := &Alert{ + ID: wearoutAlertID, + Type: "disk-wearout", + Level: AlertLevelWarning, + ResourceID: fmt.Sprintf("%s-%s", node, disk.DevPath), + ResourceName: fmt.Sprintf("%s (%s)", disk.Model, disk.DevPath), + Node: node, + Instance: instance, + Message: fmt.Sprintf("SSD has less than 10%% life remaining (%d%% wearout)", disk.Wearout), + Value: float64(disk.Wearout), + Threshold: 10.0, + StartTime: time.Now(), + LastSeen: time.Now(), + Metadata: map[string]interface{}{ + "disk_path": disk.DevPath, + "disk_model": disk.Model, + "disk_serial": disk.Serial, + "disk_type": disk.Type, + "disk_wearout": disk.Wearout, + }, + } + + m.activeAlerts[wearoutAlertID] = alert + m.recentAlerts[wearoutAlertID] = alert + m.historyManager.AddAlert(*alert) + + if m.onAlert != nil { + m.onAlert(alert) + } + + log.Warn(). + Str("node", node). + Str("disk", disk.DevPath). + Str("model", disk.Model). + Int("wearout", disk.Wearout). + Msg("Disk wearout alert created") + } + } else if disk.Wearout >= 10 { + // Wearout is acceptable, clear alert if it exists + wearoutAlertID := fmt.Sprintf("disk-wearout-%s-%s-%s", instance, node, disk.DevPath) + m.clearAlertNoLock(wearoutAlertID) + } +} + +// clearAlertNoLock clears an alert without locking (must be called with lock held) +func (m *Manager) clearAlertNoLock(alertID string) { + if alert, exists := m.activeAlerts[alertID]; exists { + delete(m.activeAlerts, alertID) + + // Add to recently resolved + resolvedAlert := &ResolvedAlert{ + Alert: alert, + ResolvedTime: time.Now(), + } + m.recentlyResolved[alertID] = resolvedAlert + + // Send recovery notification + if m.onResolved != nil { + m.onResolved(alertID) + } + + log.Info(). + Str("alertID", alertID). + Msg("Alert cleared") + } +} diff --git a/internal/models/converters.go b/internal/models/converters.go index 002b069b6..2f211de28 100644 --- a/internal/models/converters.go +++ b/internal/models/converters.go @@ -35,6 +35,7 @@ func (s *State) ToFrontend() StateFrontend { VMs: vms, Containers: containers, Storage: storage, + PhysicalDisks: s.PhysicalDisks, PBS: s.PBSInstances, ActiveAlerts: s.ActiveAlerts, Metrics: make(map[string]any), diff --git a/internal/models/models.go b/internal/models/models.go index 0f13f3e8d..ff2a8f3aa 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -14,6 +14,7 @@ type State struct { VMs []VM `json:"vms"` Containers []Container `json:"containers"` Storage []Storage `json:"storage"` + PhysicalDisks []PhysicalDisk `json:"physicalDisks"` PBSInstances []PBSInstance `json:"pbs"` PBSBackups []PBSBackup `json:"pbsBackups"` Metrics []Metric `json:"metrics"` @@ -162,6 +163,24 @@ type ZFSDevice struct { ChecksumErrors int64 `json:"checksumErrors"` } +// PhysicalDisk represents a physical disk on a node +type PhysicalDisk struct { + ID string `json:"id"` // "{instance}-{node}-{devpath}" + Node string `json:"node"` + Instance string `json:"instance"` + DevPath string `json:"devPath"` // /dev/nvme0n1, /dev/sda + Model string `json:"model"` + Serial string `json:"serial"` + Type string `json:"type"` // nvme, sata, sas + Size int64 `json:"size"` // bytes + Health string `json:"health"` // PASSED, FAILED, UNKNOWN + Wearout int `json:"wearout"` // SSD life remaining percentage (0-100, 100 is best) + Temperature int `json:"temperature"` // Celsius (if available) + RPM int `json:"rpm"` // 0 for SSDs + Used string `json:"used"` // Filesystem or partition usage + LastChecked time.Time `json:"lastChecked"` +} + // PBSInstance represents a Proxmox Backup Server instance type PBSInstance struct { ID string `json:"id"` @@ -384,6 +403,7 @@ func NewState() *State { VMs: make([]VM, 0), Containers: make([]Container, 0), Storage: make([]Storage, 0), + PhysicalDisks: make([]PhysicalDisk, 0), PBSInstances: make([]PBSInstance, 0), PBSBackups: make([]PBSBackup, 0), Metrics: make([]Metric, 0), @@ -551,6 +571,42 @@ func (s *State) UpdateStorage(storage []Storage) { s.LastUpdate = time.Now() } +// UpdatePhysicalDisks updates physical disks for a specific instance +func (s *State) UpdatePhysicalDisks(instanceName string, disks []PhysicalDisk) { + s.mu.Lock() + defer s.mu.Unlock() + + // Create a map of existing disks, excluding those from this instance + diskMap := make(map[string]PhysicalDisk) + for _, disk := range s.PhysicalDisks { + if disk.Instance != instanceName { + diskMap[disk.ID] = disk + } + } + + // Add or update disks from this instance + for _, disk := range disks { + diskMap[disk.ID] = disk + } + + // Convert map back to slice + newDisks := make([]PhysicalDisk, 0, len(diskMap)) + for _, disk := range diskMap { + newDisks = append(newDisks, disk) + } + + // Sort by node and dev path for consistent ordering + sort.Slice(newDisks, func(i, j int) bool { + if newDisks[i].Node != newDisks[j].Node { + return newDisks[i].Node < newDisks[j].Node + } + return newDisks[i].DevPath < newDisks[j].DevPath + }) + + s.PhysicalDisks = newDisks + s.LastUpdate = time.Now() +} + // UpdateStorageForInstance updates storage for a specific instance, merging with existing storage func (s *State) UpdateStorageForInstance(instanceName string, storage []Storage) { s.mu.Lock() diff --git a/internal/models/models_frontend.go b/internal/models/models_frontend.go index 40a554d3e..cbd849097 100644 --- a/internal/models/models_frontend.go +++ b/internal/models/models_frontend.go @@ -105,6 +105,7 @@ type StateFrontend struct { VMs []VMFrontend `json:"vms"` Containers []ContainerFrontend `json:"containers"` Storage []StorageFrontend `json:"storage"` + PhysicalDisks []PhysicalDisk `json:"physicalDisks"` PBS []PBSInstance `json:"pbs"` // Keep as is ActiveAlerts []Alert `json:"activeAlerts"` // Active alerts Metrics map[string]any `json:"metrics"` // Empty object for now diff --git a/internal/models/state_snapshot.go b/internal/models/state_snapshot.go index 9b578305f..1a7c917b3 100644 --- a/internal/models/state_snapshot.go +++ b/internal/models/state_snapshot.go @@ -8,6 +8,7 @@ type StateSnapshot struct { VMs []VM `json:"vms"` Containers []Container `json:"containers"` Storage []Storage `json:"storage"` + PhysicalDisks []PhysicalDisk `json:"physicalDisks"` PBSInstances []PBSInstance `json:"pbs"` PBSBackups []PBSBackup `json:"pbsBackups"` Metrics []Metric `json:"metrics"` @@ -31,6 +32,7 @@ func (s *State) GetSnapshot() StateSnapshot { VMs: append([]VM{}, s.VMs...), Containers: append([]Container{}, s.Containers...), Storage: append([]Storage{}, s.Storage...), + PhysicalDisks: append([]PhysicalDisk{}, s.PhysicalDisks...), PBSInstances: append([]PBSInstance{}, s.PBSInstances...), PBSBackups: append([]PBSBackup{}, s.PBSBackups...), Metrics: append([]Metric{}, s.Metrics...), @@ -86,6 +88,7 @@ func (s StateSnapshot) ToFrontend() StateFrontend { VMs: vms, Containers: containers, Storage: storage, + PhysicalDisks: s.PhysicalDisks, PBS: s.PBSInstances, ActiveAlerts: s.ActiveAlerts, Metrics: make(map[string]any), diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index ad2d34a45..738ffc275 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -45,6 +45,7 @@ type PVEClientInterface interface { GetVMFSInfo(ctx context.Context, node string, vmid int) ([]proxmox.VMFileSystem, error) GetZFSPoolStatus(ctx context.Context, node string) ([]proxmox.ZFSPoolStatus, error) GetZFSPoolsWithDetails(ctx context.Context, node string) ([]proxmox.ZFSPoolInfo, error) + GetDisks(ctx context.Context, node string) ([]proxmox.Disk, error) } // Monitor handles all monitoring operations @@ -901,6 +902,97 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie } } + // Poll physical disks for health monitoring + log.Debug().Int("nodeCount", len(nodes)).Msg("Starting disk health polling") + var allDisks []models.PhysicalDisk + for _, node := range nodes { + // Skip offline nodes + if node.Status != "online" { + log.Debug().Str("node", node.Node).Msg("Skipping disk poll for offline node") + continue + } + + // Get disk list for this node + log.Debug().Str("node", node.Node).Msg("Getting disk list for node") + disks, err := client.GetDisks(ctx, node.Node) + if err != nil { + // Log but don't fail - disk monitoring is optional + log.Debug(). + Str("node", node.Node). + Err(err). + Msg("Failed to get disk list - disk monitoring may not be available") + continue + } + + log.Debug(). + Str("node", node.Node). + Int("diskCount", len(disks)). + Msg("Got disk list for node") + + // Check each disk for health issues and add to state + for _, disk := range disks { + // Create PhysicalDisk model + diskID := fmt.Sprintf("%s-%s-%s", instanceName, node.Node, strings.ReplaceAll(disk.DevPath, "/", "-")) + physicalDisk := models.PhysicalDisk{ + ID: diskID, + Node: node.Node, + Instance: instanceName, + DevPath: disk.DevPath, + Model: disk.Model, + Serial: disk.Serial, + Type: disk.Type, + Size: disk.Size, + Health: disk.Health, + Wearout: disk.Wearout, + RPM: disk.RPM, + Used: disk.Used, + LastChecked: time.Now(), + } + + allDisks = append(allDisks, physicalDisk) + + log.Debug(). + Str("node", node.Node). + Str("disk", disk.DevPath). + Str("model", disk.Model). + Str("health", disk.Health). + Int("wearout", disk.Wearout). + Msg("Checking disk health") + + if disk.Health != "PASSED" && disk.Health != "" { + // Disk has failed or is failing - alert manager will handle this + log.Warn(). + Str("node", node.Node). + Str("disk", disk.DevPath). + Str("model", disk.Model). + Str("health", disk.Health). + Int("wearout", disk.Wearout). + Msg("Disk health issue detected") + + // Pass disk info to alert manager + m.alertManager.CheckDiskHealth(instanceName, node.Node, disk) + } else if disk.Wearout > 0 && disk.Wearout < 10 { + // Low wearout warning (less than 10% life remaining) + log.Warn(). + Str("node", node.Node). + Str("disk", disk.DevPath). + Str("model", disk.Model). + Int("wearout", disk.Wearout). + Msg("SSD wearout critical - less than 10% life remaining") + + // Pass to alert manager for wearout alert + m.alertManager.CheckDiskHealth(instanceName, node.Node, disk) + } + } + } + + // Update physical disks in state + log.Debug(). + Str("instance", instanceName). + Int("diskCount", len(allDisks)). + Msg("Updating physical disks in state") + m.state.UpdatePhysicalDisks(instanceName, allDisks) + // Update nodes with storage fallback if rootfs was not available for i := range modelNodes { if modelNodes[i].Disk.Total == 0 { diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.go index 5c766183e..cf67ebf3d 100644 --- a/pkg/proxmox/client.go +++ b/pkg/proxmox/client.go @@ -1150,5 +1150,68 @@ func (c *Client) GetZFSPoolDetail(ctx context.Context, node, pool string) (*ZFSP return nil, err } + return &result.Data, nil +} + +// Disk represents a physical disk on a Proxmox node +type Disk struct { + DevPath string `json:"devpath"` + Model string `json:"model"` + Serial string `json:"serial"` + Type string `json:"type"` // nvme, sata, sas + Health string `json:"health"` // PASSED, FAILED, UNKNOWN + Wearout int `json:"wearout"` // SSD wear percentage (0-100, 100 is best) + Size int64 `json:"size"` // Size in bytes + RPM int `json:"rpm"` // 0 for SSDs + Used string `json:"used"` // Filesystem or partition usage + Vendor string `json:"vendor"` + WWN string `json:"wwn"` // World Wide Name +} + +// DiskSmart represents SMART data for a disk +type DiskSmart struct { + Health string `json:"health"` // PASSED, FAILED, UNKNOWN + Wearout int `json:"wearout"` // SSD wear percentage + Type string `json:"type"` // Type of response (text, attributes) + Text string `json:"text"` // Raw SMART output text +} + +// GetDisks returns the list of physical disks on a node +func (c *Client) GetDisks(ctx context.Context, node string) ([]Disk, error) { + resp, err := c.request(ctx, "GET", fmt.Sprintf("/nodes/%s/disks/list", node), nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var result struct { + Data []Disk `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + + return result.Data, nil +} + +// GetDiskSmart returns SMART data for a specific disk +func (c *Client) GetDiskSmart(ctx context.Context, node, disk string) (*DiskSmart, error) { + params := url.Values{ + "disk": {disk}, + } + + resp, err := c.request(ctx, "GET", fmt.Sprintf("/nodes/%s/disks/smart?%s", node, params.Encode()), nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var result struct { + Data DiskSmart `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + return &result.Data, nil } \ No newline at end of file diff --git a/pkg/proxmox/cluster_client.go b/pkg/proxmox/cluster_client.go index 437c682ee..df05cce81 100644 --- a/pkg/proxmox/cluster_client.go +++ b/pkg/proxmox/cluster_client.go @@ -807,6 +807,30 @@ func (cc *ClusterClient) GetClusterHealthInfo() models.ClusterHealth { } // Helper to check if error is auth-related +func (cc *ClusterClient) GetDisks(ctx context.Context, node string) ([]Disk, error) { + var result []Disk + err := cc.executeWithFailover(ctx, func(client *Client) error { + disks, err := client.GetDisks(ctx, node) + if err != nil { + return err + } + result = disks + return nil + }) + + // Don't return error for transient connectivity issues + if err != nil && strings.Contains(err.Error(), "no healthy nodes available") { + log.Debug(). + Str("cluster", cc.name). + Str("node", node). + Err(err). + Msg("No healthy nodes for GetDisks - returning empty list") + return []Disk{}, nil + } + + return result, err +} + func IsAuthError(err error) bool { if err == nil { return false