feat: implement S.M.A.R.T. disk monitoring for Proxmox nodes (addresses #429)

- Added disk polling to monitoring cycle using Proxmox API
- Created CheckDiskHealth() alert manager for failing drives and low SSD life
- Added PhysicalDisk model to state with proper serialization
- Implemented DiskList component with health indicators and SSD wearout bars
- Added Physical Disks tab to Storage page with toggle between pools and disks
- Added ZFS health badges to storage cards for degraded/failed pools
- Alerts trigger for health != PASSED and SSD wearout < 10%
- Frontend displays disk model, type, temperature, and usage information
This commit is contained in:
Pulse Monitor
2025-09-08 16:40:05 +00:00
parent 6e41e75748
commit a1eecff088
11 changed files with 655 additions and 26 deletions
@@ -0,0 +1,167 @@
import { Component, For, Show, createMemo } from 'solid-js';
import { formatBytes } from '@/utils/format';
import type { PhysicalDisk } from '@/types/api';
interface DiskListProps {
disks: PhysicalDisk[];
selectedNode: string | null;
searchTerm: string;
}
export const DiskList: Component<DiskListProps> = (props) => {
// Filter disks based on selected node and search term
const filteredDisks = createMemo(() => {
let disks = props.disks || [];
// Filter by node if selected
if (props.selectedNode) {
disks = disks.filter(d => d.node === props.selectedNode);
}
// Filter by search term
if (props.searchTerm) {
const term = props.searchTerm.toLowerCase();
disks = disks.filter(d =>
d.model.toLowerCase().includes(term) ||
d.devPath.toLowerCase().includes(term) ||
d.serial.toLowerCase().includes(term) ||
d.node.toLowerCase().includes(term)
);
}
// Sort by node and devPath
return disks.sort((a, b) => {
if (a.node !== b.node) return a.node.localeCompare(b.node);
return a.devPath.localeCompare(b.devPath);
});
});
// Get health status color and icon
const getHealthStatus = (disk: PhysicalDisk) => {
if (disk.health === 'PASSED') {
// Check wearout for SSDs
if (disk.wearout > 0 && disk.wearout < 10) {
return { color: 'text-yellow-500', icon: '⚠️', text: 'Low Life' };
}
return { color: 'text-green-500', icon: '✅', text: 'Healthy' };
} else if (disk.health === 'FAILED') {
return { color: 'text-red-500', icon: '❌', text: 'Failed' };
}
return { color: 'text-gray-500', icon: '❓', text: 'Unknown' };
};
// Get disk type badge color
const getDiskTypeBadge = (type: string) => {
switch (type.toLowerCase()) {
case 'nvme':
return 'bg-purple-100 text-purple-800';
case 'sata':
return 'bg-blue-100 text-blue-800';
case 'sas':
return 'bg-indigo-100 text-indigo-800';
default:
return 'bg-gray-100 text-gray-800';
}
};
return (
<div class="space-y-4">
<Show when={filteredDisks().length === 0}>
<div class="text-center py-8 text-gray-500">
No physical disks found
{props.selectedNode && ` for node ${props.selectedNode}`}
{props.searchTerm && ` matching "${props.searchTerm}"`}
</div>
</Show>
<For each={filteredDisks()}>
{(disk) => {
const health = getHealthStatus(disk);
return (
<div class="bg-white rounded-lg shadow p-4 hover:shadow-md transition-shadow">
<div class="flex items-start justify-between">
<div class="flex-1">
{/* Header with model and health */}
<div class="flex items-center gap-3 mb-2">
<span class={`text-lg ${health.color}`} title={health.text}>
{health.icon}
</span>
<h3 class="text-lg font-semibold text-gray-900">
{disk.model || 'Unknown Model'}
</h3>
<span class={`px-2 py-1 text-xs font-medium rounded-full ${getDiskTypeBadge(disk.type)}`}>
{disk.type.toUpperCase()}
</span>
</div>
{/* Disk details */}
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 text-sm">
<div>
<span class="text-gray-500">Node:</span>
<span class="ml-2 font-medium">{disk.node}</span>
</div>
<div>
<span class="text-gray-500">Path:</span>
<span class="ml-2 font-mono text-xs">{disk.devPath}</span>
</div>
<div>
<span class="text-gray-500">Size:</span>
<span class="ml-2 font-medium">{formatBytes(disk.size)}</span>
</div>
<div>
<span class="text-gray-500">Usage:</span>
<span class="ml-2">{disk.used || 'Unknown'}</span>
</div>
</div>
{/* Additional metrics for SSDs */}
<Show when={disk.wearout > 0}>
<div class="mt-3 space-y-2">
<div class="flex items-center gap-2">
<span class="text-sm text-gray-500">SSD Life Remaining:</span>
<div class="flex-1 max-w-xs">
<div class="bg-gray-200 rounded-full h-2">
<div
class={`h-2 rounded-full transition-all ${
disk.wearout >= 50 ? 'bg-green-500' :
disk.wearout >= 20 ? 'bg-yellow-500' :
disk.wearout >= 10 ? 'bg-orange-500' :
'bg-red-500'
}`}
style={`width: ${disk.wearout}%`}
/>
</div>
</div>
<span class="text-sm font-medium">{disk.wearout}%</span>
</div>
</div>
</Show>
{/* Temperature if available */}
<Show when={disk.temperature > 0}>
<div class="mt-2 text-sm">
<span class="text-gray-500">Temperature:</span>
<span class={`ml-2 font-medium ${
disk.temperature > 70 ? 'text-red-500' :
disk.temperature > 60 ? 'text-yellow-500' :
'text-green-500'
}`}>
{disk.temperature}°C
</span>
</div>
</Show>
{/* Serial number (smaller, muted) */}
<div class="mt-2 text-xs text-gray-400">
Serial: {disk.serial}
</div>
</div>
</div>
</div>
);
}}
</For>
</div>
);
};
@@ -7,11 +7,13 @@ import type { Storage as StorageType } from '@/types/api';
import { ComponentErrorBoundary } from '@/components/ErrorBoundary';
import { UnifiedNodeSelector } from '@/components/shared/UnifiedNodeSelector';
import { StorageFilter } from './StorageFilter';
import { DiskList } from './DiskList';
const Storage: Component = () => {
const { state, connected, activeAlerts, initialDataReceived } = useWebSocket();
const [viewMode, setViewMode] = createSignal<'node' | 'storage'>('node');
const [tabView, setTabView] = createSignal<'pools' | 'disks'>('pools');
const [searchTerm, setSearchTerm] = createSignal('');
const [selectedNode, setSelectedNode] = createSignal<string | null>(null);
// TODO: Implement sorting in sortedStorage function
@@ -220,16 +222,62 @@ const Storage: Component = () => {
searchTerm={searchTerm()}
/>
{/* Storage Filter */}
<StorageFilter
search={searchTerm}
setSearch={setSearchTerm}
groupBy={viewMode}
setGroupBy={setViewMode}
setSortKey={() => {}}
setSortDirection={() => {}}
searchInputRef={(el) => searchInputRef = el}
/>
{/* Tab Toggle */}
<div class="mb-4 border-b border-gray-200">
<nav class="-mb-px flex space-x-8" aria-label="Tabs">
<button
onClick={() => setTabView('pools')}
class={`py-2 px-1 border-b-2 font-medium text-sm transition-colors ${
tabView() === 'pools'
? 'border-blue-500 text-blue-600'
: 'border-transparent text-gray-500 hover:text-gray-700 hover:border-gray-300'
}`}
>
Storage Pools
</button>
<button
onClick={() => setTabView('disks')}
class={`py-2 px-1 border-b-2 font-medium text-sm transition-colors ${
tabView() === 'disks'
? 'border-blue-500 text-blue-600'
: 'border-transparent text-gray-500 hover:text-gray-700 hover:border-gray-300'
}`}
>
Physical Disks
<Show when={state.physicalDisks?.length > 0}>
<span class="ml-2 bg-gray-100 text-gray-600 px-2 py-0.5 rounded-full text-xs">
{state.physicalDisks.length}
</span>
</Show>
</button>
</nav>
</div>
{/* Show Storage Filter only for pools */}
<Show when={tabView() === 'pools'}>
<StorageFilter
search={searchTerm}
setSearch={setSearchTerm}
groupBy={viewMode}
setGroupBy={setViewMode}
setSortKey={() => {}}
setSortDirection={() => {}}
searchInputRef={(el) => searchInputRef = el}
/>
</Show>
{/* Show simple search for disks */}
<Show when={tabView() === 'disks'}>
<div class="mb-4">
<input
type="text"
placeholder="Search disks by model, path, or serial..."
value={searchTerm()}
onInput={(e) => setSearchTerm(e.currentTarget.value)}
class="w-full px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500"
/>
</div>
</Show>
{/* Loading State */}
<Show when={connected() && !initialDataReceived()}>
@@ -267,21 +315,23 @@ const Storage: Component = () => {
</div>
</Show>
{/* No results found message */}
<Show when={connected() && initialDataReceived() && sortedStorage().length === 0 && searchTerm().trim() !== ''}>
<div class="bg-white dark:bg-gray-800 border border-gray-200 dark:border-gray-700 rounded-lg p-8">
<div class="text-center">
<svg class="mx-auto h-12 w-12 text-gray-400 mb-4" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z" />
</svg>
<h3 class="text-sm font-medium text-gray-900 dark:text-gray-100 mb-2">No storage found</h3>
<p class="text-xs text-gray-600 dark:text-gray-400">No storage matches your search "{searchTerm()}"</p>
{/* Conditional rendering based on tab */}
<Show when={tabView() === 'pools'}>
{/* No results found message for storage pools */}
<Show when={connected() && initialDataReceived() && sortedStorage().length === 0 && searchTerm().trim() !== ''}>
<div class="bg-white dark:bg-gray-800 border border-gray-200 dark:border-gray-700 rounded-lg p-8">
<div class="text-center">
<svg class="mx-auto h-12 w-12 text-gray-400 mb-4" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z" />
</svg>
<h3 class="text-sm font-medium text-gray-900 dark:text-gray-100 mb-2">No storage found</h3>
<p class="text-xs text-gray-600 dark:text-gray-400">No storage matches your search "{searchTerm()}"</p>
</div>
</div>
</div>
</Show>
{/* Storage Table - shows for both PVE and PBS storage */}
<Show when={connected() && initialDataReceived() && sortedStorage().length > 0}>
</Show>
{/* Storage Table - shows for both PVE and PBS storage */}
<Show when={connected() && initialDataReceived() && sortedStorage().length > 0}>
<ComponentErrorBoundary name="Storage Table">
<div class="mb-4 bg-white dark:bg-gray-800 rounded-lg shadow-sm border border-gray-200 dark:border-gray-700">
<div class="overflow-x-auto" style="scrollbar-width: none; -ms-overflow-style: none;">
@@ -356,6 +406,22 @@ const Storage: Component = () => {
<span class="text-sm font-medium text-gray-900 dark:text-gray-100">
{storage.name}
</span>
{/* ZFS Health Badge */}
<Show when={storage.zfsPool && storage.zfsPool.state !== 'ONLINE'}>
<span class={`px-1.5 py-0.5 rounded text-[10px] font-medium ${
storage.zfsPool.state === 'DEGRADED'
? 'bg-yellow-100 text-yellow-700 dark:bg-yellow-900 dark:text-yellow-300'
: 'bg-red-100 text-red-700 dark:bg-red-900 dark:text-red-300'
}`}>
{storage.zfsPool.state}
</span>
</Show>
{/* ZFS Error Badge */}
<Show when={storage.zfsPool && (storage.zfsPool.readErrors > 0 || storage.zfsPool.writeErrors > 0 || storage.zfsPool.checksumErrors > 0)}>
<span class="px-1.5 py-0.5 rounded text-[10px] font-medium bg-red-100 text-red-700 dark:bg-red-900 dark:text-red-300">
ERRORS
</span>
</Show>
<Show when={viewMode() === 'storage'}>
<Show when={storage.pbsNames}>
<span class="text-xs text-gray-500 dark:text-gray-400">
@@ -475,10 +541,18 @@ const Storage: Component = () => {
</div>
</div>
</ComponentErrorBoundary>
</Show>
</Show>
{/* Physical Disks Tab */}
<Show when={tabView() === 'disks'}>
<DiskList
disks={state.physicalDisks || []}
selectedNode={selectedNode()}
searchTerm={searchTerm()}
/>
</Show>
{/* Tooltip System */}
<TooltipComponent />
</div>
);
};
+18
View File
@@ -5,6 +5,7 @@ export interface State {
vms: VM[];
containers: Container[];
storage: Storage[];
physicalDisks: PhysicalDisk[];
pbs: PBSInstance[];
pbsBackups: PBSBackup[];
metrics: Metric[];
@@ -248,6 +249,23 @@ export interface Disk {
usage: number;
}
export interface PhysicalDisk {
id: string;
node: string;
instance: string;
devPath: string;
model: string;
serial: string;
type: 'nvme' | 'sata' | 'sas' | string;
size: number;
health: 'PASSED' | 'FAILED' | 'UNKNOWN' | string;
wearout: number; // 0-100, 100 is best (percentage life remaining)
temperature: number;
rpm: number;
used: string;
lastChecked: string;
}
export interface CPUInfo {
model: string;
cores: number;
+130
View File
@@ -11,6 +11,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/utils"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
"github.com/rs/zerolog/log"
)
@@ -2242,3 +2243,132 @@ func (m *Manager) periodicSaveAlerts() {
}
}
}
// CheckDiskHealth checks disk health and creates alerts if needed
func (m *Manager) CheckDiskHealth(instance, node string, disk proxmox.Disk) {
// Create unique alert ID for this disk
alertID := fmt.Sprintf("disk-health-%s-%s-%s", instance, node, disk.DevPath)
m.mu.Lock()
defer m.mu.Unlock()
// Check if disk health is not PASSED
if disk.Health != "PASSED" && disk.Health != "" {
// Check if alert already exists
if _, exists := m.activeAlerts[alertID]; !exists {
// Create new health alert
alert := &Alert{
ID: alertID,
Type: "disk-health",
Level: AlertLevelCritical,
ResourceID: fmt.Sprintf("%s-%s", node, disk.DevPath),
ResourceName: fmt.Sprintf("%s (%s)", disk.Model, disk.DevPath),
Node: node,
Instance: instance,
Message: fmt.Sprintf("Disk health check failed: %s", disk.Health),
Value: 0, // Not applicable for health status
Threshold: 0,
StartTime: time.Now(),
LastSeen: time.Now(),
Metadata: map[string]interface{}{
"disk_path": disk.DevPath,
"disk_model": disk.Model,
"disk_serial": disk.Serial,
"disk_type": disk.Type,
"disk_health": disk.Health,
"disk_size": disk.Size,
},
}
m.activeAlerts[alertID] = alert
m.recentAlerts[alertID] = alert
m.historyManager.AddAlert(*alert)
if m.onAlert != nil {
m.onAlert(alert)
}
log.Error().
Str("node", node).
Str("disk", disk.DevPath).
Str("model", disk.Model).
Str("health", disk.Health).
Msg("Disk health alert created")
}
} else {
// Disk is healthy, clear alert if it exists
m.clearAlertNoLock(alertID)
}
// Check for low wearout (SSD life remaining)
if disk.Wearout > 0 && disk.Wearout < 10 {
wearoutAlertID := fmt.Sprintf("disk-wearout-%s-%s-%s", instance, node, disk.DevPath)
if _, exists := m.activeAlerts[wearoutAlertID]; !exists {
// Create wearout alert
alert := &Alert{
ID: wearoutAlertID,
Type: "disk-wearout",
Level: AlertLevelWarning,
ResourceID: fmt.Sprintf("%s-%s", node, disk.DevPath),
ResourceName: fmt.Sprintf("%s (%s)", disk.Model, disk.DevPath),
Node: node,
Instance: instance,
Message: fmt.Sprintf("SSD has less than 10%% life remaining (%d%% wearout)", disk.Wearout),
Value: float64(disk.Wearout),
Threshold: 10.0,
StartTime: time.Now(),
LastSeen: time.Now(),
Metadata: map[string]interface{}{
"disk_path": disk.DevPath,
"disk_model": disk.Model,
"disk_serial": disk.Serial,
"disk_type": disk.Type,
"disk_wearout": disk.Wearout,
},
}
m.activeAlerts[wearoutAlertID] = alert
m.recentAlerts[wearoutAlertID] = alert
m.historyManager.AddAlert(*alert)
if m.onAlert != nil {
m.onAlert(alert)
}
log.Warn().
Str("node", node).
Str("disk", disk.DevPath).
Str("model", disk.Model).
Int("wearout", disk.Wearout).
Msg("Disk wearout alert created")
}
} else if disk.Wearout >= 10 {
// Wearout is acceptable, clear alert if it exists
wearoutAlertID := fmt.Sprintf("disk-wearout-%s-%s-%s", instance, node, disk.DevPath)
m.clearAlertNoLock(wearoutAlertID)
}
}
// clearAlertNoLock clears an alert without locking (must be called with lock held)
func (m *Manager) clearAlertNoLock(alertID string) {
if alert, exists := m.activeAlerts[alertID]; exists {
delete(m.activeAlerts, alertID)
// Add to recently resolved
resolvedAlert := &ResolvedAlert{
Alert: alert,
ResolvedTime: time.Now(),
}
m.recentlyResolved[alertID] = resolvedAlert
// Send recovery notification
if m.onResolved != nil {
m.onResolved(alertID)
}
log.Info().
Str("alertID", alertID).
Msg("Alert cleared")
}
}
+1
View File
@@ -35,6 +35,7 @@ func (s *State) ToFrontend() StateFrontend {
VMs: vms,
Containers: containers,
Storage: storage,
PhysicalDisks: s.PhysicalDisks,
PBS: s.PBSInstances,
ActiveAlerts: s.ActiveAlerts,
Metrics: make(map[string]any),
+56
View File
@@ -14,6 +14,7 @@ type State struct {
VMs []VM `json:"vms"`
Containers []Container `json:"containers"`
Storage []Storage `json:"storage"`
PhysicalDisks []PhysicalDisk `json:"physicalDisks"`
PBSInstances []PBSInstance `json:"pbs"`
PBSBackups []PBSBackup `json:"pbsBackups"`
Metrics []Metric `json:"metrics"`
@@ -162,6 +163,24 @@ type ZFSDevice struct {
ChecksumErrors int64 `json:"checksumErrors"`
}
// PhysicalDisk represents a physical disk on a node
type PhysicalDisk struct {
ID string `json:"id"` // "{instance}-{node}-{devpath}"
Node string `json:"node"`
Instance string `json:"instance"`
DevPath string `json:"devPath"` // /dev/nvme0n1, /dev/sda
Model string `json:"model"`
Serial string `json:"serial"`
Type string `json:"type"` // nvme, sata, sas
Size int64 `json:"size"` // bytes
Health string `json:"health"` // PASSED, FAILED, UNKNOWN
Wearout int `json:"wearout"` // SSD life remaining percentage (0-100, 100 is best)
Temperature int `json:"temperature"` // Celsius (if available)
RPM int `json:"rpm"` // 0 for SSDs
Used string `json:"used"` // Filesystem or partition usage
LastChecked time.Time `json:"lastChecked"`
}
// PBSInstance represents a Proxmox Backup Server instance
type PBSInstance struct {
ID string `json:"id"`
@@ -384,6 +403,7 @@ func NewState() *State {
VMs: make([]VM, 0),
Containers: make([]Container, 0),
Storage: make([]Storage, 0),
PhysicalDisks: make([]PhysicalDisk, 0),
PBSInstances: make([]PBSInstance, 0),
PBSBackups: make([]PBSBackup, 0),
Metrics: make([]Metric, 0),
@@ -551,6 +571,42 @@ func (s *State) UpdateStorage(storage []Storage) {
s.LastUpdate = time.Now()
}
// UpdatePhysicalDisks updates physical disks for a specific instance
func (s *State) UpdatePhysicalDisks(instanceName string, disks []PhysicalDisk) {
s.mu.Lock()
defer s.mu.Unlock()
// Create a map of existing disks, excluding those from this instance
diskMap := make(map[string]PhysicalDisk)
for _, disk := range s.PhysicalDisks {
if disk.Instance != instanceName {
diskMap[disk.ID] = disk
}
}
// Add or update disks from this instance
for _, disk := range disks {
diskMap[disk.ID] = disk
}
// Convert map back to slice
newDisks := make([]PhysicalDisk, 0, len(diskMap))
for _, disk := range diskMap {
newDisks = append(newDisks, disk)
}
// Sort by node and dev path for consistent ordering
sort.Slice(newDisks, func(i, j int) bool {
if newDisks[i].Node != newDisks[j].Node {
return newDisks[i].Node < newDisks[j].Node
}
return newDisks[i].DevPath < newDisks[j].DevPath
})
s.PhysicalDisks = newDisks
s.LastUpdate = time.Now()
}
// UpdateStorageForInstance updates storage for a specific instance, merging with existing storage
func (s *State) UpdateStorageForInstance(instanceName string, storage []Storage) {
s.mu.Lock()
+1
View File
@@ -105,6 +105,7 @@ type StateFrontend struct {
VMs []VMFrontend `json:"vms"`
Containers []ContainerFrontend `json:"containers"`
Storage []StorageFrontend `json:"storage"`
PhysicalDisks []PhysicalDisk `json:"physicalDisks"`
PBS []PBSInstance `json:"pbs"` // Keep as is
ActiveAlerts []Alert `json:"activeAlerts"` // Active alerts
Metrics map[string]any `json:"metrics"` // Empty object for now
+3
View File
@@ -8,6 +8,7 @@ type StateSnapshot struct {
VMs []VM `json:"vms"`
Containers []Container `json:"containers"`
Storage []Storage `json:"storage"`
PhysicalDisks []PhysicalDisk `json:"physicalDisks"`
PBSInstances []PBSInstance `json:"pbs"`
PBSBackups []PBSBackup `json:"pbsBackups"`
Metrics []Metric `json:"metrics"`
@@ -31,6 +32,7 @@ func (s *State) GetSnapshot() StateSnapshot {
VMs: append([]VM{}, s.VMs...),
Containers: append([]Container{}, s.Containers...),
Storage: append([]Storage{}, s.Storage...),
PhysicalDisks: append([]PhysicalDisk{}, s.PhysicalDisks...),
PBSInstances: append([]PBSInstance{}, s.PBSInstances...),
PBSBackups: append([]PBSBackup{}, s.PBSBackups...),
Metrics: append([]Metric{}, s.Metrics...),
@@ -86,6 +88,7 @@ func (s StateSnapshot) ToFrontend() StateFrontend {
VMs: vms,
Containers: containers,
Storage: storage,
PhysicalDisks: s.PhysicalDisks,
PBS: s.PBSInstances,
ActiveAlerts: s.ActiveAlerts,
Metrics: make(map[string]any),
+92
View File
@@ -45,6 +45,7 @@ type PVEClientInterface interface {
GetVMFSInfo(ctx context.Context, node string, vmid int) ([]proxmox.VMFileSystem, error)
GetZFSPoolStatus(ctx context.Context, node string) ([]proxmox.ZFSPoolStatus, error)
GetZFSPoolsWithDetails(ctx context.Context, node string) ([]proxmox.ZFSPoolInfo, error)
GetDisks(ctx context.Context, node string) ([]proxmox.Disk, error)
}
// Monitor handles all monitoring operations
@@ -901,6 +902,97 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
}
}
// Poll physical disks for health monitoring
log.Debug().Int("nodeCount", len(nodes)).Msg("Starting disk health polling")
var allDisks []models.PhysicalDisk
for _, node := range nodes {
// Skip offline nodes
if node.Status != "online" {
log.Debug().Str("node", node.Node).Msg("Skipping disk poll for offline node")
continue
}
// Get disk list for this node
log.Debug().Str("node", node.Node).Msg("Getting disk list for node")
disks, err := client.GetDisks(ctx, node.Node)
if err != nil {
// Log but don't fail - disk monitoring is optional
log.Debug().
Str("node", node.Node).
Err(err).
Msg("Failed to get disk list - disk monitoring may not be available")
continue
}
log.Debug().
Str("node", node.Node).
Int("diskCount", len(disks)).
Msg("Got disk list for node")
// Check each disk for health issues and add to state
for _, disk := range disks {
// Create PhysicalDisk model
diskID := fmt.Sprintf("%s-%s-%s", instanceName, node.Node, strings.ReplaceAll(disk.DevPath, "/", "-"))
physicalDisk := models.PhysicalDisk{
ID: diskID,
Node: node.Node,
Instance: instanceName,
DevPath: disk.DevPath,
Model: disk.Model,
Serial: disk.Serial,
Type: disk.Type,
Size: disk.Size,
Health: disk.Health,
Wearout: disk.Wearout,
RPM: disk.RPM,
Used: disk.Used,
LastChecked: time.Now(),
}
allDisks = append(allDisks, physicalDisk)
log.Debug().
Str("node", node.Node).
Str("disk", disk.DevPath).
Str("model", disk.Model).
Str("health", disk.Health).
Int("wearout", disk.Wearout).
Msg("Checking disk health")
if disk.Health != "PASSED" && disk.Health != "" {
// Disk has failed or is failing - alert manager will handle this
log.Warn().
Str("node", node.Node).
Str("disk", disk.DevPath).
Str("model", disk.Model).
Str("health", disk.Health).
Int("wearout", disk.Wearout).
Msg("Disk health issue detected")
// Pass disk info to alert manager
m.alertManager.CheckDiskHealth(instanceName, node.Node, disk)
} else if disk.Wearout > 0 && disk.Wearout < 10 {
// Low wearout warning (less than 10% life remaining)
log.Warn().
Str("node", node.Node).
Str("disk", disk.DevPath).
Str("model", disk.Model).
Int("wearout", disk.Wearout).
Msg("SSD wearout critical - less than 10% life remaining")
// Pass to alert manager for wearout alert
m.alertManager.CheckDiskHealth(instanceName, node.Node, disk)
}
}
}
// Update physical disks in state
log.Debug().
Str("instance", instanceName).
Int("diskCount", len(allDisks)).
Msg("Updating physical disks in state")
m.state.UpdatePhysicalDisks(instanceName, allDisks)
// Update nodes with storage fallback if rootfs was not available
for i := range modelNodes {
if modelNodes[i].Disk.Total == 0 {
+63
View File
@@ -1150,5 +1150,68 @@ func (c *Client) GetZFSPoolDetail(ctx context.Context, node, pool string) (*ZFSP
return nil, err
}
return &result.Data, nil
}
// Disk represents a physical disk on a Proxmox node
type Disk struct {
DevPath string `json:"devpath"`
Model string `json:"model"`
Serial string `json:"serial"`
Type string `json:"type"` // nvme, sata, sas
Health string `json:"health"` // PASSED, FAILED, UNKNOWN
Wearout int `json:"wearout"` // SSD wear percentage (0-100, 100 is best)
Size int64 `json:"size"` // Size in bytes
RPM int `json:"rpm"` // 0 for SSDs
Used string `json:"used"` // Filesystem or partition usage
Vendor string `json:"vendor"`
WWN string `json:"wwn"` // World Wide Name
}
// DiskSmart represents SMART data for a disk
type DiskSmart struct {
Health string `json:"health"` // PASSED, FAILED, UNKNOWN
Wearout int `json:"wearout"` // SSD wear percentage
Type string `json:"type"` // Type of response (text, attributes)
Text string `json:"text"` // Raw SMART output text
}
// GetDisks returns the list of physical disks on a node
func (c *Client) GetDisks(ctx context.Context, node string) ([]Disk, error) {
resp, err := c.request(ctx, "GET", fmt.Sprintf("/nodes/%s/disks/list", node), nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
var result struct {
Data []Disk `json:"data"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, err
}
return result.Data, nil
}
// GetDiskSmart returns SMART data for a specific disk
func (c *Client) GetDiskSmart(ctx context.Context, node, disk string) (*DiskSmart, error) {
params := url.Values{
"disk": {disk},
}
resp, err := c.request(ctx, "GET", fmt.Sprintf("/nodes/%s/disks/smart?%s", node, params.Encode()), nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
var result struct {
Data DiskSmart `json:"data"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, err
}
return &result.Data, nil
}
+24
View File
@@ -807,6 +807,30 @@ func (cc *ClusterClient) GetClusterHealthInfo() models.ClusterHealth {
}
// Helper to check if error is auth-related
func (cc *ClusterClient) GetDisks(ctx context.Context, node string) ([]Disk, error) {
var result []Disk
err := cc.executeWithFailover(ctx, func(client *Client) error {
disks, err := client.GetDisks(ctx, node)
if err != nil {
return err
}
result = disks
return nil
})
// Don't return error for transient connectivity issues
if err != nil && strings.Contains(err.Error(), "no healthy nodes available") {
log.Debug().
Str("cluster", cc.name).
Str("node", node).
Err(err).
Msg("No healthy nodes for GetDisks - returning empty list")
return []Disk{}, nil
}
return result, err
}
func IsAuthError(err error) bool {
if err == nil {
return false