mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-22 03:04:03 +00:00
feat: implement S.M.A.R.T. disk monitoring for Proxmox nodes (addresses #429)
- Added disk polling to monitoring cycle using Proxmox API - Created CheckDiskHealth() alert manager for failing drives and low SSD life - Added PhysicalDisk model to state with proper serialization - Implemented DiskList component with health indicators and SSD wearout bars - Added Physical Disks tab to Storage page with toggle between pools and disks - Added ZFS health badges to storage cards for degraded/failed pools - Alerts trigger for health != PASSED and SSD wearout < 10% - Frontend displays disk model, type, temperature, and usage information
This commit is contained in:
@@ -0,0 +1,167 @@
|
||||
import { Component, For, Show, createMemo } from 'solid-js';
|
||||
import { formatBytes } from '@/utils/format';
|
||||
import type { PhysicalDisk } from '@/types/api';
|
||||
|
||||
interface DiskListProps {
|
||||
disks: PhysicalDisk[];
|
||||
selectedNode: string | null;
|
||||
searchTerm: string;
|
||||
}
|
||||
|
||||
export const DiskList: Component<DiskListProps> = (props) => {
|
||||
// Filter disks based on selected node and search term
|
||||
const filteredDisks = createMemo(() => {
|
||||
let disks = props.disks || [];
|
||||
|
||||
// Filter by node if selected
|
||||
if (props.selectedNode) {
|
||||
disks = disks.filter(d => d.node === props.selectedNode);
|
||||
}
|
||||
|
||||
// Filter by search term
|
||||
if (props.searchTerm) {
|
||||
const term = props.searchTerm.toLowerCase();
|
||||
disks = disks.filter(d =>
|
||||
d.model.toLowerCase().includes(term) ||
|
||||
d.devPath.toLowerCase().includes(term) ||
|
||||
d.serial.toLowerCase().includes(term) ||
|
||||
d.node.toLowerCase().includes(term)
|
||||
);
|
||||
}
|
||||
|
||||
// Sort by node and devPath
|
||||
return disks.sort((a, b) => {
|
||||
if (a.node !== b.node) return a.node.localeCompare(b.node);
|
||||
return a.devPath.localeCompare(b.devPath);
|
||||
});
|
||||
});
|
||||
|
||||
// Get health status color and icon
|
||||
const getHealthStatus = (disk: PhysicalDisk) => {
|
||||
if (disk.health === 'PASSED') {
|
||||
// Check wearout for SSDs
|
||||
if (disk.wearout > 0 && disk.wearout < 10) {
|
||||
return { color: 'text-yellow-500', icon: '⚠️', text: 'Low Life' };
|
||||
}
|
||||
return { color: 'text-green-500', icon: '✅', text: 'Healthy' };
|
||||
} else if (disk.health === 'FAILED') {
|
||||
return { color: 'text-red-500', icon: '❌', text: 'Failed' };
|
||||
}
|
||||
return { color: 'text-gray-500', icon: '❓', text: 'Unknown' };
|
||||
};
|
||||
|
||||
// Get disk type badge color
|
||||
const getDiskTypeBadge = (type: string) => {
|
||||
switch (type.toLowerCase()) {
|
||||
case 'nvme':
|
||||
return 'bg-purple-100 text-purple-800';
|
||||
case 'sata':
|
||||
return 'bg-blue-100 text-blue-800';
|
||||
case 'sas':
|
||||
return 'bg-indigo-100 text-indigo-800';
|
||||
default:
|
||||
return 'bg-gray-100 text-gray-800';
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div class="space-y-4">
|
||||
<Show when={filteredDisks().length === 0}>
|
||||
<div class="text-center py-8 text-gray-500">
|
||||
No physical disks found
|
||||
{props.selectedNode && ` for node ${props.selectedNode}`}
|
||||
{props.searchTerm && ` matching "${props.searchTerm}"`}
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
<For each={filteredDisks()}>
|
||||
{(disk) => {
|
||||
const health = getHealthStatus(disk);
|
||||
|
||||
return (
|
||||
<div class="bg-white rounded-lg shadow p-4 hover:shadow-md transition-shadow">
|
||||
<div class="flex items-start justify-between">
|
||||
<div class="flex-1">
|
||||
{/* Header with model and health */}
|
||||
<div class="flex items-center gap-3 mb-2">
|
||||
<span class={`text-lg ${health.color}`} title={health.text}>
|
||||
{health.icon}
|
||||
</span>
|
||||
<h3 class="text-lg font-semibold text-gray-900">
|
||||
{disk.model || 'Unknown Model'}
|
||||
</h3>
|
||||
<span class={`px-2 py-1 text-xs font-medium rounded-full ${getDiskTypeBadge(disk.type)}`}>
|
||||
{disk.type.toUpperCase()}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Disk details */}
|
||||
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 text-sm">
|
||||
<div>
|
||||
<span class="text-gray-500">Node:</span>
|
||||
<span class="ml-2 font-medium">{disk.node}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span class="text-gray-500">Path:</span>
|
||||
<span class="ml-2 font-mono text-xs">{disk.devPath}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span class="text-gray-500">Size:</span>
|
||||
<span class="ml-2 font-medium">{formatBytes(disk.size)}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span class="text-gray-500">Usage:</span>
|
||||
<span class="ml-2">{disk.used || 'Unknown'}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Additional metrics for SSDs */}
|
||||
<Show when={disk.wearout > 0}>
|
||||
<div class="mt-3 space-y-2">
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="text-sm text-gray-500">SSD Life Remaining:</span>
|
||||
<div class="flex-1 max-w-xs">
|
||||
<div class="bg-gray-200 rounded-full h-2">
|
||||
<div
|
||||
class={`h-2 rounded-full transition-all ${
|
||||
disk.wearout >= 50 ? 'bg-green-500' :
|
||||
disk.wearout >= 20 ? 'bg-yellow-500' :
|
||||
disk.wearout >= 10 ? 'bg-orange-500' :
|
||||
'bg-red-500'
|
||||
}`}
|
||||
style={`width: ${disk.wearout}%`}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<span class="text-sm font-medium">{disk.wearout}%</span>
|
||||
</div>
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
{/* Temperature if available */}
|
||||
<Show when={disk.temperature > 0}>
|
||||
<div class="mt-2 text-sm">
|
||||
<span class="text-gray-500">Temperature:</span>
|
||||
<span class={`ml-2 font-medium ${
|
||||
disk.temperature > 70 ? 'text-red-500' :
|
||||
disk.temperature > 60 ? 'text-yellow-500' :
|
||||
'text-green-500'
|
||||
}`}>
|
||||
{disk.temperature}°C
|
||||
</span>
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
{/* Serial number (smaller, muted) */}
|
||||
<div class="mt-2 text-xs text-gray-400">
|
||||
Serial: {disk.serial}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}}
|
||||
</For>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -7,11 +7,13 @@ import type { Storage as StorageType } from '@/types/api';
|
||||
import { ComponentErrorBoundary } from '@/components/ErrorBoundary';
|
||||
import { UnifiedNodeSelector } from '@/components/shared/UnifiedNodeSelector';
|
||||
import { StorageFilter } from './StorageFilter';
|
||||
import { DiskList } from './DiskList';
|
||||
|
||||
|
||||
const Storage: Component = () => {
|
||||
const { state, connected, activeAlerts, initialDataReceived } = useWebSocket();
|
||||
const [viewMode, setViewMode] = createSignal<'node' | 'storage'>('node');
|
||||
const [tabView, setTabView] = createSignal<'pools' | 'disks'>('pools');
|
||||
const [searchTerm, setSearchTerm] = createSignal('');
|
||||
const [selectedNode, setSelectedNode] = createSignal<string | null>(null);
|
||||
// TODO: Implement sorting in sortedStorage function
|
||||
@@ -220,16 +222,62 @@ const Storage: Component = () => {
|
||||
searchTerm={searchTerm()}
|
||||
/>
|
||||
|
||||
{/* Storage Filter */}
|
||||
<StorageFilter
|
||||
search={searchTerm}
|
||||
setSearch={setSearchTerm}
|
||||
groupBy={viewMode}
|
||||
setGroupBy={setViewMode}
|
||||
setSortKey={() => {}}
|
||||
setSortDirection={() => {}}
|
||||
searchInputRef={(el) => searchInputRef = el}
|
||||
/>
|
||||
{/* Tab Toggle */}
|
||||
<div class="mb-4 border-b border-gray-200">
|
||||
<nav class="-mb-px flex space-x-8" aria-label="Tabs">
|
||||
<button
|
||||
onClick={() => setTabView('pools')}
|
||||
class={`py-2 px-1 border-b-2 font-medium text-sm transition-colors ${
|
||||
tabView() === 'pools'
|
||||
? 'border-blue-500 text-blue-600'
|
||||
: 'border-transparent text-gray-500 hover:text-gray-700 hover:border-gray-300'
|
||||
}`}
|
||||
>
|
||||
Storage Pools
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setTabView('disks')}
|
||||
class={`py-2 px-1 border-b-2 font-medium text-sm transition-colors ${
|
||||
tabView() === 'disks'
|
||||
? 'border-blue-500 text-blue-600'
|
||||
: 'border-transparent text-gray-500 hover:text-gray-700 hover:border-gray-300'
|
||||
}`}
|
||||
>
|
||||
Physical Disks
|
||||
<Show when={state.physicalDisks?.length > 0}>
|
||||
<span class="ml-2 bg-gray-100 text-gray-600 px-2 py-0.5 rounded-full text-xs">
|
||||
{state.physicalDisks.length}
|
||||
</span>
|
||||
</Show>
|
||||
</button>
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
{/* Show Storage Filter only for pools */}
|
||||
<Show when={tabView() === 'pools'}>
|
||||
<StorageFilter
|
||||
search={searchTerm}
|
||||
setSearch={setSearchTerm}
|
||||
groupBy={viewMode}
|
||||
setGroupBy={setViewMode}
|
||||
setSortKey={() => {}}
|
||||
setSortDirection={() => {}}
|
||||
searchInputRef={(el) => searchInputRef = el}
|
||||
/>
|
||||
</Show>
|
||||
|
||||
{/* Show simple search for disks */}
|
||||
<Show when={tabView() === 'disks'}>
|
||||
<div class="mb-4">
|
||||
<input
|
||||
type="text"
|
||||
placeholder="Search disks by model, path, or serial..."
|
||||
value={searchTerm()}
|
||||
onInput={(e) => setSearchTerm(e.currentTarget.value)}
|
||||
class="w-full px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500"
|
||||
/>
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
{/* Loading State */}
|
||||
<Show when={connected() && !initialDataReceived()}>
|
||||
@@ -267,21 +315,23 @@ const Storage: Component = () => {
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
{/* No results found message */}
|
||||
<Show when={connected() && initialDataReceived() && sortedStorage().length === 0 && searchTerm().trim() !== ''}>
|
||||
<div class="bg-white dark:bg-gray-800 border border-gray-200 dark:border-gray-700 rounded-lg p-8">
|
||||
<div class="text-center">
|
||||
<svg class="mx-auto h-12 w-12 text-gray-400 mb-4" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z" />
|
||||
</svg>
|
||||
<h3 class="text-sm font-medium text-gray-900 dark:text-gray-100 mb-2">No storage found</h3>
|
||||
<p class="text-xs text-gray-600 dark:text-gray-400">No storage matches your search "{searchTerm()}"</p>
|
||||
{/* Conditional rendering based on tab */}
|
||||
<Show when={tabView() === 'pools'}>
|
||||
{/* No results found message for storage pools */}
|
||||
<Show when={connected() && initialDataReceived() && sortedStorage().length === 0 && searchTerm().trim() !== ''}>
|
||||
<div class="bg-white dark:bg-gray-800 border border-gray-200 dark:border-gray-700 rounded-lg p-8">
|
||||
<div class="text-center">
|
||||
<svg class="mx-auto h-12 w-12 text-gray-400 mb-4" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z" />
|
||||
</svg>
|
||||
<h3 class="text-sm font-medium text-gray-900 dark:text-gray-100 mb-2">No storage found</h3>
|
||||
<p class="text-xs text-gray-600 dark:text-gray-400">No storage matches your search "{searchTerm()}"</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
{/* Storage Table - shows for both PVE and PBS storage */}
|
||||
<Show when={connected() && initialDataReceived() && sortedStorage().length > 0}>
|
||||
</Show>
|
||||
|
||||
{/* Storage Table - shows for both PVE and PBS storage */}
|
||||
<Show when={connected() && initialDataReceived() && sortedStorage().length > 0}>
|
||||
<ComponentErrorBoundary name="Storage Table">
|
||||
<div class="mb-4 bg-white dark:bg-gray-800 rounded-lg shadow-sm border border-gray-200 dark:border-gray-700">
|
||||
<div class="overflow-x-auto" style="scrollbar-width: none; -ms-overflow-style: none;">
|
||||
@@ -356,6 +406,22 @@ const Storage: Component = () => {
|
||||
<span class="text-sm font-medium text-gray-900 dark:text-gray-100">
|
||||
{storage.name}
|
||||
</span>
|
||||
{/* ZFS Health Badge */}
|
||||
<Show when={storage.zfsPool && storage.zfsPool.state !== 'ONLINE'}>
|
||||
<span class={`px-1.5 py-0.5 rounded text-[10px] font-medium ${
|
||||
storage.zfsPool.state === 'DEGRADED'
|
||||
? 'bg-yellow-100 text-yellow-700 dark:bg-yellow-900 dark:text-yellow-300'
|
||||
: 'bg-red-100 text-red-700 dark:bg-red-900 dark:text-red-300'
|
||||
}`}>
|
||||
{storage.zfsPool.state}
|
||||
</span>
|
||||
</Show>
|
||||
{/* ZFS Error Badge */}
|
||||
<Show when={storage.zfsPool && (storage.zfsPool.readErrors > 0 || storage.zfsPool.writeErrors > 0 || storage.zfsPool.checksumErrors > 0)}>
|
||||
<span class="px-1.5 py-0.5 rounded text-[10px] font-medium bg-red-100 text-red-700 dark:bg-red-900 dark:text-red-300">
|
||||
ERRORS
|
||||
</span>
|
||||
</Show>
|
||||
<Show when={viewMode() === 'storage'}>
|
||||
<Show when={storage.pbsNames}>
|
||||
<span class="text-xs text-gray-500 dark:text-gray-400">
|
||||
@@ -475,10 +541,18 @@ const Storage: Component = () => {
|
||||
</div>
|
||||
</div>
|
||||
</ComponentErrorBoundary>
|
||||
</Show>
|
||||
</Show>
|
||||
|
||||
{/* Physical Disks Tab */}
|
||||
<Show when={tabView() === 'disks'}>
|
||||
<DiskList
|
||||
disks={state.physicalDisks || []}
|
||||
selectedNode={selectedNode()}
|
||||
searchTerm={searchTerm()}
|
||||
/>
|
||||
</Show>
|
||||
|
||||
{/* Tooltip System */}
|
||||
<TooltipComponent />
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -5,6 +5,7 @@ export interface State {
|
||||
vms: VM[];
|
||||
containers: Container[];
|
||||
storage: Storage[];
|
||||
physicalDisks: PhysicalDisk[];
|
||||
pbs: PBSInstance[];
|
||||
pbsBackups: PBSBackup[];
|
||||
metrics: Metric[];
|
||||
@@ -248,6 +249,23 @@ export interface Disk {
|
||||
usage: number;
|
||||
}
|
||||
|
||||
export interface PhysicalDisk {
|
||||
id: string;
|
||||
node: string;
|
||||
instance: string;
|
||||
devPath: string;
|
||||
model: string;
|
||||
serial: string;
|
||||
type: 'nvme' | 'sata' | 'sas' | string;
|
||||
size: number;
|
||||
health: 'PASSED' | 'FAILED' | 'UNKNOWN' | string;
|
||||
wearout: number; // 0-100, 100 is best (percentage life remaining)
|
||||
temperature: number;
|
||||
rpm: number;
|
||||
used: string;
|
||||
lastChecked: string;
|
||||
}
|
||||
|
||||
export interface CPUInfo {
|
||||
model: string;
|
||||
cores: number;
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/utils"
|
||||
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
@@ -2242,3 +2243,132 @@ func (m *Manager) periodicSaveAlerts() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CheckDiskHealth checks disk health and creates alerts if needed
|
||||
func (m *Manager) CheckDiskHealth(instance, node string, disk proxmox.Disk) {
|
||||
// Create unique alert ID for this disk
|
||||
alertID := fmt.Sprintf("disk-health-%s-%s-%s", instance, node, disk.DevPath)
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
// Check if disk health is not PASSED
|
||||
if disk.Health != "PASSED" && disk.Health != "" {
|
||||
// Check if alert already exists
|
||||
if _, exists := m.activeAlerts[alertID]; !exists {
|
||||
// Create new health alert
|
||||
alert := &Alert{
|
||||
ID: alertID,
|
||||
Type: "disk-health",
|
||||
Level: AlertLevelCritical,
|
||||
ResourceID: fmt.Sprintf("%s-%s", node, disk.DevPath),
|
||||
ResourceName: fmt.Sprintf("%s (%s)", disk.Model, disk.DevPath),
|
||||
Node: node,
|
||||
Instance: instance,
|
||||
Message: fmt.Sprintf("Disk health check failed: %s", disk.Health),
|
||||
Value: 0, // Not applicable for health status
|
||||
Threshold: 0,
|
||||
StartTime: time.Now(),
|
||||
LastSeen: time.Now(),
|
||||
Metadata: map[string]interface{}{
|
||||
"disk_path": disk.DevPath,
|
||||
"disk_model": disk.Model,
|
||||
"disk_serial": disk.Serial,
|
||||
"disk_type": disk.Type,
|
||||
"disk_health": disk.Health,
|
||||
"disk_size": disk.Size,
|
||||
},
|
||||
}
|
||||
|
||||
m.activeAlerts[alertID] = alert
|
||||
m.recentAlerts[alertID] = alert
|
||||
m.historyManager.AddAlert(*alert)
|
||||
|
||||
if m.onAlert != nil {
|
||||
m.onAlert(alert)
|
||||
}
|
||||
|
||||
log.Error().
|
||||
Str("node", node).
|
||||
Str("disk", disk.DevPath).
|
||||
Str("model", disk.Model).
|
||||
Str("health", disk.Health).
|
||||
Msg("Disk health alert created")
|
||||
}
|
||||
} else {
|
||||
// Disk is healthy, clear alert if it exists
|
||||
m.clearAlertNoLock(alertID)
|
||||
}
|
||||
|
||||
// Check for low wearout (SSD life remaining)
|
||||
if disk.Wearout > 0 && disk.Wearout < 10 {
|
||||
wearoutAlertID := fmt.Sprintf("disk-wearout-%s-%s-%s", instance, node, disk.DevPath)
|
||||
|
||||
if _, exists := m.activeAlerts[wearoutAlertID]; !exists {
|
||||
// Create wearout alert
|
||||
alert := &Alert{
|
||||
ID: wearoutAlertID,
|
||||
Type: "disk-wearout",
|
||||
Level: AlertLevelWarning,
|
||||
ResourceID: fmt.Sprintf("%s-%s", node, disk.DevPath),
|
||||
ResourceName: fmt.Sprintf("%s (%s)", disk.Model, disk.DevPath),
|
||||
Node: node,
|
||||
Instance: instance,
|
||||
Message: fmt.Sprintf("SSD has less than 10%% life remaining (%d%% wearout)", disk.Wearout),
|
||||
Value: float64(disk.Wearout),
|
||||
Threshold: 10.0,
|
||||
StartTime: time.Now(),
|
||||
LastSeen: time.Now(),
|
||||
Metadata: map[string]interface{}{
|
||||
"disk_path": disk.DevPath,
|
||||
"disk_model": disk.Model,
|
||||
"disk_serial": disk.Serial,
|
||||
"disk_type": disk.Type,
|
||||
"disk_wearout": disk.Wearout,
|
||||
},
|
||||
}
|
||||
|
||||
m.activeAlerts[wearoutAlertID] = alert
|
||||
m.recentAlerts[wearoutAlertID] = alert
|
||||
m.historyManager.AddAlert(*alert)
|
||||
|
||||
if m.onAlert != nil {
|
||||
m.onAlert(alert)
|
||||
}
|
||||
|
||||
log.Warn().
|
||||
Str("node", node).
|
||||
Str("disk", disk.DevPath).
|
||||
Str("model", disk.Model).
|
||||
Int("wearout", disk.Wearout).
|
||||
Msg("Disk wearout alert created")
|
||||
}
|
||||
} else if disk.Wearout >= 10 {
|
||||
// Wearout is acceptable, clear alert if it exists
|
||||
wearoutAlertID := fmt.Sprintf("disk-wearout-%s-%s-%s", instance, node, disk.DevPath)
|
||||
m.clearAlertNoLock(wearoutAlertID)
|
||||
}
|
||||
}
|
||||
|
||||
// clearAlertNoLock clears an alert without locking (must be called with lock held)
|
||||
func (m *Manager) clearAlertNoLock(alertID string) {
|
||||
if alert, exists := m.activeAlerts[alertID]; exists {
|
||||
delete(m.activeAlerts, alertID)
|
||||
|
||||
// Add to recently resolved
|
||||
resolvedAlert := &ResolvedAlert{
|
||||
Alert: alert,
|
||||
ResolvedTime: time.Now(),
|
||||
}
|
||||
m.recentlyResolved[alertID] = resolvedAlert
|
||||
|
||||
// Send recovery notification
|
||||
if m.onResolved != nil {
|
||||
m.onResolved(alertID)
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Str("alertID", alertID).
|
||||
Msg("Alert cleared")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,6 +35,7 @@ func (s *State) ToFrontend() StateFrontend {
|
||||
VMs: vms,
|
||||
Containers: containers,
|
||||
Storage: storage,
|
||||
PhysicalDisks: s.PhysicalDisks,
|
||||
PBS: s.PBSInstances,
|
||||
ActiveAlerts: s.ActiveAlerts,
|
||||
Metrics: make(map[string]any),
|
||||
|
||||
@@ -14,6 +14,7 @@ type State struct {
|
||||
VMs []VM `json:"vms"`
|
||||
Containers []Container `json:"containers"`
|
||||
Storage []Storage `json:"storage"`
|
||||
PhysicalDisks []PhysicalDisk `json:"physicalDisks"`
|
||||
PBSInstances []PBSInstance `json:"pbs"`
|
||||
PBSBackups []PBSBackup `json:"pbsBackups"`
|
||||
Metrics []Metric `json:"metrics"`
|
||||
@@ -162,6 +163,24 @@ type ZFSDevice struct {
|
||||
ChecksumErrors int64 `json:"checksumErrors"`
|
||||
}
|
||||
|
||||
// PhysicalDisk represents a physical disk on a node
|
||||
type PhysicalDisk struct {
|
||||
ID string `json:"id"` // "{instance}-{node}-{devpath}"
|
||||
Node string `json:"node"`
|
||||
Instance string `json:"instance"`
|
||||
DevPath string `json:"devPath"` // /dev/nvme0n1, /dev/sda
|
||||
Model string `json:"model"`
|
||||
Serial string `json:"serial"`
|
||||
Type string `json:"type"` // nvme, sata, sas
|
||||
Size int64 `json:"size"` // bytes
|
||||
Health string `json:"health"` // PASSED, FAILED, UNKNOWN
|
||||
Wearout int `json:"wearout"` // SSD life remaining percentage (0-100, 100 is best)
|
||||
Temperature int `json:"temperature"` // Celsius (if available)
|
||||
RPM int `json:"rpm"` // 0 for SSDs
|
||||
Used string `json:"used"` // Filesystem or partition usage
|
||||
LastChecked time.Time `json:"lastChecked"`
|
||||
}
|
||||
|
||||
// PBSInstance represents a Proxmox Backup Server instance
|
||||
type PBSInstance struct {
|
||||
ID string `json:"id"`
|
||||
@@ -384,6 +403,7 @@ func NewState() *State {
|
||||
VMs: make([]VM, 0),
|
||||
Containers: make([]Container, 0),
|
||||
Storage: make([]Storage, 0),
|
||||
PhysicalDisks: make([]PhysicalDisk, 0),
|
||||
PBSInstances: make([]PBSInstance, 0),
|
||||
PBSBackups: make([]PBSBackup, 0),
|
||||
Metrics: make([]Metric, 0),
|
||||
@@ -551,6 +571,42 @@ func (s *State) UpdateStorage(storage []Storage) {
|
||||
s.LastUpdate = time.Now()
|
||||
}
|
||||
|
||||
// UpdatePhysicalDisks updates physical disks for a specific instance
|
||||
func (s *State) UpdatePhysicalDisks(instanceName string, disks []PhysicalDisk) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
// Create a map of existing disks, excluding those from this instance
|
||||
diskMap := make(map[string]PhysicalDisk)
|
||||
for _, disk := range s.PhysicalDisks {
|
||||
if disk.Instance != instanceName {
|
||||
diskMap[disk.ID] = disk
|
||||
}
|
||||
}
|
||||
|
||||
// Add or update disks from this instance
|
||||
for _, disk := range disks {
|
||||
diskMap[disk.ID] = disk
|
||||
}
|
||||
|
||||
// Convert map back to slice
|
||||
newDisks := make([]PhysicalDisk, 0, len(diskMap))
|
||||
for _, disk := range diskMap {
|
||||
newDisks = append(newDisks, disk)
|
||||
}
|
||||
|
||||
// Sort by node and dev path for consistent ordering
|
||||
sort.Slice(newDisks, func(i, j int) bool {
|
||||
if newDisks[i].Node != newDisks[j].Node {
|
||||
return newDisks[i].Node < newDisks[j].Node
|
||||
}
|
||||
return newDisks[i].DevPath < newDisks[j].DevPath
|
||||
})
|
||||
|
||||
s.PhysicalDisks = newDisks
|
||||
s.LastUpdate = time.Now()
|
||||
}
|
||||
|
||||
// UpdateStorageForInstance updates storage for a specific instance, merging with existing storage
|
||||
func (s *State) UpdateStorageForInstance(instanceName string, storage []Storage) {
|
||||
s.mu.Lock()
|
||||
|
||||
@@ -105,6 +105,7 @@ type StateFrontend struct {
|
||||
VMs []VMFrontend `json:"vms"`
|
||||
Containers []ContainerFrontend `json:"containers"`
|
||||
Storage []StorageFrontend `json:"storage"`
|
||||
PhysicalDisks []PhysicalDisk `json:"physicalDisks"`
|
||||
PBS []PBSInstance `json:"pbs"` // Keep as is
|
||||
ActiveAlerts []Alert `json:"activeAlerts"` // Active alerts
|
||||
Metrics map[string]any `json:"metrics"` // Empty object for now
|
||||
|
||||
@@ -8,6 +8,7 @@ type StateSnapshot struct {
|
||||
VMs []VM `json:"vms"`
|
||||
Containers []Container `json:"containers"`
|
||||
Storage []Storage `json:"storage"`
|
||||
PhysicalDisks []PhysicalDisk `json:"physicalDisks"`
|
||||
PBSInstances []PBSInstance `json:"pbs"`
|
||||
PBSBackups []PBSBackup `json:"pbsBackups"`
|
||||
Metrics []Metric `json:"metrics"`
|
||||
@@ -31,6 +32,7 @@ func (s *State) GetSnapshot() StateSnapshot {
|
||||
VMs: append([]VM{}, s.VMs...),
|
||||
Containers: append([]Container{}, s.Containers...),
|
||||
Storage: append([]Storage{}, s.Storage...),
|
||||
PhysicalDisks: append([]PhysicalDisk{}, s.PhysicalDisks...),
|
||||
PBSInstances: append([]PBSInstance{}, s.PBSInstances...),
|
||||
PBSBackups: append([]PBSBackup{}, s.PBSBackups...),
|
||||
Metrics: append([]Metric{}, s.Metrics...),
|
||||
@@ -86,6 +88,7 @@ func (s StateSnapshot) ToFrontend() StateFrontend {
|
||||
VMs: vms,
|
||||
Containers: containers,
|
||||
Storage: storage,
|
||||
PhysicalDisks: s.PhysicalDisks,
|
||||
PBS: s.PBSInstances,
|
||||
ActiveAlerts: s.ActiveAlerts,
|
||||
Metrics: make(map[string]any),
|
||||
|
||||
@@ -45,6 +45,7 @@ type PVEClientInterface interface {
|
||||
GetVMFSInfo(ctx context.Context, node string, vmid int) ([]proxmox.VMFileSystem, error)
|
||||
GetZFSPoolStatus(ctx context.Context, node string) ([]proxmox.ZFSPoolStatus, error)
|
||||
GetZFSPoolsWithDetails(ctx context.Context, node string) ([]proxmox.ZFSPoolInfo, error)
|
||||
GetDisks(ctx context.Context, node string) ([]proxmox.Disk, error)
|
||||
}
|
||||
|
||||
// Monitor handles all monitoring operations
|
||||
@@ -901,6 +902,97 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
|
||||
}
|
||||
}
|
||||
|
||||
// Poll physical disks for health monitoring
|
||||
log.Debug().Int("nodeCount", len(nodes)).Msg("Starting disk health polling")
|
||||
var allDisks []models.PhysicalDisk
|
||||
for _, node := range nodes {
|
||||
// Skip offline nodes
|
||||
if node.Status != "online" {
|
||||
log.Debug().Str("node", node.Node).Msg("Skipping disk poll for offline node")
|
||||
continue
|
||||
}
|
||||
|
||||
// Get disk list for this node
|
||||
log.Debug().Str("node", node.Node).Msg("Getting disk list for node")
|
||||
disks, err := client.GetDisks(ctx, node.Node)
|
||||
if err != nil {
|
||||
// Log but don't fail - disk monitoring is optional
|
||||
log.Debug().
|
||||
Str("node", node.Node).
|
||||
Err(err).
|
||||
Msg("Failed to get disk list - disk monitoring may not be available")
|
||||
continue
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Str("node", node.Node).
|
||||
Int("diskCount", len(disks)).
|
||||
Msg("Got disk list for node")
|
||||
|
||||
// Check each disk for health issues and add to state
|
||||
for _, disk := range disks {
|
||||
// Create PhysicalDisk model
|
||||
diskID := fmt.Sprintf("%s-%s-%s", instanceName, node.Node, strings.ReplaceAll(disk.DevPath, "/", "-"))
|
||||
physicalDisk := models.PhysicalDisk{
|
||||
ID: diskID,
|
||||
Node: node.Node,
|
||||
Instance: instanceName,
|
||||
DevPath: disk.DevPath,
|
||||
Model: disk.Model,
|
||||
Serial: disk.Serial,
|
||||
Type: disk.Type,
|
||||
Size: disk.Size,
|
||||
Health: disk.Health,
|
||||
Wearout: disk.Wearout,
|
||||
RPM: disk.RPM,
|
||||
Used: disk.Used,
|
||||
LastChecked: time.Now(),
|
||||
}
|
||||
|
||||
allDisks = append(allDisks, physicalDisk)
|
||||
|
||||
log.Debug().
|
||||
Str("node", node.Node).
|
||||
Str("disk", disk.DevPath).
|
||||
Str("model", disk.Model).
|
||||
Str("health", disk.Health).
|
||||
Int("wearout", disk.Wearout).
|
||||
Msg("Checking disk health")
|
||||
|
||||
if disk.Health != "PASSED" && disk.Health != "" {
|
||||
// Disk has failed or is failing - alert manager will handle this
|
||||
log.Warn().
|
||||
Str("node", node.Node).
|
||||
Str("disk", disk.DevPath).
|
||||
Str("model", disk.Model).
|
||||
Str("health", disk.Health).
|
||||
Int("wearout", disk.Wearout).
|
||||
Msg("Disk health issue detected")
|
||||
|
||||
// Pass disk info to alert manager
|
||||
m.alertManager.CheckDiskHealth(instanceName, node.Node, disk)
|
||||
} else if disk.Wearout > 0 && disk.Wearout < 10 {
|
||||
// Low wearout warning (less than 10% life remaining)
|
||||
log.Warn().
|
||||
Str("node", node.Node).
|
||||
Str("disk", disk.DevPath).
|
||||
Str("model", disk.Model).
|
||||
Int("wearout", disk.Wearout).
|
||||
Msg("SSD wearout critical - less than 10% life remaining")
|
||||
|
||||
// Pass to alert manager for wearout alert
|
||||
m.alertManager.CheckDiskHealth(instanceName, node.Node, disk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update physical disks in state
|
||||
log.Debug().
|
||||
Str("instance", instanceName).
|
||||
Int("diskCount", len(allDisks)).
|
||||
Msg("Updating physical disks in state")
|
||||
m.state.UpdatePhysicalDisks(instanceName, allDisks)
|
||||
|
||||
// Update nodes with storage fallback if rootfs was not available
|
||||
for i := range modelNodes {
|
||||
if modelNodes[i].Disk.Total == 0 {
|
||||
|
||||
@@ -1150,5 +1150,68 @@ func (c *Client) GetZFSPoolDetail(ctx context.Context, node, pool string) (*ZFSP
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &result.Data, nil
|
||||
}
|
||||
|
||||
// Disk represents a physical disk on a Proxmox node
|
||||
type Disk struct {
|
||||
DevPath string `json:"devpath"`
|
||||
Model string `json:"model"`
|
||||
Serial string `json:"serial"`
|
||||
Type string `json:"type"` // nvme, sata, sas
|
||||
Health string `json:"health"` // PASSED, FAILED, UNKNOWN
|
||||
Wearout int `json:"wearout"` // SSD wear percentage (0-100, 100 is best)
|
||||
Size int64 `json:"size"` // Size in bytes
|
||||
RPM int `json:"rpm"` // 0 for SSDs
|
||||
Used string `json:"used"` // Filesystem or partition usage
|
||||
Vendor string `json:"vendor"`
|
||||
WWN string `json:"wwn"` // World Wide Name
|
||||
}
|
||||
|
||||
// DiskSmart represents SMART data for a disk
|
||||
type DiskSmart struct {
|
||||
Health string `json:"health"` // PASSED, FAILED, UNKNOWN
|
||||
Wearout int `json:"wearout"` // SSD wear percentage
|
||||
Type string `json:"type"` // Type of response (text, attributes)
|
||||
Text string `json:"text"` // Raw SMART output text
|
||||
}
|
||||
|
||||
// GetDisks returns the list of physical disks on a node
|
||||
func (c *Client) GetDisks(ctx context.Context, node string) ([]Disk, error) {
|
||||
resp, err := c.request(ctx, "GET", fmt.Sprintf("/nodes/%s/disks/list", node), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var result struct {
|
||||
Data []Disk `json:"data"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result.Data, nil
|
||||
}
|
||||
|
||||
// GetDiskSmart returns SMART data for a specific disk
|
||||
func (c *Client) GetDiskSmart(ctx context.Context, node, disk string) (*DiskSmart, error) {
|
||||
params := url.Values{
|
||||
"disk": {disk},
|
||||
}
|
||||
|
||||
resp, err := c.request(ctx, "GET", fmt.Sprintf("/nodes/%s/disks/smart?%s", node, params.Encode()), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var result struct {
|
||||
Data DiskSmart `json:"data"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &result.Data, nil
|
||||
}
|
||||
@@ -807,6 +807,30 @@ func (cc *ClusterClient) GetClusterHealthInfo() models.ClusterHealth {
|
||||
}
|
||||
|
||||
// Helper to check if error is auth-related
|
||||
func (cc *ClusterClient) GetDisks(ctx context.Context, node string) ([]Disk, error) {
|
||||
var result []Disk
|
||||
err := cc.executeWithFailover(ctx, func(client *Client) error {
|
||||
disks, err := client.GetDisks(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result = disks
|
||||
return nil
|
||||
})
|
||||
|
||||
// Don't return error for transient connectivity issues
|
||||
if err != nil && strings.Contains(err.Error(), "no healthy nodes available") {
|
||||
log.Debug().
|
||||
Str("cluster", cc.name).
|
||||
Str("node", node).
|
||||
Err(err).
|
||||
Msg("No healthy nodes for GetDisks - returning empty list")
|
||||
return []Disk{}, nil
|
||||
}
|
||||
|
||||
return result, err
|
||||
}
|
||||
|
||||
func IsAuthError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
|
||||
Reference in New Issue
Block a user