gsadmin/pulse
1
0
Fork 0
mirror of https://github.com/rcourtman/Pulse.git synced 2026-09-10 02:25:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(alerts): add disk temperature alerts for host agents

Browse Source 
- Add DiskTemperature threshold to ThresholdConfig (default: 55°C trigger, 50°C clear)
- Process host SMART sensor data in CheckHost to generate disk_temperature alerts
- Add 'Disk Temp °C' column to Host Agents thresholds table in UI
- Make temperature tooltip interactive and scrollable to fix overflow issues
- Update AlertThresholds type to include diskTemperature field

Closes: #941
This commit is contained in:
rcourtman
2026-01-01 16:31:34 +00:00
parent 5b7a68bcc0
commit 9abe9c47a2
5 changed files with 94 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend-modern/src/components/Alerts/ThresholdsTable.tsx
+4 -2
View File
@@ -92,7 +92,8 @@ const normalizeThresholdLabel = (label: string): string =>
.replace('disk r', 'diskRead')
.replace('disk w', 'diskWrite')
.replace('net in', 'networkIn')
.replace('net out', 'networkOut');
.replace('net out', 'networkOut')
.replace('disk temp', 'diskTemperature');
const pmgColumn = (key: keyof PMGThresholdDefaults, label: string) => ({
key,
@@ -150,6 +151,7 @@ interface SimpleThresholds {
networkIn?: number;
networkOut?: number;
temperature?: number; // For nodes only
diskTemperature?: number; // For host agents
[key: string]: number | undefined; // Add index signature for compatibility
}
@@ -3137,7 +3139,7 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
<ResourceTable
title="Host Agents"
resources={hostAgentsWithOverrides()}
columns={['CPU %', 'Memory %', 'Disk %']}
columns={['CPU %', 'Memory %', 'Disk %', 'Disk Temp °C']}
activeAlerts={props.activeAlerts}
emptyMessage="No host agents match the current filters."
onEdit={startEditing}
frontend-modern/src/components/Hosts/HostsOverview.tsx
+20 -6
View File
@@ -174,6 +174,7 @@ function HostTemperatureCell(props: { sensors: HostSensorSummaryForCell | null |
const [tooltipPos, setTooltipPos] = createSignal({ x: 0, y: 0 });
const alertsActivation = useAlertsActivation();
const threshold = createMemo(() => alertsActivation.getTemperatureThreshold());
let closeTimeout: number | undefined;
// Get the primary (highest) temperature for display
const primaryTemp = createMemo(() => {
@@ -236,12 +237,23 @@ function HostTemperatureCell(props: { sensors: HostSensorSummaryForCell | null |
}
const handleMouseEnter = (e: MouseEvent) => {
if (closeTimeout) window.clearTimeout(closeTimeout);
const rect = (e.currentTarget as HTMLElement).getBoundingClientRect();
setTooltipPos({ x: rect.left + rect.width / 2, y: rect.top });
setShowTooltip(true);
};
const handleMouseLeave = () => {
closeTimeout = window.setTimeout(() => {
setShowTooltip(false);
}, 150);
};
const handleTooltipEnter = () => {
if (closeTimeout) window.clearTimeout(closeTimeout);
};
const handleTooltipLeave = () => {
setShowTooltip(false);
};
@@ -307,17 +319,19 @@ function HostTemperatureCell(props: { sensors: HostSensorSummaryForCell | null |
<Show when={showTooltip() && hasSensors()}>
<Portal mount={document.body}>
<div
class="fixed z-[9999] pointer-events-none"
class="fixed z-[9999]"
style={{
left: `${tooltipPos().x}px`,
top: `${tooltipPos().y - 8}px`,
transform: 'translate(-50%, -100%)',
}}
onMouseEnter={handleTooltipEnter}
onMouseLeave={handleTooltipLeave}
>
<div class="bg-gray-900 dark:bg-gray-800 text-white text-[10px] rounded-md shadow-lg px-2 py-1.5 min-w-[160px] max-w-[280px] border border-gray-700">
<div class="bg-gray-900 dark:bg-gray-800 text-white text-[10px] rounded-md shadow-lg px-2 py-1.5 min-w-[160px] max-w-[280px] border border-gray-700 max-h-[400px] overflow-y-auto custom-scrollbar">
{/* Temperature section */}
<Show when={sortedTemps().length > 0}>
<div class="font-medium mb-1 text-gray-300 border-b border-gray-700 pb-1">
<div class="font-medium mb-1 text-gray-300 border-b border-gray-700 pb-1 sticky top-0 bg-gray-900 dark:bg-gray-800 z-10">
Temperatures
</div>
<div class="space-y-0.5 mb-2">
@@ -336,7 +350,7 @@ function HostTemperatureCell(props: { sensors: HostSensorSummaryForCell | null |
{/* Disk temperatures section (SMART) */}
<Show when={sortedSmart().length > 0}>
<div class="font-medium mb-1 text-gray-300 border-b border-gray-700 pb-1">
<div class="font-medium mb-1 text-gray-300 border-b border-gray-700 pb-1 sticky top-0 bg-gray-900 dark:bg-gray-800 z-10">
Disk Temperatures
</div>
<div class="space-y-0.5 mb-2">
@@ -370,7 +384,7 @@ function HostTemperatureCell(props: { sensors: HostSensorSummaryForCell | null |
{/* Fan speeds section */}
<Show when={sortedFans().length > 0}>
<div class="font-medium mb-1 text-gray-300 border-b border-gray-700 pb-1">
<div class="font-medium mb-1 text-gray-300 border-b border-gray-700 pb-1 sticky top-0 bg-gray-900 dark:bg-gray-800 z-10">
Fan Speeds
</div>
<div class="space-y-0.5 mb-2">
@@ -387,7 +401,7 @@ function HostTemperatureCell(props: { sensors: HostSensorSummaryForCell | null |
{/* Additional sensors section */}
<Show when={sortedAdditional().length > 0}>
<div class="font-medium mb-1 text-gray-300 border-b border-gray-700 pb-1">
<div class="font-medium mb-1 text-gray-300 border-b border-gray-700 pb-1 sticky top-0 bg-gray-900 dark:bg-gray-800 z-10">
Other Sensors
</div>
<div class="space-y-0.5">
frontend-modern/src/pages/Alerts.tsx
+3
View File
@@ -1023,6 +1023,7 @@ export function Alerts() {
cpu: getTriggerValue(config.hostDefaults.cpu) ?? 80,
memory: getTriggerValue(config.hostDefaults.memory) ?? 85,
disk: getTriggerValue(config.hostDefaults.disk) ?? 90,
diskTemperature: getTriggerValue(config.hostDefaults.diskTemperature) ?? 55,
});
} else {
setHostDefaults({ ...FACTORY_HOST_DEFAULTS });
@@ -1428,6 +1429,7 @@ export function Alerts() {
cpu: 80,
memory: 85,
disk: 90,
diskTemperature: 55,
};
const FACTORY_DOCKER_DEFAULTS = {
@@ -1774,6 +1776,7 @@ export function Alerts() {
cpu: createHysteresisThreshold(hostDefaults().cpu),
memory: createHysteresisThreshold(hostDefaults().memory),
disk: createHysteresisThreshold(hostDefaults().disk),
diskTemperature: createHysteresisThreshold(hostDefaults().diskTemperature),
},
dockerDefaults: {
cpu: createHysteresisThreshold(dockerDefaultsValue.cpu),
frontend-modern/src/types/alerts.ts
+1
View File
@@ -14,6 +14,7 @@ export interface AlertThresholds {
networkIn?: HysteresisThreshold;
networkOut?: HysteresisThreshold;
temperature?: HysteresisThreshold;
diskTemperature?: HysteresisThreshold;
disableConnectivity?: boolean; // Disable connectivity/powered-off alerts
poweredOffSeverity?: 'warning' | 'critical';
// Legacy support for backward compatibility
internal/alerts/alerts.go
+66 -24
View File
@@ -203,8 +203,9 @@ type ThresholdConfig struct {
DiskWrite *HysteresisThreshold `json:"diskWrite,omitempty"`
NetworkIn *HysteresisThreshold `json:"networkIn,omitempty"`
NetworkOut *HysteresisThreshold `json:"networkOut,omitempty"`
Usage *HysteresisThreshold `json:"usage,omitempty"` // For storage devices
Temperature *HysteresisThreshold `json:"temperature,omitempty"` // For node CPU temperature
Usage *HysteresisThreshold `json:"usage,omitempty"` // For storage devices
Temperature *HysteresisThreshold `json:"temperature,omitempty"` // For node CPU temperature
DiskTemperature *HysteresisThreshold `json:"diskTemperature,omitempty"` // For host SMART temperatures
Backup *BackupAlertConfig `json:"backup,omitempty"`
Snapshot *SnapshotAlertConfig `json:"snapshot,omitempty"`
Note *string `json:"note,omitempty"`
@@ -515,9 +516,9 @@ type Manager struct {
offlineConfirmations map[string]int // Track consecutive offline counts for all resources
dockerOfflineCount map[string]int // Track consecutive offline counts for Docker hosts
dockerStateConfirm map[string]int // Track consecutive state confirmations for Docker containers
dockerRestartTracking map[string]*dockerRestartRecord // Track restart counts and times for restart loop detection
dockerLastExitCode map[string]int // Track last exit code for OOM detection
dockerUpdateFirstSeen map[string]time.Time // Track when image updates were first detected for alert delay
dockerRestartTracking map[string]*dockerRestartRecord // Track restart counts and times for restart loop detection
dockerLastExitCode map[string]int // Track last exit code for OOM detection
dockerUpdateFirstSeen map[string]time.Time // Track when image updates were first detected for alert delay
// PMG quarantine growth tracking
pmgQuarantineHistory map[string][]pmgQuarantineSnapshot // Track quarantine snapshots for growth detection
// PMG anomaly detection tracking
@@ -540,8 +541,8 @@ type Manager struct {
type ackRecord struct {
acknowledged bool
user string
time time.Time // When the alert was acknowledged
inactiveAt time.Time // When the alert was removed (zero if still active)
time time.Time // When the alert was acknowledged
inactiveAt time.Time // When the alert was removed (zero if still active)
}
type dockerRestartRecord struct {
@@ -558,18 +559,18 @@ func NewManager() *Manager {
activeAlerts: make(map[string]*Alert),
historyManager: NewHistoryManager(alertsDir),
escalationStop: make(chan struct{}),
alertRateLimit: make(map[string][]time.Time),
recentAlerts: make(map[string]*Alert),
suppressedUntil: make(map[string]time.Time),
recentlyResolved: make(map[string]*ResolvedAlert),
pendingAlerts: make(map[string]time.Time),
nodeOfflineCount: make(map[string]int),
offlineConfirmations: make(map[string]int),
dockerOfflineCount: make(map[string]int),
dockerStateConfirm: make(map[string]int),
dockerRestartTracking: make(map[string]*dockerRestartRecord),
dockerLastExitCode: make(map[string]int),
dockerUpdateFirstSeen: make(map[string]time.Time),
alertRateLimit: make(map[string][]time.Time),
recentAlerts: make(map[string]*Alert),
suppressedUntil: make(map[string]time.Time),
recentlyResolved: make(map[string]*ResolvedAlert),
pendingAlerts: make(map[string]time.Time),
nodeOfflineCount: make(map[string]int),
offlineConfirmations: make(map[string]int),
dockerOfflineCount: make(map[string]int),
dockerStateConfirm: make(map[string]int),
dockerRestartTracking: make(map[string]*dockerRestartRecord),
dockerLastExitCode: make(map[string]int),
dockerUpdateFirstSeen: make(map[string]time.Time),
pmgQuarantineHistory: make(map[string][]pmgQuarantineSnapshot),
pmgAnomalyTrackers: make(map[string]*pmgAnomalyTracker),
ackState: make(map[string]ackRecord),
@@ -598,9 +599,10 @@ func NewManager() *Manager {
Temperature: &HysteresisThreshold{Trigger: 80, Clear: 75}, // Warning at 80°C, clear at 75°C
},
HostDefaults: ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
DiskTemperature: &HysteresisThreshold{Trigger: 55, Clear: 50},
},
DockerDefaults: DockerThresholdConfig{
CPU: HysteresisThreshold{Trigger: 80, Clear: 75},
@@ -1332,6 +1334,18 @@ func normalizeHostDefaults(config *AlertConfig) {
config.HostDefaults.Disk.Clear = 85
}
}
if config.HostDefaults.DiskTemperature == nil || config.HostDefaults.DiskTemperature.Trigger < 0 {
config.HostDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 55, Clear: 50}
} else if config.HostDefaults.DiskTemperature.Trigger == 0 {
config.HostDefaults.DiskTemperature.Clear = 0
} else if config.HostDefaults.DiskTemperature.Clear <= 0 {
config.HostDefaults.DiskTemperature.Clear = config.HostDefaults.DiskTemperature.Trigger - 5
if config.HostDefaults.DiskTemperature.Clear <= 0 {
config.HostDefaults.DiskTemperature.Clear = 50
}
}
ensureValidHysteresis(config.HostDefaults.DiskTemperature, "host.diskTemperature")
}
// normalizeGeneralSettings ensures general alert settings have valid values
@@ -2202,8 +2216,6 @@ func (m *Manager) CheckGuest(guest interface{}, instanceName string) {
return
}
// Check ignored prefixes
for _, prefix := range ignoredGuestPrefixes {
if prefix != "" && strings.HasPrefix(name, prefix) {
@@ -2771,6 +2783,31 @@ func (m *Manager) CheckHost(host models.Host) {
m.clearHostMetricAlerts(host.ID, "memory")
}
if thresholds.DiskTemperature != nil && thresholds.DiskTemperature.Trigger > 0 {
if len(host.Sensors.SMART) > 0 {
for _, disk := range host.Sensors.SMART {
if disk.Temperature > 0 && !disk.Standby {
// Use specific resource ID for the disk: hostID/disk-temp:device
tempResourceID := fmt.Sprintf("%s/disk_temp:%s", hostResourceID(host.ID), sanitizeHostComponent(disk.Device))
tempResourceName := fmt.Sprintf("%s (%s Temp)", host.DisplayName, disk.Device)
diskTempMetadata := cloneMetadata(baseMetadata)
diskTempMetadata["metric"] = "disk_temperature"
diskTempMetadata["device"] = disk.Device
diskTempMetadata["temperature"] = disk.Temperature
diskTempMetadata["model"] = disk.Model
m.checkMetric(tempResourceID, tempResourceName, nodeName, disk.Device, "Host", "disk_temperature", float64(disk.Temperature), thresholds.DiskTemperature, &metricOptions{Metadata: diskTempMetadata})
}
}
}
} else {
// We can't easily clear all disk temp alerts without tracking them,
// but checkMetric logic handles auto-resolution if value drops.
// If feature is disabled, ideally we should clear existing alerts.
// For now simple implementation.
}
seenDisks := make(map[string]struct{}, len(host.Disks))
if thresholds.Disk != nil && thresholds.Disk.Trigger > 0 {
for _, disk := range host.Disks {
@@ -8220,6 +8257,7 @@ func cloneThresholdConfig(cfg ThresholdConfig) ThresholdConfig {
clone.NetworkIn = cloneThreshold(cfg.NetworkIn)
clone.NetworkOut = cloneThreshold(cfg.NetworkOut)
clone.Temperature = cloneThreshold(cfg.Temperature)
clone.DiskTemperature = cloneThreshold(cfg.DiskTemperature)
clone.Usage = cloneThreshold(cfg.Usage)
clone.Note = cloneStringPtr(cfg.Note)
return clone
@@ -8281,6 +8319,10 @@ func (m *Manager) applyThresholdOverride(base ThresholdConfig, override Threshol
result.Temperature = ensureHysteresisThreshold(cloneThreshold(override.Temperature))
}
if override.DiskTemperature != nil {
result.DiskTemperature = ensureHysteresisThreshold(cloneThreshold(override.DiskTemperature))
}
if override.Usage != nil {
result.Usage = ensureHysteresisThreshold(cloneThreshold(override.Usage))
}