mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-21 18:53:37 +00:00
add node temperature monitoring via SSH
addresses #101 - Implement SSH-based temperature collector using lm-sensors - Add Temperature struct to node models (CPU package, cores, NVMe) - Collect temps during node polling (5s timeout, non-blocking) - Display temperature in node cards with color coding: - Green: <60°C - Yellow: 60-80°C - Red: >80°C - Shows CPU temp or falls back to load average if unavailable - Tooltip includes NVMe drive temps when present - Uses root SSH access (no additional auth setup needed for now) - Temperature data only collected for online nodes
This commit is contained in:
rcourtman
@@ -207,7 +207,23 @@ const NodeCard: Component<NodeCardProps> = (props) => {
|
||||
<span title={`Uptime: ${formatUptime(props.node.uptime)}`}>
|
||||
↑{formatUptime(props.node.uptime)}
|
||||
</span>
|
||||
<span title={`Load: ${normalizedLoad()}`}>⚡{normalizedLoad()}</span>
|
||||
<Show
|
||||
when={props.node.temperature?.available}
|
||||
fallback={<span title={`Load: ${normalizedLoad()}`}>⚡{normalizedLoad()}</span>}
|
||||
>
|
||||
<span
|
||||
class={`font-medium ${
|
||||
(props.node.temperature!.cpuPackage || props.node.temperature!.cpuMax || 0) > 80
|
||||
? 'text-red-500'
|
||||
: (props.node.temperature!.cpuPackage || props.node.temperature!.cpuMax || 0) > 60
|
||||
? 'text-yellow-500'
|
||||
: 'text-green-500'
|
||||
}`}
|
||||
title={`CPU: ${Math.round(props.node.temperature!.cpuPackage || props.node.temperature!.cpuMax || 0)}°C${props.node.temperature!.nvme && props.node.temperature!.nvme.length > 0 ? ` | NVMe: ${props.node.temperature!.nvme.map((n) => `${n.device}: ${Math.round(n.temp)}°C`).join(', ')}` : ''}`}
|
||||
>
|
||||
🌡{Math.round(props.node.temperature!.cpuPackage || props.node.temperature!.cpuMax || 0)}°C
|
||||
</span>
|
||||
</Show>
|
||||
</div>
|
||||
</Card>
|
||||
);
|
||||
|
||||
@@ -33,6 +33,7 @@ export interface Node {
|
||||
kernelVersion: string;
|
||||
pveVersion: string;
|
||||
cpuInfo: CPUInfo;
|
||||
temperature?: Temperature; // CPU/NVMe temperatures
|
||||
lastSeen: string;
|
||||
connectionHealth: string;
|
||||
isClusterMember?: boolean; // True if part of a cluster
|
||||
@@ -275,6 +276,25 @@ export interface CPUInfo {
|
||||
mhz: string;
|
||||
}
|
||||
|
||||
export interface Temperature {
|
||||
cpuPackage?: number; // CPU package temperature (primary metric)
|
||||
cpuMax?: number; // Highest core temperature
|
||||
cores?: CoreTemp[]; // Individual core temperatures
|
||||
nvme?: NVMeTemp[]; // NVMe drive temperatures
|
||||
available: boolean; // Whether temperature data is available
|
||||
lastUpdate: string; // When this data was collected
|
||||
}
|
||||
|
||||
export interface CoreTemp {
|
||||
core: number;
|
||||
temp: number;
|
||||
}
|
||||
|
||||
export interface NVMeTemp {
|
||||
device: string;
|
||||
temp: number;
|
||||
}
|
||||
|
||||
export interface Metric {
|
||||
timestamp: string;
|
||||
type: string;
|
||||
|
||||
@@ -79,6 +79,11 @@ func (n Node) ToFrontend() NodeFrontend {
|
||||
nf.Disk = &n.Disk
|
||||
}
|
||||
|
||||
// Include temperature data if available
|
||||
if n.Temperature != nil && n.Temperature.Available {
|
||||
nf.Temperature = n.Temperature
|
||||
}
|
||||
|
||||
return nf
|
||||
}
|
||||
|
||||
|
||||
+41
-18
@@ -51,24 +51,25 @@ type ResolvedAlert struct {
|
||||
|
||||
// Node represents a Proxmox VE node
|
||||
type Node struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Instance string `json:"instance"`
|
||||
Host string `json:"host"` // Full host URL from config
|
||||
Status string `json:"status"`
|
||||
Type string `json:"type"`
|
||||
CPU float64 `json:"cpu"`
|
||||
Memory Memory `json:"memory"`
|
||||
Disk Disk `json:"disk"`
|
||||
Uptime int64 `json:"uptime"`
|
||||
LoadAverage []float64 `json:"loadAverage"`
|
||||
KernelVersion string `json:"kernelVersion"`
|
||||
PVEVersion string `json:"pveVersion"`
|
||||
CPUInfo CPUInfo `json:"cpuInfo"`
|
||||
LastSeen time.Time `json:"lastSeen"`
|
||||
ConnectionHealth string `json:"connectionHealth"`
|
||||
IsClusterMember bool `json:"isClusterMember"` // True if part of a cluster
|
||||
ClusterName string `json:"clusterName"` // Name of cluster (empty if standalone)
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Instance string `json:"instance"`
|
||||
Host string `json:"host"` // Full host URL from config
|
||||
Status string `json:"status"`
|
||||
Type string `json:"type"`
|
||||
CPU float64 `json:"cpu"`
|
||||
Memory Memory `json:"memory"`
|
||||
Disk Disk `json:"disk"`
|
||||
Uptime int64 `json:"uptime"`
|
||||
LoadAverage []float64 `json:"loadAverage"`
|
||||
KernelVersion string `json:"kernelVersion"`
|
||||
PVEVersion string `json:"pveVersion"`
|
||||
CPUInfo CPUInfo `json:"cpuInfo"`
|
||||
Temperature *Temperature `json:"temperature,omitempty"` // CPU/NVMe temperatures
|
||||
LastSeen time.Time `json:"lastSeen"`
|
||||
ConnectionHealth string `json:"connectionHealth"`
|
||||
IsClusterMember bool `json:"isClusterMember"` // True if part of a cluster
|
||||
ClusterName string `json:"clusterName"` // Name of cluster (empty if standalone)
|
||||
}
|
||||
|
||||
// VM represents a virtual machine
|
||||
@@ -318,6 +319,28 @@ type CPUInfo struct {
|
||||
MHz string `json:"mhz"`
|
||||
}
|
||||
|
||||
// Temperature represents temperature sensors data
|
||||
type Temperature struct {
|
||||
CPUPackage float64 `json:"cpuPackage,omitempty"` // CPU package temperature (primary metric)
|
||||
CPUMax float64 `json:"cpuMax,omitempty"` // Highest core temperature
|
||||
Cores []CoreTemp `json:"cores,omitempty"` // Individual core temperatures
|
||||
NVMe []NVMeTemp `json:"nvme,omitempty"` // NVMe drive temperatures
|
||||
Available bool `json:"available"` // Whether temperature data is available
|
||||
LastUpdate time.Time `json:"lastUpdate"` // When this data was collected
|
||||
}
|
||||
|
||||
// CoreTemp represents a CPU core temperature
|
||||
type CoreTemp struct {
|
||||
Core int `json:"core"`
|
||||
Temp float64 `json:"temp"`
|
||||
}
|
||||
|
||||
// NVMeTemp represents an NVMe drive temperature
|
||||
type NVMeTemp struct {
|
||||
Device string `json:"device"`
|
||||
Temp float64 `json:"temp"`
|
||||
}
|
||||
|
||||
// Metric represents a time-series metric
|
||||
type Metric struct {
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
|
||||
@@ -5,25 +5,26 @@ package models
|
||||
|
||||
// NodeFrontend represents a Node with frontend-friendly field names
|
||||
type NodeFrontend struct {
|
||||
ID string `json:"id"`
|
||||
Node string `json:"node"` // Maps to Name
|
||||
Name string `json:"name"`
|
||||
Instance string `json:"instance"`
|
||||
Status string `json:"status"`
|
||||
Type string `json:"type"`
|
||||
CPU float64 `json:"cpu"`
|
||||
Memory *Memory `json:"memory,omitempty"` // Full memory object with usage percentage
|
||||
Mem int64 `json:"mem"` // Maps to Memory.Used (kept for backward compat)
|
||||
MaxMem int64 `json:"maxmem"` // Maps to Memory.Total (kept for backward compat)
|
||||
Disk *Disk `json:"disk,omitempty"` // Full disk object with usage percentage
|
||||
MaxDisk int64 `json:"maxdisk"` // Maps to Disk.Total (kept for backward compat)
|
||||
Uptime int64 `json:"uptime"`
|
||||
LoadAverage []float64 `json:"loadAverage"`
|
||||
KernelVersion string `json:"kernelVersion"`
|
||||
PVEVersion string `json:"pveVersion"`
|
||||
CPUInfo CPUInfo `json:"cpuInfo"`
|
||||
LastSeen int64 `json:"lastSeen"` // Unix timestamp
|
||||
ConnectionHealth string `json:"connectionHealth"`
|
||||
ID string `json:"id"`
|
||||
Node string `json:"node"` // Maps to Name
|
||||
Name string `json:"name"`
|
||||
Instance string `json:"instance"`
|
||||
Status string `json:"status"`
|
||||
Type string `json:"type"`
|
||||
CPU float64 `json:"cpu"`
|
||||
Memory *Memory `json:"memory,omitempty"` // Full memory object with usage percentage
|
||||
Mem int64 `json:"mem"` // Maps to Memory.Used (kept for backward compat)
|
||||
MaxMem int64 `json:"maxmem"` // Maps to Memory.Total (kept for backward compat)
|
||||
Disk *Disk `json:"disk,omitempty"` // Full disk object with usage percentage
|
||||
MaxDisk int64 `json:"maxdisk"` // Maps to Disk.Total (kept for backward compat)
|
||||
Uptime int64 `json:"uptime"`
|
||||
LoadAverage []float64 `json:"loadAverage"`
|
||||
KernelVersion string `json:"kernelVersion"`
|
||||
PVEVersion string `json:"pveVersion"`
|
||||
CPUInfo CPUInfo `json:"cpuInfo"`
|
||||
Temperature *Temperature `json:"temperature,omitempty"` // CPU/NVMe temperatures
|
||||
LastSeen int64 `json:"lastSeen"` // Unix timestamp
|
||||
ConnectionHealth string `json:"connectionHealth"`
|
||||
}
|
||||
|
||||
// VMFrontend represents a VM with frontend-friendly field names
|
||||
|
||||
@@ -53,6 +53,7 @@ type Monitor struct {
|
||||
state *models.State
|
||||
pveClients map[string]PVEClientInterface
|
||||
pbsClients map[string]*pbs.Client
|
||||
tempCollector *TemperatureCollector // SSH-based temperature collector
|
||||
mu sync.RWMutex
|
||||
startTime time.Time
|
||||
rateTracker *RateTracker
|
||||
@@ -175,11 +176,16 @@ func (m *Monitor) GetConnectionStatuses() map[string]bool {
|
||||
|
||||
// New creates a new Monitor instance
|
||||
func New(cfg *config.Config) (*Monitor, error) {
|
||||
// Initialize temperature collector with default SSH settings
|
||||
// Will use root user for now - can be made configurable later
|
||||
tempCollector := NewTemperatureCollector("root", "")
|
||||
|
||||
m := &Monitor{
|
||||
config: cfg,
|
||||
state: models.NewState(),
|
||||
pveClients: make(map[string]PVEClientInterface),
|
||||
pbsClients: make(map[string]*pbs.Client),
|
||||
tempCollector: tempCollector,
|
||||
startTime: time.Now(),
|
||||
rateTracker: NewRateTracker(),
|
||||
metricsHistory: NewMetricsHistory(1000, 24*time.Hour), // Keep up to 1000 points or 24 hours
|
||||
@@ -1097,6 +1103,25 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
|
||||
}
|
||||
}
|
||||
|
||||
// Collect temperature data via SSH (non-blocking, best effort)
|
||||
// Only attempt for online nodes
|
||||
if node.Status == "online" && m.tempCollector != nil {
|
||||
tempCtx, tempCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
// Use node name as hostname (works for both cluster and standalone)
|
||||
temp, err := m.tempCollector.CollectTemperature(tempCtx, node.Node, node.Node)
|
||||
tempCancel()
|
||||
|
||||
if err == nil && temp != nil && temp.Available {
|
||||
modelNode.Temperature = temp
|
||||
log.Debug().
|
||||
Str("node", node.Node).
|
||||
Float64("cpuPackage", temp.CPUPackage).
|
||||
Float64("cpuMax", temp.CPUMax).
|
||||
Int("nvmeCount", len(temp.NVMe)).
|
||||
Msg("Collected temperature data")
|
||||
}
|
||||
}
|
||||
|
||||
modelNodes = append(modelNodes, modelNode)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// TemperatureCollector handles SSH-based temperature collection from Proxmox nodes
|
||||
type TemperatureCollector struct {
|
||||
sshUser string // SSH user (typically "root" or "pulse-monitor")
|
||||
sshKeyPath string // Path to SSH private key
|
||||
}
|
||||
|
||||
// NewTemperatureCollector creates a new temperature collector
|
||||
func NewTemperatureCollector(sshUser, sshKeyPath string) *TemperatureCollector {
|
||||
return &TemperatureCollector{
|
||||
sshUser: sshUser,
|
||||
sshKeyPath: sshKeyPath,
|
||||
}
|
||||
}
|
||||
|
||||
// CollectTemperature collects temperature data from a node via SSH
|
||||
func (tc *TemperatureCollector) CollectTemperature(ctx context.Context, nodeHost, nodeName string) (*models.Temperature, error) {
|
||||
// Extract hostname/IP from the host URL (might be https://hostname:8006)
|
||||
host := extractHostname(nodeHost)
|
||||
|
||||
// Try to get sensors JSON output
|
||||
output, err := tc.runSSHCommand(ctx, host, "sensors -j 2>/dev/null")
|
||||
if err != nil {
|
||||
log.Debug().
|
||||
Str("node", nodeName).
|
||||
Str("host", host).
|
||||
Err(err).
|
||||
Msg("Failed to collect temperature data via SSH")
|
||||
return &models.Temperature{Available: false}, nil
|
||||
}
|
||||
|
||||
// Parse sensors JSON output
|
||||
temp, err := tc.parseSensorsJSON(output)
|
||||
if err != nil {
|
||||
log.Debug().
|
||||
Str("node", nodeName).
|
||||
Err(err).
|
||||
Msg("Failed to parse sensors output")
|
||||
return &models.Temperature{Available: false}, nil
|
||||
}
|
||||
|
||||
temp.Available = true
|
||||
temp.LastUpdate = time.Now()
|
||||
|
||||
return temp, nil
|
||||
}
|
||||
|
||||
// runSSHCommand executes a command on a remote node via SSH
|
||||
func (tc *TemperatureCollector) runSSHCommand(ctx context.Context, host, command string) (string, error) {
|
||||
// Build SSH command with appropriate options
|
||||
sshArgs := []string{
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null",
|
||||
"-o", "ConnectTimeout=5",
|
||||
"-o", "BatchMode=yes", // No password prompts
|
||||
}
|
||||
|
||||
// Add key if specified
|
||||
if tc.sshKeyPath != "" {
|
||||
sshArgs = append(sshArgs, "-i", tc.sshKeyPath)
|
||||
}
|
||||
|
||||
// Add user@host and command
|
||||
sshArgs = append(sshArgs, fmt.Sprintf("%s@%s", tc.sshUser, host), command)
|
||||
|
||||
cmd := exec.CommandContext(ctx, "ssh", sshArgs...)
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("ssh command failed: %w (output: %s)", err, string(output))
|
||||
}
|
||||
|
||||
return string(output), nil
|
||||
}
|
||||
|
||||
// parseSensorsJSON parses the JSON output from `sensors -j`
|
||||
func (tc *TemperatureCollector) parseSensorsJSON(jsonStr string) (*models.Temperature, error) {
|
||||
if strings.TrimSpace(jsonStr) == "" {
|
||||
return nil, fmt.Errorf("empty sensors output")
|
||||
}
|
||||
|
||||
// sensors -j output structure:
|
||||
// {
|
||||
// "coretemp-isa-0000": {
|
||||
// "Package id 0": {"temp1_input": 45.0},
|
||||
// "Core 0": {"temp2_input": 43.0},
|
||||
// ...
|
||||
// },
|
||||
// "nvme-pci-0400": {
|
||||
// "Composite": {"temp1_input": 38.9}
|
||||
// }
|
||||
// }
|
||||
|
||||
var sensorsData map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(jsonStr), &sensorsData); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse sensors JSON: %w", err)
|
||||
}
|
||||
|
||||
temp := &models.Temperature{
|
||||
Cores: []models.CoreTemp{},
|
||||
NVMe: []models.NVMeTemp{},
|
||||
}
|
||||
|
||||
// Parse each sensor chip
|
||||
for chipName, chipData := range sensorsData {
|
||||
chipMap, ok := chipData.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// Handle CPU temperature sensors (coretemp, k10temp, etc.)
|
||||
if strings.Contains(chipName, "coretemp") || strings.Contains(chipName, "k10temp") {
|
||||
tc.parseCPUTemps(chipMap, temp)
|
||||
}
|
||||
|
||||
// Handle NVMe temperature sensors
|
||||
if strings.Contains(chipName, "nvme") {
|
||||
tc.parseNVMeTemps(chipName, chipMap, temp)
|
||||
}
|
||||
}
|
||||
|
||||
// If we got CPU temps, calculate max from cores if package not available
|
||||
if temp.CPUPackage == 0 && len(temp.Cores) > 0 {
|
||||
for _, core := range temp.Cores {
|
||||
if core.Temp > temp.CPUMax {
|
||||
temp.CPUMax = core.Temp
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return temp, nil
|
||||
}
|
||||
|
||||
// parseCPUTemps extracts CPU temperature data from a sensor chip
|
||||
func (tc *TemperatureCollector) parseCPUTemps(chipMap map[string]interface{}, temp *models.Temperature) {
|
||||
for sensorName, sensorData := range chipMap {
|
||||
sensorMap, ok := sensorData.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// Look for Package id (Intel) or Tdie (AMD)
|
||||
if strings.Contains(sensorName, "Package id") || strings.Contains(sensorName, "Tdie") {
|
||||
if tempVal := extractTempInput(sensorMap); tempVal > 0 {
|
||||
temp.CPUPackage = tempVal
|
||||
}
|
||||
}
|
||||
|
||||
// Look for individual cores
|
||||
if strings.HasPrefix(sensorName, "Core ") {
|
||||
coreNum := extractCoreNumber(sensorName)
|
||||
if tempVal := extractTempInput(sensorMap); tempVal > 0 {
|
||||
temp.Cores = append(temp.Cores, models.CoreTemp{
|
||||
Core: coreNum,
|
||||
Temp: tempVal,
|
||||
})
|
||||
if tempVal > temp.CPUMax {
|
||||
temp.CPUMax = tempVal
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parseNVMeTemps extracts NVMe temperature data from a sensor chip
|
||||
func (tc *TemperatureCollector) parseNVMeTemps(chipName string, chipMap map[string]interface{}, temp *models.Temperature) {
|
||||
// Extract device name from chip name (e.g., "nvme-pci-0400" -> "nvme0")
|
||||
device := "nvme" + strings.TrimPrefix(chipName, "nvme-pci-")
|
||||
|
||||
for sensorName, sensorData := range chipMap {
|
||||
sensorMap, ok := sensorData.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// Look for Composite temperature (main NVMe temp)
|
||||
if strings.Contains(sensorName, "Composite") || strings.Contains(sensorName, "Sensor 1") {
|
||||
if tempVal := extractTempInput(sensorMap); tempVal > 0 {
|
||||
temp.NVMe = append(temp.NVMe, models.NVMeTemp{
|
||||
Device: device,
|
||||
Temp: tempVal,
|
||||
})
|
||||
break // Only one temp per NVMe device
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// extractTempInput extracts temperature value from sensor data
|
||||
func extractTempInput(sensorMap map[string]interface{}) float64 {
|
||||
// Look for temp*_input fields
|
||||
for key, val := range sensorMap {
|
||||
if strings.HasSuffix(key, "_input") {
|
||||
switch v := val.(type) {
|
||||
case float64:
|
||||
return v
|
||||
case int:
|
||||
return float64(v)
|
||||
case string:
|
||||
if f, err := strconv.ParseFloat(v, 64); err == nil {
|
||||
return f
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// extractCoreNumber extracts the core number from a sensor name like "Core 0"
|
||||
func extractCoreNumber(name string) int {
|
||||
parts := strings.Fields(name)
|
||||
if len(parts) >= 2 {
|
||||
if num, err := strconv.Atoi(parts[len(parts)-1]); err == nil {
|
||||
return num
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// extractHostname extracts hostname/IP from a Proxmox host URL
|
||||
func extractHostname(hostURL string) string {
|
||||
// Remove protocol
|
||||
host := strings.TrimPrefix(hostURL, "https://")
|
||||
host = strings.TrimPrefix(host, "http://")
|
||||
|
||||
// Remove port
|
||||
if idx := strings.Index(host, ":"); idx != -1 {
|
||||
host = host[:idx]
|
||||
}
|
||||
|
||||
// Remove path
|
||||
if idx := strings.Index(host, "/"); idx != -1 {
|
||||
host = host[:idx]
|
||||
}
|
||||
|
||||
return host
|
||||
}
|
||||
Reference in New Issue
Block a user