Files
pulse/internal/monitoring/temperature.go
T
courtmanr@gmail.com b45bd66b94 Route discovery policy DNS through the cached resolver (#1638)
The discovery-policy check resolved endpoint hostnames with a bare
net.LookupIP while the actual dials went through pkg/tlsutil's process-global
cached resolver, so the policy and the connection reasoned about two different
DNS views. That split is why 108aa4e20 had to skip resolution entirely for the
default policy, which left the injected 169.254.0.0/16 blocklist enforced only
against literal IPs: a hostname endpoint pointed at the metadata range walked
straight through.

Resolve through tlsutil.LookupHostCached instead. The shared resolver caches
answers and lookup failures alike until its next refresh, so repeat poll cycles
cost a cache hit rather than a query and the per-poll DNS volume that opened
#1638 stays gone. With that in place the default-policy skip is removed and the
blocklist applies to resolved addresses again, and the five-minute decision
cache is dropped rather than kept: it bought nothing on top of the resolver
cache, made the verdict trail the configuration, and memoized the fail-open
"resolution failed, allow" outcome for minutes even with an explicit allowlist
configured. Its claim to match a DNS refresh interval that operators configure
through DNS_CACHE_TIMEOUT goes with it.

The SSH backoffs now only escalate for work that ran. A knownhosts manager
suppressing a call inside its own window reports ErrKeyscanSuppressed, and the
temperature layer neither records a failure nor pays for the RPi fallback in
that case. An expired collection deadline is our own budget rather than
evidence about the host, so it holds the window at the floor. Both backoffs
decay once a retry deadline is more than one window past, and replacing the
temperature SSH key on disk clears both maps so a repaired key is tried on the
next cycle instead of after fifteen minutes.

Refs discussion #1638.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-28 12:08:10 +01:00

1116 lines
34 KiB
Go

package monitoring
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"math"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/system"
"github.com/rs/zerolog/log"
)
const maxTemperatureCommandOutputSize = 1 << 20 // 1 MiB
const defaultSSHCommandTimeout = 15 * time.Second
const pulseSensorsSSHCommand = "if command -v /usr/local/sbin/pulse-sensors >/dev/null 2>&1; then /usr/local/sbin/pulse-sensors; else sensors -j 2>/dev/null || true; fi"
var errTemperatureCommandOutputTooLarge = errors.New("temperature command output exceeded limit")
// CommandRunner abstracts command execution for testing
type CommandRunner interface {
Run(ctx context.Context, name string, args ...string) ([]byte, error)
}
type defaultCommandRunner struct{}
type limitedTemperatureBuffer struct {
buf bytes.Buffer
maxBytes int
exceeded bool
}
func (b *limitedTemperatureBuffer) Write(p []byte) (int, error) {
remaining := b.maxBytes - b.buf.Len()
if remaining <= 0 {
b.exceeded = true
return 0, errTemperatureCommandOutputTooLarge
}
if len(p) > remaining {
b.exceeded = true
written, _ := b.buf.Write(p[:remaining])
return written, errTemperatureCommandOutputTooLarge
}
return b.buf.Write(p)
}
func (b *limitedTemperatureBuffer) Bytes() []byte {
return b.buf.Bytes()
}
func (r *defaultCommandRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) {
cmd := exec.CommandContext(ctx, name, args...)
stdout := limitedTemperatureBuffer{maxBytes: maxTemperatureCommandOutputSize}
stderr := limitedTemperatureBuffer{maxBytes: maxTemperatureCommandOutputSize}
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
if stdout.exceeded || stderr.exceeded {
return nil, errTemperatureCommandOutputTooLarge
}
if err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
// Preserve bounded stderr details for upstream error wrapping.
exitErr.Stderr = append([]byte(nil), stderr.Bytes()...)
}
return stdout.Bytes(), err
}
return stdout.Bytes(), nil
}
// TemperatureCollector handles SSH-based temperature collection from Proxmox nodes
type TemperatureCollector struct {
sshUser string // SSH user (typically "root" or "pulse-monitor")
sshKeyPath string // Path to SSH private key
sshPort int // SSH port (default 22)
hostKeys KnownHostsManager
missingKeyWarned atomic.Bool
runner CommandRunner
sshFailureMu sync.Mutex
sshFailures map[string]*temperatureSSHFailure
sshKeyIdentity string
}
// temperatureSSHFailure remembers a host whose SSH collection failed so
// subsequent 10s poll cycles don't re-exec ssh (twice, with the RPi fallback)
// against a host that keeps failing identically (#1638). Backoff doubles per
// consecutive failure; any successful collection clears it.
type temperatureSSHFailure struct {
retryAt time.Time
backoff time.Duration
}
const (
temperatureSSHFailureInitialBackoff = 30 * time.Second
temperatureSSHFailureMaxBackoff = 15 * time.Minute
)
// sshFailureOutcome describes what a failed collection attempt tells us about
// the host, which decides how the per-host backoff moves.
type sshFailureOutcome int
const (
// sshFailureHard is a refusal, auth failure, or unusable output: the host
// answered (or actively did not) and the backoff should compound.
sshFailureHard sshFailureOutcome = iota
// sshFailureTransient is our own deadline expiring. That says nothing about
// the host, so hold the backoff at the floor instead of doubling it.
sshFailureTransient
// sshFailureNotAttempted means no ssh ran at all — the knownhosts manager
// suppressed the call inside its own backoff. Nothing was learned and
// nothing was spent, so the backoff must not move.
sshFailureNotAttempted
)
// classifySSHFailure maps a failed attempt onto its backoff treatment (#1638).
func classifySSHFailure(err error) sshFailureOutcome {
switch {
case err == nil:
return sshFailureHard
case errors.Is(err, ErrKeyscanSuppressed):
return sshFailureNotAttempted
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
return sshFailureTransient
default:
return sshFailureHard
}
}
func (tc *TemperatureCollector) inSSHFailureBackoff(host string) bool {
tc.sshFailureMu.Lock()
defer tc.sshFailureMu.Unlock()
failure := tc.sshFailures[host]
return failure != nil && sshBackoffNow().Before(failure.retryAt)
}
func (tc *TemperatureCollector) recordSSHFailure(host string, outcome sshFailureOutcome) {
if outcome == sshFailureNotAttempted {
return
}
tc.sshFailureMu.Lock()
defer tc.sshFailureMu.Unlock()
if tc.sshFailures == nil {
tc.sshFailures = make(map[string]*temperatureSSHFailure)
}
now := sshBackoffNow()
var previousRetryAt time.Time
var previousBackoff time.Duration
if existing := tc.sshFailures[host]; existing != nil {
previousRetryAt = existing.retryAt
previousBackoff = existing.backoff
}
backoff := nextSSHBackoff(now, previousRetryAt, previousBackoff, temperatureSSHFailureInitialBackoff, temperatureSSHFailureMaxBackoff, outcome == sshFailureHard)
tc.sshFailures[host] = &temperatureSSHFailure{
retryAt: now.Add(backoff),
backoff: backoff,
}
}
func (tc *TemperatureCollector) clearSSHFailure(host string) {
tc.sshFailureMu.Lock()
defer tc.sshFailureMu.Unlock()
delete(tc.sshFailures, host)
}
// ResetSSHFailures drops every recorded SSH backoff, including the knownhosts
// manager's keyscan backoff, so an operator who has just repaired an SSH key or
// host key sees the next poll cycle retry rather than waiting out a window that
// has compounded to a quarter of an hour (#1638).
func (tc *TemperatureCollector) ResetSSHFailures() {
if tc == nil {
return
}
tc.sshFailureMu.Lock()
tc.sshFailures = nil
hostKeys := tc.hostKeys
tc.sshFailureMu.Unlock()
if hostKeys != nil {
hostKeys.ResetFailures()
}
}
// resetSSHFailuresOnKeyChange clears the backoff maps when the SSH key on disk
// has been replaced since the last cycle.
//
// Repairing that key is the usual fix for the failures that opened these
// backoff windows in the first place, and nothing else notices it happening, so
// without this the operator waits out a window that has already compounded
// toward fifteen minutes before Pulse tries the repaired key (#1638).
func (tc *TemperatureCollector) resetSSHFailuresOnKeyChange() {
identity := tc.currentSSHKeyIdentity()
tc.sshFailureMu.Lock()
changed := tc.sshKeyIdentity != "" && tc.sshKeyIdentity != identity
tc.sshKeyIdentity = identity
tc.sshFailureMu.Unlock()
if changed {
log.Info().
Str("sshKeyPath", tc.sshKeyPath).
Msg("Temperature SSH key changed on disk; clearing per-host SSH backoff")
tc.ResetSSHFailures()
}
}
// currentSSHKeyIdentity fingerprints the key file cheaply enough to run every
// poll cycle. Content is deliberately not read: this is change detection, not
// validation.
func (tc *TemperatureCollector) currentSSHKeyIdentity() string {
info, err := os.Stat(strings.TrimSpace(tc.sshKeyPath))
if err != nil {
return ""
}
return fmt.Sprintf("%d:%d", info.ModTime().UnixNano(), info.Size())
}
// NewTemperatureCollectorWithPort creates a new temperature collector with custom SSH port
func NewTemperatureCollectorWithPort(sshUser, sshKeyPath string, sshPort int) *TemperatureCollector {
if sshPort <= 0 {
sshPort = 22 // Default to standard SSH port
}
tc := &TemperatureCollector{
sshUser: sshUser,
sshKeyPath: sshKeyPath,
sshPort: sshPort,
runner: &defaultCommandRunner{},
}
homeDir := os.Getenv("HOME")
if homeDir == "" {
homeDir = "/home/pulse"
}
knownHostsPath := filepath.Join(homeDir, ".ssh", "known_hosts_sensors")
if manager, err := NewKnownHostsManager(knownHostsPath); err != nil {
log.Warn().Err(err).Str("path", knownHostsPath).Msg("failed to initialize temperature known_hosts manager")
} else {
tc.hostKeys = manager
}
return tc
}
// CollectTemperature collects temperature data from a node via SSH
func (tc *TemperatureCollector) CollectTemperature(ctx context.Context, nodeHost, nodeName string) (*models.Temperature, error) {
// Extract hostname/IP from the host URL (might be https://hostname:8006)
host := extractHostname(nodeHost)
// SECURITY: Block SSH fallback when running in containers (unless dev mode)
// Container compromise = SSH key compromise = root access to infrastructure
devModeAllowSSH := os.Getenv("PULSE_DEV_ALLOW_CONTAINER_SSH") == "true"
isContainer := os.Getenv("PULSE_DOCKER") == "true" || system.InContainer()
if isContainer && devModeAllowSSH {
// Log when dev override is active so operators understand the security posture
log.Info().
Str("node", nodeName).
Msg("Temperature collection using direct SSH (dev mode override active - not for production)")
}
if isContainer && !devModeAllowSSH {
// Warn but allow if key is present (legacy behavior restoration)
// We don't return here, allowing the code to fall through to the SSH key check
log.Warn().
Str("node", nodeName).
Msg("Temperature collection using direct SSH from container. This is insecure for production deployments.")
}
if strings.TrimSpace(tc.sshKeyPath) == "" {
tc.logMissingSSHKey(nil)
return &models.Temperature{Available: false}, nil
}
if keyErr := tc.validateSSHKeyPath(); keyErr != nil {
tc.logMissingSSHKey(keyErr)
return &models.Temperature{Available: false}, nil
}
tc.resetSSHFailuresOnKeyChange()
if tc.inSSHFailureBackoff(host) {
log.Debug().
Str("node", nodeName).
Str("host", host).
Msg("Skipping SSH temperature collection while failure backoff is active")
return &models.Temperature{Available: false}, nil
}
// Direct SSH (legacy method for non-containerized deployments).
// New setup scripts restrict the key to /usr/local/sbin/pulse-sensors, which emits
// the canonical {sensors, smart} payload. Older keys ignore the requested command
// and still force sensors -j, so keep the parser backward-compatible.
output, err := tc.runSSHCommand(ctx, host, pulseSensorsSSHCommand)
if err != nil || strings.TrimSpace(output) == "" {
// A suppressed keyscan or an expired deadline would fail the RPi
// fallback identically, so stop here rather than paying for a second
// probe, and do not compound a backoff the host has not earned (#1638).
if outcome := classifySSHFailure(err); outcome != sshFailureHard {
tc.logSkippedSSHAttempt(outcome, nodeName, host, err)
tc.recordSSHFailure(host, outcome)
return &models.Temperature{Available: false}, nil
}
if tc.disableLegacySSHOnAuthFailure(err, nodeName, host) {
tc.recordSSHFailure(host, sshFailureHard)
return &models.Temperature{Available: false}, nil
}
// Try Raspberry Pi temperature method
output, err = tc.runSSHCommand(ctx, host, "cat /sys/class/thermal/thermal_zone0/temp 2>/dev/null")
if err == nil && strings.TrimSpace(output) != "" {
// Parse RPi temperature format
temp, parseErr := tc.parseRPiTemperature(output)
if parseErr == nil {
tc.clearSSHFailure(host)
return temp, nil
}
}
if outcome := classifySSHFailure(err); outcome != sshFailureHard {
tc.logSkippedSSHAttempt(outcome, nodeName, host, err)
tc.recordSSHFailure(host, outcome)
return &models.Temperature{Available: false}, nil
}
if tc.disableLegacySSHOnAuthFailure(err, nodeName, host) {
tc.recordSSHFailure(host, sshFailureHard)
return &models.Temperature{Available: false}, nil
}
log.Debug().
Str("node", nodeName).
Str("host", host).
Err(err).
Msg("Failed to collect temperature data via SSH (tried both lm-sensors and RPi methods)")
tc.recordSSHFailure(host, sshFailureHard)
return &models.Temperature{Available: false}, nil
}
tc.clearSSHFailure(host)
// Parse sensors JSON output
temp, err := tc.parseSensorsJSON(output)
if err != nil {
log.Debug().
Str("node", nodeName).
Err(err).
Msg("Failed to parse sensors output")
return &models.Temperature{Available: false}, nil
}
if !temp.Available {
return temp, nil
}
temp.LastUpdate = time.Now()
return temp, nil
}
// logSkippedSSHAttempt records why an attempt is being abandoned without
// escalating the per-host backoff.
func (tc *TemperatureCollector) logSkippedSSHAttempt(outcome sshFailureOutcome, nodeName, host string, err error) {
event := log.Debug().Str("node", nodeName).Str("host", host).Err(err)
if outcome == sshFailureNotAttempted {
event.Msg("Skipping SSH temperature collection; host key scan is in its own backoff window")
return
}
event.Msg("Abandoning SSH temperature collection after the collection deadline expired")
}
func (tc *TemperatureCollector) runSSHCommand(ctx context.Context, host, command string) (string, error) {
if err := tc.validateSSHKeyPath(); err != nil {
return "", err
}
runCtx, cancel := withDefaultTimeout(ctx, defaultSSHCommandTimeout)
defer cancel()
if err := tc.ensureHostKey(runCtx, host); err != nil {
return "", err
}
// Build SSH command with appropriate options
sshArgs := []string{
"-o", "StrictHostKeyChecking=yes",
"-o", "BatchMode=yes",
"-o", "LogLevel=ERROR", // Suppress host key warnings that break JSON parsing
"-o", "ConnectTimeout=5",
"-p", strconv.Itoa(tc.sshPort), // Use configured SSH port
}
if tc.hostKeys != nil && tc.hostKeys.Path() != "" {
sshArgs = append(sshArgs,
"-o", fmt.Sprintf("UserKnownHostsFile=%s", tc.hostKeys.Path()),
"-o", "GlobalKnownHostsFile=/dev/null",
)
}
// Explicitly use SSH config file if it exists (for ProxyJump configuration)
homeDir := os.Getenv("HOME")
if homeDir == "" {
homeDir = "/home/pulse"
}
sshConfigPath := filepath.Join(homeDir, ".ssh/config")
if _, err := os.Stat(sshConfigPath); err == nil {
sshArgs = append(sshArgs, "-F", sshConfigPath)
}
// Add key if specified
if tc.sshKeyPath != "" {
sshArgs = append(sshArgs, "-i", tc.sshKeyPath)
}
// Add user@host and command
sshArgs = append(sshArgs, fmt.Sprintf("%s@%s", tc.sshUser, host), command)
output, err := tc.runner.Run(runCtx, "ssh", sshArgs...)
if err != nil {
if errors.Is(err, errTemperatureCommandOutputTooLarge) {
return "", fmt.Errorf("ssh command output exceeded %d bytes", maxTemperatureCommandOutputSize)
}
// Our own budget running out is not evidence about the host, and
// sanitizeSSHCommandError would flatten the signal into an exit status,
// so report the context error directly (#1638).
if ctxErr := runCtx.Err(); ctxErr != nil {
return "", fmt.Errorf("ssh command aborted: %w", ctxErr)
}
return "", sanitizeSSHCommandError(err)
}
outputStr := strings.TrimSpace(string(output))
// Strip any leading SSH noise (e.g., "Warning: Permanently added ...") so sensors JSON parses cleanly.
if idx := strings.Index(outputStr, "{"); idx > 0 {
outputStr = outputStr[idx:]
}
if idx := strings.LastIndex(outputStr, "}"); idx != -1 && idx < len(outputStr)-1 {
outputStr = outputStr[:idx+1]
}
return outputStr, nil
}
func withDefaultTimeout(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) {
if ctx == nil {
return context.WithTimeout(context.Background(), timeout)
}
if _, hasDeadline := ctx.Deadline(); hasDeadline {
return ctx, func() {}
}
return context.WithTimeout(ctx, timeout)
}
func sanitizeSSHCommandError(err error) error {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
stderrLower := strings.ToLower(string(exitErr.Stderr))
switch {
case strings.Contains(stderrLower, "permission denied"),
strings.Contains(stderrLower, "publickey"),
strings.Contains(stderrLower, "authentication failed"):
return errors.New("ssh command failed: authentication failed")
case strings.Contains(stderrLower, "host key verification failed"):
return errors.New("ssh command failed: host key verification failed")
case strings.Contains(stderrLower, "connection timed out"),
strings.Contains(stderrLower, "operation timed out"),
strings.Contains(stderrLower, "connection timeout"):
return errors.New("ssh command failed: timeout")
default:
if code := exitErr.ExitCode(); code >= 0 {
return fmt.Errorf("ssh command failed with exit status %d", code)
}
return errors.New("ssh command failed")
}
}
return fmt.Errorf("ssh command failed: %w", err)
}
func (tc *TemperatureCollector) validateSSHKeyPath() error {
keyPath := strings.TrimSpace(tc.sshKeyPath)
if keyPath == "" {
return errors.New("temperature SSH key unavailable")
}
info, err := os.Stat(keyPath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return os.ErrNotExist
}
return errors.New("temperature SSH key unavailable")
}
if info.IsDir() {
return errors.New("temperature SSH key path is not a file")
}
if err := validatePrivateKeyPermissions(info.Mode().Perm()); err != nil {
return err
}
return nil
}
func validatePrivateKeyPermissions(mode os.FileMode) error {
perm := mode.Perm()
if perm&0o077 != 0 {
return fmt.Errorf("temperature SSH key has insecure permissions (%03o): expected no group/other access", perm)
}
return nil
}
func (tc *TemperatureCollector) logMissingSSHKey(cause error) {
if tc.missingKeyWarned.Load() {
return
}
if tc.missingKeyWarned.CompareAndSwap(false, true) {
event := log.Debug().
Str("sshKeyPath", tc.sshKeyPath)
if cause != nil && !errors.Is(cause, os.ErrNotExist) {
event = event.Err(cause)
}
event.Msg("temperature SSH key not available; skipping legacy SSH collection")
}
}
func (tc *TemperatureCollector) disableLegacySSHOnAuthFailure(err error, nodeName, host string) bool {
if err == nil {
return false
}
msg := strings.ToLower(err.Error())
authFailure := strings.Contains(msg, "permission denied") ||
strings.Contains(msg, "authentication failed") ||
strings.Contains(msg, "publickey")
if !authFailure {
return false
}
// Do not disable globally on single node failure
log.Warn().
Str("node", nodeName).
Str("host", host).
Err(err).
Msg("SSH temperature collection failed due to authentication error; check SSH keys")
return true
}
type smartEntryRaw struct {
Device string `json:"device"`
Serial string `json:"serial,omitempty"`
WWN string `json:"wwn,omitempty"`
Model string `json:"model,omitempty"`
Type string `json:"type,omitempty"`
Temperature *int `json:"temperature"`
LastUpdated string `json:"lastUpdated,omitempty"`
StandbySkipped bool `json:"standbySkipped,omitempty"`
}
func normalizeSMARTEntries(raw []smartEntryRaw) []models.DiskTemp {
if len(raw) == 0 {
return nil
}
normalized := make([]models.DiskTemp, 0, len(raw))
for _, entry := range raw {
dev := strings.TrimSpace(entry.Device)
if dev == "" {
continue
}
var lastUpdated time.Time
if entry.LastUpdated != "" {
if parsed, err := time.Parse(time.RFC3339, entry.LastUpdated); err == nil {
lastUpdated = parsed
}
}
tempVal := 0
if entry.Temperature != nil {
tempVal = *entry.Temperature
}
normalized = append(normalized, models.DiskTemp{
Device: dev,
Serial: strings.TrimSpace(entry.Serial),
WWN: strings.TrimSpace(entry.WWN),
Model: strings.TrimSpace(entry.Model),
Type: strings.TrimSpace(entry.Type),
Temperature: tempVal,
LastUpdated: lastUpdated,
StandbySkipped: entry.StandbySkipped,
})
}
return normalized
}
// parseSensorsJSON parses the JSON output from the sensor wrapper
func (tc *TemperatureCollector) parseSensorsJSON(jsonStr string) (*models.Temperature, error) {
if strings.TrimSpace(jsonStr) == "" {
return nil, fmt.Errorf("empty sensors output")
}
// Try to parse as wrapper format first: {sensors: {...}, smart: [...]}
// Fall back to legacy format for backward compatibility
var wrapperData struct {
Sensors map[string]interface{} `json:"sensors"`
SMART []smartEntryRaw `json:"smart"`
}
var sensorsData map[string]interface{}
var smartRaw []smartEntryRaw
var parsedWrapper bool
if err := json.Unmarshal([]byte(jsonStr), &wrapperData); err == nil && wrapperData.Sensors != nil {
// New wrapper format
sensorsData = wrapperData.Sensors
smartRaw = wrapperData.SMART
parsedWrapper = true
} else {
// Legacy format: direct sensors -j output
if err := json.Unmarshal([]byte(jsonStr), &sensorsData); err != nil {
return nil, fmt.Errorf("failed to parse sensors JSON: %w", err)
}
log.Debug().Msg("parsed legacy sensors format (no SMART data)")
}
smartData := normalizeSMARTEntries(smartRaw)
if parsedWrapper {
log.Debug().
Int("smartDisks", len(smartData)).
Msg("Parsed new wrapper format with SMART data")
}
temp := &models.Temperature{
Cores: []models.CoreTemp{},
NVMe: []models.NVMeTemp{},
SMART: smartData,
LegacySensorsFormat: !parsedWrapper,
}
foundCPUChip := false
// Parse each sensor chip
for chipName, chipData := range sensorsData {
chipMap, ok := chipData.(map[string]interface{})
if !ok {
continue
}
// Handle CPU temperature sensors
chipLower := strings.ToLower(chipName)
if strings.Contains(chipLower, "coretemp") ||
strings.Contains(chipLower, "k10temp") ||
strings.Contains(chipLower, "zenpower") ||
strings.Contains(chipLower, "k8temp") ||
strings.Contains(chipLower, "acpitz") ||
strings.Contains(chipLower, "it87") ||
strings.Contains(chipLower, "nct6687") || // Nuvoton NCT6687 SuperIO
strings.Contains(chipLower, "nct6775") || // Nuvoton NCT6775 SuperIO
strings.Contains(chipLower, "nct6776") || // Nuvoton NCT6776 SuperIO
strings.Contains(chipLower, "nct6779") || // Nuvoton NCT6779 SuperIO
strings.Contains(chipLower, "nct6791") || // Nuvoton NCT6791 SuperIO
strings.Contains(chipLower, "nct6792") || // Nuvoton NCT6792 SuperIO
strings.Contains(chipLower, "nct6793") || // Nuvoton NCT6793 SuperIO
strings.Contains(chipLower, "nct6795") || // Nuvoton NCT6795 SuperIO
strings.Contains(chipLower, "nct6796") || // Nuvoton NCT6796 SuperIO
strings.Contains(chipLower, "nct6797") || // Nuvoton NCT6797 SuperIO
strings.Contains(chipLower, "nct6798") || // Nuvoton NCT6798 SuperIO
strings.Contains(chipLower, "w83627") || // Winbond W83627 SuperIO series
strings.Contains(chipLower, "f71882") || // Fintek F71882 SuperIO
strings.Contains(chipLower, "cpu_thermal") || // Raspberry Pi CPU temperature
strings.Contains(chipLower, "rp1_adc") || // Raspberry Pi RP1 ADC
strings.Contains(chipLower, "rpitemp") {
foundCPUChip = true
log.Debug().
Str("chip", chipName).
Msg("Detected CPU temperature chip")
tc.parseCPUTemps(chipMap, temp)
}
// Handle NVMe temperature sensors
if strings.Contains(chipName, "nvme") {
tc.parseNVMeTemps(chipName, chipMap, temp)
}
// Handle GPU temperature sensors
if strings.Contains(chipLower, "amdgpu") {
log.Debug().
Str("chip", chipName).
Msg("Detected AMD GPU temperature chip")
tc.parseGPUTemps(chipName, chipMap, temp)
}
// Handle NVIDIA GPU temperature sensors (nouveau driver)
if strings.Contains(chipLower, "nouveau") {
log.Debug().
Str("chip", chipName).
Msg("Detected NVIDIA GPU temperature chip (nouveau)")
tc.parseNouveauGPUTemps(chipName, chipMap, temp)
}
}
// If we got CPU temps, calculate max from cores if package not available
if temp.CPUPackage == 0 && len(temp.Cores) > 0 {
for _, core := range temp.Cores {
if core.Temp > temp.CPUMax {
temp.CPUMax = core.Temp
}
}
}
// Set individual sensor type flags based on chip presence, not value thresholds
// This prevents false negatives when sensors report 0°C during resets or temporarily
temp.HasCPU = foundCPUChip
temp.HasNVMe = len(temp.NVMe) > 0
temp.HasGPU = len(temp.GPU) > 0
temp.HasSMART = len(temp.SMART) > 0
// Available means any temperature data exists (backward compatibility)
temp.Available = temp.HasCPU || temp.HasNVMe || temp.HasGPU || temp.HasSMART
// Log summary of what was detected
if !foundCPUChip {
// List all chip names found for debugging
chipNames := make([]string, 0, len(sensorsData))
for chipName := range sensorsData {
chipNames = append(chipNames, chipName)
}
log.Debug().
Strs("chips", chipNames).
Msg("No recognized CPU temperature chip found in sensors output")
} else {
log.Debug().
Bool("hasCPU", temp.HasCPU).
Bool("hasNVMe", temp.HasNVMe).
Bool("hasGPU", temp.HasGPU).
Bool("hasSMART", temp.HasSMART).
Float64("cpuPackage", temp.CPUPackage).
Float64("cpuMax", temp.CPUMax).
Int("coreCount", len(temp.Cores)).
Int("nvmeCount", len(temp.NVMe)).
Int("gpuCount", len(temp.GPU)).
Int("smartCount", len(temp.SMART)).
Msg("Temperature data parsed successfully")
}
return temp, nil
}
// parseCPUTemps extracts CPU temperature data from a sensor chip
func (tc *TemperatureCollector) parseCPUTemps(chipMap map[string]interface{}, temp *models.Temperature) {
foundPackageTemp := false
var chipletTemps []float64 // Store AMD Tccd chiplet temps for fallback
for sensorName, sensorData := range chipMap {
sensorMap, ok := sensorData.(map[string]interface{})
if !ok {
continue
}
sensorNameLower := strings.ToLower(sensorName)
// Look for Package id (Intel) or Tdie/Tctl (AMD control loop temperature)
if strings.Contains(sensorName, "Package id") ||
strings.Contains(sensorName, "Tdie") ||
strings.Contains(sensorNameLower, "tctl") {
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) {
temp.CPUPackage = tempVal
foundPackageTemp = true
if tempVal > temp.CPUMax {
temp.CPUMax = tempVal
}
log.Debug().
Str("sensor", sensorName).
Float64("temp", tempVal).
Msg("Found CPU package temperature")
}
}
// Look for AMD chiplet temperatures (Tccd1, Tccd2, etc.) as fallback
if strings.HasPrefix(sensorName, "Tccd") {
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
chipletTemps = append(chipletTemps, tempVal)
if tempVal > temp.CPUMax {
temp.CPUMax = tempVal
}
log.Debug().
Str("sensor", sensorName).
Float64("temp", tempVal).
Msg("Found AMD chiplet temperature")
}
}
// Look for SuperIO chip CPU temperature fields (CPUTIN, CPU Temperature, etc.)
if strings.Contains(sensorNameLower, "cputin") ||
strings.Contains(sensorNameLower, "cpu temperature") ||
(strings.Contains(sensorNameLower, "temp") && strings.Contains(sensorNameLower, "cpu")) {
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
if !foundPackageTemp {
temp.CPUPackage = tempVal
foundPackageTemp = true
}
if tempVal > temp.CPUMax {
temp.CPUMax = tempVal
}
log.Debug().
Str("sensor", sensorName).
Float64("temp", tempVal).
Msg("Found SuperIO CPU temperature")
}
}
// Look for individual cores
if strings.HasPrefix(sensorName, "Core ") {
coreNum := extractCoreNumber(sensorName)
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) {
temp.Cores = append(temp.Cores, models.CoreTemp{
Core: coreNum,
Temp: tempVal,
})
if tempVal > temp.CPUMax {
temp.CPUMax = tempVal
}
log.Debug().
Str("sensor", sensorName).
Int("core", coreNum).
Float64("temp", tempVal).
Msg("Found core temperature")
}
}
}
// If no package temperature found, use highest chiplet temp (AMD Ryzen)
if !foundPackageTemp && len(chipletTemps) > 0 {
for _, chipletTemp := range chipletTemps {
if chipletTemp > temp.CPUPackage {
temp.CPUPackage = chipletTemp
}
}
foundPackageTemp = true
log.Debug().
Float64("temp", temp.CPUPackage).
Msg("Using highest chiplet temperature as CPU package temperature")
}
// If no package temperature was found (e.g., Raspberry Pi), look for generic temp sensors
if !foundPackageTemp {
for sensorName, sensorData := range chipMap {
sensorMap, ok := sensorData.(map[string]interface{})
if !ok {
continue
}
// Look for generic temperature sensors (e.g., "temp1" on Raspberry Pi)
if strings.HasPrefix(sensorName, "temp") || strings.HasPrefix(sensorName, "Temp") {
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
if temp.CPUPackage <= 0 {
temp.CPUPackage = tempVal
}
if tempVal > temp.CPUMax {
temp.CPUMax = tempVal
}
break // Use the first valid generic temp sensor
}
}
}
}
}
// parseNVMeTemps extracts NVMe temperature data from a sensor chip
func (tc *TemperatureCollector) parseNVMeTemps(chipName string, chipMap map[string]interface{}, temp *models.Temperature) {
// Extract device name from chip name (e.g., "nvme-pci-0400" -> "nvme0")
device := "nvme" + strings.TrimPrefix(chipName, "nvme-pci-")
// Try "Composite" first (preferred sensor name for NVMe temps)
for sensorName, sensorData := range chipMap {
if !strings.Contains(sensorName, "Composite") {
continue
}
sensorMap, ok := sensorData.(map[string]interface{})
if !ok {
continue
}
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
temp.NVMe = append(temp.NVMe, models.NVMeTemp{
Device: device,
Temp: tempVal,
})
return
}
}
// Fall back to "Sensor 1" if no valid Composite found
for sensorName, sensorData := range chipMap {
if !strings.Contains(sensorName, "Sensor 1") {
continue
}
sensorMap, ok := sensorData.(map[string]interface{})
if !ok {
continue
}
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
temp.NVMe = append(temp.NVMe, models.NVMeTemp{
Device: device,
Temp: tempVal,
})
return
}
}
}
// parseGPUTemps extracts GPU temperature data from a sensor chip
func (tc *TemperatureCollector) parseGPUTemps(chipName string, chipMap map[string]interface{}, temp *models.Temperature) {
gpuTemp := models.GPUTemp{
Device: chipName,
}
// AMD GPU sensors typically have: edge, junction (hotspot), mem
for sensorName, sensorData := range chipMap {
sensorMap, ok := sensorData.(map[string]interface{})
if !ok {
continue
}
sensorLower := strings.ToLower(sensorName)
tempVal := extractTempInput(sensorMap)
if math.IsNaN(tempVal) || tempVal <= 0 {
continue
}
// Map sensor names to struct fields
if strings.Contains(sensorLower, "edge") {
gpuTemp.Edge = tempVal
} else if strings.Contains(sensorLower, "junction") || strings.Contains(sensorLower, "hotspot") {
gpuTemp.Junction = tempVal
} else if strings.Contains(sensorLower, "mem") {
gpuTemp.Mem = tempVal
}
}
// Only add GPU entry if we got at least one valid temperature
if gpuTemp.Edge > 0 || gpuTemp.Junction > 0 || gpuTemp.Mem > 0 {
temp.GPU = append(temp.GPU, gpuTemp)
log.Debug().
Str("device", chipName).
Float64("edge", gpuTemp.Edge).
Float64("junction", gpuTemp.Junction).
Float64("mem", gpuTemp.Mem).
Msg("Parsed GPU temperatures")
}
}
// parseNouveauGPUTemps extracts NVIDIA GPU temperature data from nouveau driver sensors
func (tc *TemperatureCollector) parseNouveauGPUTemps(chipName string, chipMap map[string]interface{}, temp *models.Temperature) {
gpuTemp := models.GPUTemp{
Device: chipName,
}
// Nouveau driver typically exposes "GPU core" sensor
for sensorName, sensorData := range chipMap {
sensorMap, ok := sensorData.(map[string]interface{})
if !ok {
continue
}
sensorLower := strings.ToLower(sensorName)
tempVal := extractTempInput(sensorMap)
if math.IsNaN(tempVal) || tempVal <= 0 {
continue
}
// Nouveau typically has "GPU core" sensor - map to edge temperature
if strings.Contains(sensorLower, "gpu") || strings.Contains(sensorLower, "core") {
gpuTemp.Edge = tempVal
}
}
// Only add GPU entry if we got a valid temperature
if gpuTemp.Edge > 0 {
temp.GPU = append(temp.GPU, gpuTemp)
log.Debug().
Str("device", chipName).
Float64("edge", gpuTemp.Edge).
Msg("Parsed NVIDIA GPU (nouveau) temperature")
}
}
// extractTempInput extracts temperature value from sensor data
func extractTempInput(sensorMap map[string]interface{}) float64 {
// Look for temp*_input fields
for key, val := range sensorMap {
if strings.HasSuffix(key, "_input") {
switch v := val.(type) {
case float64:
return v
case int:
return float64(v)
case string:
if parsed, ok := parseStringTemperature(v); ok {
return parsed
}
}
}
}
return math.NaN()
}
func parseStringTemperature(value string) (float64, bool) {
value = strings.TrimSpace(value)
if value == "" {
return 0, false
}
parsed, err := strconv.ParseFloat(value, 64)
if err != nil {
if _, scanErr := fmt.Sscanf(value, "%f", &parsed); scanErr != nil {
return 0, false
}
}
if math.Abs(parsed) >= 1000 {
parsed = parsed / 1000.0
}
return parsed, true
}
// extractCoreNumber extracts the core number from a sensor name like "Core 0"
func extractCoreNumber(name string) int {
parts := strings.Fields(name)
if len(parts) >= 2 {
if num, err := strconv.Atoi(parts[len(parts)-1]); err == nil {
return num
}
}
return 0
}
// parseRPiTemperature parses Raspberry Pi temperature from /sys/class/thermal/thermal_zone0/temp
// Format: integer representing millidegrees Celsius (e.g., "45678" = 45.678°C)
func (tc *TemperatureCollector) parseRPiTemperature(output string) (*models.Temperature, error) {
millidegrees := strings.TrimSpace(output)
if millidegrees == "" {
return nil, fmt.Errorf("empty RPi temperature output")
}
tempMilliC, err := strconv.ParseFloat(millidegrees, 64)
if err != nil {
return nil, fmt.Errorf("failed to parse RPi temperature: %w", err)
}
// Convert millidegrees to degrees Celsius
tempC := tempMilliC / 1000.0
temp := &models.Temperature{
Available: true,
HasCPU: true,
CPUPackage: tempC,
CPUMax: tempC,
Cores: []models.CoreTemp{},
NVMe: []models.NVMeTemp{},
LastUpdate: time.Now(),
}
return temp, nil
}
// extractHostname extracts hostname/IP from a Proxmox host URL
func extractHostname(hostURL string) string {
// Remove protocol
host := strings.TrimPrefix(hostURL, "https://")
host = strings.TrimPrefix(host, "http://")
// Remove port
if idx := strings.Index(host, ":"); idx != -1 {
host = host[:idx]
}
// Remove path
if idx := strings.Index(host, "/"); idx != -1 {
host = host[:idx]
}
return host
}
func (tc *TemperatureCollector) ensureHostKey(ctx context.Context, host string) error {
if tc.hostKeys == nil {
return nil
}
if ctx == nil {
ctx = context.Background()
}
return tc.hostKeys.EnsureWithPort(ctx, host, tc.sshPort)
}