mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 10:35:51 +00:00
b45bd66b94
The discovery-policy check resolved endpoint hostnames with a bare
net.LookupIP while the actual dials went through pkg/tlsutil's process-global
cached resolver, so the policy and the connection reasoned about two different
DNS views. That split is why 108aa4e20 had to skip resolution entirely for the
default policy, which left the injected 169.254.0.0/16 blocklist enforced only
against literal IPs: a hostname endpoint pointed at the metadata range walked
straight through.
Resolve through tlsutil.LookupHostCached instead. The shared resolver caches
answers and lookup failures alike until its next refresh, so repeat poll cycles
cost a cache hit rather than a query and the per-poll DNS volume that opened
#1638 stays gone. With that in place the default-policy skip is removed and the
blocklist applies to resolved addresses again, and the five-minute decision
cache is dropped rather than kept: it bought nothing on top of the resolver
cache, made the verdict trail the configuration, and memoized the fail-open
"resolution failed, allow" outcome for minutes even with an explicit allowlist
configured. Its claim to match a DNS refresh interval that operators configure
through DNS_CACHE_TIMEOUT goes with it.
The SSH backoffs now only escalate for work that ran. A knownhosts manager
suppressing a call inside its own window reports ErrKeyscanSuppressed, and the
temperature layer neither records a failure nor pays for the RPi fallback in
that case. An expired collection deadline is our own budget rather than
evidence about the host, so it holds the window at the floor. Both backoffs
decay once a retry deadline is more than one window past, and replacing the
temperature SSH key on disk clears both maps so a repaired key is tried on the
next cycle instead of after fifteen minutes.
Refs discussion #1638.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1116 lines
34 KiB
Go
1116 lines
34 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"math"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/system"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
const maxTemperatureCommandOutputSize = 1 << 20 // 1 MiB
|
|
const defaultSSHCommandTimeout = 15 * time.Second
|
|
const pulseSensorsSSHCommand = "if command -v /usr/local/sbin/pulse-sensors >/dev/null 2>&1; then /usr/local/sbin/pulse-sensors; else sensors -j 2>/dev/null || true; fi"
|
|
|
|
var errTemperatureCommandOutputTooLarge = errors.New("temperature command output exceeded limit")
|
|
|
|
// CommandRunner abstracts command execution for testing
|
|
type CommandRunner interface {
|
|
Run(ctx context.Context, name string, args ...string) ([]byte, error)
|
|
}
|
|
|
|
type defaultCommandRunner struct{}
|
|
|
|
type limitedTemperatureBuffer struct {
|
|
buf bytes.Buffer
|
|
maxBytes int
|
|
exceeded bool
|
|
}
|
|
|
|
func (b *limitedTemperatureBuffer) Write(p []byte) (int, error) {
|
|
remaining := b.maxBytes - b.buf.Len()
|
|
if remaining <= 0 {
|
|
b.exceeded = true
|
|
return 0, errTemperatureCommandOutputTooLarge
|
|
}
|
|
if len(p) > remaining {
|
|
b.exceeded = true
|
|
written, _ := b.buf.Write(p[:remaining])
|
|
return written, errTemperatureCommandOutputTooLarge
|
|
}
|
|
return b.buf.Write(p)
|
|
}
|
|
|
|
func (b *limitedTemperatureBuffer) Bytes() []byte {
|
|
return b.buf.Bytes()
|
|
}
|
|
|
|
func (r *defaultCommandRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) {
|
|
cmd := exec.CommandContext(ctx, name, args...)
|
|
stdout := limitedTemperatureBuffer{maxBytes: maxTemperatureCommandOutputSize}
|
|
stderr := limitedTemperatureBuffer{maxBytes: maxTemperatureCommandOutputSize}
|
|
cmd.Stdout = &stdout
|
|
cmd.Stderr = &stderr
|
|
|
|
err := cmd.Run()
|
|
if stdout.exceeded || stderr.exceeded {
|
|
return nil, errTemperatureCommandOutputTooLarge
|
|
}
|
|
if err != nil {
|
|
if exitErr, ok := err.(*exec.ExitError); ok {
|
|
// Preserve bounded stderr details for upstream error wrapping.
|
|
exitErr.Stderr = append([]byte(nil), stderr.Bytes()...)
|
|
}
|
|
return stdout.Bytes(), err
|
|
}
|
|
|
|
return stdout.Bytes(), nil
|
|
}
|
|
|
|
// TemperatureCollector handles SSH-based temperature collection from Proxmox nodes
|
|
type TemperatureCollector struct {
|
|
sshUser string // SSH user (typically "root" or "pulse-monitor")
|
|
sshKeyPath string // Path to SSH private key
|
|
sshPort int // SSH port (default 22)
|
|
hostKeys KnownHostsManager
|
|
missingKeyWarned atomic.Bool
|
|
runner CommandRunner
|
|
sshFailureMu sync.Mutex
|
|
sshFailures map[string]*temperatureSSHFailure
|
|
sshKeyIdentity string
|
|
}
|
|
|
|
// temperatureSSHFailure remembers a host whose SSH collection failed so
|
|
// subsequent 10s poll cycles don't re-exec ssh (twice, with the RPi fallback)
|
|
// against a host that keeps failing identically (#1638). Backoff doubles per
|
|
// consecutive failure; any successful collection clears it.
|
|
type temperatureSSHFailure struct {
|
|
retryAt time.Time
|
|
backoff time.Duration
|
|
}
|
|
|
|
const (
|
|
temperatureSSHFailureInitialBackoff = 30 * time.Second
|
|
temperatureSSHFailureMaxBackoff = 15 * time.Minute
|
|
)
|
|
|
|
// sshFailureOutcome describes what a failed collection attempt tells us about
|
|
// the host, which decides how the per-host backoff moves.
|
|
type sshFailureOutcome int
|
|
|
|
const (
|
|
// sshFailureHard is a refusal, auth failure, or unusable output: the host
|
|
// answered (or actively did not) and the backoff should compound.
|
|
sshFailureHard sshFailureOutcome = iota
|
|
// sshFailureTransient is our own deadline expiring. That says nothing about
|
|
// the host, so hold the backoff at the floor instead of doubling it.
|
|
sshFailureTransient
|
|
// sshFailureNotAttempted means no ssh ran at all — the knownhosts manager
|
|
// suppressed the call inside its own backoff. Nothing was learned and
|
|
// nothing was spent, so the backoff must not move.
|
|
sshFailureNotAttempted
|
|
)
|
|
|
|
// classifySSHFailure maps a failed attempt onto its backoff treatment (#1638).
|
|
func classifySSHFailure(err error) sshFailureOutcome {
|
|
switch {
|
|
case err == nil:
|
|
return sshFailureHard
|
|
case errors.Is(err, ErrKeyscanSuppressed):
|
|
return sshFailureNotAttempted
|
|
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
|
|
return sshFailureTransient
|
|
default:
|
|
return sshFailureHard
|
|
}
|
|
}
|
|
|
|
func (tc *TemperatureCollector) inSSHFailureBackoff(host string) bool {
|
|
tc.sshFailureMu.Lock()
|
|
defer tc.sshFailureMu.Unlock()
|
|
failure := tc.sshFailures[host]
|
|
return failure != nil && sshBackoffNow().Before(failure.retryAt)
|
|
}
|
|
|
|
func (tc *TemperatureCollector) recordSSHFailure(host string, outcome sshFailureOutcome) {
|
|
if outcome == sshFailureNotAttempted {
|
|
return
|
|
}
|
|
|
|
tc.sshFailureMu.Lock()
|
|
defer tc.sshFailureMu.Unlock()
|
|
if tc.sshFailures == nil {
|
|
tc.sshFailures = make(map[string]*temperatureSSHFailure)
|
|
}
|
|
|
|
now := sshBackoffNow()
|
|
var previousRetryAt time.Time
|
|
var previousBackoff time.Duration
|
|
if existing := tc.sshFailures[host]; existing != nil {
|
|
previousRetryAt = existing.retryAt
|
|
previousBackoff = existing.backoff
|
|
}
|
|
backoff := nextSSHBackoff(now, previousRetryAt, previousBackoff, temperatureSSHFailureInitialBackoff, temperatureSSHFailureMaxBackoff, outcome == sshFailureHard)
|
|
|
|
tc.sshFailures[host] = &temperatureSSHFailure{
|
|
retryAt: now.Add(backoff),
|
|
backoff: backoff,
|
|
}
|
|
}
|
|
|
|
func (tc *TemperatureCollector) clearSSHFailure(host string) {
|
|
tc.sshFailureMu.Lock()
|
|
defer tc.sshFailureMu.Unlock()
|
|
delete(tc.sshFailures, host)
|
|
}
|
|
|
|
// ResetSSHFailures drops every recorded SSH backoff, including the knownhosts
|
|
// manager's keyscan backoff, so an operator who has just repaired an SSH key or
|
|
// host key sees the next poll cycle retry rather than waiting out a window that
|
|
// has compounded to a quarter of an hour (#1638).
|
|
func (tc *TemperatureCollector) ResetSSHFailures() {
|
|
if tc == nil {
|
|
return
|
|
}
|
|
|
|
tc.sshFailureMu.Lock()
|
|
tc.sshFailures = nil
|
|
hostKeys := tc.hostKeys
|
|
tc.sshFailureMu.Unlock()
|
|
|
|
if hostKeys != nil {
|
|
hostKeys.ResetFailures()
|
|
}
|
|
}
|
|
|
|
// resetSSHFailuresOnKeyChange clears the backoff maps when the SSH key on disk
|
|
// has been replaced since the last cycle.
|
|
//
|
|
// Repairing that key is the usual fix for the failures that opened these
|
|
// backoff windows in the first place, and nothing else notices it happening, so
|
|
// without this the operator waits out a window that has already compounded
|
|
// toward fifteen minutes before Pulse tries the repaired key (#1638).
|
|
func (tc *TemperatureCollector) resetSSHFailuresOnKeyChange() {
|
|
identity := tc.currentSSHKeyIdentity()
|
|
|
|
tc.sshFailureMu.Lock()
|
|
changed := tc.sshKeyIdentity != "" && tc.sshKeyIdentity != identity
|
|
tc.sshKeyIdentity = identity
|
|
tc.sshFailureMu.Unlock()
|
|
|
|
if changed {
|
|
log.Info().
|
|
Str("sshKeyPath", tc.sshKeyPath).
|
|
Msg("Temperature SSH key changed on disk; clearing per-host SSH backoff")
|
|
tc.ResetSSHFailures()
|
|
}
|
|
}
|
|
|
|
// currentSSHKeyIdentity fingerprints the key file cheaply enough to run every
|
|
// poll cycle. Content is deliberately not read: this is change detection, not
|
|
// validation.
|
|
func (tc *TemperatureCollector) currentSSHKeyIdentity() string {
|
|
info, err := os.Stat(strings.TrimSpace(tc.sshKeyPath))
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return fmt.Sprintf("%d:%d", info.ModTime().UnixNano(), info.Size())
|
|
}
|
|
|
|
// NewTemperatureCollectorWithPort creates a new temperature collector with custom SSH port
|
|
func NewTemperatureCollectorWithPort(sshUser, sshKeyPath string, sshPort int) *TemperatureCollector {
|
|
if sshPort <= 0 {
|
|
sshPort = 22 // Default to standard SSH port
|
|
}
|
|
|
|
tc := &TemperatureCollector{
|
|
sshUser: sshUser,
|
|
sshKeyPath: sshKeyPath,
|
|
sshPort: sshPort,
|
|
runner: &defaultCommandRunner{},
|
|
}
|
|
|
|
homeDir := os.Getenv("HOME")
|
|
if homeDir == "" {
|
|
homeDir = "/home/pulse"
|
|
}
|
|
knownHostsPath := filepath.Join(homeDir, ".ssh", "known_hosts_sensors")
|
|
if manager, err := NewKnownHostsManager(knownHostsPath); err != nil {
|
|
log.Warn().Err(err).Str("path", knownHostsPath).Msg("failed to initialize temperature known_hosts manager")
|
|
} else {
|
|
tc.hostKeys = manager
|
|
}
|
|
|
|
return tc
|
|
}
|
|
|
|
// CollectTemperature collects temperature data from a node via SSH
|
|
func (tc *TemperatureCollector) CollectTemperature(ctx context.Context, nodeHost, nodeName string) (*models.Temperature, error) {
|
|
// Extract hostname/IP from the host URL (might be https://hostname:8006)
|
|
host := extractHostname(nodeHost)
|
|
|
|
// SECURITY: Block SSH fallback when running in containers (unless dev mode)
|
|
// Container compromise = SSH key compromise = root access to infrastructure
|
|
devModeAllowSSH := os.Getenv("PULSE_DEV_ALLOW_CONTAINER_SSH") == "true"
|
|
isContainer := os.Getenv("PULSE_DOCKER") == "true" || system.InContainer()
|
|
|
|
if isContainer && devModeAllowSSH {
|
|
// Log when dev override is active so operators understand the security posture
|
|
log.Info().
|
|
Str("node", nodeName).
|
|
Msg("Temperature collection using direct SSH (dev mode override active - not for production)")
|
|
}
|
|
|
|
if isContainer && !devModeAllowSSH {
|
|
// Warn but allow if key is present (legacy behavior restoration)
|
|
// We don't return here, allowing the code to fall through to the SSH key check
|
|
log.Warn().
|
|
Str("node", nodeName).
|
|
Msg("Temperature collection using direct SSH from container. This is insecure for production deployments.")
|
|
}
|
|
|
|
if strings.TrimSpace(tc.sshKeyPath) == "" {
|
|
tc.logMissingSSHKey(nil)
|
|
return &models.Temperature{Available: false}, nil
|
|
}
|
|
|
|
if keyErr := tc.validateSSHKeyPath(); keyErr != nil {
|
|
tc.logMissingSSHKey(keyErr)
|
|
return &models.Temperature{Available: false}, nil
|
|
}
|
|
|
|
tc.resetSSHFailuresOnKeyChange()
|
|
|
|
if tc.inSSHFailureBackoff(host) {
|
|
log.Debug().
|
|
Str("node", nodeName).
|
|
Str("host", host).
|
|
Msg("Skipping SSH temperature collection while failure backoff is active")
|
|
return &models.Temperature{Available: false}, nil
|
|
}
|
|
|
|
// Direct SSH (legacy method for non-containerized deployments).
|
|
// New setup scripts restrict the key to /usr/local/sbin/pulse-sensors, which emits
|
|
// the canonical {sensors, smart} payload. Older keys ignore the requested command
|
|
// and still force sensors -j, so keep the parser backward-compatible.
|
|
output, err := tc.runSSHCommand(ctx, host, pulseSensorsSSHCommand)
|
|
if err != nil || strings.TrimSpace(output) == "" {
|
|
// A suppressed keyscan or an expired deadline would fail the RPi
|
|
// fallback identically, so stop here rather than paying for a second
|
|
// probe, and do not compound a backoff the host has not earned (#1638).
|
|
if outcome := classifySSHFailure(err); outcome != sshFailureHard {
|
|
tc.logSkippedSSHAttempt(outcome, nodeName, host, err)
|
|
tc.recordSSHFailure(host, outcome)
|
|
return &models.Temperature{Available: false}, nil
|
|
}
|
|
|
|
if tc.disableLegacySSHOnAuthFailure(err, nodeName, host) {
|
|
tc.recordSSHFailure(host, sshFailureHard)
|
|
return &models.Temperature{Available: false}, nil
|
|
}
|
|
|
|
// Try Raspberry Pi temperature method
|
|
output, err = tc.runSSHCommand(ctx, host, "cat /sys/class/thermal/thermal_zone0/temp 2>/dev/null")
|
|
if err == nil && strings.TrimSpace(output) != "" {
|
|
// Parse RPi temperature format
|
|
temp, parseErr := tc.parseRPiTemperature(output)
|
|
if parseErr == nil {
|
|
tc.clearSSHFailure(host)
|
|
return temp, nil
|
|
}
|
|
}
|
|
|
|
if outcome := classifySSHFailure(err); outcome != sshFailureHard {
|
|
tc.logSkippedSSHAttempt(outcome, nodeName, host, err)
|
|
tc.recordSSHFailure(host, outcome)
|
|
return &models.Temperature{Available: false}, nil
|
|
}
|
|
|
|
if tc.disableLegacySSHOnAuthFailure(err, nodeName, host) {
|
|
tc.recordSSHFailure(host, sshFailureHard)
|
|
return &models.Temperature{Available: false}, nil
|
|
}
|
|
|
|
log.Debug().
|
|
Str("node", nodeName).
|
|
Str("host", host).
|
|
Err(err).
|
|
Msg("Failed to collect temperature data via SSH (tried both lm-sensors and RPi methods)")
|
|
tc.recordSSHFailure(host, sshFailureHard)
|
|
return &models.Temperature{Available: false}, nil
|
|
}
|
|
|
|
tc.clearSSHFailure(host)
|
|
|
|
// Parse sensors JSON output
|
|
temp, err := tc.parseSensorsJSON(output)
|
|
if err != nil {
|
|
log.Debug().
|
|
Str("node", nodeName).
|
|
Err(err).
|
|
Msg("Failed to parse sensors output")
|
|
return &models.Temperature{Available: false}, nil
|
|
}
|
|
|
|
if !temp.Available {
|
|
return temp, nil
|
|
}
|
|
|
|
temp.LastUpdate = time.Now()
|
|
|
|
return temp, nil
|
|
}
|
|
|
|
// logSkippedSSHAttempt records why an attempt is being abandoned without
|
|
// escalating the per-host backoff.
|
|
func (tc *TemperatureCollector) logSkippedSSHAttempt(outcome sshFailureOutcome, nodeName, host string, err error) {
|
|
event := log.Debug().Str("node", nodeName).Str("host", host).Err(err)
|
|
if outcome == sshFailureNotAttempted {
|
|
event.Msg("Skipping SSH temperature collection; host key scan is in its own backoff window")
|
|
return
|
|
}
|
|
event.Msg("Abandoning SSH temperature collection after the collection deadline expired")
|
|
}
|
|
|
|
func (tc *TemperatureCollector) runSSHCommand(ctx context.Context, host, command string) (string, error) {
|
|
if err := tc.validateSSHKeyPath(); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
runCtx, cancel := withDefaultTimeout(ctx, defaultSSHCommandTimeout)
|
|
defer cancel()
|
|
|
|
if err := tc.ensureHostKey(runCtx, host); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
// Build SSH command with appropriate options
|
|
sshArgs := []string{
|
|
"-o", "StrictHostKeyChecking=yes",
|
|
"-o", "BatchMode=yes",
|
|
"-o", "LogLevel=ERROR", // Suppress host key warnings that break JSON parsing
|
|
"-o", "ConnectTimeout=5",
|
|
"-p", strconv.Itoa(tc.sshPort), // Use configured SSH port
|
|
}
|
|
|
|
if tc.hostKeys != nil && tc.hostKeys.Path() != "" {
|
|
sshArgs = append(sshArgs,
|
|
"-o", fmt.Sprintf("UserKnownHostsFile=%s", tc.hostKeys.Path()),
|
|
"-o", "GlobalKnownHostsFile=/dev/null",
|
|
)
|
|
}
|
|
|
|
// Explicitly use SSH config file if it exists (for ProxyJump configuration)
|
|
homeDir := os.Getenv("HOME")
|
|
if homeDir == "" {
|
|
homeDir = "/home/pulse"
|
|
}
|
|
sshConfigPath := filepath.Join(homeDir, ".ssh/config")
|
|
if _, err := os.Stat(sshConfigPath); err == nil {
|
|
sshArgs = append(sshArgs, "-F", sshConfigPath)
|
|
}
|
|
|
|
// Add key if specified
|
|
if tc.sshKeyPath != "" {
|
|
sshArgs = append(sshArgs, "-i", tc.sshKeyPath)
|
|
}
|
|
|
|
// Add user@host and command
|
|
sshArgs = append(sshArgs, fmt.Sprintf("%s@%s", tc.sshUser, host), command)
|
|
|
|
output, err := tc.runner.Run(runCtx, "ssh", sshArgs...)
|
|
if err != nil {
|
|
if errors.Is(err, errTemperatureCommandOutputTooLarge) {
|
|
return "", fmt.Errorf("ssh command output exceeded %d bytes", maxTemperatureCommandOutputSize)
|
|
}
|
|
// Our own budget running out is not evidence about the host, and
|
|
// sanitizeSSHCommandError would flatten the signal into an exit status,
|
|
// so report the context error directly (#1638).
|
|
if ctxErr := runCtx.Err(); ctxErr != nil {
|
|
return "", fmt.Errorf("ssh command aborted: %w", ctxErr)
|
|
}
|
|
return "", sanitizeSSHCommandError(err)
|
|
}
|
|
|
|
outputStr := strings.TrimSpace(string(output))
|
|
|
|
// Strip any leading SSH noise (e.g., "Warning: Permanently added ...") so sensors JSON parses cleanly.
|
|
if idx := strings.Index(outputStr, "{"); idx > 0 {
|
|
outputStr = outputStr[idx:]
|
|
}
|
|
if idx := strings.LastIndex(outputStr, "}"); idx != -1 && idx < len(outputStr)-1 {
|
|
outputStr = outputStr[:idx+1]
|
|
}
|
|
|
|
return outputStr, nil
|
|
}
|
|
|
|
func withDefaultTimeout(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) {
|
|
if ctx == nil {
|
|
return context.WithTimeout(context.Background(), timeout)
|
|
}
|
|
if _, hasDeadline := ctx.Deadline(); hasDeadline {
|
|
return ctx, func() {}
|
|
}
|
|
return context.WithTimeout(ctx, timeout)
|
|
}
|
|
|
|
func sanitizeSSHCommandError(err error) error {
|
|
var exitErr *exec.ExitError
|
|
if errors.As(err, &exitErr) {
|
|
stderrLower := strings.ToLower(string(exitErr.Stderr))
|
|
switch {
|
|
case strings.Contains(stderrLower, "permission denied"),
|
|
strings.Contains(stderrLower, "publickey"),
|
|
strings.Contains(stderrLower, "authentication failed"):
|
|
return errors.New("ssh command failed: authentication failed")
|
|
case strings.Contains(stderrLower, "host key verification failed"):
|
|
return errors.New("ssh command failed: host key verification failed")
|
|
case strings.Contains(stderrLower, "connection timed out"),
|
|
strings.Contains(stderrLower, "operation timed out"),
|
|
strings.Contains(stderrLower, "connection timeout"):
|
|
return errors.New("ssh command failed: timeout")
|
|
default:
|
|
if code := exitErr.ExitCode(); code >= 0 {
|
|
return fmt.Errorf("ssh command failed with exit status %d", code)
|
|
}
|
|
return errors.New("ssh command failed")
|
|
}
|
|
}
|
|
|
|
return fmt.Errorf("ssh command failed: %w", err)
|
|
}
|
|
|
|
func (tc *TemperatureCollector) validateSSHKeyPath() error {
|
|
keyPath := strings.TrimSpace(tc.sshKeyPath)
|
|
if keyPath == "" {
|
|
return errors.New("temperature SSH key unavailable")
|
|
}
|
|
|
|
info, err := os.Stat(keyPath)
|
|
if err != nil {
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return os.ErrNotExist
|
|
}
|
|
return errors.New("temperature SSH key unavailable")
|
|
}
|
|
if info.IsDir() {
|
|
return errors.New("temperature SSH key path is not a file")
|
|
}
|
|
if err := validatePrivateKeyPermissions(info.Mode().Perm()); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validatePrivateKeyPermissions(mode os.FileMode) error {
|
|
perm := mode.Perm()
|
|
if perm&0o077 != 0 {
|
|
return fmt.Errorf("temperature SSH key has insecure permissions (%03o): expected no group/other access", perm)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (tc *TemperatureCollector) logMissingSSHKey(cause error) {
|
|
if tc.missingKeyWarned.Load() {
|
|
return
|
|
}
|
|
if tc.missingKeyWarned.CompareAndSwap(false, true) {
|
|
event := log.Debug().
|
|
Str("sshKeyPath", tc.sshKeyPath)
|
|
if cause != nil && !errors.Is(cause, os.ErrNotExist) {
|
|
event = event.Err(cause)
|
|
}
|
|
event.Msg("temperature SSH key not available; skipping legacy SSH collection")
|
|
}
|
|
}
|
|
|
|
func (tc *TemperatureCollector) disableLegacySSHOnAuthFailure(err error, nodeName, host string) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
|
|
msg := strings.ToLower(err.Error())
|
|
authFailure := strings.Contains(msg, "permission denied") ||
|
|
strings.Contains(msg, "authentication failed") ||
|
|
strings.Contains(msg, "publickey")
|
|
|
|
if !authFailure {
|
|
return false
|
|
}
|
|
|
|
// Do not disable globally on single node failure
|
|
log.Warn().
|
|
Str("node", nodeName).
|
|
Str("host", host).
|
|
Err(err).
|
|
Msg("SSH temperature collection failed due to authentication error; check SSH keys")
|
|
|
|
return true
|
|
}
|
|
|
|
type smartEntryRaw struct {
|
|
Device string `json:"device"`
|
|
Serial string `json:"serial,omitempty"`
|
|
WWN string `json:"wwn,omitempty"`
|
|
Model string `json:"model,omitempty"`
|
|
Type string `json:"type,omitempty"`
|
|
Temperature *int `json:"temperature"`
|
|
LastUpdated string `json:"lastUpdated,omitempty"`
|
|
StandbySkipped bool `json:"standbySkipped,omitempty"`
|
|
}
|
|
|
|
func normalizeSMARTEntries(raw []smartEntryRaw) []models.DiskTemp {
|
|
if len(raw) == 0 {
|
|
return nil
|
|
}
|
|
|
|
normalized := make([]models.DiskTemp, 0, len(raw))
|
|
for _, entry := range raw {
|
|
dev := strings.TrimSpace(entry.Device)
|
|
if dev == "" {
|
|
continue
|
|
}
|
|
|
|
var lastUpdated time.Time
|
|
if entry.LastUpdated != "" {
|
|
if parsed, err := time.Parse(time.RFC3339, entry.LastUpdated); err == nil {
|
|
lastUpdated = parsed
|
|
}
|
|
}
|
|
|
|
tempVal := 0
|
|
if entry.Temperature != nil {
|
|
tempVal = *entry.Temperature
|
|
}
|
|
|
|
normalized = append(normalized, models.DiskTemp{
|
|
Device: dev,
|
|
Serial: strings.TrimSpace(entry.Serial),
|
|
WWN: strings.TrimSpace(entry.WWN),
|
|
Model: strings.TrimSpace(entry.Model),
|
|
Type: strings.TrimSpace(entry.Type),
|
|
Temperature: tempVal,
|
|
LastUpdated: lastUpdated,
|
|
StandbySkipped: entry.StandbySkipped,
|
|
})
|
|
}
|
|
|
|
return normalized
|
|
}
|
|
|
|
// parseSensorsJSON parses the JSON output from the sensor wrapper
|
|
func (tc *TemperatureCollector) parseSensorsJSON(jsonStr string) (*models.Temperature, error) {
|
|
if strings.TrimSpace(jsonStr) == "" {
|
|
return nil, fmt.Errorf("empty sensors output")
|
|
}
|
|
|
|
// Try to parse as wrapper format first: {sensors: {...}, smart: [...]}
|
|
// Fall back to legacy format for backward compatibility
|
|
var wrapperData struct {
|
|
Sensors map[string]interface{} `json:"sensors"`
|
|
SMART []smartEntryRaw `json:"smart"`
|
|
}
|
|
|
|
var sensorsData map[string]interface{}
|
|
var smartRaw []smartEntryRaw
|
|
var parsedWrapper bool
|
|
|
|
if err := json.Unmarshal([]byte(jsonStr), &wrapperData); err == nil && wrapperData.Sensors != nil {
|
|
// New wrapper format
|
|
sensorsData = wrapperData.Sensors
|
|
smartRaw = wrapperData.SMART
|
|
parsedWrapper = true
|
|
} else {
|
|
// Legacy format: direct sensors -j output
|
|
if err := json.Unmarshal([]byte(jsonStr), &sensorsData); err != nil {
|
|
return nil, fmt.Errorf("failed to parse sensors JSON: %w", err)
|
|
}
|
|
log.Debug().Msg("parsed legacy sensors format (no SMART data)")
|
|
}
|
|
|
|
smartData := normalizeSMARTEntries(smartRaw)
|
|
if parsedWrapper {
|
|
log.Debug().
|
|
Int("smartDisks", len(smartData)).
|
|
Msg("Parsed new wrapper format with SMART data")
|
|
}
|
|
|
|
temp := &models.Temperature{
|
|
Cores: []models.CoreTemp{},
|
|
NVMe: []models.NVMeTemp{},
|
|
SMART: smartData,
|
|
LegacySensorsFormat: !parsedWrapper,
|
|
}
|
|
|
|
foundCPUChip := false
|
|
|
|
// Parse each sensor chip
|
|
for chipName, chipData := range sensorsData {
|
|
chipMap, ok := chipData.(map[string]interface{})
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
// Handle CPU temperature sensors
|
|
chipLower := strings.ToLower(chipName)
|
|
if strings.Contains(chipLower, "coretemp") ||
|
|
strings.Contains(chipLower, "k10temp") ||
|
|
strings.Contains(chipLower, "zenpower") ||
|
|
strings.Contains(chipLower, "k8temp") ||
|
|
strings.Contains(chipLower, "acpitz") ||
|
|
strings.Contains(chipLower, "it87") ||
|
|
strings.Contains(chipLower, "nct6687") || // Nuvoton NCT6687 SuperIO
|
|
strings.Contains(chipLower, "nct6775") || // Nuvoton NCT6775 SuperIO
|
|
strings.Contains(chipLower, "nct6776") || // Nuvoton NCT6776 SuperIO
|
|
strings.Contains(chipLower, "nct6779") || // Nuvoton NCT6779 SuperIO
|
|
strings.Contains(chipLower, "nct6791") || // Nuvoton NCT6791 SuperIO
|
|
strings.Contains(chipLower, "nct6792") || // Nuvoton NCT6792 SuperIO
|
|
strings.Contains(chipLower, "nct6793") || // Nuvoton NCT6793 SuperIO
|
|
strings.Contains(chipLower, "nct6795") || // Nuvoton NCT6795 SuperIO
|
|
strings.Contains(chipLower, "nct6796") || // Nuvoton NCT6796 SuperIO
|
|
strings.Contains(chipLower, "nct6797") || // Nuvoton NCT6797 SuperIO
|
|
strings.Contains(chipLower, "nct6798") || // Nuvoton NCT6798 SuperIO
|
|
strings.Contains(chipLower, "w83627") || // Winbond W83627 SuperIO series
|
|
strings.Contains(chipLower, "f71882") || // Fintek F71882 SuperIO
|
|
strings.Contains(chipLower, "cpu_thermal") || // Raspberry Pi CPU temperature
|
|
strings.Contains(chipLower, "rp1_adc") || // Raspberry Pi RP1 ADC
|
|
strings.Contains(chipLower, "rpitemp") {
|
|
foundCPUChip = true
|
|
log.Debug().
|
|
Str("chip", chipName).
|
|
Msg("Detected CPU temperature chip")
|
|
tc.parseCPUTemps(chipMap, temp)
|
|
}
|
|
|
|
// Handle NVMe temperature sensors
|
|
if strings.Contains(chipName, "nvme") {
|
|
tc.parseNVMeTemps(chipName, chipMap, temp)
|
|
}
|
|
|
|
// Handle GPU temperature sensors
|
|
if strings.Contains(chipLower, "amdgpu") {
|
|
log.Debug().
|
|
Str("chip", chipName).
|
|
Msg("Detected AMD GPU temperature chip")
|
|
tc.parseGPUTemps(chipName, chipMap, temp)
|
|
}
|
|
|
|
// Handle NVIDIA GPU temperature sensors (nouveau driver)
|
|
if strings.Contains(chipLower, "nouveau") {
|
|
log.Debug().
|
|
Str("chip", chipName).
|
|
Msg("Detected NVIDIA GPU temperature chip (nouveau)")
|
|
tc.parseNouveauGPUTemps(chipName, chipMap, temp)
|
|
}
|
|
}
|
|
|
|
// If we got CPU temps, calculate max from cores if package not available
|
|
if temp.CPUPackage == 0 && len(temp.Cores) > 0 {
|
|
for _, core := range temp.Cores {
|
|
if core.Temp > temp.CPUMax {
|
|
temp.CPUMax = core.Temp
|
|
}
|
|
}
|
|
}
|
|
|
|
// Set individual sensor type flags based on chip presence, not value thresholds
|
|
// This prevents false negatives when sensors report 0°C during resets or temporarily
|
|
temp.HasCPU = foundCPUChip
|
|
temp.HasNVMe = len(temp.NVMe) > 0
|
|
temp.HasGPU = len(temp.GPU) > 0
|
|
temp.HasSMART = len(temp.SMART) > 0
|
|
|
|
// Available means any temperature data exists (backward compatibility)
|
|
temp.Available = temp.HasCPU || temp.HasNVMe || temp.HasGPU || temp.HasSMART
|
|
|
|
// Log summary of what was detected
|
|
if !foundCPUChip {
|
|
// List all chip names found for debugging
|
|
chipNames := make([]string, 0, len(sensorsData))
|
|
for chipName := range sensorsData {
|
|
chipNames = append(chipNames, chipName)
|
|
}
|
|
log.Debug().
|
|
Strs("chips", chipNames).
|
|
Msg("No recognized CPU temperature chip found in sensors output")
|
|
} else {
|
|
log.Debug().
|
|
Bool("hasCPU", temp.HasCPU).
|
|
Bool("hasNVMe", temp.HasNVMe).
|
|
Bool("hasGPU", temp.HasGPU).
|
|
Bool("hasSMART", temp.HasSMART).
|
|
Float64("cpuPackage", temp.CPUPackage).
|
|
Float64("cpuMax", temp.CPUMax).
|
|
Int("coreCount", len(temp.Cores)).
|
|
Int("nvmeCount", len(temp.NVMe)).
|
|
Int("gpuCount", len(temp.GPU)).
|
|
Int("smartCount", len(temp.SMART)).
|
|
Msg("Temperature data parsed successfully")
|
|
}
|
|
|
|
return temp, nil
|
|
}
|
|
|
|
// parseCPUTemps extracts CPU temperature data from a sensor chip
|
|
func (tc *TemperatureCollector) parseCPUTemps(chipMap map[string]interface{}, temp *models.Temperature) {
|
|
foundPackageTemp := false
|
|
var chipletTemps []float64 // Store AMD Tccd chiplet temps for fallback
|
|
|
|
for sensorName, sensorData := range chipMap {
|
|
sensorMap, ok := sensorData.(map[string]interface{})
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
sensorNameLower := strings.ToLower(sensorName)
|
|
|
|
// Look for Package id (Intel) or Tdie/Tctl (AMD control loop temperature)
|
|
if strings.Contains(sensorName, "Package id") ||
|
|
strings.Contains(sensorName, "Tdie") ||
|
|
strings.Contains(sensorNameLower, "tctl") {
|
|
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) {
|
|
temp.CPUPackage = tempVal
|
|
foundPackageTemp = true
|
|
if tempVal > temp.CPUMax {
|
|
temp.CPUMax = tempVal
|
|
}
|
|
log.Debug().
|
|
Str("sensor", sensorName).
|
|
Float64("temp", tempVal).
|
|
Msg("Found CPU package temperature")
|
|
}
|
|
}
|
|
|
|
// Look for AMD chiplet temperatures (Tccd1, Tccd2, etc.) as fallback
|
|
if strings.HasPrefix(sensorName, "Tccd") {
|
|
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
|
|
chipletTemps = append(chipletTemps, tempVal)
|
|
if tempVal > temp.CPUMax {
|
|
temp.CPUMax = tempVal
|
|
}
|
|
log.Debug().
|
|
Str("sensor", sensorName).
|
|
Float64("temp", tempVal).
|
|
Msg("Found AMD chiplet temperature")
|
|
}
|
|
}
|
|
|
|
// Look for SuperIO chip CPU temperature fields (CPUTIN, CPU Temperature, etc.)
|
|
if strings.Contains(sensorNameLower, "cputin") ||
|
|
strings.Contains(sensorNameLower, "cpu temperature") ||
|
|
(strings.Contains(sensorNameLower, "temp") && strings.Contains(sensorNameLower, "cpu")) {
|
|
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
|
|
if !foundPackageTemp {
|
|
temp.CPUPackage = tempVal
|
|
foundPackageTemp = true
|
|
}
|
|
if tempVal > temp.CPUMax {
|
|
temp.CPUMax = tempVal
|
|
}
|
|
log.Debug().
|
|
Str("sensor", sensorName).
|
|
Float64("temp", tempVal).
|
|
Msg("Found SuperIO CPU temperature")
|
|
}
|
|
}
|
|
|
|
// Look for individual cores
|
|
if strings.HasPrefix(sensorName, "Core ") {
|
|
coreNum := extractCoreNumber(sensorName)
|
|
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) {
|
|
temp.Cores = append(temp.Cores, models.CoreTemp{
|
|
Core: coreNum,
|
|
Temp: tempVal,
|
|
})
|
|
if tempVal > temp.CPUMax {
|
|
temp.CPUMax = tempVal
|
|
}
|
|
log.Debug().
|
|
Str("sensor", sensorName).
|
|
Int("core", coreNum).
|
|
Float64("temp", tempVal).
|
|
Msg("Found core temperature")
|
|
}
|
|
}
|
|
}
|
|
|
|
// If no package temperature found, use highest chiplet temp (AMD Ryzen)
|
|
if !foundPackageTemp && len(chipletTemps) > 0 {
|
|
for _, chipletTemp := range chipletTemps {
|
|
if chipletTemp > temp.CPUPackage {
|
|
temp.CPUPackage = chipletTemp
|
|
}
|
|
}
|
|
foundPackageTemp = true
|
|
log.Debug().
|
|
Float64("temp", temp.CPUPackage).
|
|
Msg("Using highest chiplet temperature as CPU package temperature")
|
|
}
|
|
|
|
// If no package temperature was found (e.g., Raspberry Pi), look for generic temp sensors
|
|
if !foundPackageTemp {
|
|
for sensorName, sensorData := range chipMap {
|
|
sensorMap, ok := sensorData.(map[string]interface{})
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
// Look for generic temperature sensors (e.g., "temp1" on Raspberry Pi)
|
|
if strings.HasPrefix(sensorName, "temp") || strings.HasPrefix(sensorName, "Temp") {
|
|
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
|
|
if temp.CPUPackage <= 0 {
|
|
temp.CPUPackage = tempVal
|
|
}
|
|
if tempVal > temp.CPUMax {
|
|
temp.CPUMax = tempVal
|
|
}
|
|
break // Use the first valid generic temp sensor
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// parseNVMeTemps extracts NVMe temperature data from a sensor chip
|
|
func (tc *TemperatureCollector) parseNVMeTemps(chipName string, chipMap map[string]interface{}, temp *models.Temperature) {
|
|
// Extract device name from chip name (e.g., "nvme-pci-0400" -> "nvme0")
|
|
device := "nvme" + strings.TrimPrefix(chipName, "nvme-pci-")
|
|
|
|
// Try "Composite" first (preferred sensor name for NVMe temps)
|
|
for sensorName, sensorData := range chipMap {
|
|
if !strings.Contains(sensorName, "Composite") {
|
|
continue
|
|
}
|
|
sensorMap, ok := sensorData.(map[string]interface{})
|
|
if !ok {
|
|
continue
|
|
}
|
|
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
|
|
temp.NVMe = append(temp.NVMe, models.NVMeTemp{
|
|
Device: device,
|
|
Temp: tempVal,
|
|
})
|
|
return
|
|
}
|
|
}
|
|
|
|
// Fall back to "Sensor 1" if no valid Composite found
|
|
for sensorName, sensorData := range chipMap {
|
|
if !strings.Contains(sensorName, "Sensor 1") {
|
|
continue
|
|
}
|
|
sensorMap, ok := sensorData.(map[string]interface{})
|
|
if !ok {
|
|
continue
|
|
}
|
|
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
|
|
temp.NVMe = append(temp.NVMe, models.NVMeTemp{
|
|
Device: device,
|
|
Temp: tempVal,
|
|
})
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// parseGPUTemps extracts GPU temperature data from a sensor chip
|
|
func (tc *TemperatureCollector) parseGPUTemps(chipName string, chipMap map[string]interface{}, temp *models.Temperature) {
|
|
gpuTemp := models.GPUTemp{
|
|
Device: chipName,
|
|
}
|
|
|
|
// AMD GPU sensors typically have: edge, junction (hotspot), mem
|
|
for sensorName, sensorData := range chipMap {
|
|
sensorMap, ok := sensorData.(map[string]interface{})
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
sensorLower := strings.ToLower(sensorName)
|
|
tempVal := extractTempInput(sensorMap)
|
|
|
|
if math.IsNaN(tempVal) || tempVal <= 0 {
|
|
continue
|
|
}
|
|
|
|
// Map sensor names to struct fields
|
|
if strings.Contains(sensorLower, "edge") {
|
|
gpuTemp.Edge = tempVal
|
|
} else if strings.Contains(sensorLower, "junction") || strings.Contains(sensorLower, "hotspot") {
|
|
gpuTemp.Junction = tempVal
|
|
} else if strings.Contains(sensorLower, "mem") {
|
|
gpuTemp.Mem = tempVal
|
|
}
|
|
}
|
|
|
|
// Only add GPU entry if we got at least one valid temperature
|
|
if gpuTemp.Edge > 0 || gpuTemp.Junction > 0 || gpuTemp.Mem > 0 {
|
|
temp.GPU = append(temp.GPU, gpuTemp)
|
|
log.Debug().
|
|
Str("device", chipName).
|
|
Float64("edge", gpuTemp.Edge).
|
|
Float64("junction", gpuTemp.Junction).
|
|
Float64("mem", gpuTemp.Mem).
|
|
Msg("Parsed GPU temperatures")
|
|
}
|
|
}
|
|
|
|
// parseNouveauGPUTemps extracts NVIDIA GPU temperature data from nouveau driver sensors
|
|
func (tc *TemperatureCollector) parseNouveauGPUTemps(chipName string, chipMap map[string]interface{}, temp *models.Temperature) {
|
|
gpuTemp := models.GPUTemp{
|
|
Device: chipName,
|
|
}
|
|
|
|
// Nouveau driver typically exposes "GPU core" sensor
|
|
for sensorName, sensorData := range chipMap {
|
|
sensorMap, ok := sensorData.(map[string]interface{})
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
sensorLower := strings.ToLower(sensorName)
|
|
tempVal := extractTempInput(sensorMap)
|
|
|
|
if math.IsNaN(tempVal) || tempVal <= 0 {
|
|
continue
|
|
}
|
|
|
|
// Nouveau typically has "GPU core" sensor - map to edge temperature
|
|
if strings.Contains(sensorLower, "gpu") || strings.Contains(sensorLower, "core") {
|
|
gpuTemp.Edge = tempVal
|
|
}
|
|
}
|
|
|
|
// Only add GPU entry if we got a valid temperature
|
|
if gpuTemp.Edge > 0 {
|
|
temp.GPU = append(temp.GPU, gpuTemp)
|
|
log.Debug().
|
|
Str("device", chipName).
|
|
Float64("edge", gpuTemp.Edge).
|
|
Msg("Parsed NVIDIA GPU (nouveau) temperature")
|
|
}
|
|
}
|
|
|
|
// extractTempInput extracts temperature value from sensor data
|
|
func extractTempInput(sensorMap map[string]interface{}) float64 {
|
|
// Look for temp*_input fields
|
|
for key, val := range sensorMap {
|
|
if strings.HasSuffix(key, "_input") {
|
|
switch v := val.(type) {
|
|
case float64:
|
|
return v
|
|
case int:
|
|
return float64(v)
|
|
case string:
|
|
if parsed, ok := parseStringTemperature(v); ok {
|
|
return parsed
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return math.NaN()
|
|
}
|
|
|
|
func parseStringTemperature(value string) (float64, bool) {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return 0, false
|
|
}
|
|
|
|
parsed, err := strconv.ParseFloat(value, 64)
|
|
if err != nil {
|
|
if _, scanErr := fmt.Sscanf(value, "%f", &parsed); scanErr != nil {
|
|
return 0, false
|
|
}
|
|
}
|
|
|
|
if math.Abs(parsed) >= 1000 {
|
|
parsed = parsed / 1000.0
|
|
}
|
|
|
|
return parsed, true
|
|
}
|
|
|
|
// extractCoreNumber extracts the core number from a sensor name like "Core 0"
|
|
func extractCoreNumber(name string) int {
|
|
parts := strings.Fields(name)
|
|
if len(parts) >= 2 {
|
|
if num, err := strconv.Atoi(parts[len(parts)-1]); err == nil {
|
|
return num
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// parseRPiTemperature parses Raspberry Pi temperature from /sys/class/thermal/thermal_zone0/temp
|
|
// Format: integer representing millidegrees Celsius (e.g., "45678" = 45.678°C)
|
|
func (tc *TemperatureCollector) parseRPiTemperature(output string) (*models.Temperature, error) {
|
|
millidegrees := strings.TrimSpace(output)
|
|
if millidegrees == "" {
|
|
return nil, fmt.Errorf("empty RPi temperature output")
|
|
}
|
|
|
|
tempMilliC, err := strconv.ParseFloat(millidegrees, 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse RPi temperature: %w", err)
|
|
}
|
|
|
|
// Convert millidegrees to degrees Celsius
|
|
tempC := tempMilliC / 1000.0
|
|
|
|
temp := &models.Temperature{
|
|
Available: true,
|
|
HasCPU: true,
|
|
CPUPackage: tempC,
|
|
CPUMax: tempC,
|
|
Cores: []models.CoreTemp{},
|
|
NVMe: []models.NVMeTemp{},
|
|
LastUpdate: time.Now(),
|
|
}
|
|
|
|
return temp, nil
|
|
}
|
|
|
|
// extractHostname extracts hostname/IP from a Proxmox host URL
|
|
func extractHostname(hostURL string) string {
|
|
// Remove protocol
|
|
host := strings.TrimPrefix(hostURL, "https://")
|
|
host = strings.TrimPrefix(host, "http://")
|
|
|
|
// Remove port
|
|
if idx := strings.Index(host, ":"); idx != -1 {
|
|
host = host[:idx]
|
|
}
|
|
|
|
// Remove path
|
|
if idx := strings.Index(host, "/"); idx != -1 {
|
|
host = host[:idx]
|
|
}
|
|
|
|
return host
|
|
}
|
|
|
|
func (tc *TemperatureCollector) ensureHostKey(ctx context.Context, host string) error {
|
|
if tc.hostKeys == nil {
|
|
return nil
|
|
}
|
|
if ctx == nil {
|
|
ctx = context.Background()
|
|
}
|
|
return tc.hostKeys.EnsureWithPort(ctx, host, tc.sshPort)
|
|
}
|