From 9c772a72007efbfc6f5f7e4e3418fd204126f47f Mon Sep 17 00:00:00 2001 From: "courtmanr@gmail.com" Date: Thu, 30 Jul 2026 18:01:37 +0100 Subject: [PATCH] feat(agent): collect NVIDIA telemetry on Windows --- docs/TEMPERATURE_MONITORING.md | 11 +++ .../v6/internal/subsystems/agent-lifecycle.md | 23 ++++--- internal/hostagent/agent.go | 5 ++ internal/hostagent/agent_metrics_test.go | 67 +++++++++++++++++++ internal/hostagent/agent_sensors_test.go | 38 ++++++++++- 5 files changed, 133 insertions(+), 11 deletions(-) diff --git a/docs/TEMPERATURE_MONITORING.md b/docs/TEMPERATURE_MONITORING.md index 92693a4b4..0b31a7412 100644 --- a/docs/TEMPERATURE_MONITORING.md +++ b/docs/TEMPERATURE_MONITORING.md @@ -21,6 +21,17 @@ Notes: - Temperatures appear automatically once the agent reports. - When a Proxmox host has recent usable agent temperature data, Pulse treats the agent as the source of truth and does not also try SSH temperature collection for that host. +## Windows NVIDIA GPU temperatures + +On Windows hosts with an NVIDIA driver, the unified agent uses the driver's +`nvidia-smi` executable to report GPU temperature, utilization, and VRAM +usage. No extra Pulse configuration is required; `nvidia-smi.exe` must be +available on the agent service's `PATH`. + +Windows does not provide a dependable built-in API for CPU, motherboard, or +storage temperature sensors. Pulse does not report those readings on Windows +unless and until a qualified driver-backed provider is available. + ## SSH-Based Collection (Fallback) Pulse can also collect temperatures by SSHing into each host that does not have usable agent temperature data. The SSH path runs the Pulse sensor wrapper when present, falls back to `sensors -j`, and can fall back again to `/sys/class/thermal/thermal_zone0/temp` when available (for example, on Raspberry Pi). diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 8a4bc660c..26a4dda42 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -1487,16 +1487,19 @@ the intentionally sparse public response. it must report Darwin `pmset` thermal and performance pressure as `sensors.thermalState` instead of inventing Celsius readings from unavailable Apple silicon sensor values. - Linux NVIDIA GPU telemetry belongs in that same host-agent sensor contract: - the runtime may supplement `lm-sensors` with a bounded `nvidia-smi` query - for direct GPU temperature, utilization, and VRAM readings, and may use that - query as a best-effort fallback when `lm-sensors` is unavailable. The report - must keep typed GPU readings in `sensors.gpu` while mapping only direct - `temperature.gpu` readings into existing `sensors.temperatureCelsius` - `gpu_nvidia_` keys for compatibility. Authenticated server ingest may - reduce those typed samples into bounded host-level GPU utilization, VRAM - pressure, and GPU temperature history on the existing agent identity, but - that monitoring projection does not change the report or command protocol. + Linux and Windows NVIDIA GPU telemetry belongs in that same host-agent + sensor contract: the runtime may use a bounded `nvidia-smi` query for direct + GPU temperature, utilization, and VRAM readings. Linux may supplement + `lm-sensors` with that query or use it as a best-effort fallback when + `lm-sensors` is unavailable; Windows may use it as its supported direct-GPU + provider without implying support for CPU, motherboard, or storage + temperatures. The report must keep typed GPU readings in `sensors.gpu` + while mapping only direct `temperature.gpu` readings into existing + `sensors.temperatureCelsius` `gpu_nvidia_` keys for compatibility. + Authenticated server ingest may reduce those typed samples into bounded + host-level GPU utilization, VRAM pressure, and GPU temperature history on + the existing agent identity, but that monitoring projection does not change + the report or command protocol. Neither side may infer lifecycle health, command authority, enrollment state, or GPU workload/process inventory from `nvidia-smi` output. Runtime RAID collection uses `/proc/mdstat` as the canonical discovery diff --git a/internal/hostagent/agent.go b/internal/hostagent/agent.go index ca5018302..5b01b9e53 100644 --- a/internal/hostagent/agent.go +++ b/internal/hostagent/agent.go @@ -1532,6 +1532,11 @@ func (a *Agent) collectTemperatures(ctx context.Context) agentshost.Sensors { return a.collectDarwinThermalState(ctx) case "freebsd": return a.collectFreeBSDTemperatures(ctx) + case "windows": + // Windows does not expose a reliable built-in CPU or motherboard + // temperature API. NVIDIA's driver does ship nvidia-smi, however, and + // the bounded query used on Linux is portable to Windows. + return a.collectNVIDIATemperatureSensors(ctx) default: return agentshost.Sensors{} } diff --git a/internal/hostagent/agent_metrics_test.go b/internal/hostagent/agent_metrics_test.go index e848f5732..d3da30cbd 100644 --- a/internal/hostagent/agent_metrics_test.go +++ b/internal/hostagent/agent_metrics_test.go @@ -574,6 +574,73 @@ func TestBuildReportIncludesNVIDIASMITemperaturesWhenLMSensorsUnavailable(t *tes } } +func TestBuildReportIncludesNVIDIASMITelemetryOnWindows(t *testing.T) { + mc := &mockCollector{ + goos: "windows", + nowFn: func() time.Time { return time.Date(2026, 7, 30, 18, 0, 0, 0, time.UTC) }, + hostInfoFn: func(context.Context) (*gohost.InfoStat, error) { + return &gohost.InfoStat{ + Hostname: "windows-gpu-node", + OS: "windows", + Platform: "Microsoft Windows 11 Pro", + HostID: "windows-gpu-node-id", + }, nil + }, + hostUptimeFn: func(context.Context) (uint64, error) { + return 3600, nil + }, + metricsFn: func(context.Context, []string) (hostmetrics.Snapshot, error) { + return hostmetrics.Snapshot{}, nil + }, + lookPathFn: func(file string) (string, error) { + if file == "nvidia-smi" { + return `C:\Windows\System32\nvidia-smi.exe`, nil + } + return "", os.ErrNotExist + }, + commandCombinedOutputFn: func(_ context.Context, name string, arg ...string) (string, error) { + if name != `C:\Windows\System32\nvidia-smi.exe` { + t.Fatalf("command name = %q, want Windows nvidia-smi path", name) + } + if len(arg) != 2 || arg[0] != "--query-gpu=index,name,temperature.gpu,utilization.gpu,memory.used,memory.total" || arg[1] != "--format=csv,noheader,nounits" { + t.Fatalf("command args = %#v, want NVIDIA stats query", arg) + } + return "0, NVIDIA GeForce RTX 3070, 57, 31, 2048, 8192\r\n", nil + }, + } + + agent, err := New(Config{ + AgentID: "windows-gpu-agent", + APIToken: "token", + LogLevel: -1, + Collector: mc, + }) + if err != nil { + t.Fatalf("New() failed: %v", err) + } + + report, err := agent.buildReport(context.Background()) + if err != nil { + t.Fatalf("buildReport failed: %v", err) + } + + if report.Host.Platform != "windows" { + t.Fatalf("report host platform = %q, want windows", report.Host.Platform) + } + if report.Sensors.TemperatureCelsius["gpu_nvidia_0"] != 57 { + t.Fatalf("Windows NVIDIA GPU temp = %v, want 57", report.Sensors.TemperatureCelsius["gpu_nvidia_0"]) + } + if len(report.Sensors.GPU) != 1 { + t.Fatalf("Windows GPU stats = %d, want 1: %+v", len(report.Sensors.GPU), report.Sensors.GPU) + } + if report.Sensors.GPU[0].UtilizationPercent == nil || *report.Sensors.GPU[0].UtilizationPercent != 31 { + t.Fatalf("Windows GPU utilization = %#v, want 31", report.Sensors.GPU[0].UtilizationPercent) + } + if report.Sensors.GPU[0].MemoryTotalBytes == nil || *report.Sensors.GPU[0].MemoryTotalBytes != 8192*1024*1024 { + t.Fatalf("Windows GPU memory total = %#v, want 8192 MiB", report.Sensors.GPU[0].MemoryTotalBytes) + } +} + func TestBuildReportUsesResolvedNASOSIdentity(t *testing.T) { fixedTime := time.Date(2026, time.April, 15, 12, 0, 0, 0, time.UTC) diff --git a/internal/hostagent/agent_sensors_test.go b/internal/hostagent/agent_sensors_test.go index 5643a843c..5e7a91e87 100644 --- a/internal/hostagent/agent_sensors_test.go +++ b/internal/hostagent/agent_sensors_test.go @@ -246,8 +246,44 @@ func TestAgent_queryNVIDIASMITemperatures_CommandNotInstalled(t *testing.T) { } } +func TestAgent_collectTemperatures_CollectsNVIDIAOnWindows(t *testing.T) { + mc := &mockCollector{ + goos: "windows", + lookPathFn: func(file string) (string, error) { + if file != "nvidia-smi" { + t.Fatalf("look path file = %q, want nvidia-smi", file) + } + return `C:\Windows\System32\nvidia-smi.exe`, nil + }, + commandCombinedOutputFn: func(_ context.Context, name string, arg ...string) (string, error) { + if name != `C:\Windows\System32\nvidia-smi.exe` { + t.Fatalf("command name = %q, want Windows nvidia-smi path", name) + } + if len(arg) != 2 || arg[0] != "--query-gpu=index,name,temperature.gpu,utilization.gpu,memory.used,memory.total" || arg[1] != "--format=csv,noheader,nounits" { + t.Fatalf("command args = %#v, want NVIDIA stats query", arg) + } + return "0, NVIDIA GeForce RTX 3070, 57, 31, 2048, 8192\r\n", nil + }, + } + a := &Agent{logger: zerolog.Nop(), collector: mc} + + got := a.collectTemperatures(context.Background()) + if got.TemperatureCelsius["gpu_nvidia_0"] != 57 { + t.Fatalf("Windows NVIDIA GPU temp = %v, want 57", got.TemperatureCelsius["gpu_nvidia_0"]) + } + if len(got.GPU) != 1 { + t.Fatalf("Windows GPU stats = %d, want 1: %+v", len(got.GPU), got.GPU) + } + if got.GPU[0].UtilizationPercent == nil || *got.GPU[0].UtilizationPercent != 31 { + t.Fatalf("Windows GPU utilization = %#v, want 31", got.GPU[0].UtilizationPercent) + } + if got.GPU[0].MemoryTotalBytes == nil || *got.GPU[0].MemoryTotalBytes != 8192*1024*1024 { + t.Fatalf("Windows GPU memory total = %#v, want 8192 MiB", got.GPU[0].MemoryTotalBytes) + } +} + func TestAgent_collectTemperatures_SkipsUnsupportedOS(t *testing.T) { - mc := &mockCollector{goos: "windows"} + mc := &mockCollector{goos: "plan9"} a := &Agent{logger: zerolog.Nop(), collector: mc} got := a.collectTemperatures(context.Background())