feat(agent): collect NVIDIA telemetry on Windows

This commit is contained in:
courtmanr@gmail.com
2026-07-30 18:01:37 +01:00
parent e13f42667e
commit 9c772a7200
5 changed files with 133 additions and 11 deletions
+11
View File
@@ -21,6 +21,17 @@ Notes:
- Temperatures appear automatically once the agent reports.
- When a Proxmox host has recent usable agent temperature data, Pulse treats the agent as the source of truth and does not also try SSH temperature collection for that host.
## Windows NVIDIA GPU temperatures
On Windows hosts with an NVIDIA driver, the unified agent uses the driver's
`nvidia-smi` executable to report GPU temperature, utilization, and VRAM
usage. No extra Pulse configuration is required; `nvidia-smi.exe` must be
available on the agent service's `PATH`.
Windows does not provide a dependable built-in API for CPU, motherboard, or
storage temperature sensors. Pulse does not report those readings on Windows
unless and until a qualified driver-backed provider is available.
## SSH-Based Collection (Fallback)
Pulse can also collect temperatures by SSHing into each host that does not have usable agent temperature data. The SSH path runs the Pulse sensor wrapper when present, falls back to `sensors -j`, and can fall back again to `/sys/class/thermal/thermal_zone0/temp` when available (for example, on Raspberry Pi).
@@ -1487,16 +1487,19 @@ the intentionally sparse public response.
it must report Darwin `pmset` thermal and performance pressure as
`sensors.thermalState` instead of inventing Celsius readings from unavailable
Apple silicon sensor values.
Linux NVIDIA GPU telemetry belongs in that same host-agent sensor contract:
the runtime may supplement `lm-sensors` with a bounded `nvidia-smi` query
for direct GPU temperature, utilization, and VRAM readings, and may use that
query as a best-effort fallback when `lm-sensors` is unavailable. The report
must keep typed GPU readings in `sensors.gpu` while mapping only direct
`temperature.gpu` readings into existing `sensors.temperatureCelsius`
`gpu_nvidia_<index>` keys for compatibility. Authenticated server ingest may
reduce those typed samples into bounded host-level GPU utilization, VRAM
pressure, and GPU temperature history on the existing agent identity, but
that monitoring projection does not change the report or command protocol.
Linux and Windows NVIDIA GPU telemetry belongs in that same host-agent
sensor contract: the runtime may use a bounded `nvidia-smi` query for direct
GPU temperature, utilization, and VRAM readings. Linux may supplement
`lm-sensors` with that query or use it as a best-effort fallback when
`lm-sensors` is unavailable; Windows may use it as its supported direct-GPU
provider without implying support for CPU, motherboard, or storage
temperatures. The report must keep typed GPU readings in `sensors.gpu`
while mapping only direct `temperature.gpu` readings into existing
`sensors.temperatureCelsius` `gpu_nvidia_<index>` keys for compatibility.
Authenticated server ingest may reduce those typed samples into bounded
host-level GPU utilization, VRAM pressure, and GPU temperature history on
the existing agent identity, but that monitoring projection does not change
the report or command protocol.
Neither side may infer lifecycle health, command authority, enrollment
state, or GPU workload/process inventory from `nvidia-smi` output.
Runtime RAID collection uses `/proc/mdstat` as the canonical discovery
+5
View File
@@ -1532,6 +1532,11 @@ func (a *Agent) collectTemperatures(ctx context.Context) agentshost.Sensors {
return a.collectDarwinThermalState(ctx)
case "freebsd":
return a.collectFreeBSDTemperatures(ctx)
case "windows":
// Windows does not expose a reliable built-in CPU or motherboard
// temperature API. NVIDIA's driver does ship nvidia-smi, however, and
// the bounded query used on Linux is portable to Windows.
return a.collectNVIDIATemperatureSensors(ctx)
default:
return agentshost.Sensors{}
}
+67
View File
@@ -574,6 +574,73 @@ func TestBuildReportIncludesNVIDIASMITemperaturesWhenLMSensorsUnavailable(t *tes
}
}
func TestBuildReportIncludesNVIDIASMITelemetryOnWindows(t *testing.T) {
mc := &mockCollector{
goos: "windows",
nowFn: func() time.Time { return time.Date(2026, 7, 30, 18, 0, 0, 0, time.UTC) },
hostInfoFn: func(context.Context) (*gohost.InfoStat, error) {
return &gohost.InfoStat{
Hostname: "windows-gpu-node",
OS: "windows",
Platform: "Microsoft Windows 11 Pro",
HostID: "windows-gpu-node-id",
}, nil
},
hostUptimeFn: func(context.Context) (uint64, error) {
return 3600, nil
},
metricsFn: func(context.Context, []string) (hostmetrics.Snapshot, error) {
return hostmetrics.Snapshot{}, nil
},
lookPathFn: func(file string) (string, error) {
if file == "nvidia-smi" {
return `C:\Windows\System32\nvidia-smi.exe`, nil
}
return "", os.ErrNotExist
},
commandCombinedOutputFn: func(_ context.Context, name string, arg ...string) (string, error) {
if name != `C:\Windows\System32\nvidia-smi.exe` {
t.Fatalf("command name = %q, want Windows nvidia-smi path", name)
}
if len(arg) != 2 || arg[0] != "--query-gpu=index,name,temperature.gpu,utilization.gpu,memory.used,memory.total" || arg[1] != "--format=csv,noheader,nounits" {
t.Fatalf("command args = %#v, want NVIDIA stats query", arg)
}
return "0, NVIDIA GeForce RTX 3070, 57, 31, 2048, 8192\r\n", nil
},
}
agent, err := New(Config{
AgentID: "windows-gpu-agent",
APIToken: "token",
LogLevel: -1,
Collector: mc,
})
if err != nil {
t.Fatalf("New() failed: %v", err)
}
report, err := agent.buildReport(context.Background())
if err != nil {
t.Fatalf("buildReport failed: %v", err)
}
if report.Host.Platform != "windows" {
t.Fatalf("report host platform = %q, want windows", report.Host.Platform)
}
if report.Sensors.TemperatureCelsius["gpu_nvidia_0"] != 57 {
t.Fatalf("Windows NVIDIA GPU temp = %v, want 57", report.Sensors.TemperatureCelsius["gpu_nvidia_0"])
}
if len(report.Sensors.GPU) != 1 {
t.Fatalf("Windows GPU stats = %d, want 1: %+v", len(report.Sensors.GPU), report.Sensors.GPU)
}
if report.Sensors.GPU[0].UtilizationPercent == nil || *report.Sensors.GPU[0].UtilizationPercent != 31 {
t.Fatalf("Windows GPU utilization = %#v, want 31", report.Sensors.GPU[0].UtilizationPercent)
}
if report.Sensors.GPU[0].MemoryTotalBytes == nil || *report.Sensors.GPU[0].MemoryTotalBytes != 8192*1024*1024 {
t.Fatalf("Windows GPU memory total = %#v, want 8192 MiB", report.Sensors.GPU[0].MemoryTotalBytes)
}
}
func TestBuildReportUsesResolvedNASOSIdentity(t *testing.T) {
fixedTime := time.Date(2026, time.April, 15, 12, 0, 0, 0, time.UTC)
+37 -1
View File
@@ -246,8 +246,44 @@ func TestAgent_queryNVIDIASMITemperatures_CommandNotInstalled(t *testing.T) {
}
}
func TestAgent_collectTemperatures_CollectsNVIDIAOnWindows(t *testing.T) {
mc := &mockCollector{
goos: "windows",
lookPathFn: func(file string) (string, error) {
if file != "nvidia-smi" {
t.Fatalf("look path file = %q, want nvidia-smi", file)
}
return `C:\Windows\System32\nvidia-smi.exe`, nil
},
commandCombinedOutputFn: func(_ context.Context, name string, arg ...string) (string, error) {
if name != `C:\Windows\System32\nvidia-smi.exe` {
t.Fatalf("command name = %q, want Windows nvidia-smi path", name)
}
if len(arg) != 2 || arg[0] != "--query-gpu=index,name,temperature.gpu,utilization.gpu,memory.used,memory.total" || arg[1] != "--format=csv,noheader,nounits" {
t.Fatalf("command args = %#v, want NVIDIA stats query", arg)
}
return "0, NVIDIA GeForce RTX 3070, 57, 31, 2048, 8192\r\n", nil
},
}
a := &Agent{logger: zerolog.Nop(), collector: mc}
got := a.collectTemperatures(context.Background())
if got.TemperatureCelsius["gpu_nvidia_0"] != 57 {
t.Fatalf("Windows NVIDIA GPU temp = %v, want 57", got.TemperatureCelsius["gpu_nvidia_0"])
}
if len(got.GPU) != 1 {
t.Fatalf("Windows GPU stats = %d, want 1: %+v", len(got.GPU), got.GPU)
}
if got.GPU[0].UtilizationPercent == nil || *got.GPU[0].UtilizationPercent != 31 {
t.Fatalf("Windows GPU utilization = %#v, want 31", got.GPU[0].UtilizationPercent)
}
if got.GPU[0].MemoryTotalBytes == nil || *got.GPU[0].MemoryTotalBytes != 8192*1024*1024 {
t.Fatalf("Windows GPU memory total = %#v, want 8192 MiB", got.GPU[0].MemoryTotalBytes)
}
}
func TestAgent_collectTemperatures_SkipsUnsupportedOS(t *testing.T) {
mc := &mockCollector{goos: "windows"}
mc := &mockCollector{goos: "plan9"}
a := &Agent{logger: zerolog.Nop(), collector: mc}
got := a.collectTemperatures(context.Background())