diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 6b57a74b6..d1072d0d4 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -425,12 +425,36 @@ func mergeHostAgentSMARTIntoDisks(disks []models.PhysicalDisk, nodes []models.No // Always merge SMART attributes from host agent if matched.Attributes != nil { updated[i].SmartAttributes = matched.Attributes + if updated[i].Wearout < 0 { + if derivedWearout := deriveWearoutFromSMARTAttributes(matched.Attributes); derivedWearout >= 0 { + updated[i].Wearout = derivedWearout + } + } + } + + if (updated[i].Health == "" || strings.EqualFold(updated[i].Health, "unknown")) && matched.Health != "" { + updated[i].Health = matched.Health } } return updated } +func deriveWearoutFromSMARTAttributes(attrs *models.SMARTAttributes) int { + if attrs == nil || attrs.PercentageUsed == nil { + return -1 + } + + used := *attrs.PercentageUsed + if used < 0 { + used = 0 + } + if used > 100 { + used = 100 + } + return 100 - used +} + func proxmoxDiskMatchesExclude(disk proxmox.Disk, excludePatterns []string) bool { if len(excludePatterns) == 0 { return false @@ -6990,15 +7014,30 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie // Mark this node as successfully polled polledNodes[node.Node] = true - allDisks = append(allDisks, - m.buildPhysicalDisksForNode( - inst, - node.Node, - disks, - diskExcludeByNode[node.Node], - time.Now(), - )..., + nodeDisks := m.buildPhysicalDisksForNode( + inst, + node.Node, + disks, + diskExcludeByNode[node.Node], + time.Now(), ) + nodeDisks = mergeHostAgentSMARTIntoDisks(nodeDisks, currentState.Nodes, currentState.Hosts) + for _, disk := range nodeDisks { + m.alertManager.CheckDiskHealth(inst, disk.Node, proxmox.Disk{ + DevPath: disk.DevPath, + Model: disk.Model, + Serial: disk.Serial, + Type: disk.Type, + Health: disk.Health, + Wearout: disk.Wearout, + Size: disk.Size, + RPM: disk.RPM, + Used: disk.Used, + WWN: disk.WWN, + }) + } + + allDisks = append(allDisks, nodeDisks...) } // Preserve existing disk data for nodes that weren't polled (offline or error) diff --git a/internal/monitoring/monitor_extra_coverage_test.go b/internal/monitoring/monitor_extra_coverage_test.go index 9e7d397b6..ba7116405 100644 --- a/internal/monitoring/monitor_extra_coverage_test.go +++ b/internal/monitoring/monitor_extra_coverage_test.go @@ -77,6 +77,54 @@ func TestMonitor_GetStateRefreshesAlertSnapshots(t *testing.T) { } } +func TestMergeHostAgentSMARTIntoDisksDerivesWearoutAndHealth(t *testing.T) { + disks := []models.PhysicalDisk{{ + ID: "disk-1", + Node: "node1", + Instance: "inst", + DevPath: "/dev/sda", + Model: "RAID SSD", + Serial: "raid-serial-1", + Health: "UNKNOWN", + Wearout: -1, + }} + nodes := []models.Node{{ + ID: "node-1", + Name: "node1", + Instance: "inst", + LinkedHostAgentID: "host-1", + }} + used := 7 + hosts := []models.Host{{ + ID: "host-1", + Sensors: models.HostSensorSummary{ + SMART: []models.HostDiskSMART{{ + Device: "/dev/bsg/sssraid0 [sssraid,0,1]", + Model: "RAID SSD", + Serial: "raid-serial-1", + Health: "PASSED", + Attributes: &models.SMARTAttributes{ + PercentageUsed: &used, + }, + }}, + }, + }} + + merged := mergeHostAgentSMARTIntoDisks(disks, nodes, hosts) + if len(merged) != 1 { + t.Fatalf("expected 1 merged disk, got %#v", merged) + } + if merged[0].Wearout != 93 { + t.Fatalf("expected derived wearout 93, got %#v", merged[0]) + } + if merged[0].Health != "PASSED" { + t.Fatalf("expected SMART health to fill UNKNOWN disk health, got %#v", merged[0]) + } + if merged[0].SmartAttributes == nil || merged[0].SmartAttributes.PercentageUsed == nil || *merged[0].SmartAttributes.PercentageUsed != 7 { + t.Fatalf("expected SMART attributes to be merged, got %#v", merged[0].SmartAttributes) + } +} + func TestMonitor_Stop_Extra(t *testing.T) { m := &Monitor{} m.Stop() diff --git a/internal/smartctl/collector.go b/internal/smartctl/collector.go index ab2ae7bbe..614c18820 100644 --- a/internal/smartctl/collector.go +++ b/internal/smartctl/collector.go @@ -128,6 +128,22 @@ type lsblkDevice struct { Subsystems string `json:"subsystems"` } +type smartctlTarget struct { + Path string + DeviceType string +} + +func (t smartctlTarget) displayName() string { + name := filepath.Base(strings.TrimSpace(t.Path)) + if name == "" || name == "." || name == string(filepath.Separator) { + name = strings.TrimSpace(t.Path) + } + if t.DeviceType == "" { + return name + } + return name + " [" + t.DeviceType + "]" +} + var linuxSMARTVirtualPrefixes = []string{ "dm-", "drbd", @@ -167,22 +183,21 @@ var linuxSMARTVirtualSubsystemTokens = []string{ // CollectLocal collects S.M.A.R.T. data from all local block devices. // The diskExclude parameter specifies patterns for devices to skip (e.g., "sda", "/dev/nvme*", "*cache*"). func CollectLocal(ctx context.Context, diskExclude []string) ([]DiskSMART, error) { - // List block devices - devices, err := listBlockDevices(ctx, diskExclude) + targets, err := listSMARTTargets(ctx, diskExclude) if err != nil { log.Debug().Err(err).Msg("Failed to list block devices for SMART collection") return nil, err } - if len(devices) == 0 { + if len(targets) == 0 { return nil, nil } var results []DiskSMART - for _, dev := range devices { - smart, err := collectDeviceSMART(ctx, dev) + for _, target := range targets { + smart, err := collectSMARTTarget(ctx, target) if err != nil { - log.Debug().Err(err).Str("device", dev).Msg("Failed to collect SMART data for device") + log.Debug().Err(err).Str("device", target.displayName()).Msg("Failed to collect SMART data for device") continue } if smart != nil { @@ -193,6 +208,129 @@ func CollectLocal(ctx context.Context, diskExclude []string) ([]DiskSMART, error return results, nil } +func listSMARTTargets(ctx context.Context, diskExclude []string) ([]smartctlTarget, error) { + if runtimeGOOS == "linux" { + return listSMARTTargetsLinux(ctx, diskExclude) + } + + devices, err := listBlockDevices(ctx, diskExclude) + if err != nil { + return nil, err + } + return smartctlTargetsFromDevices(devices), nil +} + +func smartctlTargetsFromDevices(devices []string) []smartctlTarget { + if len(devices) == 0 { + return nil + } + targets := make([]smartctlTarget, 0, len(devices)) + for _, device := range devices { + targets = append(targets, smartctlTarget{Path: device}) + } + return targets +} + +func listSMARTTargetsLinux(ctx context.Context, diskExclude []string) ([]smartctlTarget, error) { + targets, err := listSMARTTargetsLinuxFromScanOpen(ctx, diskExclude) + if err == nil && len(targets) > 0 { + return targets, nil + } + if err != nil { + log.Debug().Err(err).Msg("Failed to enumerate Linux SMART targets via smartctl --scan-open, falling back to block device discovery") + } + + devices, fallbackErr := listBlockDevicesLinux(ctx, diskExclude) + if fallbackErr != nil { + return nil, fallbackErr + } + return smartctlTargetsFromDevices(devices), nil +} + +func listSMARTTargetsLinuxFromScanOpen(ctx context.Context, diskExclude []string) ([]smartctlTarget, error) { + smartctlPath, err := execLookPath("smartctl") + if err != nil { + return nil, err + } + + output, err := runCommandOutput(ctx, smartctlPath, "--scan-open") + if err != nil { + return nil, err + } + + return parseSmartctlScanOpenTargets(output, diskExclude), nil +} + +func parseSmartctlScanOpenTargets(output []byte, diskExclude []string) []smartctlTarget { + lines := strings.Split(string(output), "\n") + targets := make([]smartctlTarget, 0, len(lines)) + typedByPath := make(map[string]bool) + seen := make(map[string]struct{}) + + for _, rawLine := range lines { + line := strings.TrimSpace(rawLine) + if line == "" { + continue + } + if idx := strings.Index(line, "#"); idx >= 0 { + line = strings.TrimSpace(line[:idx]) + } + if line == "" { + continue + } + + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + + path := strings.TrimSpace(fields[0]) + if path == "" || (!strings.HasPrefix(path, "/") && !strings.HasPrefix(path, "-")) { + continue + } + + deviceType := "" + for i := 1; i < len(fields)-1; i++ { + if fields[i] == "-d" { + deviceType = strings.TrimSpace(fields[i+1]) + break + } + } + + name := filepath.Base(path) + if matchesDeviceExclude(name, path, diskExclude) { + continue + } + + key := path + "\x00" + deviceType + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + if deviceType != "" { + typedByPath[path] = true + } + + targets = append(targets, smartctlTarget{ + Path: path, + DeviceType: deviceType, + }) + } + + if len(targets) == 0 { + return nil + } + + filtered := make([]smartctlTarget, 0, len(targets)) + for _, target := range targets { + if target.DeviceType == "" && typedByPath[target.Path] { + continue + } + filtered = append(filtered, target) + } + return filtered +} + // listBlockDevices returns a list of block devices suitable for SMART queries. // Devices matching any of the diskExclude patterns are skipped. func listBlockDevices(ctx context.Context, diskExclude []string) ([]string, error) { @@ -540,6 +678,10 @@ func matchesDeviceExclude(name, devicePath string, excludePatterns []string) boo // collectDeviceSMART runs smartctl on a single device and parses the result. func collectDeviceSMART(ctx context.Context, device string) (*DiskSMART, error) { + return collectSMARTTarget(ctx, smartctlTarget{Path: device}) +} + +func collectSMARTTarget(ctx context.Context, target smartctlTarget) (*DiskSMART, error) { // Use timeout to avoid hanging on slow/unresponsive disks cmdCtx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() @@ -550,7 +692,7 @@ func collectDeviceSMART(ctx context.Context, device string) (*DiskSMART, error) return nil, err } - attempts := smartctlProbeAttempts(device) + attempts := smartctlProbeAttempts(target) var firstParsed *DiskSMART var firstStandby *DiskSMART var lastErr error @@ -563,11 +705,11 @@ func collectDeviceSMART(ctx context.Context, device string) (*DiskSMART, error) if exitErr, ok := err.(*exec.ExitError); ok { if exitErr.ExitCode() == smartctlStandbyExitStatus && len(output) == 0 { standbyResult := &DiskSMART{ - Device: filepath.Base(device), + Device: target.displayName(), Standby: true, LastUpdated: timeNow(), } - if runtimeGOOS == "freebsd" && i < len(attempts)-1 { + if runtimeGOOS == "freebsd" && i < len(attempts)-1 && target.DeviceType == "" { if firstStandby == nil { firstStandby = standbyResult } @@ -585,7 +727,7 @@ func collectDeviceSMART(ctx context.Context, device string) (*DiskSMART, error) } } - result, parseErr := parseSMARTOutput(output, device) + result, parseErr := parseSMARTOutput(output, target) if parseErr != nil { lastErr = parseErr continue @@ -593,7 +735,7 @@ func collectDeviceSMART(ctx context.Context, device string) (*DiskSMART, error) if firstParsed == nil { firstParsed = result } - if !shouldRetryFreeBSDSMART(device, result, i, len(attempts)) { + if !shouldRetryFreeBSDSMART(target.Path, result, i, len(attempts)) { log.Debug(). Str("device", result.Device). Str("model", result.Model). @@ -628,7 +770,14 @@ func collectDeviceSMART(ctx context.Context, device string) (*DiskSMART, error) return nil, nil } -func smartctlProbeAttempts(device string) [][]string { +func smartctlProbeAttempts(target smartctlTarget) [][]string { + device := target.Path + if target.DeviceType != "" { + return [][]string{ + smartctlArgs(device, target.DeviceType), + } + } + if runtimeGOOS == "freebsd" { deviceTypes := freeBSDSmartctlDeviceTypes(filepath.Base(device)) if len(deviceTypes) > 0 { @@ -681,14 +830,14 @@ func shouldRetryFreeBSDSMART(device string, result *DiskSMART, attemptIndex, att return len(freeBSDSmartctlDeviceTypes(filepath.Base(device))) > 0 } -func parseSMARTOutput(output []byte, device string) (*DiskSMART, error) { +func parseSMARTOutput(output []byte, target smartctlTarget) (*DiskSMART, error) { var smartData smartctlJSON if err := json.Unmarshal(output, &smartData); err != nil { return nil, err } result := &DiskSMART{ - Device: filepath.Base(device), + Device: target.displayName(), Model: smartData.ModelName, Serial: smartData.SerialNumber, Type: detectDiskType(smartData), diff --git a/internal/smartctl/collector_coverage_test.go b/internal/smartctl/collector_coverage_test.go index 38838d3a4..f795f7d19 100644 --- a/internal/smartctl/collector_coverage_test.go +++ b/internal/smartctl/collector_coverage_test.go @@ -117,6 +117,91 @@ func TestListBlockDevices(t *testing.T) { } } +func TestParseSmartctlScanOpenTargets(t *testing.T) { + output := []byte(strings.Join([]string{ + `/dev/sda -d sat # /dev/sda, ATA device`, + `/dev/sda # duplicate plain path should be ignored when typed target exists`, + `/dev/bsg/sssraid0 -d sssraid,0,1 # controller-backed disk`, + `/dev/sdb -d megaraid,0 # megaraid slot 0`, + `/dev/sdb -d megaraid,1 # megaraid slot 1`, + `/dev/sdc # plain disk`, + ``, + }, "\n")) + + targets := parseSmartctlScanOpenTargets(output, []string{"sdc"}) + if len(targets) != 4 { + t.Fatalf("expected 4 targets, got %#v", targets) + } + + if targets[0].Path != "/dev/sda" || targets[0].DeviceType != "sat" { + t.Fatalf("unexpected first target: %#v", targets[0]) + } + if targets[1].Path != "/dev/bsg/sssraid0" || targets[1].DeviceType != "sssraid,0,1" { + t.Fatalf("unexpected second target: %#v", targets[1]) + } + if targets[2].DeviceType != "megaraid,0" || targets[3].DeviceType != "megaraid,1" { + t.Fatalf("expected megaraid targets to be preserved separately, got %#v", targets) + } +} + +func TestCollectLocalUsesSmartctlScanOpenTargetsOnLinux(t *testing.T) { + origRun := runCommandOutput + origLook := execLookPath + origGOOS := runtimeGOOS + t.Cleanup(func() { + runCommandOutput = origRun + execLookPath = origLook + runtimeGOOS = origGOOS + }) + + runtimeGOOS = "linux" + execLookPath = func(string) (string, error) { return "smartctl", nil } + + var seenArgs [][]string + runCommandOutput = func(ctx context.Context, name string, args ...string) ([]byte, error) { + seenArgs = append(seenArgs, append([]string(nil), args...)) + + if len(args) == 1 && args[0] == "--scan-open" { + return []byte("/dev/sda -d megaraid,7 # RAID-backed SSD\n"), nil + } + + payload := smartctlJSON{ + ModelName: "RAID SSD", + SerialNumber: "raid-ssd-1", + } + payload.Device.Protocol = "NVMe" + payload.SmartStatus = &struct { + Passed bool `json:"passed"` + }{Passed: true} + payload.NVMeSmartHealthInformationLog.PercentageUsed = 6 + payload.NVMeSmartHealthInformationLog.AvailableSpare = 94 + out, _ := json.Marshal(payload) + return out, nil + } + + result, err := CollectLocal(context.Background(), nil) + if err != nil { + t.Fatalf("CollectLocal error: %v", err) + } + if len(result) != 1 { + t.Fatalf("expected 1 result, got %#v", result) + } + if result[0].Device != "sda [megaraid,7]" { + t.Fatalf("unexpected device label: %#v", result[0]) + } + if result[0].Attributes == nil || result[0].Attributes.PercentageUsed == nil || *result[0].Attributes.PercentageUsed != 6 { + t.Fatalf("expected percentage_used SMART data, got %#v", result[0].Attributes) + } + + if len(seenArgs) < 2 { + t.Fatalf("expected scan and probe calls, got %v", seenArgs) + } + probe := seenArgs[1] + if len(probe) < 3 || probe[0] != "-d" || probe[1] != "megaraid,7" || probe[len(probe)-1] != "/dev/sda" { + t.Fatalf("expected typed smartctl probe, got %v", probe) + } +} + func TestListBlockDevicesFreeBSD(t *testing.T) { origRun := runCommandOutput origReadDir := readDir diff --git a/internal/smartctl/collector_test.go b/internal/smartctl/collector_test.go index 4e44159cd..1a48220cc 100644 --- a/internal/smartctl/collector_test.go +++ b/internal/smartctl/collector_test.go @@ -445,7 +445,7 @@ func TestParseSMARTOutputStandbyPowerMode(t *testing.T) { t.Fatalf("marshal payload: %v", err) } - result, err := parseSMARTOutput(out, "/dev/ada0") + result, err := parseSMARTOutput(out, smartctlTarget{Path: "/dev/ada0"}) if err != nil { t.Fatalf("unexpected error: %v", err) }