Discover controller-backed SMART wearout paths (#1368)

internal/monitoring/monitor.go

@@ -425,12 +425,36 @@ func mergeHostAgentSMARTIntoDisks(disks []models.PhysicalDisk, nodes []models.No
 		// Always merge SMART attributes from host agent
 		if matched.Attributes != nil {
 			updated[i].SmartAttributes = matched.Attributes
+			if updated[i].Wearout < 0 {
+				if derivedWearout := deriveWearoutFromSMARTAttributes(matched.Attributes); derivedWearout >= 0 {
+					updated[i].Wearout = derivedWearout
+				}
+			}
+		}
+
+		if (updated[i].Health == "" || strings.EqualFold(updated[i].Health, "unknown")) && matched.Health != "" {
+			updated[i].Health = matched.Health
 		}
 	}
 
 	return updated
 }
 
+func deriveWearoutFromSMARTAttributes(attrs *models.SMARTAttributes) int {
+	if attrs == nil || attrs.PercentageUsed == nil {
+		return -1
+	}
+
+	used := *attrs.PercentageUsed
+	if used < 0 {
+		used = 0
+	}
+	if used > 100 {
+		used = 100
+	}
+	return 100 - used
+}
+
 func proxmoxDiskMatchesExclude(disk proxmox.Disk, excludePatterns []string) bool {
 	if len(excludePatterns) == 0 {
 		return false
@@ -6990,15 +7014,30 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
 					// Mark this node as successfully polled
 					polledNodes[node.Node] = true
 
-					allDisks = append(allDisks,
-						m.buildPhysicalDisksForNode(
-							inst,
-							node.Node,
-							disks,
-							diskExcludeByNode[node.Node],
-							time.Now(),
-						)...,
+					nodeDisks := m.buildPhysicalDisksForNode(
+						inst,
+						node.Node,
+						disks,
+						diskExcludeByNode[node.Node],
+						time.Now(),
 					)
+					nodeDisks = mergeHostAgentSMARTIntoDisks(nodeDisks, currentState.Nodes, currentState.Hosts)
+					for _, disk := range nodeDisks {
+						m.alertManager.CheckDiskHealth(inst, disk.Node, proxmox.Disk{
+							DevPath: disk.DevPath,
+							Model:   disk.Model,
+							Serial:  disk.Serial,
+							Type:    disk.Type,
+							Health:  disk.Health,
+							Wearout: disk.Wearout,
+							Size:    disk.Size,
+							RPM:     disk.RPM,
+							Used:    disk.Used,
+							WWN:     disk.WWN,
+						})
+					}
+
+					allDisks = append(allDisks, nodeDisks...)
 				}
 
 				// Preserve existing disk data for nodes that weren't polled (offline or error)

internal/monitoring/monitor_extra_coverage_test.go

@@ -77,6 +77,54 @@ func TestMonitor_GetStateRefreshesAlertSnapshots(t *testing.T) {
 	}
 }
 
+func TestMergeHostAgentSMARTIntoDisksDerivesWearoutAndHealth(t *testing.T) {
+	disks := []models.PhysicalDisk{{
+		ID:       "disk-1",
+		Node:     "node1",
+		Instance: "inst",
+		DevPath:  "/dev/sda",
+		Model:    "RAID SSD",
+		Serial:   "raid-serial-1",
+		Health:   "UNKNOWN",
+		Wearout:  -1,
+	}}
+	nodes := []models.Node{{
+		ID:                "node-1",
+		Name:              "node1",
+		Instance:          "inst",
+		LinkedHostAgentID: "host-1",
+	}}
+	used := 7
+	hosts := []models.Host{{
+		ID: "host-1",
+		Sensors: models.HostSensorSummary{
+			SMART: []models.HostDiskSMART{{
+				Device: "/dev/bsg/sssraid0 [sssraid,0,1]",
+				Model:  "RAID SSD",
+				Serial: "raid-serial-1",
+				Health: "PASSED",
+				Attributes: &models.SMARTAttributes{
+					PercentageUsed: &used,
+				},
+			}},
+		},
+	}}
+
+	merged := mergeHostAgentSMARTIntoDisks(disks, nodes, hosts)
+	if len(merged) != 1 {
+		t.Fatalf("expected 1 merged disk, got %#v", merged)
+	}
+	if merged[0].Wearout != 93 {
+		t.Fatalf("expected derived wearout 93, got %#v", merged[0])
+	}
+	if merged[0].Health != "PASSED" {
+		t.Fatalf("expected SMART health to fill UNKNOWN disk health, got %#v", merged[0])
+	}
+	if merged[0].SmartAttributes == nil || merged[0].SmartAttributes.PercentageUsed == nil || *merged[0].SmartAttributes.PercentageUsed != 7 {
+		t.Fatalf("expected SMART attributes to be merged, got %#v", merged[0].SmartAttributes)
+	}
+}
+
 func TestMonitor_Stop_Extra(t *testing.T) {
 	m := &Monitor{}
 	m.Stop()

internal/smartctl/collector.go

@@ -128,6 +128,22 @@ type lsblkDevice struct {
 	Subsystems string `json:"subsystems"`
 }
 
+type smartctlTarget struct {
+	Path       string
+	DeviceType string
+}
+
+func (t smartctlTarget) displayName() string {
+	name := filepath.Base(strings.TrimSpace(t.Path))
+	if name == "" || name == "." || name == string(filepath.Separator) {
+		name = strings.TrimSpace(t.Path)
+	}
+	if t.DeviceType == "" {
+		return name
+	}
+	return name + " [" + t.DeviceType + "]"
+}
+
 var linuxSMARTVirtualPrefixes = []string{
 	"dm-",
 	"drbd",
@@ -167,22 +183,21 @@ var linuxSMARTVirtualSubsystemTokens = []string{
 // CollectLocal collects S.M.A.R.T. data from all local block devices.
 // The diskExclude parameter specifies patterns for devices to skip (e.g., "sda", "/dev/nvme*", "*cache*").
 func CollectLocal(ctx context.Context, diskExclude []string) ([]DiskSMART, error) {
-	// List block devices
-	devices, err := listBlockDevices(ctx, diskExclude)
+	targets, err := listSMARTTargets(ctx, diskExclude)
 	if err != nil {
 		log.Debug().Err(err).Msg("Failed to list block devices for SMART collection")
 		return nil, err
 	}
 
-	if len(devices) == 0 {
+	if len(targets) == 0 {
 		return nil, nil
 	}
 
 	var results []DiskSMART
-	for _, dev := range devices {
-		smart, err := collectDeviceSMART(ctx, dev)
+	for _, target := range targets {
+		smart, err := collectSMARTTarget(ctx, target)
 		if err != nil {
-			log.Debug().Err(err).Str("device", dev).Msg("Failed to collect SMART data for device")
+			log.Debug().Err(err).Str("device", target.displayName()).Msg("Failed to collect SMART data for device")
 			continue
 		}
 		if smart != nil {
@@ -193,6 +208,129 @@ func CollectLocal(ctx context.Context, diskExclude []string) ([]DiskSMART, error
 	return results, nil
 }
 
+func listSMARTTargets(ctx context.Context, diskExclude []string) ([]smartctlTarget, error) {
+	if runtimeGOOS == "linux" {
+		return listSMARTTargetsLinux(ctx, diskExclude)
+	}
+
+	devices, err := listBlockDevices(ctx, diskExclude)
+	if err != nil {
+		return nil, err
+	}
+	return smartctlTargetsFromDevices(devices), nil
+}
+
+func smartctlTargetsFromDevices(devices []string) []smartctlTarget {
+	if len(devices) == 0 {
+		return nil
+	}
+	targets := make([]smartctlTarget, 0, len(devices))
+	for _, device := range devices {
+		targets = append(targets, smartctlTarget{Path: device})
+	}
+	return targets
+}
+
+func listSMARTTargetsLinux(ctx context.Context, diskExclude []string) ([]smartctlTarget, error) {
+	targets, err := listSMARTTargetsLinuxFromScanOpen(ctx, diskExclude)
+	if err == nil && len(targets) > 0 {
+		return targets, nil
+	}
+	if err != nil {
+		log.Debug().Err(err).Msg("Failed to enumerate Linux SMART targets via smartctl --scan-open, falling back to block device discovery")
+	}
+
+	devices, fallbackErr := listBlockDevicesLinux(ctx, diskExclude)
+	if fallbackErr != nil {
+		return nil, fallbackErr
+	}
+	return smartctlTargetsFromDevices(devices), nil
+}
+
+func listSMARTTargetsLinuxFromScanOpen(ctx context.Context, diskExclude []string) ([]smartctlTarget, error) {
+	smartctlPath, err := execLookPath("smartctl")
+	if err != nil {
+		return nil, err
+	}
+
+	output, err := runCommandOutput(ctx, smartctlPath, "--scan-open")
+	if err != nil {
+		return nil, err
+	}
+
+	return parseSmartctlScanOpenTargets(output, diskExclude), nil
+}
+
+func parseSmartctlScanOpenTargets(output []byte, diskExclude []string) []smartctlTarget {
+	lines := strings.Split(string(output), "\n")
+	targets := make([]smartctlTarget, 0, len(lines))
+	typedByPath := make(map[string]bool)
+	seen := make(map[string]struct{})
+
+	for _, rawLine := range lines {
+		line := strings.TrimSpace(rawLine)
+		if line == "" {
+			continue
+		}
+		if idx := strings.Index(line, "#"); idx >= 0 {
+			line = strings.TrimSpace(line[:idx])
+		}
+		if line == "" {
+			continue
+		}
+
+		fields := strings.Fields(line)
+		if len(fields) == 0 {
+			continue
+		}
+
+		path := strings.TrimSpace(fields[0])
+		if path == "" || (!strings.HasPrefix(path, "/") && !strings.HasPrefix(path, "-")) {
+			continue
+		}
+
+		deviceType := ""
+		for i := 1; i < len(fields)-1; i++ {
+			if fields[i] == "-d" {
+				deviceType = strings.TrimSpace(fields[i+1])
+				break
+			}
+		}
+
+		name := filepath.Base(path)
+		if matchesDeviceExclude(name, path, diskExclude) {
+			continue
+		}
+
+		key := path + "\x00" + deviceType
+		if _, ok := seen[key]; ok {
+			continue
+		}
+		seen[key] = struct{}{}
+		if deviceType != "" {
+			typedByPath[path] = true
+		}
+
+		targets = append(targets, smartctlTarget{
+			Path:       path,
+			DeviceType: deviceType,
+		})
+	}
+
+	if len(targets) == 0 {
+		return nil
+	}
+
+	filtered := make([]smartctlTarget, 0, len(targets))
+	for _, target := range targets {
+		if target.DeviceType == "" && typedByPath[target.Path] {
+			continue
+		}
+		filtered = append(filtered, target)
+	}
+	return filtered
+}
+
 // listBlockDevices returns a list of block devices suitable for SMART queries.
 // Devices matching any of the diskExclude patterns are skipped.
 func listBlockDevices(ctx context.Context, diskExclude []string) ([]string, error) {
@@ -540,6 +678,10 @@ func matchesDeviceExclude(name, devicePath string, excludePatterns []string) boo
 
 // collectDeviceSMART runs smartctl on a single device and parses the result.
 func collectDeviceSMART(ctx context.Context, device string) (*DiskSMART, error) {
+	return collectSMARTTarget(ctx, smartctlTarget{Path: device})
+}
+
+func collectSMARTTarget(ctx context.Context, target smartctlTarget) (*DiskSMART, error) {
 	// Use timeout to avoid hanging on slow/unresponsive disks
 	cmdCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
 	defer cancel()
@@ -550,7 +692,7 @@ func collectDeviceSMART(ctx context.Context, device string) (*DiskSMART, error)
 		return nil, err
 	}
 
-	attempts := smartctlProbeAttempts(device)
+	attempts := smartctlProbeAttempts(target)
 	var firstParsed *DiskSMART
 	var firstStandby *DiskSMART
 	var lastErr error
@@ -563,11 +705,11 @@ func collectDeviceSMART(ctx context.Context, device string) (*DiskSMART, error)
 			if exitErr, ok := err.(*exec.ExitError); ok {
 				if exitErr.ExitCode() == smartctlStandbyExitStatus && len(output) == 0 {
 					standbyResult := &DiskSMART{
-						Device:      filepath.Base(device),
+						Device:      target.displayName(),
 						Standby:     true,
 						LastUpdated: timeNow(),
 					}
-					if runtimeGOOS == "freebsd" && i < len(attempts)-1 {
+					if runtimeGOOS == "freebsd" && i < len(attempts)-1 && target.DeviceType == "" {
 						if firstStandby == nil {
 							firstStandby = standbyResult
 						}
@@ -585,7 +727,7 @@ func collectDeviceSMART(ctx context.Context, device string) (*DiskSMART, error)
 			}
 		}
 
-		result, parseErr := parseSMARTOutput(output, device)
+		result, parseErr := parseSMARTOutput(output, target)
 		if parseErr != nil {
 			lastErr = parseErr
 			continue
@@ -593,7 +735,7 @@ func collectDeviceSMART(ctx context.Context, device string) (*DiskSMART, error)
 		if firstParsed == nil {
 			firstParsed = result
 		}
-		if !shouldRetryFreeBSDSMART(device, result, i, len(attempts)) {
+		if !shouldRetryFreeBSDSMART(target.Path, result, i, len(attempts)) {
 			log.Debug().
 				Str("device", result.Device).
 				Str("model", result.Model).
@@ -628,7 +770,14 @@ func collectDeviceSMART(ctx context.Context, device string) (*DiskSMART, error)
 	return nil, nil
 }
 
-func smartctlProbeAttempts(device string) [][]string {
+func smartctlProbeAttempts(target smartctlTarget) [][]string {
+	device := target.Path
+	if target.DeviceType != "" {
+		return [][]string{
+			smartctlArgs(device, target.DeviceType),
+		}
+	}
+
 	if runtimeGOOS == "freebsd" {
 		deviceTypes := freeBSDSmartctlDeviceTypes(filepath.Base(device))
 		if len(deviceTypes) > 0 {
@@ -681,14 +830,14 @@ func shouldRetryFreeBSDSMART(device string, result *DiskSMART, attemptIndex, att
 	return len(freeBSDSmartctlDeviceTypes(filepath.Base(device))) > 0
 }
 
-func parseSMARTOutput(output []byte, device string) (*DiskSMART, error) {
+func parseSMARTOutput(output []byte, target smartctlTarget) (*DiskSMART, error) {
 	var smartData smartctlJSON
 	if err := json.Unmarshal(output, &smartData); err != nil {
 		return nil, err
 	}
 
 	result := &DiskSMART{
-		Device:      filepath.Base(device),
+		Device:      target.displayName(),
 		Model:       smartData.ModelName,
 		Serial:      smartData.SerialNumber,
 		Type:        detectDiskType(smartData),

internal/smartctl/collector_coverage_test.go

@@ -117,6 +117,91 @@ func TestListBlockDevices(t *testing.T) {
 	}
 }
 
+func TestParseSmartctlScanOpenTargets(t *testing.T) {
+	output := []byte(strings.Join([]string{
+		`/dev/sda -d sat # /dev/sda, ATA device`,
+		`/dev/sda # duplicate plain path should be ignored when typed target exists`,
+		`/dev/bsg/sssraid0 -d sssraid,0,1 # controller-backed disk`,
+		`/dev/sdb -d megaraid,0 # megaraid slot 0`,
+		`/dev/sdb -d megaraid,1 # megaraid slot 1`,
+		`/dev/sdc # plain disk`,
+		``,
+	}, "\n"))
+
+	targets := parseSmartctlScanOpenTargets(output, []string{"sdc"})
+	if len(targets) != 4 {
+		t.Fatalf("expected 4 targets, got %#v", targets)
+	}
+
+	if targets[0].Path != "/dev/sda" || targets[0].DeviceType != "sat" {
+		t.Fatalf("unexpected first target: %#v", targets[0])
+	}
+	if targets[1].Path != "/dev/bsg/sssraid0" || targets[1].DeviceType != "sssraid,0,1" {
+		t.Fatalf("unexpected second target: %#v", targets[1])
+	}
+	if targets[2].DeviceType != "megaraid,0" || targets[3].DeviceType != "megaraid,1" {
+		t.Fatalf("expected megaraid targets to be preserved separately, got %#v", targets)
+	}
+}
+
+func TestCollectLocalUsesSmartctlScanOpenTargetsOnLinux(t *testing.T) {
+	origRun := runCommandOutput
+	origLook := execLookPath
+	origGOOS := runtimeGOOS
+	t.Cleanup(func() {
+		runCommandOutput = origRun
+		execLookPath = origLook
+		runtimeGOOS = origGOOS
+	})
+
+	runtimeGOOS = "linux"
+	execLookPath = func(string) (string, error) { return "smartctl", nil }
+
+	var seenArgs [][]string
+	runCommandOutput = func(ctx context.Context, name string, args ...string) ([]byte, error) {
+		seenArgs = append(seenArgs, append([]string(nil), args...))
+
+		if len(args) == 1 && args[0] == "--scan-open" {
+			return []byte("/dev/sda -d megaraid,7 # RAID-backed SSD\n"), nil
+		}
+
+		payload := smartctlJSON{
+			ModelName:    "RAID SSD",
+			SerialNumber: "raid-ssd-1",
+		}
+		payload.Device.Protocol = "NVMe"
+		payload.SmartStatus = &struct {
+			Passed bool `json:"passed"`
+		}{Passed: true}
+		payload.NVMeSmartHealthInformationLog.PercentageUsed = 6
+		payload.NVMeSmartHealthInformationLog.AvailableSpare = 94
+		out, _ := json.Marshal(payload)
+		return out, nil
+	}
+
+	result, err := CollectLocal(context.Background(), nil)
+	if err != nil {
+		t.Fatalf("CollectLocal error: %v", err)
+	}
+	if len(result) != 1 {
+		t.Fatalf("expected 1 result, got %#v", result)
+	}
+	if result[0].Device != "sda [megaraid,7]" {
+		t.Fatalf("unexpected device label: %#v", result[0])
+	}
+	if result[0].Attributes == nil || result[0].Attributes.PercentageUsed == nil || *result[0].Attributes.PercentageUsed != 6 {
+		t.Fatalf("expected percentage_used SMART data, got %#v", result[0].Attributes)
+	}
+
+	if len(seenArgs) < 2 {
+		t.Fatalf("expected scan and probe calls, got %v", seenArgs)
+	}
+	probe := seenArgs[1]
+	if len(probe) < 3 || probe[0] != "-d" || probe[1] != "megaraid,7" || probe[len(probe)-1] != "/dev/sda" {
+		t.Fatalf("expected typed smartctl probe, got %v", probe)
+	}
+}
+
 func TestListBlockDevicesFreeBSD(t *testing.T) {
 	origRun := runCommandOutput
 	origReadDir := readDir

internal/smartctl/collector_test.go

@@ -445,7 +445,7 @@ func TestParseSMARTOutputStandbyPowerMode(t *testing.T) {
 		t.Fatalf("marshal payload: %v", err)
 	}
 
-	result, err := parseSMARTOutput(out, "/dev/ada0")
+	result, err := parseSMARTOutput(out, smartctlTarget{Path: "/dev/ada0"})
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}