From 6e93cb3b5cf9171bacfb2e98c14352c421cc823e Mon Sep 17 00:00:00 2001 From: rcourtman Date: Fri, 24 Jul 2026 09:38:39 +0100 Subject: [PATCH] Fix Unraid SMART probing and disk authority --- .../v6/internal/subsystems/agent-lifecycle.md | 31 +- .../v6/internal/subsystems/api-contracts.md | 20 +- .../internal/subsystems/storage-recovery.md | 19 +- .../update_hostagent_integration_test.go | 2 +- .../api/config_handlers_setup_script_test.go | 4 +- internal/api/contract_test.go | 44 +- internal/api/setup_script_render.go | 63 ++- internal/hostagent/agent.go | 33 +- internal/hostagent/agent_metrics_test.go | 4 +- ...issue1516_physical_disk_regression_test.go | 4 +- .../issue1595_sas_collection_test.go | 2 +- .../hostagent/issue1612_unraid_smart_test.go | 384 ++++++++++++++++++ internal/hostagent/mock_collector_test.go | 6 +- internal/hostagent/smartctl.go | 270 +++++++++--- internal/hostagent/smartctl_coverage_test.go | 8 +- internal/hostagent/smartctl_discovery_test.go | 20 +- internal/hostagent/sysinfo.go | 6 +- internal/hostagent/sysinfo_test.go | 2 +- internal/hostagent/unraid.go | 78 +++- internal/storagehealth/topology_test.go | 24 +- testdata/issue1612_unraid_28tb.json | 9 + 21 files changed, 887 insertions(+), 146 deletions(-) create mode 100644 internal/hostagent/issue1612_unraid_smart_test.go create mode 100644 testdata/issue1612_unraid_28tb.json diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 571baacf6..e169860e3 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -26,7 +26,11 @@ smartctl permission/open failures are not standby; only the explicit must retain sysfs model, serial, WWID, size, and transport when available, smartctl retry modes merge rather than replace evidence, and partial NVMe JSON must preserve field presence so omitted health counters are not fabricated as -zero. Direct SATA, SAS, NVMe, and controller-member inventory must all survive +zero. On Unraid, native `disks.ini` spin state may suppress SMART commands +entirely and produce an identity-only standby row without touching the device. +Unsupported SMART commands, permission failures, and timeouts must never alter +native array membership or turn a present disk into a missing disk. Direct +SATA, SAS, USB-bridge, NVMe, and controller-member inventory must all survive one compressed unified-agent report without a collector-side suffix cap. `internal/monitoring/monitor.go` also serializes shared unified-resource websocket payloads. Carrying plural availability facets through that serializer @@ -899,10 +903,14 @@ generic block-device inference. The Unified Agent should best-effort merge `/var/local/emhttp/disks.ini` into the `mdcmd status` view and carry disk device, model, transport, filesystem, size, used/free capacity, temperature, spin state, read/write counters, and error counters in the report contract. -Failure to read the native file must degrade to the existing mdcmd view without -blocking host reporting, but successful native collection is the canonical -source for Unraid array/cache membership; SMART rows are supplemental hardware -telemetry, not the owner of Unraid storage topology. +The runtime must read that native membership before optional SMART work and +preserve it when `mdcmd status` times out or is unavailable. When structured +native per-disk states exist, their disabled/invalid/missing counts override +stale aggregate mdcmd counters while assigned `DISK_NP` evidence still remains +a genuine missing disk. Failure to read the native file must degrade to the +existing mdcmd view without blocking host reporting. SMART collection uses an +independent deadline, receives native transport/spin hints, and is supplemental +hardware telemetry rather than the owner of Unraid storage topology. First-class platform hosts that also run the Pulse Agent must keep the same operator-facing system identity split: a Proxmox VE node may report a Debian runtime platform underneath, but the host-agent OS identity and infrastructure @@ -1355,11 +1363,16 @@ the intentionally sparse public response. loading solely because no API token was supplied. That runtime-side ownership includes local disk telemetry collection in `internal/hostagent/smartctl.go`. Linux SMART discovery must prefer - `smartctl --scan-open` typed targets before generic block-device fallback so - controller-backed disks keep their canonical SMART and wearout coverage. + non-opening `smartctl --scan` typed targets before generic block-device + fallback so controller-backed disks keep their canonical SMART and wearout + coverage without opening every enumerated disk during discovery. Direct Linux SATA/SAT-style block devices that return health but no - temperature through smartctl auto-detection must retry explicit `-d sat` and - `-d scsi` probes before settling on a no-temperature result. + temperature through smartctl auto-detection may retry explicit `-d sat` + before settling on a no-temperature result. The runtime must not infer + `-d scsi` from an `sdX` basename: direct ATA may use SAT, direct SAS may use + SCSI only from native/sysfs/typed-scan evidence, USB bridges keep their + explicit scan mode, and multiplexed HBA members keep their exact controller + target without an untyped fallback. Those retries must accumulate model, serial, WWN, health, temperature, and SMART attributes across attempts. A later healthy result must not erase an earlier explicit failure, and an omitted NVMe counter must stay absent while diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 0bad864c8..4ce2166ca 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -3727,10 +3727,12 @@ route requires `monitoring:read`, returns `404` for unknown active alerts, and must not send notifications or mutate delivery tracking state. The generated PVE setup-script temperature wrapper is part of the API contract -for legacy SSH sensor collection. Direct Linux SATA/SAT-style disks that return -health but no temperature through smartctl auto-detection must retry explicit -`-d sat` and `-d scsi` probes before the rendered wrapper reports an active -disk with no temperature. +for legacy SSH sensor collection. Discovery uses non-opening `smartctl --scan` +and merges typed scan targets with `lsblk` transport evidence. Direct Linux +SATA/SAT-style disks that return health but no temperature through smartctl +auto-detection may retry explicit `-d sat`; `-d scsi` is reserved for +SAS/SCSI evidence, USB bridges retain their typed scan mode, and multiplexed +HBA members retain their exact controller target. Manifest-backed Patrol finding lifecycle schemas are the API source of truth for Assistant provider-tool optionality as well as MCP/API discovery. Legacy @@ -6659,10 +6661,12 @@ raw `sensors -j`. The wrapper is the setup-script API contract for legacy SSH temperature collection: it must emit a bounded JSON object with `sensors` and `smart` members, install or verify `smartmontools` for SATA/SAS/HDD disk temperatures, and keep `sensors -j` only as a compatibility fallback inside -the wrapper/runtime collector path. Direct Linux SATA/SAT-style disks that -return health but no temperature through smartctl auto-detection must retry -explicit `-d sat` and `-d scsi` probes before the wrapper reports an active -disk with no temperature. +the wrapper/runtime collector path. The wrapper must use non-opening +`smartctl --scan`, carry `lsblk` transport evidence into probe selection, and +never infer `-d scsi` from an `sdX` basename. Direct SATA may retry explicit +`-d sat`, SAS/SCSI evidence may select `-d scsi`, USB bridges retain typed scan +modes, and multiplexed HBA members retain their exact controller target before +the wrapper reports an active disk with no temperature. That same generated-script payload must also preserve the canonical encoded rerun URL contract: embedded `SETUP_SCRIPT_URL` values must carry the exact selected `host`, `pulse_url`, and `backup_perms` query state instead of diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 8d5119c60..5007a56e7 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -2020,9 +2020,9 @@ api-contract/security owned and create no storage, recovery-point, or backup-surface semantics. The shared PVE setup-script SMART wrapper remains a storage/recovery dependency -only for disk-temperature evidence. Storage surfaces may depend on its explicit -`-d sat` and `-d scsi` retries for active direct Linux SATA/SAT-style disks, but -they must not fork a storage-local disk-temperature collector or replace the +only for disk-temperature evidence. Storage surfaces may depend on its +non-opening scan and transport-aware SAT/SCSI/USB/HBA probe selection, but they +must not fork a storage-local disk-temperature collector or replace the API-owned setup-script contract. Storage physical-disk rows also depend on the unified-resource disk contract preserving Proxmox node/instance metadata and SMART capacity when Proxmox inventory and host-agent SMART telemetry merge; @@ -3915,12 +3915,13 @@ script renderer, but they must not replace the symlink path with a local file when filtering Pulse-managed `# pulse-` SSH key entries. That same dependency also assumes the shared PVE setup script binds temperature-monitoring SSH keys to `/usr/local/sbin/pulse-sensors` and emits -SMART disk temperatures in the wrapper payload, including explicit `-d sat` -and `-d scsi` retries for direct Linux SATA/SAT-style disks whose smartctl -auto-detection returns no temperature. Storage and recovery disk temperature -surfaces may depend on that monitoring-owned SMART merge path, but they must -not reintroduce raw `sensors -j` as the setup contract or build a storage-local -disk-temperature collector. +SMART disk temperatures in the wrapper payload. The wrapper uses non-opening +`smartctl --scan`, merges `lsblk` transport evidence, permits explicit `-d sat` +for direct SATA and `-d scsi` only for SAS/SCSI evidence, retains typed USB and +multiplexed HBA modes, and never infers SCSI from an `sdX` basename. Storage and +recovery disk temperature surfaces may depend on that monitoring-owned SMART +merge path, but they must not reintroduce raw `sensors -j` as the setup contract +or build a storage-local disk-temperature collector. Pressure-only host-agent telemetry remains outside that storage collector contract: storage surfaces may read the shared host context, but may not add a parallel macOS thermal collector or fold `thermalState` into disk SMART state. diff --git a/internal/agentupdate/update_hostagent_integration_test.go b/internal/agentupdate/update_hostagent_integration_test.go index a54854496..ad8b4c0ee 100644 --- a/internal/agentupdate/update_hostagent_integration_test.go +++ b/internal/agentupdate/update_hostagent_integration_test.go @@ -78,7 +78,7 @@ func (c *integrationCollector) CephStatus(context.Context) (*hostagent.CephClust return nil, nil } -func (c *integrationCollector) SMARTLocal(context.Context, []string) ([]hostagent.DiskSMART, error) { +func (c *integrationCollector) SMARTLocal(context.Context, []string, *agentshost.UnraidStorage) ([]hostagent.DiskSMART, error) { return nil, nil } diff --git a/internal/api/config_handlers_setup_script_test.go b/internal/api/config_handlers_setup_script_test.go index 8a4eb1f62..7147bf972 100644 --- a/internal/api/config_handlers_setup_script_test.go +++ b/internal/api/config_handlers_setup_script_test.go @@ -179,9 +179,9 @@ func TestPVESetupScriptRestrictsTemperatureMonitoringToPulseSensorWrapper(t *tes `def block_device_targets():`, `["lsblk", "-J", "-d", "-o", "NAME,TYPE,TRAN,MODEL,VENDOR,SUBSYSTEMS"]`, `def union_smart_targets(scan_targets, block_devices):`, - `def smart_probe_attempts(device, device_type):`, + `def smart_probe_attempts(device, device_type, transport):`, `def inferred_smart_device_types(device):`, - `for dtype in inferred_smart_device_types(device):`, + `for dtype in inferred_transport_device_types(device, transport):`, `for attempt_index, (attempt_device, attempt_type) in enumerate(attempts):`, `attempt_index == len(attempts) - 1`, `"smart": collect_smart(),`, diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go index f63bca046..c9e420a4f 100644 --- a/internal/api/contract_test.go +++ b/internal/api/contract_test.go @@ -1214,6 +1214,41 @@ fi` } } +func TestContract_PVESetupScriptUsesNonOpeningTransportAwareSMARTProbes(t *testing.T) { + script := renderSetupScript("pve", setupScriptRenderContext{ + ServerName: "pve-example", + PulseURL: "https://pulse.example", + ServerHost: "https://pve.example:8006", + SetupToken: "setup-token-123", + TokenName: "pulse-example", + TokenMatchPrefix: "pulse-example", + SensorsPublicKey: "ssh-ed25519 AAAATEST pulse@test", + }) + + for _, required := range []string{ + `run_command([path, "--scan"])`, + `def smart_probe_attempts(device, device_type, transport):`, + `if device_type and smart_device_type_matches_transport(device_type, transport):`, + `if transport == "sata":`, + `return ["sat"]`, + `if transport == "sas":`, + `return ["scsi"]`, + `if transport == "usb":`, + } { + if !strings.Contains(script, required) { + t.Fatalf("PVE setup script must contain %q", required) + } + } + for _, forbidden := range []string{ + `run_command([path, "--scan-open"])`, + `return ["sat", "scsi"]`, + } { + if strings.Contains(script, forbidden) { + t.Fatalf("PVE setup script must not contain unsafe SMART probe %q", forbidden) + } + } +} + func TestContract_NodeConfigUpdateTracksOptionalConnectionFieldsAndRedactedSecrets(t *testing.T) { var preserveReq NodeConfigRequest if err := json.Unmarshal([]byte(`{"name":"cluster","tokenName":"pulse-monitor@pve!pulse-pve","tokenValue":"********"}`), &preserveReq); err != nil { @@ -6629,11 +6664,16 @@ func TestContract_SetupScriptEmbedsFailFastGuidance(t *testing.T) { } if !strings.Contains(script, `"smart": collect_smart(),`) || !strings.Contains(script, `apt-get install -y smartmontools`) || - !strings.Contains(script, `def inferred_smart_device_types(device):`) || - !strings.Contains(script, `for dtype in inferred_smart_device_types(device):`) || + !strings.Contains(script, `run_command([path, "--scan"])`) || + !strings.Contains(script, `def inferred_transport_device_types(device, transport):`) || + !strings.Contains(script, `for dtype in inferred_transport_device_types(device, transport):`) || !strings.Contains(script, `attempt_index == len(attempts) - 1`) { t.Fatalf("setup script missing SMART temperature wrapper contract: %s", script) } + if strings.Contains(script, `run_command([path, "--scan-open"])`) || + strings.Contains(script, `return ["sat", "scsi"]`) { + t.Fatalf("setup script preserved opening or transport-blind SMART discovery: %s", script) + } if strings.Contains(script, `SSH_SENSORS_KEY_ENTRY="command=\"sensors -j\"`) { t.Fatalf("setup script preserved stale raw sensors forced command: %s", script) } diff --git a/internal/api/setup_script_render.go b/internal/api/setup_script_render.go index 68e06498f..732b51449 100644 --- a/internal/api/setup_script_render.go +++ b/internal/api/setup_script_render.go @@ -951,10 +951,40 @@ def is_non_rotational_block_device(device): def inferred_smart_device_types(device): name = os.path.basename(str(device or "").strip()).lower() if re.fullmatch(r"(sd|hd)[a-z]+", name or ""): - return ["sat", "scsi"] + return ["sat"] return [] +def normalize_smart_transport(transport): + value = str(transport or "").strip().lower() + if value == "ata": + return "sata" + if value in ("sata", "sas", "usb", "nvme"): + return value + return "" + + +def smart_device_type_matches_transport(device_type, transport): + dtype = str(device_type or "").strip().lower() + transport = normalize_smart_transport(transport) + if transport == "sata": + return not dtype.startswith("scsi") + if transport == "sas": + return not dtype.startswith("sat") + return True + + +def inferred_transport_device_types(device, transport): + transport = normalize_smart_transport(transport) + if transport == "sata": + return ["sat"] + if transport == "sas": + return ["scsi"] + if transport == "usb": + return [] + return inferred_smart_device_types(device) + + def is_physical_block_device(device): name = str(device.get("name") or "").strip() dtype = str(device.get("type") or "").strip().lower() @@ -995,7 +1025,7 @@ def block_device_targets(): if not name or name in seen: continue seen.add(name) - devices.append("/dev/" + name) + devices.append(("/dev/" + name, normalize_smart_transport(device.get("tran")))) return devices @@ -1003,19 +1033,24 @@ def union_smart_targets(scan_targets, block_devices): targets = [] seen_targets = set() covered_blocks = set() + transport_by_block = { + canonical_block_for_device(device): transport + for device, transport in block_devices + if canonical_block_for_device(device) + } for device, device_type in scan_targets: key = (device, device_type) if key in seen_targets: continue seen_targets.add(key) - targets.append(key) + block = canonical_block_for_device(device) + targets.append((device, device_type, transport_by_block.get(block, ""))) if not is_multiplexed_device_type(device_type): - block = canonical_block_for_device(device) if block: covered_blocks.add(block) - for device in block_devices: + for device, transport in block_devices: block = canonical_block_for_device(device) if not block or block in covered_blocks: continue @@ -1023,14 +1058,14 @@ def union_smart_targets(scan_targets, block_devices): if key in seen_targets: continue seen_targets.add(key) - targets.append(key) + targets.append((key[0], key[1], transport)) covered_blocks.add(block) return targets def smart_targets(path): - result = run_command([path, "--scan-open"]) + result = run_command([path, "--scan"]) targets = [] seen = set() @@ -1069,7 +1104,7 @@ def smart_targets(path): return union_smart_targets(scan_targets, block_device_targets()) -def smart_probe_attempts(device, device_type): +def smart_probe_attempts(device, device_type, transport): attempts = [] seen = set() @@ -1080,15 +1115,15 @@ def smart_probe_attempts(device, device_type): seen.add(key) attempts.append(key) - if device_type: + if device_type and smart_device_type_matches_transport(device_type, transport): add(device_type) - else: - add("") if device_type and not is_multiplexed_device_type(device_type): add("") + elif not device_type: + add("") if not is_multiplexed_device_type(device_type): - for dtype in inferred_smart_device_types(device): + for dtype in inferred_transport_device_types(device, transport): add(dtype) return attempts @@ -1190,9 +1225,9 @@ def collect_smart(): entries = [] observed_at = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") - for device, device_type in smart_targets(path): + for device, device_type, transport in smart_targets(path): best_entry = None - attempts = smart_probe_attempts(device, device_type) + attempts = smart_probe_attempts(device, device_type, transport) for attempt_index, (attempt_device, attempt_type) in enumerate(attempts): args = [path] if attempt_type: diff --git a/internal/hostagent/agent.go b/internal/hostagent/agent.go index bcceccf11..f011da9cf 100644 --- a/internal/hostagent/agent.go +++ b/internal/hostagent/agent.go @@ -1032,6 +1032,13 @@ func (a *Agent) loadPersistedReportQueue(queue *utils.Queue[agentshost.Report], } func (a *Agent) buildReport(ctx context.Context) (agentshost.Report, error) { + // Unraid's native inventory is the authority for array membership. Collect + // it before optional disk diagnostics and on an independent deadline so a + // slow or unsupported SMART target cannot starve mdcmd/disks.ini. + unraidCtx, cancelUnraid := context.WithTimeout(ctx, 5*time.Second) + unraidData := a.collectUnraidStorage(unraidCtx) + cancelUnraid() + collectCtx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() @@ -1048,22 +1055,22 @@ func (a *Agent) buildReport(ctx context.Context) (agentshost.Report, error) { // Collect temperature data (best effort - don't fail if unavailable) sensorData := a.collectTemperatures(collectCtx) - // Collect S.M.A.R.T. disk data (best effort - don't fail if unavailable) - smartData := a.collectSMARTData(collectCtx, runtimeConfig.diskExclude) + // Collect RAID array data (best effort - don't fail if unavailable) + raidData := a.collectRAIDArrays(collectCtx) + + // Collect Ceph cluster data (best effort - only on Ceph nodes) + cephData := a.collectCephStatus(collectCtx, runtimeConfig.disableCeph) + + // Collect S.M.A.R.T. disk data after topology owners and on its own + // deadline. Unsupported diagnostics must not starve RAID/Unraid/Ceph state. + smartCtx, cancelSMART := context.WithTimeout(ctx, 10*time.Second) + smartData := a.collectSMARTData(smartCtx, runtimeConfig.diskExclude, unraidData) + cancelSMART() if len(smartData) > 0 { annotateSMARTWithDiskIO(smartData, snapshot.DiskIO) sensorData.SMART = smartData } - // Collect RAID array data (best effort - don't fail if unavailable) - raidData := a.collectRAIDArrays(collectCtx) - - // Collect Unraid array topology (best effort - only on Unraid hosts). - unraidData := a.collectUnraidStorage(collectCtx) - - // Collect Ceph cluster data (best effort - only on Ceph nodes) - cephData := a.collectCephStatus(collectCtx, runtimeConfig.disableCeph) - // Collect temperature data from Proxmox cluster peers via SSH (best effort). // Uses parent ctx, not collectCtx — cluster SSH has its own 15s budget that // would be capped by collectCtx's 10s timeout. @@ -1975,13 +1982,13 @@ func (a *Agent) collectCephStatus(ctx context.Context, disableCeph bool) *agents // collectSMARTData collects S.M.A.R.T. data from local disks. // Returns nil if smartctl is not available or no disks are found. -func (a *Agent) collectSMARTData(ctx context.Context, diskExclude []string) []agentshost.DiskSMART { +func (a *Agent) collectSMARTData(ctx context.Context, diskExclude []string, unraid *agentshost.UnraidStorage) []agentshost.DiskSMART { goos := a.collector.GOOS() if goos != "linux" && goos != "freebsd" { return nil } - smartData, err := a.collector.SMARTLocal(ctx, diskExclude) + smartData, err := a.collector.SMARTLocal(ctx, diskExclude, unraid) if err != nil { a.logger.Debug().Err(err).Msg("Failed to collect S.M.A.R.T. data (smartctl may not be installed)") return nil diff --git a/internal/hostagent/agent_metrics_test.go b/internal/hostagent/agent_metrics_test.go index ef17656b4..e848f5732 100644 --- a/internal/hostagent/agent_metrics_test.go +++ b/internal/hostagent/agent_metrics_test.go @@ -379,7 +379,7 @@ rdevId.29= // Test case 6: SMART collection t.Run("SMART collection", func(t *testing.T) { - mc.smartLocalFn = func(_ context.Context, _ []string) ([]DiskSMART, error) { + mc.smartLocalFn = func(_ context.Context, _ []string, _ *agentshost.UnraidStorage) ([]DiskSMART, error) { return []DiskSMART{ { Device: "/dev/sda", @@ -407,7 +407,7 @@ rdevId.29= t.Run("SMART collection preserves typed controller-backed attributes", func(t *testing.T) { used := 6 spare := 94 - mc.smartLocalFn = func(_ context.Context, _ []string) ([]DiskSMART, error) { + mc.smartLocalFn = func(_ context.Context, _ []string, _ *agentshost.UnraidStorage) ([]DiskSMART, error) { return []DiskSMART{ { Device: "/dev/sda [megaraid,7]", diff --git a/internal/hostagent/issue1516_physical_disk_regression_test.go b/internal/hostagent/issue1516_physical_disk_regression_test.go index 53340c07d..5a92215a6 100644 --- a/internal/hostagent/issue1516_physical_disk_regression_test.go +++ b/internal/hostagent/issue1516_physical_disk_regression_test.go @@ -31,7 +31,7 @@ func TestIssue1516PermissionFailureRemainsUnknownAndKeepsSysfsIdentity(t *testin }) execLookPath = func(string) (string, error) { return "smartctl", nil } smartRunCommandOutput = func(ctx context.Context, name string, args ...string) ([]byte, error) { - if len(args) == 1 && args[0] == "--scan-open" { + if len(args) == 1 && args[0] == "--scan" { return nil, nil } return exec.CommandContext(ctx, "sh", "-c", "exit 2").Output() @@ -83,7 +83,7 @@ func TestIssue1516StandbyDiskKeepsStableIdentityWithoutSMARTClaims(t *testing.T) }) execLookPath = func(string) (string, error) { return "smartctl", nil } smartRunCommandOutput = func(ctx context.Context, name string, args ...string) ([]byte, error) { - if len(args) == 1 && args[0] == "--scan-open" { + if len(args) == 1 && args[0] == "--scan" { return []byte("/dev/sdb -d scsi # sleeping SAS disk\n"), nil } return exec.CommandContext(ctx, "sh", "-c", "exit 3").Output() diff --git a/internal/hostagent/issue1595_sas_collection_test.go b/internal/hostagent/issue1595_sas_collection_test.go index 8b8c59227..1fdd11c44 100644 --- a/internal/hostagent/issue1595_sas_collection_test.go +++ b/internal/hostagent/issue1595_sas_collection_test.go @@ -155,7 +155,7 @@ func TestIssue1595CollectSMARTLocalPreservesTwentyFourSASDisksAcrossTwoHBAs(t *t var active atomic.Int32 var maxActive atomic.Int32 smartRunCommandOutput = func(ctx context.Context, name string, args ...string) ([]byte, error) { - if len(args) == 1 && args[0] == "--scan-open" { + if len(args) == 1 && args[0] == "--scan" { return []byte(scan.String()), nil } device := strings.TrimPrefix(args[len(args)-1], "/dev/") diff --git a/internal/hostagent/issue1612_unraid_smart_test.go b/internal/hostagent/issue1612_unraid_smart_test.go new file mode 100644 index 000000000..a56abbf05 --- /dev/null +++ b/internal/hostagent/issue1612_unraid_smart_test.go @@ -0,0 +1,384 @@ +//go:build !windows + +package hostagent + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/agentupdate" + "github.com/rcourtman/pulse-go-rewrite/internal/hostmetrics" + agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" +) + +type issue1612Fixture struct { + SmartctlVersion string `json:"smartctlVersion"` + Model string `json:"model"` + SizeBytes int64 `json:"sizeBytes"` + MdcmdStatus string `json:"mdcmdStatus"` + DisksINI string `json:"disksINI"` + SmartctlScan string `json:"smartctlScan"` + UnsupportedATAJSON string `json:"unsupportedATAJSON"` +} + +func loadIssue1612Fixture(t *testing.T) issue1612Fixture { + t.Helper() + data, err := os.ReadFile(filepath.Join("..", "..", "testdata", "issue1612_unraid_28tb.json")) + if err != nil { + t.Fatalf("read issue #1612 fixture: %v", err) + } + var fixture issue1612Fixture + if err := json.Unmarshal(data, &fixture); err != nil { + t.Fatalf("decode issue #1612 fixture: %v", err) + } + if fixture.SmartctlVersion != "smartctl 7.5 2025-04-30 r6178" || + fixture.Model != "ST28000NT000-4AB103" || + fixture.SizeBytes != 28_001_039_286_272 { + t.Fatalf("fixture lost reporter hardware identity: %+v", fixture) + } + return fixture +} + +func issue1612NativeStorage(t *testing.T, fixture issue1612Fixture) *agentshost.UnraidStorage { + t.Helper() + storage, err := parseUnraidStatusOutput(fixture.MdcmdStatus) + if err != nil { + t.Fatalf("parse mdcmd fixture: %v", err) + } + return reconcileUnraidDiskCounts(mergeUnraidDiskINI(storage, parseUnraidDisksINI(fixture.DisksINI))) +} + +func TestIssue1612NativeInventoryOverridesFalseAggregateMissingAndPreservesRealMissing(t *testing.T) { + fixture := loadIssue1612Fixture(t) + mdcmdPath := filepath.Join(t.TempDir(), "mdcmd") + if err := os.WriteFile(mdcmdPath, nil, 0o600); err != nil { + t.Fatalf("write mdcmd fixture: %v", err) + } + stat, err := os.Stat(mdcmdPath) + if err != nil { + t.Fatalf("stat mdcmd fixture: %v", err) + } + + collector := &mockCollector{ + goos: "linux", + statFn: func(name string) (os.FileInfo, error) { + switch name { + case hostAgentUnraidVersionPath, mdcmdPath: + return stat, nil + default: + return nil, fs.ErrNotExist + } + }, + lookPathFn: func(string) (string, error) { return mdcmdPath, nil }, + readFileFn: func(name string) ([]byte, error) { + if name == hostAgentUnraidDisksINIPath { + return []byte(fixture.DisksINI), nil + } + return nil, fs.ErrNotExist + }, + commandCombinedOutputFn: func(_ context.Context, name string, args ...string) (string, error) { + if name != mdcmdPath || len(args) != 1 || args[0] != "status" { + t.Fatalf("unexpected native command: %s %v", name, args) + } + return fixture.MdcmdStatus, nil + }, + } + + storage, err := CollectUnraidStorage(context.Background(), collector) + if err != nil { + t.Fatalf("CollectUnraidStorage() error = %v", err) + } + if storage == nil || !storage.ArrayStarted || storage.SyncAction != "check" || storage.SyncProgress != 50 { + t.Fatalf("native parity-check state was not preserved: %+v", storage) + } + if storage.NumMissing != 1 || storage.NumDisabled != 0 || storage.NumInvalid != 0 { + t.Fatalf("native counts = missing:%d disabled:%d invalid:%d, want only one genuine missing member", + storage.NumMissing, storage.NumDisabled, storage.NumInvalid) + } + if len(storage.Disks) != 7 { + t.Fatalf("native membership count = %d, want 7: %+v", len(storage.Disks), storage.Disks) + } + for _, disk := range storage.Disks { + if disk.Name == "disk1" && (disk.Model != fixture.Model || disk.SizeBytes != fixture.SizeBytes || disk.Status != "online") { + t.Fatalf("28 TB native member lost identity: %+v", disk) + } + } +} + +func TestIssue1612DisksINIRemainsAvailableWhenMdcmdTimesOut(t *testing.T) { + fixture := loadIssue1612Fixture(t) + mdcmdPath := filepath.Join(t.TempDir(), "mdcmd") + if err := os.WriteFile(mdcmdPath, nil, 0o600); err != nil { + t.Fatalf("write mdcmd fixture: %v", err) + } + stat, err := os.Stat(mdcmdPath) + if err != nil { + t.Fatalf("stat mdcmd fixture: %v", err) + } + + collector := &mockCollector{ + goos: "linux", + statFn: func(name string) (os.FileInfo, error) { + switch name { + case hostAgentUnraidVersionPath, mdcmdPath: + return stat, nil + default: + return nil, fs.ErrNotExist + } + }, + lookPathFn: func(string) (string, error) { return mdcmdPath, nil }, + readFileFn: func(name string) ([]byte, error) { + if name == hostAgentUnraidDisksINIPath { + return []byte(fixture.DisksINI), nil + } + return nil, fs.ErrNotExist + }, + commandCombinedOutputFn: func(context.Context, string, ...string) (string, error) { + return "", context.DeadlineExceeded + }, + } + + storage, err := CollectUnraidStorage(context.Background(), collector) + if err != nil { + t.Fatalf("native disks.ini fallback returned error: %v", err) + } + if storage == nil || len(storage.Disks) != 7 || storage.NumMissing != 1 { + t.Fatalf("native inventory was lost after mdcmd timeout: %+v", storage) + } + if storage.ArrayStarted || storage.SyncAction != "" { + t.Fatalf("mdcmd-only runtime state was fabricated: %+v", storage) + } +} + +func TestIssue1612SMARTCommandsAreTransportAwareAndSkipNativeStandby(t *testing.T) { + fixture := loadIssue1612Fixture(t) + entries := []string{"sda", "sdb", "sdc", "sdd", "sde", "sdf", "sdg", "sdh"} + files := make(map[string]string) + for _, block := range entries { + files["/sys/block/"+block+"/size"] = "54689529856\n" + files["/sys/block/"+block+"/queue/rotational"] = "1\n" + files["/sys/block/"+block+"/device/model"] = fixture.Model + "\n" + files["/sys/block/"+block+"/device/vendor"] = "ATA\n" + files["/sys/block/"+block+"/device/serial"] = "SERIAL-" + block + "\n" + } + files["/sys/block/sdf/device/protocol"] = "SAS\n" + delete(files, "/sys/block/sdf/device/vendor") + stubLinuxSysfs(t, entries, files) + + origRun := smartRunCommandOutput + origLook := execLookPath + t.Cleanup(func() { + smartRunCommandOutput = origRun + execLookPath = origLook + }) + execLookPath = func(string) (string, error) { return "smartctl", nil } + + var mu sync.Mutex + var commands [][]string + smartRunCommandOutput = func(_ context.Context, _ string, args ...string) ([]byte, error) { + mu.Lock() + commands = append(commands, append([]string(nil), args...)) + mu.Unlock() + if len(args) == 1 && args[0] == "--scan" { + return []byte(fixture.SmartctlScan), nil + } + device := args[len(args)-1] + switch device { + case "/dev/sdb", "/dev/sdc": + return []byte(fixture.UnsupportedATAJSON), nil + case "/dev/sdg": + return nil, fs.ErrPermission + case "/dev/sdh": + return nil, context.DeadlineExceeded + default: + deviceType := "sat" + protocol := "ATA" + if device == "/dev/sdf" || device == "/dev/bus/0" { + deviceType = "scsi" + protocol = "SCSI" + } + return []byte(fmt.Sprintf( + `{"device":{"name":%q,"type":%q,"protocol":%q},"model_name":%q,"serial_number":%q,"user_capacity":{"bytes":%d},"smart_status":{"passed":true},"temperature":{"current":31}}`, + device, deviceType, protocol, fixture.Model, "SERIAL-"+filepath.Base(device), fixture.SizeBytes, + )), nil + } + } + + native := issue1612NativeStorage(t, fixture) + results, err := CollectSMARTLocalWithUnraid(context.Background(), nil, native) + if err != nil { + t.Fatalf("CollectSMARTLocalWithUnraid() error = %v", err) + } + + byDevice := make(map[string]DiskSMART, len(results)) + for _, disk := range results { + byDevice[disk.Device] = disk + } + for _, device := range []string{"sda", "sdb", "sdc", "sdd", "sde", "sdf", "sdg", "sdh"} { + if _, ok := byDevice[device]; !ok { + t.Fatalf("present disk %s disappeared after unsupported SMART response: %+v", device, results) + } + } + if disk := byDevice["sdd"]; !disk.Standby || disk.Model != fixture.Model || disk.SizeBytes != fixture.SizeBytes { + t.Fatalf("native standby identity = %+v", disk) + } + for _, device := range []string{"sdb", "sdc", "sdg", "sdh"} { + disk := byDevice[device] + if disk.SizeBytes != fixture.SizeBytes || disk.Health != "UNKNOWN" { + t.Fatalf("unsupported/failed disk %s was not retained as unknown identity: %+v", device, disk) + } + } + if byDevice["sdb"].Type != "sata" || byDevice["sdc"].Type != "sata" || byDevice["sde"].Type != "usb" || byDevice["sdf"].Type != "sas" { + t.Fatalf("native transport evidence was not preserved: sdb=%q sdc=%q sde=%q sdf=%q", + byDevice["sdb"].Type, byDevice["sdc"].Type, byDevice["sde"].Type, byDevice["sdf"].Type) + } + if native.NumMissing != 1 || native.Disks[6].Status != "missing" { + t.Fatalf("SMART collection mutated genuine native missing evidence: %+v", native) + } + + mu.Lock() + defer mu.Unlock() + for _, args := range commands { + joined := strings.Join(args, " ") + if joined == "--scan-open" { + t.Fatalf("opening discovery command was issued: %v", commands) + } + if strings.HasSuffix(joined, " /dev/sdd") || joined == "/dev/sdd" { + t.Fatalf("spun-down native member received a SMART command: %v", args) + } + if (strings.HasSuffix(joined, " /dev/sda") || + strings.HasSuffix(joined, " /dev/sdb") || + strings.HasSuffix(joined, " /dev/sdc")) && + strings.Contains(" "+joined+" ", " -d scsi ") { + t.Fatalf("direct ATA member was forced through SCSI: %v", args) + } + } + assertIssue1612Command(t, commands, "/dev/sdf", "scsi") + assertIssue1612Command(t, commands, "/dev/bus/0", "megaraid,0") +} + +func TestIssue1612USBPathKeepsExplicitBridgeMode(t *testing.T) { + stubLinuxSysfs(t, []string{"sde"}, map[string]string{ + "/sys/block/sde/device/vendor": "ATA\n", + }) + origEval := smartctlEvalSymlinks + t.Cleanup(func() { smartctlEvalSymlinks = origEval }) + smartctlEvalSymlinks = func(name string) (string, error) { + if name == "/sys/block/sde/device" { + return "/sys/devices/pci0000:00/0000:00:14.0/usb1/1-2/1-2:1.0/host8/target8:0:0/8:0:0:0", nil + } + return "", fs.ErrNotExist + } + + attempts := smartctlProbeAttempts(smartctlTarget{Path: "/dev/sde", DeviceType: "scsi"}) + if len(attempts) != 2 || attempts[0][0] != "-d" || attempts[0][1] != "scsi" || attempts[1][0] == "-d" { + t.Fatalf("USB bridge attempts = %v, want typed scan mode then auto-detection", attempts) + } + for _, args := range attempts { + if strings.Contains(" "+strings.Join(args, " ")+" ", " -d sat ") { + t.Fatalf("USB path was reclassified as direct ATA: %v", attempts) + } + } +} + +func assertIssue1612Command(t *testing.T, commands [][]string, device, deviceType string) { + t.Helper() + for _, args := range commands { + if len(args) >= 3 && args[0] == "-d" && args[1] == deviceType && args[len(args)-1] == device { + return + } + } + t.Fatalf("missing smartctl -d %s command for %s: %v", deviceType, device, commands) +} + +func TestIssue1612BuildReportCollectsNativeInventoryBeforeSMARTTimeout(t *testing.T) { + native := &agentshost.UnraidStorage{ + ArrayStarted: true, + Disks: []agentshost.UnraidDisk{ + {Name: "parity", Device: "/dev/sda", Role: "parity", Status: "online"}, + {Name: "disk1", Device: "/dev/sdb", Role: "data", Status: "online"}, + }, + } + nativeCalled := make(chan struct{}) + raidCalled := false + cephCalled := false + collector := &mockCollector{ + goos: "linux", + unraidStorageFn: func(context.Context) (*agentshost.UnraidStorage, error) { + close(nativeCalled) + return native, nil + }, + metricsFn: func(context.Context, []string) (hostmetrics.Snapshot, error) { + return hostmetrics.Snapshot{}, nil + }, + raidArraysFn: func(context.Context) ([]agentshost.RAIDArray, error) { + raidCalled = true + return nil, nil + }, + cephStatusFn: func(context.Context) (*CephClusterStatus, error) { + cephCalled = true + return nil, nil + }, + smartLocalFn: func(ctx context.Context, _ []string, got *agentshost.UnraidStorage) ([]DiskSMART, error) { + select { + case <-nativeCalled: + default: + t.Fatal("SMART collection ran before native Unraid inventory") + } + if got != native { + t.Fatalf("SMART inventory pointer = %p, want %p", got, native) + } + if !raidCalled || !cephCalled { + t.Fatalf("optional SMART ran before topology collectors: raid=%v ceph=%v", raidCalled, cephCalled) + } + <-ctx.Done() + return nil, ctx.Err() + }, + } + agent, err := New(Config{ + AgentID: "issue-1612", + APIToken: "redacted-test-token", + LogLevel: -1, + Collector: collector, + UpdateStatus: func() agentupdate.Status { + return agentupdate.Status{State: agentupdate.UpdateStateIdle} + }, + }) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Millisecond) + defer cancel() + report, err := agent.buildReport(ctx) + if err != nil { + t.Fatalf("buildReport() error = %v", err) + } + if report.Unraid == nil || len(report.Unraid.Disks) != 2 || report.Unraid.NumMissing != 0 { + t.Fatalf("native inventory was lost when SMART timed out: %+v", report.Unraid) + } +} + +func TestIssue1612AggregateFallbackStillPreservesErrorWhenNoNativeInventoryExists(t *testing.T) { + collector := &mockCollector{ + goos: "linux", + statFn: func(name string) (os.FileInfo, error) { + if name == hostAgentUnraidVersionPath { + return nil, errors.New("permission denied") + } + return nil, fs.ErrNotExist + }, + } + if storage, err := CollectUnraidStorage(context.Background(), collector); err == nil || storage != nil { + t.Fatalf("unreadable Unraid identity = (%+v, %v), want explicit error", storage, err) + } +} diff --git a/internal/hostagent/mock_collector_test.go b/internal/hostagent/mock_collector_test.go index 9be9c7d5f..49a408714 100644 --- a/internal/hostagent/mock_collector_test.go +++ b/internal/hostagent/mock_collector_test.go @@ -22,7 +22,7 @@ type mockCollector struct { raidArraysFn func(ctx context.Context) ([]agentshost.RAIDArray, error) unraidStorageFn func(ctx context.Context) (*agentshost.UnraidStorage, error) cephStatusFn func(ctx context.Context) (*CephClusterStatus, error) - smartLocalFn func(ctx context.Context, exclude []string) ([]DiskSMART, error) + smartLocalFn func(ctx context.Context, exclude []string, unraid *agentshost.UnraidStorage) ([]DiskSMART, error) nowFn func() time.Time goos string readFileFn func(name string) ([]byte, error) @@ -102,9 +102,9 @@ func (m *mockCollector) CephStatus(ctx context.Context) (*CephClusterStatus, err return nil, nil } -func (m *mockCollector) SMARTLocal(ctx context.Context, exclude []string) ([]DiskSMART, error) { +func (m *mockCollector) SMARTLocal(ctx context.Context, exclude []string, unraid *agentshost.UnraidStorage) ([]DiskSMART, error) { if m.smartLocalFn != nil { - return m.smartLocalFn(ctx, exclude) + return m.smartLocalFn(ctx, exclude, unraid) } return nil, nil } diff --git a/internal/hostagent/smartctl.go b/internal/hostagent/smartctl.go index a31080939..f72bd5711 100644 --- a/internal/hostagent/smartctl.go +++ b/internal/hostagent/smartctl.go @@ -20,6 +20,7 @@ import ( "github.com/rs/zerolog/log" + agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" "github.com/rcourtman/pulse-go-rewrite/pkg/fsfilters" ) @@ -225,8 +226,9 @@ var ( ) type smartctlTarget struct { - Path string - DeviceType string + Path string + DeviceType string + NativeTransport string } func (t smartctlTarget) displayName() string { @@ -243,49 +245,56 @@ func (t smartctlTarget) displayName() string { // CollectSMARTLocal collects S.M.A.R.T. data from all local block devices. // The diskExclude parameter specifies patterns for devices to skip (e.g., "sda", "/dev/nvme*", "*cache*"). func CollectSMARTLocal(ctx context.Context, diskExclude []string) ([]DiskSMART, error) { + return CollectSMARTLocalWithUnraid(ctx, diskExclude, nil) +} + +// CollectSMARTLocalWithUnraid collects local SMART data while treating native +// Unraid membership, transport, and spin state as authoritative hints. Native +// array state is never derived from SMART success or failure. +func CollectSMARTLocalWithUnraid(ctx context.Context, diskExclude []string, unraid *agentshost.UnraidStorage) ([]DiskSMART, error) { targets, err := listSMARTTargets(ctx, diskExclude) if err != nil { log.Debug().Err(err).Msg("failed to list block devices for SMART collection") return nil, fmt.Errorf("list block devices for SMART collection: %w", err) } - if len(targets) == 0 { - return nil, nil - } + targets, nativeStandby := applyUnraidSMARTInventory(targets, diskExclude, unraid) type smartOutcome struct { smart *DiskSMART err error } outcomes := make([]smartOutcome, len(targets)) - workerCount := smartCollectionConcurrency - if len(targets) < smartCollectionParallelThreshold { - workerCount = 1 + if len(targets) > 0 { + workerCount := smartCollectionConcurrency + if len(targets) < smartCollectionParallelThreshold { + workerCount = 1 + } + if workerCount < 1 { + workerCount = 1 + } + if workerCount > len(targets) { + workerCount = len(targets) + } + jobs := make(chan int) + var workers sync.WaitGroup + workers.Add(workerCount) + for worker := 0; worker < workerCount; worker++ { + go func() { + defer workers.Done() + for index := range jobs { + outcomes[index].smart, outcomes[index].err = collectSMARTTarget(ctx, targets[index]) + } + }() + } + for index := range targets { + jobs <- index + } + close(jobs) + workers.Wait() } - if workerCount < 1 { - workerCount = 1 - } - if workerCount > len(targets) { - workerCount = len(targets) - } - jobs := make(chan int) - var workers sync.WaitGroup - workers.Add(workerCount) - for worker := 0; worker < workerCount; worker++ { - go func() { - defer workers.Done() - for index := range jobs { - outcomes[index].smart, outcomes[index].err = collectSMARTTarget(ctx, targets[index]) - } - }() - } - for index := range targets { - jobs <- index - } - close(jobs) - workers.Wait() - var results []DiskSMART + results := append([]DiskSMART(nil), nativeStandby...) var missed []smartctlTarget collected := make(map[string]struct{}, len(targets)) multiplexed := make(map[string]struct{}) @@ -352,6 +361,90 @@ func CollectSMARTLocal(ctx context.Context, diskExclude []string) ([]DiskSMART, return results, nil } +func applyUnraidSMARTInventory(targets []smartctlTarget, diskExclude []string, unraid *agentshost.UnraidStorage) ([]smartctlTarget, []DiskSMART) { + if unraid == nil || len(unraid.Disks) == 0 { + return targets, nil + } + + nativeByBlock := make(map[string]agentshost.UnraidDisk, len(unraid.Disks)) + for _, disk := range unraid.Disks { + block := canonicalBlockDeviceForScanPath(disk.Device) + if block == "" || matchesDeviceExclude(block, "/dev/"+block, diskExclude) { + continue + } + nativeByBlock[block] = disk + } + + filtered := make([]smartctlTarget, 0, len(targets)) + standbyByBlock := make(map[string]DiskSMART) + for _, target := range targets { + block := canonicalBlockDeviceForScanPath(target.Path) + native, ok := nativeByBlock[block] + if !ok { + filtered = append(filtered, target) + continue + } + target.NativeTransport = normalizeSMARTTransport(native.Transport) + if !native.SpunDown { + filtered = append(filtered, target) + continue + } + standbyByBlock[block] = nativeStandbySMARTDisk(block, native) + log.Debug(). + Str("component", smartctlComponent). + Str("action", "skip_native_standby"). + Str("device", block). + Msg("Skipping SMART commands for disk reported spun down by Unraid") + } + + // A spun-down native member can be absent from smartctl's non-opening scan. + // Preserve its identity without touching the device. + for block, native := range nativeByBlock { + if native.SpunDown { + standbyByBlock[block] = nativeStandbySMARTDisk(block, native) + } + } + + standby := make([]DiskSMART, 0, len(standbyByBlock)) + for _, disk := range standbyByBlock { + standby = append(standby, disk) + } + sort.Slice(standby, func(i, j int) bool { return standby[i].Device < standby[j].Device }) + return filtered, standby +} + +func nativeStandbySMARTDisk(block string, disk agentshost.UnraidDisk) DiskSMART { + serialStatus := diskinventory.Missing("unraid", "disk serial was not reported") + if strings.TrimSpace(disk.Serial) != "" { + serialStatus = diskinventory.Available("unraid") + } + return DiskSMART{ + Device: block, + Model: strings.TrimSpace(disk.Model), + Serial: strings.TrimSpace(disk.Serial), + Type: normalizeSMARTTransport(disk.Transport), + SizeBytes: disk.SizeBytes, + Health: "UNKNOWN", + Standby: true, + Collection: &diskinventory.CollectionStatus{ + Serial: serialStatus, + Temperature: diskinventory.Unavailable("unraid", "disk is reported spun down"), + }, + LastUpdated: timeNow(), + } +} + +func normalizeSMARTTransport(transport string) string { + switch normalized := strings.ToLower(strings.TrimSpace(transport)); normalized { + case "ata": + return "sata" + case "sata", "sas", "usb", "nvme": + return normalized + default: + return "" + } +} + func listSMARTTargets(ctx context.Context, diskExclude []string) ([]smartctlTarget, error) { if runtimeGOOS == "linux" { return listSMARTTargetsLinux(ctx, diskExclude) @@ -376,27 +469,25 @@ func smartctlTargetsFromDevices(devices []string) []smartctlTarget { } func listSMARTTargetsLinux(ctx context.Context, diskExclude []string) ([]smartctlTarget, error) { - scanTargets, scanErr := listSMARTTargetsLinuxFromScanOpen(ctx, diskExclude) + scanTargets, scanErr := listSMARTTargetsLinuxFromScan(ctx, diskExclude) if scanErr != nil { log.Debug(). Str("component", smartctlComponent). Err(scanErr). - Msg("Failed to enumerate Linux SMART targets via smartctl --scan-open, relying on block device discovery") + Msg("Failed to enumerate Linux SMART targets via smartctl --scan, relying on block device discovery") } - // smartctl --scan-open silently omits any device it fails to open at scan - // time (the failure is only a #-comment in its output), so the scan alone - // can hide a real disk while listing its neighbours (#1483: a SATA SSD - // missing while two NVMe controllers were reported). The kernel block - // device list is the ground truth for which disks exist; scan-open only - // contributes device-type hints. Union the two. + // smartctl's scan alone can omit devices it cannot classify (#1483: a SATA + // SSD missing while two NVMe controllers were reported). The kernel block + // device list is the ground truth for which disks exist; the non-opening + // scan only contributes device-type hints. Union the two. devices, devErr := listBlockDevicesLinux(ctx, diskExclude) if devErr != nil { if len(scanTargets) > 0 { log.Debug(). Str("component", smartctlComponent). Err(devErr). - Msg("Block device discovery failed; using smartctl --scan-open targets only") + Msg("Block device discovery failed; using smartctl --scan targets only") return scanTargets, nil } if scanErr != nil { @@ -437,21 +528,21 @@ func unionSMARTTargets(scanTargets []smartctlTarget, devices []string) []smartct return targets } -func listSMARTTargetsLinuxFromScanOpen(ctx context.Context, diskExclude []string) ([]smartctlTarget, error) { +func listSMARTTargetsLinuxFromScan(ctx context.Context, diskExclude []string) ([]smartctlTarget, error) { smartctlPath, err := execLookPath("smartctl") if err != nil { return nil, fmt.Errorf("look up smartctl binary: %w", err) } - output, err := smartRunCommandOutput(ctx, smartctlPath, "--scan-open") + output, err := smartRunCommandOutput(ctx, smartctlPath, "--scan") if err != nil { return nil, err } - return parseSmartctlScanOpenTargets(output, diskExclude), nil + return parseSmartctlScanTargets(output, diskExclude), nil } -func parseSmartctlScanOpenTargets(output []byte, diskExclude []string) []smartctlTarget { +func parseSmartctlScanTargets(output []byte, diskExclude []string) []smartctlTarget { lines := strings.Split(string(output), "\n") targets := make([]smartctlTarget, 0, len(lines)) typedByPath := make(map[string]bool) @@ -493,7 +584,7 @@ func parseSmartctlScanOpenTargets(output []byte, diskExclude []string) []smartct Str("component", smartctlComponent). Str("action", "skip_virtual_device"). Str("device", path). - Msg("Skipping non-physical device reported by smartctl --scan-open") + Msg("Skipping non-physical device reported by smartctl --scan") continue } if matchesDeviceExclude(name, path, diskExclude) { @@ -843,7 +934,7 @@ func isFreeBSDDiskDeviceName(name string) bool { // refineLinuxBlockDeviceIdentity rewrites a freshly collected SMART reading so // that its device identity and size reflect the underlying block device rather -// than the smartctl scan target. smartctl --scan-open reports NVMe disks by their +// than the smartctl scan target. smartctl --scan reports NVMe disks by their // controller char device (/dev/nvme0), but the stable, user-visible identity is // the namespace block device (/dev/nvme0n1) — the same name Proxmox's disks/list // and /sys/block expose. It also backfills the capacity from /sys/block, the @@ -870,9 +961,12 @@ func refineLinuxBlockDeviceIdentity(smart *DiskSMART, target smartctlTarget) { smart.Device = block smart.Controller, smart.Target = linuxBlockDeviceTopology(block) ensureControllerCollectionStatus(smart, "sysfs") + // Unraid's native transport is authoritative for array members. Otherwise // smartctl labels SAS members with the generic SCSI protocol when the - // transport descriptor is absent; sysfs knows the real link type. - if smart.Type == "" || smart.Type == "scsi" { + // transport descriptor is absent, so prefer explicit sysfs evidence. + if native := normalizeSMARTTransport(target.NativeTransport); native != "" { + smart.Type = native + } else if smart.Type == "" || smart.Type == "scsi" { if evidence := linuxBlockDeviceTransportEvidence(block); evidence != "" { smart.Type = evidence } @@ -969,10 +1063,32 @@ func linuxBlockDeviceTransportEvidence(block string) string { return "usb" } } + resolvedTransport := "" + if resolved, err := smartctlEvalSymlinks(filepath.Join("/sys/block", block, "device")); err == nil { + normalized := strings.ToLower(filepath.ToSlash(resolved)) + switch { + case strings.Contains(normalized, "/usb"): + return "usb" + case strings.Contains(normalized, "/ata"): + resolvedTransport = "sata" + } + } + if readTrimmedFile(filepath.Join("/sys/block", block, "device", "sas_address")) != "" { + return "sas" + } + if strings.EqualFold(readTrimmedFile(filepath.Join("/sys/block", block, "device", "vendor")), "ATA") { + return "sata" + } + if resolvedTransport != "" { + return resolvedTransport + } return "" } func linuxBlockDeviceTransport(block string, target smartctlTarget) string { + if native := normalizeSMARTTransport(target.NativeTransport); native != "" { + return native + } if evidence := linuxBlockDeviceTransportEvidence(block); evidence != "" { return evidence } @@ -1460,8 +1576,11 @@ func mergeSMARTAttributes(base, incoming *SMARTAttributes) *SMARTAttributes { func smartctlProbeAttempts(target smartctlTarget) [][]string { device := target.Path if target.DeviceType != "" { - deviceTypes := []string{target.DeviceType} - // A scan-open device type is a hint, not ground truth: smartctl can + deviceTypes := []string{} + if smartctlDeviceTypeMatchesTransport(target.DeviceType, linuxSMARTTargetTransport(target)) { + deviceTypes = append(deviceTypes, target.DeviceType) + } + // A scan device type is a hint, not ground truth: smartctl can // suggest a type whose full query (-i -A -H) fails or returns no usable // data even though untyped auto-detection works (#1483: a SATA SSD // dropped after its typed probe yielded nothing). Retry untyped before @@ -1469,13 +1588,13 @@ func smartctlProbeAttempts(target smartctlTarget) [][]string { // the -d would re-probe the shared array device, not the member. if runtimeGOOS == "linux" && !isMultiplexedDeviceType(target.DeviceType) { deviceTypes = append(deviceTypes, "") - deviceTypes = append(deviceTypes, linuxInferredSmartctlDeviceTypes(device)...) + deviceTypes = append(deviceTypes, linuxInferredSmartctlDeviceTypes(target)...) } return smartctlArgsForDeviceTypes(device, deviceTypes) } if runtimeGOOS == "linux" { - deviceTypes := append([]string{""}, linuxInferredSmartctlDeviceTypes(device)...) + deviceTypes := append([]string{""}, linuxInferredSmartctlDeviceTypes(target)...) return smartctlArgsForDeviceTypes(device, deviceTypes) } @@ -1505,17 +1624,52 @@ func smartctlArgsForDeviceTypes(device string, deviceTypes []string) [][]string return attempts } -func linuxInferredSmartctlDeviceTypes(device string) []string { +func linuxInferredSmartctlDeviceTypes(target smartctlTarget) []string { if runtimeGOOS != "linux" { return nil } - name := strings.ToLower(filepath.Base(strings.TrimSpace(device))) - switch { - case linuxDirectSATDeviceRE.MatchString(name): - return []string{"sat", "scsi"} - default: + name := strings.ToLower(filepath.Base(strings.TrimSpace(target.Path))) + if !linuxDirectSATDeviceRE.MatchString(name) { return nil } + switch linuxSMARTTargetTransport(target) { + case "sata": + return []string{"sat"} + case "sas": + return []string{"scsi"} + case "usb": + return nil + default: + // An untyped sdX target is more commonly direct ATA than SAS. The SAT + // retry recovers omitted SATA scan targets (#1483), but deliberately + // never guesses SCSI: forcing -d scsi on libata can issue an unsupported + // REPORT SUPPORTED OPERATION CODES request (#1612). + return []string{"sat"} + } +} + +func linuxSMARTTargetTransport(target smartctlTarget) string { + if native := normalizeSMARTTransport(target.NativeTransport); native != "" { + return native + } + block := canonicalBlockDeviceForScanPath(target.Path) + if block == "" { + return "" + } + return linuxBlockDeviceTransportEvidence(block) +} + +func smartctlDeviceTypeMatchesTransport(deviceType, transport string) bool { + deviceType = strings.ToLower(strings.TrimSpace(deviceType)) + transport = normalizeSMARTTransport(transport) + switch transport { + case "sata": + return !strings.HasPrefix(deviceType, "scsi") + case "sas": + return !strings.HasPrefix(deviceType, "sat") + default: + return true + } } func smartctlArgs(device, deviceType string) []string { diff --git a/internal/hostagent/smartctl_coverage_test.go b/internal/hostagent/smartctl_coverage_test.go index 68e636c6e..4d868c69c 100644 --- a/internal/hostagent/smartctl_coverage_test.go +++ b/internal/hostagent/smartctl_coverage_test.go @@ -221,7 +221,7 @@ func TestParseSmartctlScanOpenTargets(t *testing.T) { ``, }, "\n")) - targets := parseSmartctlScanOpenTargets(output, []string{"sdc"}) + targets := parseSmartctlScanTargets(output, []string{"sdc"}) if len(targets) != 4 { t.Fatalf("expected 4 targets, got %#v", targets) } @@ -250,7 +250,7 @@ func TestCollectSMARTLocalUsesSmartctlScanOpenTargetsOnLinux(t *testing.T) { runtimeGOOS = "linux" execLookPath = func(string) (string, error) { return "smartctl", nil } - // Empty /sys/block: the scan-open target is the only discovery source, so + // Empty /sys/block: the scan target is the only discovery source, so // the seenArgs order below stays scan -> probe. readDir = func(string) ([]os.DirEntry, error) { return nil, nil } @@ -258,7 +258,7 @@ func TestCollectSMARTLocalUsesSmartctlScanOpenTargetsOnLinux(t *testing.T) { smartRunCommandOutput = func(ctx context.Context, name string, args ...string) ([]byte, error) { seenArgs = append(seenArgs, append([]string(nil), args...)) - if len(args) == 1 && args[0] == "--scan-open" { + if len(args) == 1 && args[0] == "--scan" { return []byte("/dev/sda -d megaraid,7 # RAID-backed SSD\n"), nil } @@ -487,7 +487,7 @@ func TestCollectSMARTLocalReportsIdentityOnlyForProbeErrors(t *testing.T) { smartRunCommandOutput = func(ctx context.Context, name string, args ...string) ([]byte, error) { if name == "smartctl" { - if len(args) == 1 && args[0] == "--scan-open" { + if len(args) == 1 && args[0] == "--scan" { return nil, nil } device := args[len(args)-1] diff --git a/internal/hostagent/smartctl_discovery_test.go b/internal/hostagent/smartctl_discovery_test.go index 194e587ed..c7e5a9e63 100644 --- a/internal/hostagent/smartctl_discovery_test.go +++ b/internal/hostagent/smartctl_discovery_test.go @@ -15,15 +15,15 @@ import ( ) // scanOpenPVENVMeOnly reproduces the #1483 discovery failure: smartctl -// --scan-open lists the two NVMe controllers but omits the SATA disk it could +// --scan lists the two NVMe controllers but omits the SATA disk it could // not open at scan time (open failures are only emitted as # comments). -// The NVMe lines are verbatim `smartctl --scan-open` output from a PVE 9.1.9 +// The NVMe lines are verbatim `smartctl --scan` output from a PVE 9.1.9 // host (delly, 2026-06-10). const scanOpenPVENVMeOnly = `/dev/nvme0 -d nvme # /dev/nvme0, NVMe device /dev/nvme1 -d nvme # /dev/nvme1, NVMe device ` -// scanOpenPVESATA is verbatim `smartctl --scan-open` output from a PVE 9.1.9 +// scanOpenPVESATA is verbatim `smartctl --scan` output from a PVE 9.1.9 // host with a single SATA SSD (minipc, 2026-06-10). const scanOpenPVESATA = `/dev/sda -d sat # /dev/sda [SAT], ATA device ` @@ -52,7 +52,7 @@ const smartctlNVMeProbeJSON = `{ }` func TestParseSmartctlScanOpenTargetsRealPVEOutput(t *testing.T) { - targets := parseSmartctlScanOpenTargets([]byte(scanOpenPVENVMeOnly+scanOpenPVESATA), nil) + targets := parseSmartctlScanTargets([]byte(scanOpenPVENVMeOnly+scanOpenPVESATA), nil) if len(targets) != 3 { t.Fatalf("expected 3 targets, got %#v", targets) } @@ -90,7 +90,7 @@ func TestUnionSMARTTargetsAddsDevicesMissingFromScanOpen(t *testing.T) { } // TestCollectSMARTLocalIncludesSATADiskOmittedByScanOpen reproduces the #1483 -// host: two NVMe controllers in the scan-open output, the SATA SSD missing +// host: two NVMe controllers in the scan output, the SATA SSD missing // from it, and all three disks present in /sys/block. The SATA disk must // still be discovered, probed untyped, and reported. func TestCollectSMARTLocalIncludesSATADiskOmittedByScanOpen(t *testing.T) { @@ -113,7 +113,7 @@ func TestCollectSMARTLocalIncludesSATADiskOmittedByScanOpen(t *testing.T) { var probed []string smartRunCommandOutput = func(ctx context.Context, name string, args ...string) ([]byte, error) { - if len(args) == 1 && args[0] == "--scan-open" { + if len(args) == 1 && args[0] == "--scan" { return []byte(scanOpenPVENVMeOnly), nil } device := args[len(args)-1] @@ -306,7 +306,7 @@ func TestCollectSMARTLocalEmitsIdentityOnlyEntryWhenSMARTUnavailable(t *testing. execLookPath = func(string) (string, error) { return "smartctl", nil } smartRunCommandOutput = func(ctx context.Context, name string, args ...string) ([]byte, error) { - if len(args) == 1 && args[0] == "--scan-open" { + if len(args) == 1 && args[0] == "--scan" { return []byte(scanOpenPVESATA), nil } return []byte(smartctlNoDataJSON), nil @@ -350,7 +350,7 @@ func TestCollectSMARTLocalSkipsIdentityOnlyEntries(t *testing.T) { execLookPath = func(string) (string, error) { return "smartctl", nil } smartRunCommandOutput = func(ctx context.Context, name string, args ...string) ([]byte, error) { - if len(args) == 1 && args[0] == "--scan-open" { + if len(args) == 1 && args[0] == "--scan" { return []byte("/dev/sda -d megaraid,0 # slot 0\n/dev/sda -d megaraid,1 # slot 1\n"), nil } return []byte(smartctlNoDataJSON), nil @@ -377,7 +377,7 @@ func TestCollectSMARTLocalSkipsIdentityOnlyEntries(t *testing.T) { execLookPath = func(string) (string, error) { return "smartctl", nil } smartRunCommandOutput = func(ctx context.Context, name string, args ...string) ([]byte, error) { - if len(args) == 1 && args[0] == "--scan-open" { + if len(args) == 1 && args[0] == "--scan" { return []byte(scanOpenPVESATA), nil } return []byte(smartctlNoDataJSON), nil @@ -411,7 +411,7 @@ func TestCollectSMARTLocalAppliesExcludeToCanonicalNamespaceName(t *testing.T) { execLookPath = func(string) (string, error) { return "smartctl", nil } smartRunCommandOutput = func(ctx context.Context, name string, args ...string) ([]byte, error) { - if len(args) == 1 && args[0] == "--scan-open" { + if len(args) == 1 && args[0] == "--scan" { return []byte("/dev/nvme0 -d nvme # /dev/nvme0, NVMe device\n"), nil } return []byte(smartctlNVMeProbeJSON), nil diff --git a/internal/hostagent/sysinfo.go b/internal/hostagent/sysinfo.go index 70aff5705..2ad60485e 100644 --- a/internal/hostagent/sysinfo.go +++ b/internal/hostagent/sysinfo.go @@ -25,7 +25,7 @@ type SystemCollector interface { RAIDArrays(ctx context.Context) ([]agentshost.RAIDArray, error) UnraidStorage(ctx context.Context) (*agentshost.UnraidStorage, error) CephStatus(ctx context.Context) (*CephClusterStatus, error) - SMARTLocal(ctx context.Context, exclude []string) ([]DiskSMART, error) + SMARTLocal(ctx context.Context, exclude []string, unraid *agentshost.UnraidStorage) ([]DiskSMART, error) Now() time.Time GOOS() string ReadFile(name string) ([]byte, error) @@ -84,8 +84,8 @@ func (c *defaultCollector) CephStatus(ctx context.Context) (*CephClusterStatus, return CollectCeph(ctx) } -func (c *defaultCollector) SMARTLocal(ctx context.Context, exclude []string) ([]DiskSMART, error) { - return CollectSMARTLocal(ctx, exclude) +func (c *defaultCollector) SMARTLocal(ctx context.Context, exclude []string, unraid *agentshost.UnraidStorage) ([]DiskSMART, error) { + return CollectSMARTLocalWithUnraid(ctx, exclude, unraid) } func (c *defaultCollector) Now() time.Time { diff --git a/internal/hostagent/sysinfo_test.go b/internal/hostagent/sysinfo_test.go index f0c6f02e5..9c4661e14 100644 --- a/internal/hostagent/sysinfo_test.go +++ b/internal/hostagent/sysinfo_test.go @@ -47,7 +47,7 @@ func TestDefaultCollector_Smoke(t *testing.T) { _, _ = c.CephStatus(ctx) // SMART - _, _ = c.SMARTLocal(ctx, nil) + _, _ = c.SMARTLocal(ctx, nil, nil) // Now if c.Now().IsZero() { diff --git a/internal/hostagent/unraid.go b/internal/hostagent/unraid.go index f4daec413..49ceb26de 100644 --- a/internal/hostagent/unraid.go +++ b/internal/hostagent/unraid.go @@ -11,6 +11,7 @@ import ( "time" agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" + "github.com/rs/zerolog/log" ) const hostAgentUnraidVersionPath = "/etc/unraid-version" @@ -39,8 +40,25 @@ func CollectUnraidStorage(ctx context.Context, collector SystemCollector) (*agen return nil, fmt.Errorf("stat %s: %w", hostAgentUnraidVersionPath, err) } + // disks.ini is Unraid's native membership inventory. Read it before mdcmd + // so assigned disks remain reportable even when the status command is + // unavailable, slow, or canceled by the caller. + var iniDisks []agentshost.UnraidDisk + if data, err := collector.ReadFile(hostAgentUnraidDisksINIPath); err == nil { + iniDisks = parseUnraidDisksINI(string(data)) + } else { + log.Debug(). + Str("component", "unraid_collector"). + Str("action", "native_inventory_unavailable"). + Err(err). + Msg("Unable to read Unraid native disk inventory; mdcmd remains available as fallback") + } + mdcmdPath, err := resolveUnraidMdcmdBinary(collector) if err != nil { + if len(iniDisks) > 0 { + return unraidNativeInventoryFallback(iniDisks, err), nil + } return nil, err } @@ -49,17 +67,71 @@ func CollectUnraidStorage(ctx context.Context, collector SystemCollector) (*agen output, err := collector.CommandCombinedOutput(statusCtx, mdcmdPath, "status") if err != nil { + if len(iniDisks) > 0 { + return unraidNativeInventoryFallback(iniDisks, fmt.Errorf("run mdcmd status: %w", err)), nil + } return nil, fmt.Errorf("run mdcmd status: %w", err) } storage, err := parseUnraidStatusOutput(output) if err != nil { + if len(iniDisks) > 0 { + return unraidNativeInventoryFallback(iniDisks, err), nil + } return nil, err } - if data, readErr := collector.ReadFile(hostAgentUnraidDisksINIPath); readErr == nil { - storage = mergeUnraidDiskINI(storage, parseUnraidDisksINI(string(data))) + storage = mergeUnraidDiskINI(storage, iniDisks) + return reconcileUnraidDiskCounts(storage), nil +} + +func unraidNativeInventoryFallback(disks []agentshost.UnraidDisk, cause error) *agentshost.UnraidStorage { + log.Debug(). + Str("component", "unraid_collector"). + Str("action", "native_inventory_fallback"). + Int("disk_count", len(disks)). + Err(cause). + Msg("Reporting Unraid native disk inventory without mdcmd runtime state") + return reconcileUnraidDiskCounts(mergeUnraidDiskINI(nil, disks)) +} + +// reconcileUnraidDiskCounts makes structured native disk states authoritative +// over aggregate mdcmd counters when those states are available. This avoids +// stale or capability-related aggregate values turning healthy assigned disks +// into false missing/disabled alerts, while retaining aggregate fallback on +// older Unraid responses that do not expose per-disk state. +func reconcileUnraidDiskCounts(storage *agentshost.UnraidStorage) *agentshost.UnraidStorage { + if storage == nil { + return nil } - return storage, nil + + hasStructuredStatus := false + disabled, invalid, missing := 0, 0, 0 + for _, disk := range storage.Disks { + if isUnraidEmptySlot(disk) { + continue + } + switch strings.ToLower(strings.TrimSpace(disk.Status)) { + case "": + continue + case "disabled": + hasStructuredStatus = true + disabled++ + case "invalid": + hasStructuredStatus = true + invalid++ + case "missing": + hasStructuredStatus = true + missing++ + default: + hasStructuredStatus = true + } + } + if hasStructuredStatus { + storage.NumDisabled = disabled + storage.NumInvalid = invalid + storage.NumMissing = missing + } + return storage } func resolveUnraidMdcmdBinary(collector SystemCollector) (string, error) { diff --git a/internal/storagehealth/topology_test.go b/internal/storagehealth/topology_test.go index 277cf84da..5e2bc990f 100644 --- a/internal/storagehealth/topology_test.go +++ b/internal/storagehealth/topology_test.go @@ -264,6 +264,7 @@ func TestAssessUnraidStorageUsesDiskStatusesOverAggregateCounters(t *testing.T) SyncAction: "check", NumDisabled: 1, NumInvalid: 1, + NumMissing: 2, Disks: []models.HostUnraidDisk{ {Name: "parity", Role: "parity", Status: "online"}, {Name: "disk1", Role: "data", Status: "online"}, @@ -274,12 +275,33 @@ func TestAssessUnraidStorageUsesDiskStatusesOverAggregateCounters(t *testing.T) t.Fatalf("Level = %q, want %q", assessment.Level, RiskWarning) } for _, reason := range assessment.Reasons { - if reason.Code == "unraid_disabled_disks" || reason.Code == "unraid_invalid_disks" { + if reason.Code == "unraid_disabled_disks" || reason.Code == "unraid_invalid_disks" || reason.Code == "unraid_missing_disks" { t.Fatalf("unexpected aggregate-count reason when structured disk state is healthy: %+v", assessment.Reasons) } } } +func TestAssessUnraidStoragePreservesGenuineStructuredMissingDisk(t *testing.T) { + assessment := AssessUnraidStorage(models.HostUnraidStorage{ + ArrayStarted: true, + Disks: []models.HostUnraidDisk{ + {Name: "parity", Role: "parity", Status: "online"}, + {Name: "disk1", Role: "data", Status: "online"}, + {Name: "disk2", Role: "data", Status: "missing", RawStatus: "DISK_NP", Serial: "EXPECTED-DISK"}, + }, + }) + + if assessment.Level != RiskCritical { + t.Fatalf("Level = %q, want %q", assessment.Level, RiskCritical) + } + for _, reason := range assessment.Reasons { + if reason.Code == "unraid_missing_disks" { + return + } + } + t.Fatalf("genuine missing member was not preserved: %+v", assessment.Reasons) +} + func TestAssessUnraidStorageFallsBackToAggregateCountersWithoutDiskStatuses(t *testing.T) { assessment := AssessUnraidStorage(models.HostUnraidStorage{ ArrayStarted: true, diff --git a/testdata/issue1612_unraid_28tb.json b/testdata/issue1612_unraid_28tb.json new file mode 100644 index 000000000..06736905a --- /dev/null +++ b/testdata/issue1612_unraid_28tb.json @@ -0,0 +1,9 @@ +{ + "smartctlVersion": "smartctl 7.5 2025-04-30 r6178", + "model": "ST28000NT000-4AB103", + "sizeBytes": 28001039286272, + "mdcmdStatus": "mdState=STARTED\nmdResyncAction=check\nmdResyncPos=14000519643136\nmdResyncSize=28001039286272\nmdNumProtected=2\nmdNumDisabled=0\nmdNumInvalid=0\nmdNumMissing=3\ndiskName.0=parity\nrdevName.0=sda\nrdevStatus.0=DISK_OK\ndiskName.1=disk1\nrdevName.1=sdb\nrdevStatus.1=DISK_OK\ndiskName.2=disk2\nrdevName.2=sdc\nrdevStatus.2=DISK_OK\ndiskName.3=disk3\nrdevName.3=sdd\nrdevStatus.3=DISK_OK\ndiskName.4=disk4\nrdevName.4=sde\nrdevStatus.4=DISK_OK\ndiskName.5=disk5\nrdevName.5=sdf\nrdevStatus.5=DISK_OK\ndiskName.6=disk6\nrdevStatus.6=DISK_NP\n", + "disksINI": "[\"parity\"]\nidx=\"0\"\nname=\"parity\"\ndevice=\"sda\"\nid=\"ST28000NT000-4AB103_ZA000000\"\nsectors=\"54689529856\"\nsector_size=\"512\"\ntransport=\"ata\"\nrotational=\"1\"\nspundown=\"0\"\nstatus=\"DISK_OK\"\ntype=\"Parity\"\n[\"disk1\"]\nidx=\"1\"\nname=\"disk1\"\ndevice=\"sdb\"\nid=\"ST28000NT000-4AB103_ZA000001\"\nsectors=\"54689529856\"\nsector_size=\"512\"\ntransport=\"ata\"\nrotational=\"1\"\nspundown=\"0\"\nstatus=\"DISK_OK\"\ntype=\"Data\"\n[\"disk2\"]\nidx=\"2\"\nname=\"disk2\"\ndevice=\"sdc\"\nid=\"ST28000NT000-4AB103_ZA000002\"\nsectors=\"54689529856\"\nsector_size=\"512\"\ntransport=\"ata\"\nrotational=\"1\"\nspundown=\"0\"\nstatus=\"DISK_OK\"\ntype=\"Data\"\n[\"disk3\"]\nidx=\"3\"\nname=\"disk3\"\ndevice=\"sdd\"\nid=\"ST28000NT000-4AB103_ZA000003\"\nsectors=\"54689529856\"\nsector_size=\"512\"\ntransport=\"ata\"\nrotational=\"1\"\nspundown=\"1\"\nstatus=\"DISK_OK\"\ntype=\"Data\"\n[\"disk4\"]\nidx=\"4\"\nname=\"disk4\"\ndevice=\"sde\"\nid=\"ST28000NT000-4AB103_USB00004\"\nsectors=\"54689529856\"\nsector_size=\"512\"\ntransport=\"usb\"\nrotational=\"1\"\nspundown=\"0\"\nstatus=\"DISK_OK\"\ntype=\"Data\"\n[\"disk5\"]\nidx=\"5\"\nname=\"disk5\"\ndevice=\"sdf\"\nid=\"ST28000NT000-4AB103_SAS00005\"\nsectors=\"54689529856\"\nsector_size=\"512\"\ntransport=\"sas\"\nrotational=\"1\"\nspundown=\"0\"\nstatus=\"DISK_OK\"\ntype=\"Data\"\n[\"disk6\"]\nidx=\"6\"\nname=\"disk6\"\ndevice=\"\"\nid=\"ST28000NT000-4AB103_MISSING6\"\nsectors=\"54689529856\"\nsector_size=\"512\"\ntransport=\"ata\"\nrotational=\"1\"\nspundown=\"0\"\nstatus=\"DISK_NP\"\ntype=\"Data\"\n", + "smartctlScan": "/dev/sda -d sat # /dev/sda, ATA device\n/dev/sdb -d sat # /dev/sdb, ATA device\n/dev/sdc -d scsi # rejected direct-ATA hint reproduced from issue path\n/dev/sdd -d sat # sleeping ATA member\n/dev/sde -d sat # SAT-capable USB bridge\n/dev/sdf -d scsi # direct SAS member\n/dev/bus/0 -d megaraid,0 # HBA member\n", + "unsupportedATAJSON": "{\"smartctl\":{\"version\":[7,5],\"svn_revision\":\"6178\",\"messages\":[{\"string\":\"Read Device Identity failed: scsi error unsupported field in scsi command\",\"severity\":\"error\"}]},\"device\":{\"name\":\"/dev/sdb\",\"type\":\"sat\",\"protocol\":\"ATA\"},\"model_name\":\"ST28000NT000-4AB103\",\"serial_number\":\"ZA000001\",\"user_capacity\":{\"bytes\":28001039286272}}" +}