From 0910542f0a03dcbc0dbe52271be712262aa2b25c Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 17:23:14 +0100 Subject: [PATCH] fix(monitoring): stop host-agent endurance raising disk life The physical-disk SMART merge let a linked host agent raise a disk's remaining life above the Proxmox inventory value. An intermittent NVMe PercentageUsed 0 reading therefore resolved a genuine low-life disk-wearout alert, which re-fired on the next physical-disk poll and emitted a fresh resolved notification every poll interval (#2112). The merge now takes the agent value only when the Proxmox value is unreported or more pessimistic, never raising a reported remaining-life value, while still recording the agent SMART attributes. This does not change agent admission, token binding or removal-block behaviour. Regression: TestMergeHostAgentSMARTIntoDisks_AgentWearoutDoesNotHideLowPVELife and TestMergeHostAgentSMARTIntoDisks_AgentWearoutFillsUnreportedPVELife in internal/monitoring/monitor_integration_test.go. The monitoring contract is updated in this commit. Change-source: pulse-maintainer Contract-Neutral: Behavioral bug fix in the physical-disk SMART merge; no public API or external contract delta. The monitoring contract is updated in this commit and the dependent agent-lifecycle contract and accepted proof file follow in the next commit. --- .../v6/internal/subsystems/monitoring.md | 19 ++++++ internal/monitoring/monitor.go | 13 +++- .../monitoring/monitor_integration_test.go | 67 +++++++++++++++++++ 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 8fa50f318f..2afeefc878 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -4101,3 +4101,22 @@ consults the base identity as well when the presenting token had no prior binding: a fresh install token clears the block and heals to the base identity, and a token already bound to its own derived identity is left alone (#1753). Focused proof lives in `internal/monitoring/monitor_host_agents_test.go`. + +### Host-agent endurance evidence cannot raise a disk's remaining life + +The physical-disk merge lets the linked host agent's SMART endurance counter +correct the Proxmox inventory, because the agent's `PercentageUsed` is more +precise than the provider's coarse `wearout`. It must not, however, make a disk +look healthier than Proxmox already reports. An intermittent agent reading +(for example an NVMe endurance log that briefly reports `PercentageUsed` 0) +previously raised the merged remaining life to 100, which resolved a genuine +low-life alert and let it re-fire on the next physical-disk poll with a fresh +resolved notification, repeating roughly every poll interval (#2112). The merge +now takes the agent value when the Proxmox value is unreported (`-1`) or when +the agent value is lower (more pessimistic); it never raises a reported +remaining-life value. `PercentageUsed` and the other merged SMART attributes are +still recorded. Focused proof lives in +`internal/monitoring/monitor_integration_test.go` +(`TestMergeHostAgentSMARTIntoDisks_AgentWearoutDoesNotHideLowPVELife` and +`TestMergeHostAgentSMARTIntoDisks_AgentWearoutFillsUnreportedPVELife`). This is +synthetic merge evidence, not reporter acceptance or hardware confirmation. diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 4be2bf5093..268b2e8be5 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -627,11 +627,20 @@ func mergeHostAgentSMARTIntoDisks(disks []models.PhysicalDisk, nodes []models.No Msg("Matched host agent SMART temperature") } - // Always merge SMART attributes from host agent + // Always merge SMART attributes from host agent. The agent's endurance + // counter is more precise than the Proxmox inventory, so it may correct + // a coarse value downward or fill an unreported one. It must not, + // however, make a disk look healthier than the Proxmox inventory already + // reports: an intermittent agent reading (for example an NVMe log page + // that briefly reports PercentageUsed 0) used to raise the remaining + // life, resolve a genuine low-life alert, and let it re-fire on the next + // poll with a fresh resolved notification (#2112). if matched.Attributes != nil { updated[i].SmartAttributes = smartAttributesCopy(matched.Attributes) if derivedWearout := deriveWearoutFromSMARTAttributes(matched.Attributes); derivedWearout >= 0 { - updated[i].Wearout = derivedWearout + if updated[i].Wearout < 0 || derivedWearout < updated[i].Wearout { + updated[i].Wearout = derivedWearout + } } } if matched.IO != nil { diff --git a/internal/monitoring/monitor_integration_test.go b/internal/monitoring/monitor_integration_test.go index db2dd0d5ad..dfd24180b5 100644 --- a/internal/monitoring/monitor_integration_test.go +++ b/internal/monitoring/monitor_integration_test.go @@ -217,6 +217,73 @@ func TestMergeHostAgentSMARTIntoDisks_SMARTFailureAndWearoutOverrideCoarsePVEVal } } +func TestMergeHostAgentSMARTIntoDisks_AgentWearoutDoesNotHideLowPVELife(t *testing.T) { + // A worn drive whose NVMe endurance log intermittently reads + // PercentageUsed 0 must not have its Proxmox-reported remaining life + // raised to 100. That resolved the low-life alert and let it re-fire on the + // next poll, producing repeated notifications (#2112). + disks := []models.PhysicalDisk{{ + ID: "d1", + Node: "pve1", + Serial: "SER1", + Type: "nvme", + Health: "PASSED", + Wearout: 0, + }} + nodes := []models.Node{{Name: "pve1", LinkedAgentID: "host-1"}} + used := 0 + hosts := []models.Host{{ + ID: "host-1", + Sensors: models.HostSensorSummary{ + SMART: []models.HostDiskSMART{{ + Device: "/dev/nvme0n1", + Serial: "SER1", + Health: "PASSED", + Attributes: &models.SMARTAttributes{ + PercentageUsed: &used, + }, + }}, + }, + }} + + result := mergeHostAgentSMARTIntoDisks(disks, nodes, hosts) + if result[0].Wearout != 0 { + t.Fatalf("agent endurance reading raised the remaining life of a worn disk: wearout=%d", result[0].Wearout) + } + if result[0].SmartAttributes == nil || result[0].SmartAttributes.PercentageUsed == nil || *result[0].SmartAttributes.PercentageUsed != 0 { + t.Fatalf("agent SMART attributes were not merged: %+v", result[0].SmartAttributes) + } +} + +func TestMergeHostAgentSMARTIntoDisks_AgentWearoutFillsUnreportedPVELife(t *testing.T) { + disks := []models.PhysicalDisk{{ + ID: "d1", + Node: "pve1", + Serial: "SER1", + Type: "nvme", + Wearout: -1, + }} + nodes := []models.Node{{Name: "pve1", LinkedAgentID: "host-1"}} + used := 40 + hosts := []models.Host{{ + ID: "host-1", + Sensors: models.HostSensorSummary{ + SMART: []models.HostDiskSMART{{ + Device: "/dev/nvme0n1", + Serial: "SER1", + Attributes: &models.SMARTAttributes{ + PercentageUsed: &used, + }, + }}, + }, + }} + + result := mergeHostAgentSMARTIntoDisks(disks, nodes, hosts) + if result[0].Wearout != 60 { + t.Fatalf("agent endurance did not fill unreported PVE life: wearout=%d", result[0].Wearout) + } +} + func TestMergeHostAgentSMARTIntoDisks_DoesNotReplaceFailureWithPassed(t *testing.T) { disks := []models.PhysicalDisk{{ID: "d1", Node: "pve1", Serial: "SER1", Health: "FAILED"}} nodes := []models.Node{{Name: "pve1", LinkedAgentID: "host-1"}}