Clear stale ZFS alerts when a storage loses its pool attachment

checkZFSPoolHealth only runs while storage.ZFSPool is attached, so its
clear paths never fire once the attachment goes away and previously raised
zfs-pool-state, zfs-pool-errors, and zfs-device alerts linger until the
multi-day stale cleanup. Clear them on the spot when a storage is checked
without a pool. This is the recovery path for the shared storages that were
wrongly matched to a node-local pool before the matcher's type gate.

Refs #1731
This commit is contained in:
rcourtman
2026-08-16 06:27:23 +01:00
parent 1ff0680c75
commit d0e7c747d9
3 changed files with 97 additions and 0 deletions
@@ -583,6 +583,11 @@ incidents. ZFS device alert labels must preserve raw device names such as
`/dev/sda4`, but must not join pool and device labels with a raw slash because
device paths can already begin with `/`; browser alert surfaces consume the
runtime `resourceName` as authored rather than patching storage labels locally.
ZFS pool and device alerts follow the storage's pool attachment lifecycle. A
storage checked without an attached ZFS pool must shed any previously raised
zfs-pool-state, zfs-pool-errors, and zfs-device alerts on that check rather
than waiting for the stale-alert cleanup, because the health path can only
clear its own alerts while the attachment exists (#1731).
Ceph pool storage threshold resolution is also source-alias aware. Storage
alerts must evaluate the normalized pool storage id while accepting legacy
`agent:<host>-ceph-pool-<name>` override keys as aliases, so operators do not
+58
View File
@@ -20162,3 +20162,61 @@ func TestMergeSnapshotOverrideInheritsZeroFields(t *testing.T) {
t.Fatalf("explicit snapshot override field lost: %+v", explicit)
}
}
// A storage that stops carrying an attached ZFS pool must shed its ZFS alerts
// on the next check instead of keeping them until the multi-day cleanup. This
// is the recovery path for shared storages that were wrongly matched to a
// node-local pool before the matcher gained its shared-type gate (#1731).
func TestCheckStorageClearsZFSAlertsWhenPoolDetaches(t *testing.T) {
m := newTestManager(t)
degraded := &models.ZFSPool{
Name: "rpool",
State: "DEGRADED",
Devices: []models.ZFSDevice{
{Name: "sda", State: "ONLINE", ReadErrors: 3},
},
}
storage := models.Storage{
ID: "nfs-share",
Name: "NFS Share",
Node: "pve-node1",
Instance: "pve-instance",
ZFSPool: degraded,
}
m.checkZFSPoolHealth(storage)
stateAlertID := buildCanonicalStateID("nfs-share/zfs-pool:rpool", "nfs-share/zfs-pool:rpool-state")
deviceAlertID := buildCanonicalStateID("nfs-share/zfs-pool:rpool/device:sda", "nfs-share/zfs-pool:rpool/device:sda-health")
m.mu.RLock()
testRequireActiveAlert(t, m, stateAlertID)
testRequireActiveAlert(t, m, deviceAlertID)
m.mu.RUnlock()
// An unrelated storage keeps its ZFS alerts.
other := storage
other.ID = "other-zfs"
other.Name = "Other ZFS"
m.checkZFSPoolHealth(other)
storage.ZFSPool = nil
m.clearStorageZFSAlerts(storage)
m.mu.RLock()
_, stateExists := testLookupActiveAlert(t, m, stateAlertID)
_, deviceExists := testLookupActiveAlert(t, m, deviceAlertID)
otherStateID := buildCanonicalStateID("other-zfs/zfs-pool:rpool", "other-zfs/zfs-pool:rpool-state")
_, otherExists := testLookupActiveAlert(t, m, otherStateID)
m.mu.RUnlock()
if stateExists {
t.Error("expected pool state alert cleared after pool detached")
}
if deviceExists {
t.Error("expected device alert cleared after pool detached")
}
if !otherExists {
t.Error("expected unrelated storage to keep its ZFS alerts")
}
}
+34
View File
@@ -107,6 +107,40 @@ func (m *Manager) CheckStorage(storage models.Storage) {
// Check ZFS pool status if this is ZFS storage
if storage.ZFSPool != nil {
m.checkZFSPoolHealth(storage)
} else {
m.clearStorageZFSAlerts(storage)
}
}
// clearStorageZFSAlerts removes ZFS pool/device alerts for a storage that no
// longer carries an attached ZFS pool. checkZFSPoolHealth can only clear its
// own alerts while the pool stays attached, so a storage whose attachment goes
// away (a shared storage that was wrongly matched to a node-local pool, or a
// pool genuinely detached from a storage) would otherwise keep its stale ZFS
// alerts until the multi-day cleanup (#1731).
func (m *Manager) clearStorageZFSAlerts(storage models.Storage) {
storageID := strings.TrimSpace(storage.ID)
if storageID == "" {
return
}
compositePrefix := storageID + "/zfs-pool:"
m.mu.Lock()
defer m.mu.Unlock()
for alertID, alert := range m.activeAlerts {
if alert == nil {
continue
}
switch alert.Type {
case "zfs-pool-state", "zfs-pool-errors", "zfs-device":
default:
continue
}
resourceID := strings.TrimSpace(alert.ResourceID)
if resourceID != storageID && !strings.HasPrefix(resourceID, compositePrefix) {
continue
}
m.clearAlertNoLock(alertID)
}
}