mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-11 14:00:29 +00:00
Clear stale ZFS alerts when a storage loses its pool attachment
checkZFSPoolHealth only runs while storage.ZFSPool is attached, so its clear paths never fire once the attachment goes away and previously raised zfs-pool-state, zfs-pool-errors, and zfs-device alerts linger until the multi-day stale cleanup. Clear them on the spot when a storage is checked without a pool. This is the recovery path for the shared storages that were wrongly matched to a node-local pool before the matcher's type gate. Refs #1731
This commit is contained in:
@@ -583,6 +583,11 @@ incidents. ZFS device alert labels must preserve raw device names such as
|
||||
`/dev/sda4`, but must not join pool and device labels with a raw slash because
|
||||
device paths can already begin with `/`; browser alert surfaces consume the
|
||||
runtime `resourceName` as authored rather than patching storage labels locally.
|
||||
ZFS pool and device alerts follow the storage's pool attachment lifecycle. A
|
||||
storage checked without an attached ZFS pool must shed any previously raised
|
||||
zfs-pool-state, zfs-pool-errors, and zfs-device alerts on that check rather
|
||||
than waiting for the stale-alert cleanup, because the health path can only
|
||||
clear its own alerts while the attachment exists (#1731).
|
||||
Ceph pool storage threshold resolution is also source-alias aware. Storage
|
||||
alerts must evaluate the normalized pool storage id while accepting legacy
|
||||
`agent:<host>-ceph-pool-<name>` override keys as aliases, so operators do not
|
||||
|
||||
@@ -20162,3 +20162,61 @@ func TestMergeSnapshotOverrideInheritsZeroFields(t *testing.T) {
|
||||
t.Fatalf("explicit snapshot override field lost: %+v", explicit)
|
||||
}
|
||||
}
|
||||
|
||||
// A storage that stops carrying an attached ZFS pool must shed its ZFS alerts
|
||||
// on the next check instead of keeping them until the multi-day cleanup. This
|
||||
// is the recovery path for shared storages that were wrongly matched to a
|
||||
// node-local pool before the matcher gained its shared-type gate (#1731).
|
||||
func TestCheckStorageClearsZFSAlertsWhenPoolDetaches(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
degraded := &models.ZFSPool{
|
||||
Name: "rpool",
|
||||
State: "DEGRADED",
|
||||
Devices: []models.ZFSDevice{
|
||||
{Name: "sda", State: "ONLINE", ReadErrors: 3},
|
||||
},
|
||||
}
|
||||
storage := models.Storage{
|
||||
ID: "nfs-share",
|
||||
Name: "NFS Share",
|
||||
Node: "pve-node1",
|
||||
Instance: "pve-instance",
|
||||
ZFSPool: degraded,
|
||||
}
|
||||
m.checkZFSPoolHealth(storage)
|
||||
|
||||
stateAlertID := buildCanonicalStateID("nfs-share/zfs-pool:rpool", "nfs-share/zfs-pool:rpool-state")
|
||||
deviceAlertID := buildCanonicalStateID("nfs-share/zfs-pool:rpool/device:sda", "nfs-share/zfs-pool:rpool/device:sda-health")
|
||||
|
||||
m.mu.RLock()
|
||||
testRequireActiveAlert(t, m, stateAlertID)
|
||||
testRequireActiveAlert(t, m, deviceAlertID)
|
||||
m.mu.RUnlock()
|
||||
|
||||
// An unrelated storage keeps its ZFS alerts.
|
||||
other := storage
|
||||
other.ID = "other-zfs"
|
||||
other.Name = "Other ZFS"
|
||||
m.checkZFSPoolHealth(other)
|
||||
|
||||
storage.ZFSPool = nil
|
||||
m.clearStorageZFSAlerts(storage)
|
||||
|
||||
m.mu.RLock()
|
||||
_, stateExists := testLookupActiveAlert(t, m, stateAlertID)
|
||||
_, deviceExists := testLookupActiveAlert(t, m, deviceAlertID)
|
||||
otherStateID := buildCanonicalStateID("other-zfs/zfs-pool:rpool", "other-zfs/zfs-pool:rpool-state")
|
||||
_, otherExists := testLookupActiveAlert(t, m, otherStateID)
|
||||
m.mu.RUnlock()
|
||||
|
||||
if stateExists {
|
||||
t.Error("expected pool state alert cleared after pool detached")
|
||||
}
|
||||
if deviceExists {
|
||||
t.Error("expected device alert cleared after pool detached")
|
||||
}
|
||||
if !otherExists {
|
||||
t.Error("expected unrelated storage to keep its ZFS alerts")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,6 +107,40 @@ func (m *Manager) CheckStorage(storage models.Storage) {
|
||||
// Check ZFS pool status if this is ZFS storage
|
||||
if storage.ZFSPool != nil {
|
||||
m.checkZFSPoolHealth(storage)
|
||||
} else {
|
||||
m.clearStorageZFSAlerts(storage)
|
||||
}
|
||||
}
|
||||
|
||||
// clearStorageZFSAlerts removes ZFS pool/device alerts for a storage that no
|
||||
// longer carries an attached ZFS pool. checkZFSPoolHealth can only clear its
|
||||
// own alerts while the pool stays attached, so a storage whose attachment goes
|
||||
// away (a shared storage that was wrongly matched to a node-local pool, or a
|
||||
// pool genuinely detached from a storage) would otherwise keep its stale ZFS
|
||||
// alerts until the multi-day cleanup (#1731).
|
||||
func (m *Manager) clearStorageZFSAlerts(storage models.Storage) {
|
||||
storageID := strings.TrimSpace(storage.ID)
|
||||
if storageID == "" {
|
||||
return
|
||||
}
|
||||
compositePrefix := storageID + "/zfs-pool:"
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
for alertID, alert := range m.activeAlerts {
|
||||
if alert == nil {
|
||||
continue
|
||||
}
|
||||
switch alert.Type {
|
||||
case "zfs-pool-state", "zfs-pool-errors", "zfs-device":
|
||||
default:
|
||||
continue
|
||||
}
|
||||
resourceID := strings.TrimSpace(alert.ResourceID)
|
||||
if resourceID != storageID && !strings.HasPrefix(resourceID, compositePrefix) {
|
||||
continue
|
||||
}
|
||||
m.clearAlertNoLock(alertID)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user