From 183fd41dd0dc58a11ab0f8fe67b3fb87baab53aa Mon Sep 17 00:00:00 2001 From: rcourtman Date: Wed, 15 Apr 2026 17:25:07 +0100 Subject: [PATCH] Resolve guest custom-rule alert paths Fix snapshot, backup, powered-off, and config-reevaluation guest threshold resolution by routing them through shared guest alert context instead of override-only fallback paths.\n\nFixes #1418 --- .../v6/internal/subsystems/alerts.md | 8 + internal/alerts/alerts.go | 144 +++++++- internal/alerts/guest_snapshot.go | 118 ++++++- .../threshold_resolution_shared_test.go | 332 ++++++++++++++++++ 4 files changed, 585 insertions(+), 17 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 423a2c851..f4937f373 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -82,6 +82,14 @@ runtime must migrate the active alert, history entry, acknowledgment record, suppression/rate-limit/flapping tracking, and guest per-disk metric identity to the current canonical state instead of reopening a duplicate alert or resolving only the stale node-scoped identity. +That same guest-threshold owner also governs guest-derived lifecycle and +posture alerts. Snapshot age, backup age, powered-off state, and +configuration-change reevaluation must all construct a canonical lightweight +guest snapshot and route threshold resolution through the shared +guest-defaults → filter-driven custom rules → guest-override chain. +Passing `nil` guest context or resolving only overrides/defaults is forbidden +because it silently bypasses custom guest rules and makes guest lifecycle +alerting diverge from running-guest metric truth. That same guest-alert owner also has to retire per-disk guest alerts when the guest stops, disk alerting is disabled, or the reported disk set changes. Canonical guest disk identity is only valid while the guest still exposes that diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index a0a58d964..a4af61ac7 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -2162,22 +2162,34 @@ func (m *Manager) reevaluateActiveAlertsLocked() { alertsToResolve = append(alertsToResolve, alertID) continue } - // We need to evaluate custom rules, but we don't have the guest object here. - // For now, we'll mark these alerts for re-evaluation by the monitor. - // The next poll cycle will properly evaluate them with custom rules. - thresholds := m.resolveGuestThresholdOverride(cloneThresholdConfig(m.config.GuestDefaults), nil, resourceID) - if thresholds.Disabled { + guestThresholds := m.getGuestThresholds(guestSnapshotFromAlert(alert, resourceID), resourceID) + if guestThresholds.Disabled { alertsToResolve = append(alertsToResolve, alertID) continue } - // If no custom rule context is available, reevaluation still uses the - // shared default+override resolution path and waits for the next poll - // to apply filter-driven guest rules. - // Note: This doesn't consider custom rules - those will be evaluated - // on the next poll cycle when we have the full guest object - threshold = getThresholdForMetric(thresholds, metricType) + switch alert.Type { + case "snapshot-age": + if !snapshotAlertStillTriggered(alert, m.resolvedSnapshotAlertConfigNoLock(guestThresholds)) { + alertsToResolve = append(alertsToResolve, alertID) + } + continue + case "backup-age": + if !backupAlertStillTriggered(alert, m.resolvedBackupAlertConfigNoLock(guestThresholds)) { + alertsToResolve = append(alertsToResolve, alertID) + } + continue + case "powered-off": + if guestThresholds.DisableConnectivity { + alertsToResolve = append(alertsToResolve, alertID) + continue + } + alert.Level = normalizePoweredOffSeverity(guestThresholds.PoweredOffSeverity) + continue + } + + threshold = getThresholdForMetric(guestThresholds, metricType) } // If no threshold found or threshold is disabled (trigger <= 0), resolve the alert @@ -2260,6 +2272,83 @@ func (m *Manager) reevaluateActiveAlertsLocked() { } } +func (m *Manager) resolvedSnapshotAlertConfigNoLock(thresholds ThresholdConfig) SnapshotAlertConfig { + cfg := m.config.SnapshotDefaults + if thresholds.Snapshot != nil { + cfg = *thresholds.Snapshot + } + return cfg +} + +func (m *Manager) resolvedBackupAlertConfigNoLock(thresholds ThresholdConfig) BackupAlertConfig { + cfg := m.config.BackupDefaults + if thresholds.Backup != nil { + cfg = *thresholds.Backup + } + if cfg.AlertOrphaned == nil { + alertOrphaned := true + cfg.AlertOrphaned = &alertOrphaned + } + return cfg +} + +func snapshotAlertStillTriggered(alert *Alert, cfg SnapshotAlertConfig) bool { + if alert == nil || !cfg.Enabled { + return false + } + + ageValue, _ := metadataFloatValue(alert.Metadata, "snapshotAgeDays") + sizeValue, _ := metadataFloatValue(alert.Metadata, "snapshotSizeGiB") + + if cfg.CriticalDays > 0 && ageValue >= float64(cfg.CriticalDays) { + return true + } + if cfg.WarningDays > 0 && ageValue >= float64(cfg.WarningDays) { + return true + } + if cfg.CriticalSizeGiB > 0 && sizeValue >= cfg.CriticalSizeGiB { + return true + } + if cfg.WarningSizeGiB > 0 && sizeValue >= cfg.WarningSizeGiB { + return true + } + + return false +} + +func backupAlertStillTriggered(alert *Alert, cfg BackupAlertConfig) bool { + if alert == nil || !cfg.Enabled { + return false + } + + vmid := metadataStringValue(alert.Metadata, "guestVmid") + if vmid == "" { + if parsed := metadataIntValue(alert.Metadata["guestVmid"]); parsed > 0 { + vmid = strconv.Itoa(parsed) + } + } + if backupIgnoreVMID(vmid, cfg.IgnoreVMIDs) { + return false + } + if metadataBoolValue(alert.Metadata, "orphaned") && cfg.AlertOrphaned != nil && !*cfg.AlertOrphaned { + return false + } + + ageValue, ok := metadataFloatValue(alert.Metadata, "ageDays") + if !ok { + ageValue = alert.Value + } + + if cfg.CriticalDays > 0 && ageValue >= float64(cfg.CriticalDays) { + return true + } + if cfg.WarningDays > 0 && ageValue >= float64(cfg.WarningDays) { + return true + } + + return false +} + // ReevaluateGuestAlert reevaluates a specific guest's alerts with full threshold resolution including custom rules // This should be called by the monitor with the current guest state func (m *Manager) ReevaluateGuestAlert(guest any, guestID string) { @@ -6094,8 +6183,10 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod // Determine thresholds for this snapshot resourceID := fmt.Sprintf("%s:%s:%d", snapshot.Instance, snapshot.Node, snapshot.VMID) + guestName := strings.TrimSpace(guestNames[BuildGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID)]) + guestContext := guestSnapshotFromIdentity(resourceID, guestName, snapshot.Node, snapshot.Instance, snapshot.Type, "") m.mu.RLock() - gh := m.getGuestThresholds(nil, resourceID) + gh := m.getGuestThresholds(guestContext, resourceID) m.mu.RUnlock() if gh.Disabled { @@ -6153,7 +6244,6 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod alertID := fmt.Sprintf("snapshot-age-%s", snapshot.ID) guestKey := BuildGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID) - guestName := strings.TrimSpace(guestNames[guestKey]) guestType := "VM" if strings.EqualFold(snapshot.Type, "lxc") { @@ -6484,9 +6574,14 @@ func (m *Manager) CheckBackups( // Determine thresholds for this backup currentBackupCfg := backupCfg - if record.lookup.ResourceID != "" { + guestContext := guestSnapshotFromLookup(record.lookup, record.fallbackName) + guestResourceID := strings.TrimSpace(record.lookup.ResourceID) + if guestResourceID == "" { + guestResourceID = guestContext.ID + } + if guestResourceID != "" { m.mu.RLock() - gh := m.getGuestThresholds(nil, record.lookup.ResourceID) + gh := m.getGuestThresholds(guestContext, guestResourceID) m.mu.RUnlock() if gh.Disabled { continue @@ -6576,6 +6671,12 @@ func (m *Manager) CheckBackups( "lastBackupTime": record.lastTime, "ageDays": ageDays, "thresholdDays": threshold, + "guestName": displayName, + "guestType": record.lookup.Type, + "guestInstance": instance, + "guestNode": node, + "guestVmid": metadataIntValue(record.vmID), + "orphaned": record.vmID != "" && guestResourceID == "", } specResourceID := canonicalBackupSubjectResourceID(alertKey, *record) specResourceType := canonicalBackupSubjectResourceType(*record) @@ -7865,6 +7966,17 @@ func metadataIntValue(value interface{}) int { return 0 } +func metadataFloatValue(metadata map[string]interface{}, key string) (float64, bool) { + if metadata == nil { + return 0, false + } + value, ok := metadata[key] + if !ok { + return 0, false + } + return numericConditionValue(value) +} + func metadataStringValue(metadata map[string]interface{}, key string) string { if metadata == nil { return "" @@ -9223,7 +9335,7 @@ func (m *Manager) clearStorageOfflineAlert(storage models.Storage) { // checkGuestPoweredOff creates an alert for powered-off guests func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestType string, monitorOnly bool) { m.mu.RLock() - thresholds := m.resolveGuestThresholdOverride(cloneThresholdConfig(m.config.GuestDefaults), nil, guestID) + thresholds := m.getGuestThresholds(guestSnapshotFromIdentity(guestID, name, node, instanceName, guestType, "stopped"), guestID) m.mu.RUnlock() m.checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType, thresholds, monitorOnly) } diff --git a/internal/alerts/guest_snapshot.go b/internal/alerts/guest_snapshot.go index a98a33040..05ab3bed7 100644 --- a/internal/alerts/guest_snapshot.go +++ b/internal/alerts/guest_snapshot.go @@ -1,6 +1,10 @@ package alerts -import "github.com/rcourtman/pulse-go-rewrite/internal/models" +import ( + "strings" + + "github.com/rcourtman/pulse-go-rewrite/internal/models" +) type guestKind uint8 @@ -126,12 +130,124 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot { }.normalizeCollections() } +func guestKindFromType(guestType string) guestKind { + switch strings.ToLower(strings.TrimSpace(guestType)) { + case "qemu", "vm": + return guestKindVM + case "lxc", "container", "system-container": + return guestKindContainer + default: + return guestKindUnknown + } +} + +func guestSnapshotFromIdentity(resourceID, name, node, instance, guestType, status string) guestSnapshot { + snapshot := guestSnapshot{ + Kind: guestKindFromType(guestType), + ID: strings.TrimSpace(resourceID), + Name: strings.TrimSpace(name), + Node: strings.TrimSpace(node), + Instance: strings.TrimSpace(instance), + Status: strings.TrimSpace(status), + } + + if ident, ok := guestOverrideIdentityFromGuestOrID(nil, resourceID); ok { + if snapshot.VMID <= 0 { + snapshot.VMID = ident.vmid + } + if snapshot.Instance == "" { + snapshot.Instance = ident.instance + } + if snapshot.Node == "" { + snapshot.Node = ident.node + } + } + + if snapshot.Instance == "" { + snapshot.Instance = snapshot.Node + } + + return snapshot.normalizeCollections() +} + +func guestSnapshotFromLookup(lookup GuestLookup, fallbackName string) guestSnapshot { + resourceID := strings.TrimSpace(lookup.ResourceID) + if resourceID == "" && lookup.Instance != "" && lookup.Node != "" && lookup.VMID > 0 { + resourceID = BuildGuestKey(lookup.Instance, lookup.Node, lookup.VMID) + } + + name := strings.TrimSpace(lookup.Name) + if name == "" { + name = strings.TrimSpace(fallbackName) + } + + snapshot := guestSnapshotFromIdentity(resourceID, name, lookup.Node, lookup.Instance, lookup.Type, "") + if snapshot.VMID <= 0 && lookup.VMID > 0 { + snapshot.VMID = lookup.VMID + } + return snapshot.normalizeCollections() +} + +func guestSnapshotFromAlert(alert *Alert, resourceID string) guestSnapshot { + if alert == nil { + return guestSnapshotFromIdentity(resourceID, "", "", "", "", "") + } + + name := metadataStringValue(alert.Metadata, "guestName") + if name == "" { + name = alert.ResourceName + } + + node := metadataStringValue(alert.Metadata, "guestNode") + if node == "" { + node = alert.Node + } + + instance := metadataStringValue(alert.Metadata, "guestInstance") + if instance == "" { + instance = alert.Instance + } + + guestType := metadataStringValue(alert.Metadata, "guestType") + if guestType == "" { + guestType = metadataStringValue(alert.Metadata, "resourceType") + } + + status := metadataStringValue(alert.Metadata, "guestStatus") + if status == "" { + status = metadataStringValue(alert.Metadata, "status") + } + + snapshot := guestSnapshotFromIdentity(resourceID, name, node, instance, guestType, status) + if snapshot.VMID <= 0 { + snapshot.VMID = metadataIntValue(alert.Metadata["guestVmid"]) + } + return snapshot.normalizeCollections() +} + func extractGuestSnapshot(guest any) (guestSnapshot, bool) { switch g := guest.(type) { case models.VM: return guestSnapshotFromVM(g), true + case *models.VM: + if g == nil { + return emptyGuestSnapshot(), false + } + return guestSnapshotFromVM(*g), true case models.Container: return guestSnapshotFromContainer(g), true + case *models.Container: + if g == nil { + return emptyGuestSnapshot(), false + } + return guestSnapshotFromContainer(*g), true + case guestSnapshot: + return g.normalizeCollections(), true + case *guestSnapshot: + if g == nil { + return emptyGuestSnapshot(), false + } + return g.normalizeCollections(), true default: return emptyGuestSnapshot(), false } diff --git a/internal/alerts/threshold_resolution_shared_test.go b/internal/alerts/threshold_resolution_shared_test.go index dfa69d2c6..8d98facf7 100644 --- a/internal/alerts/threshold_resolution_shared_test.go +++ b/internal/alerts/threshold_resolution_shared_test.go @@ -2,9 +2,11 @@ package alerts import ( "testing" + "time" alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs" "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/internal/recovery" ) func boolPtr(v bool) *bool { @@ -318,3 +320,333 @@ func TestCheckStorageOfflineUsesSharedThresholdResolution(t *testing.T) { t.Fatalf("expected storage offline confirmations to clear when shared thresholds disable connectivity") } } + +func TestCheckSnapshotsUsesGuestContextForCustomRules(t *testing.T) { + m := newTestManager(t) + m.ClearActiveAlerts() + + cfg := AlertConfig{ + Enabled: true, + SnapshotDefaults: SnapshotAlertConfig{ + Enabled: true, + WarningDays: 7, + CriticalDays: 14, + }, + CustomRules: []CustomAlertRule{ + { + Name: "db-snapshots", + Enabled: true, + Priority: 10, + FilterConditions: FilterStack{ + LogicalOperator: "AND", + Filters: []FilterCondition{ + {Type: "text", Field: "name", Value: "db"}, + }, + }, + Thresholds: ThresholdConfig{ + Snapshot: &SnapshotAlertConfig{ + Enabled: true, + WarningDays: 15, + CriticalDays: 20, + }, + }, + }, + }, + } + m.UpdateConfig(cfg) + m.mu.Lock() + m.config.TimeThresholds = map[string]int{} + m.mu.Unlock() + + now := time.Now() + snapshots := []models.GuestSnapshot{ + { + ID: "inst-node-100-weekly", + Name: "weekly", + Node: "node", + Instance: "inst", + Type: "qemu", + VMID: 100, + Time: now.Add(-10 * 24 * time.Hour), + }, + { + ID: "inst-node-101-weekly", + Name: "weekly", + Node: "node", + Instance: "inst", + Type: "qemu", + VMID: 101, + Time: now.Add(-10 * 24 * time.Hour), + }, + } + guestNames := map[string]string{ + BuildGuestKey("inst", "node", 100): "db-server", + BuildGuestKey("inst", "node", 101): "web-server", + } + + m.CheckSnapshotsForInstance("inst", snapshots, guestNames) + + m.mu.RLock() + _, dbExists := testLookupActiveAlert(t, m, "snapshot-age-inst-node-100-weekly") + _, webExists := testLookupActiveAlert(t, m, "snapshot-age-inst-node-101-weekly") + m.mu.RUnlock() + + if dbExists { + t.Fatalf("expected db snapshot alert to be suppressed by custom rule thresholds") + } + if !webExists { + t.Fatalf("expected non-matching snapshot alert to use default thresholds") + } +} + +func TestCheckBackupsUsesGuestContextForCustomRules(t *testing.T) { + m := newTestManager(t) + m.ClearActiveAlerts() + + cfg := AlertConfig{ + Enabled: true, + BackupDefaults: BackupAlertConfig{ + Enabled: true, + WarningDays: 7, + CriticalDays: 14, + }, + CustomRules: []CustomAlertRule{ + { + Name: "db-backups", + Enabled: true, + Priority: 10, + FilterConditions: FilterStack{ + LogicalOperator: "AND", + Filters: []FilterCondition{ + {Type: "text", Field: "name", Value: "db"}, + }, + }, + Thresholds: ThresholdConfig{ + Backup: &BackupAlertConfig{ + Enabled: true, + WarningDays: 15, + CriticalDays: 20, + }, + }, + }, + }, + } + m.UpdateConfig(cfg) + + now := time.Now() + rollups := []recovery.ProtectionRollup{ + { + RollupID: "db-rollup", + SubjectRef: &recovery.ExternalRef{ + Type: "proxmox-vm", + Namespace: "inst", + Name: "db-server", + ID: BuildGuestKey("inst", "node", 100), + Class: "node", + }, + LastSuccessAt: ptrTime(now.Add(-10 * 24 * time.Hour)), + LastOutcome: recovery.OutcomeSuccess, + Providers: []recovery.Provider{recovery.ProviderProxmoxPVE}, + }, + { + RollupID: "web-rollup", + SubjectRef: &recovery.ExternalRef{ + Type: "proxmox-vm", + Namespace: "inst", + Name: "web-server", + ID: BuildGuestKey("inst", "node", 101), + Class: "node", + }, + LastSuccessAt: ptrTime(now.Add(-10 * 24 * time.Hour)), + LastOutcome: recovery.OutcomeSuccess, + Providers: []recovery.Provider{recovery.ProviderProxmoxPVE}, + }, + } + + guest100 := GuestLookup{ + ResourceID: BuildGuestKey("inst", "node", 100), + Name: "db-server", + Instance: "inst", + Node: "node", + Type: "qemu", + VMID: 100, + } + guest101 := GuestLookup{ + ResourceID: BuildGuestKey("inst", "node", 101), + Name: "web-server", + Instance: "inst", + Node: "node", + Type: "qemu", + VMID: 101, + } + guestsByKey := map[string]GuestLookup{ + guest100.ResourceID: guest100, + guest101.ResourceID: guest101, + } + guestsByVMID := map[string][]GuestLookup{ + "100": {guest100}, + "101": {guest101}, + } + + m.CheckBackups(rollups, guestsByKey, guestsByVMID) + + m.mu.RLock() + _, dbExists := testLookupActiveAlert(t, m, "backup-age-"+sanitizeAlertKey(guest100.ResourceID)) + _, webExists := testLookupActiveAlert(t, m, "backup-age-"+sanitizeAlertKey(guest101.ResourceID)) + m.mu.RUnlock() + + if dbExists { + t.Fatalf("expected db backup alert to be suppressed by custom rule thresholds") + } + if !webExists { + t.Fatalf("expected non-matching backup alert to use default thresholds") + } +} + +func TestReevaluateActiveAlertsUsesGuestContextForMetricCustomRules(t *testing.T) { + m := newTestManager(t) + resourceID := BuildGuestKey("pve1", "node1", 100) + + m.mu.Lock() + m.config.Enabled = true + m.config.GuestDefaults = ThresholdConfig{ + CPU: &HysteresisThreshold{Trigger: 80, Clear: 75}, + } + m.config.CustomRules = []CustomAlertRule{ + { + Name: "db-metrics", + Enabled: true, + Priority: 10, + FilterConditions: FilterStack{ + LogicalOperator: "AND", + Filters: []FilterCondition{ + {Type: "text", Field: "name", Value: "db"}, + }, + }, + Thresholds: ThresholdConfig{ + CPU: &HysteresisThreshold{Trigger: 95, Clear: 90}, + }, + }, + } + state, alert := testNewCanonicalAlert(resourceID, canonicalMetricSpecID(resourceID, "cpu"), string(alertspecs.AlertSpecKindMetricThreshold), "cpu") + alert.Value = 90 + alert.Threshold = 80 + alert.ResourceName = "db-server" + alert.Node = "node1" + alert.Instance = "pve1" + alert.Metadata = map[string]interface{}{ + "resourceType": "vm", + } + m.setActiveAlertNoLock(state, alert) + m.reevaluateActiveAlertsLocked() + m.mu.Unlock() + + m.mu.RLock() + _, exists := m.activeAlerts[state] + m.mu.RUnlock() + + if exists { + t.Fatalf("expected guest metric alert to resolve when custom rule raises the trigger") + } +} + +func TestReevaluateActiveAlertsUsesGuestContextForBackupCustomRules(t *testing.T) { + m := newTestManager(t) + resourceID := BuildGuestKey("pve1", "node1", 100) + + m.mu.Lock() + m.config.Enabled = true + m.config.BackupDefaults = BackupAlertConfig{ + Enabled: true, + WarningDays: 7, + CriticalDays: 14, + } + m.config.CustomRules = []CustomAlertRule{ + { + Name: "db-backup-reeval", + Enabled: true, + Priority: 10, + FilterConditions: FilterStack{ + LogicalOperator: "AND", + Filters: []FilterCondition{ + {Type: "text", Field: "name", Value: "db"}, + }, + }, + Thresholds: ThresholdConfig{ + Backup: &BackupAlertConfig{ + Enabled: true, + WarningDays: 15, + CriticalDays: 20, + }, + }, + }, + } + state, alert := testNewCanonicalAlert(resourceID, resourceID+"-backup-age", string(alertspecs.AlertSpecKindPostureThreshold), "backup-age") + alert.Value = 10 + alert.Threshold = 7 + alert.ResourceName = "db-server backup" + alert.Node = "node1" + alert.Instance = "pve1" + alert.Metadata = map[string]interface{}{ + "ageDays": 10.0, + "guestName": "db-server", + "guestType": "qemu", + "guestInstance": "pve1", + "guestNode": "node1", + "guestVmid": 100, + "orphaned": false, + } + m.setActiveAlertNoLock(state, alert) + m.reevaluateActiveAlertsLocked() + m.mu.Unlock() + + m.mu.RLock() + _, exists := m.activeAlerts[state] + m.mu.RUnlock() + + if exists { + t.Fatalf("expected guest backup alert to resolve when custom rule raises backup thresholds") + } +} + +func TestReevaluateActiveAlertsUsesGuestContextForPoweredOffCustomRules(t *testing.T) { + m := newTestManager(t) + resourceID := BuildGuestKey("pve1", "node1", 100) + + m.mu.Lock() + m.config.Enabled = true + m.config.CustomRules = []CustomAlertRule{ + { + Name: "db-no-powered-off", + Enabled: true, + Priority: 10, + FilterConditions: FilterStack{ + LogicalOperator: "AND", + Filters: []FilterCondition{ + {Type: "text", Field: "name", Value: "db"}, + }, + }, + Thresholds: ThresholdConfig{ + DisableConnectivity: true, + }, + }, + } + state, alert := testNewCanonicalAlert(resourceID, canonicalPoweredStateSpecID(resourceID), string(alertspecs.AlertSpecKindPoweredState), "powered-off") + alert.ResourceName = "db-server" + alert.Node = "node1" + alert.Instance = "pve1" + alert.Metadata = map[string]interface{}{ + "resourceType": "vm", + } + m.setActiveAlertNoLock(state, alert) + m.reevaluateActiveAlertsLocked() + m.mu.Unlock() + + m.mu.RLock() + _, exists := m.activeAlerts[state] + m.mu.RUnlock() + + if exists { + t.Fatalf("expected guest powered-off alert to resolve when custom rule disables connectivity") + } +}