Resolve guest custom-rule alert paths

Fix snapshot, backup, powered-off, and config-reevaluation guest threshold resolution by routing them through shared guest alert context instead of override-only fallback paths.\n\nFixes #1418
This commit is contained in:
rcourtman
2026-04-15 17:25:07 +01:00
parent f72ff5df5c
commit 183fd41dd0
4 changed files with 585 additions and 17 deletions
@@ -82,6 +82,14 @@ runtime must migrate the active alert, history entry, acknowledgment record,
suppression/rate-limit/flapping tracking, and guest per-disk metric identity
to the current canonical state instead of reopening a duplicate alert or
resolving only the stale node-scoped identity.
That same guest-threshold owner also governs guest-derived lifecycle and
posture alerts. Snapshot age, backup age, powered-off state, and
configuration-change reevaluation must all construct a canonical lightweight
guest snapshot and route threshold resolution through the shared
guest-defaults → filter-driven custom rules → guest-override chain.
Passing `nil` guest context or resolving only overrides/defaults is forbidden
because it silently bypasses custom guest rules and makes guest lifecycle
alerting diverge from running-guest metric truth.
That same guest-alert owner also has to retire per-disk guest alerts when the
guest stops, disk alerting is disabled, or the reported disk set changes.
Canonical guest disk identity is only valid while the guest still exposes that
+128 -16
View File
@@ -2162,22 +2162,34 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
// We need to evaluate custom rules, but we don't have the guest object here.
// For now, we'll mark these alerts for re-evaluation by the monitor.
// The next poll cycle will properly evaluate them with custom rules.
thresholds := m.resolveGuestThresholdOverride(cloneThresholdConfig(m.config.GuestDefaults), nil, resourceID)
if thresholds.Disabled {
guestThresholds := m.getGuestThresholds(guestSnapshotFromAlert(alert, resourceID), resourceID)
if guestThresholds.Disabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
// If no custom rule context is available, reevaluation still uses the
// shared default+override resolution path and waits for the next poll
// to apply filter-driven guest rules.
// Note: This doesn't consider custom rules - those will be evaluated
// on the next poll cycle when we have the full guest object
threshold = getThresholdForMetric(thresholds, metricType)
switch alert.Type {
case "snapshot-age":
if !snapshotAlertStillTriggered(alert, m.resolvedSnapshotAlertConfigNoLock(guestThresholds)) {
alertsToResolve = append(alertsToResolve, alertID)
}
continue
case "backup-age":
if !backupAlertStillTriggered(alert, m.resolvedBackupAlertConfigNoLock(guestThresholds)) {
alertsToResolve = append(alertsToResolve, alertID)
}
continue
case "powered-off":
if guestThresholds.DisableConnectivity {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
alert.Level = normalizePoweredOffSeverity(guestThresholds.PoweredOffSeverity)
continue
}
threshold = getThresholdForMetric(guestThresholds, metricType)
}
// If no threshold found or threshold is disabled (trigger <= 0), resolve the alert
@@ -2260,6 +2272,83 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
}
}
func (m *Manager) resolvedSnapshotAlertConfigNoLock(thresholds ThresholdConfig) SnapshotAlertConfig {
cfg := m.config.SnapshotDefaults
if thresholds.Snapshot != nil {
cfg = *thresholds.Snapshot
}
return cfg
}
func (m *Manager) resolvedBackupAlertConfigNoLock(thresholds ThresholdConfig) BackupAlertConfig {
cfg := m.config.BackupDefaults
if thresholds.Backup != nil {
cfg = *thresholds.Backup
}
if cfg.AlertOrphaned == nil {
alertOrphaned := true
cfg.AlertOrphaned = &alertOrphaned
}
return cfg
}
func snapshotAlertStillTriggered(alert *Alert, cfg SnapshotAlertConfig) bool {
if alert == nil || !cfg.Enabled {
return false
}
ageValue, _ := metadataFloatValue(alert.Metadata, "snapshotAgeDays")
sizeValue, _ := metadataFloatValue(alert.Metadata, "snapshotSizeGiB")
if cfg.CriticalDays > 0 && ageValue >= float64(cfg.CriticalDays) {
return true
}
if cfg.WarningDays > 0 && ageValue >= float64(cfg.WarningDays) {
return true
}
if cfg.CriticalSizeGiB > 0 && sizeValue >= cfg.CriticalSizeGiB {
return true
}
if cfg.WarningSizeGiB > 0 && sizeValue >= cfg.WarningSizeGiB {
return true
}
return false
}
func backupAlertStillTriggered(alert *Alert, cfg BackupAlertConfig) bool {
if alert == nil || !cfg.Enabled {
return false
}
vmid := metadataStringValue(alert.Metadata, "guestVmid")
if vmid == "" {
if parsed := metadataIntValue(alert.Metadata["guestVmid"]); parsed > 0 {
vmid = strconv.Itoa(parsed)
}
}
if backupIgnoreVMID(vmid, cfg.IgnoreVMIDs) {
return false
}
if metadataBoolValue(alert.Metadata, "orphaned") && cfg.AlertOrphaned != nil && !*cfg.AlertOrphaned {
return false
}
ageValue, ok := metadataFloatValue(alert.Metadata, "ageDays")
if !ok {
ageValue = alert.Value
}
if cfg.CriticalDays > 0 && ageValue >= float64(cfg.CriticalDays) {
return true
}
if cfg.WarningDays > 0 && ageValue >= float64(cfg.WarningDays) {
return true
}
return false
}
// ReevaluateGuestAlert reevaluates a specific guest's alerts with full threshold resolution including custom rules
// This should be called by the monitor with the current guest state
func (m *Manager) ReevaluateGuestAlert(guest any, guestID string) {
@@ -6094,8 +6183,10 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
// Determine thresholds for this snapshot
resourceID := fmt.Sprintf("%s:%s:%d", snapshot.Instance, snapshot.Node, snapshot.VMID)
guestName := strings.TrimSpace(guestNames[BuildGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID)])
guestContext := guestSnapshotFromIdentity(resourceID, guestName, snapshot.Node, snapshot.Instance, snapshot.Type, "")
m.mu.RLock()
gh := m.getGuestThresholds(nil, resourceID)
gh := m.getGuestThresholds(guestContext, resourceID)
m.mu.RUnlock()
if gh.Disabled {
@@ -6153,7 +6244,6 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
alertID := fmt.Sprintf("snapshot-age-%s", snapshot.ID)
guestKey := BuildGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID)
guestName := strings.TrimSpace(guestNames[guestKey])
guestType := "VM"
if strings.EqualFold(snapshot.Type, "lxc") {
@@ -6484,9 +6574,14 @@ func (m *Manager) CheckBackups(
// Determine thresholds for this backup
currentBackupCfg := backupCfg
if record.lookup.ResourceID != "" {
guestContext := guestSnapshotFromLookup(record.lookup, record.fallbackName)
guestResourceID := strings.TrimSpace(record.lookup.ResourceID)
if guestResourceID == "" {
guestResourceID = guestContext.ID
}
if guestResourceID != "" {
m.mu.RLock()
gh := m.getGuestThresholds(nil, record.lookup.ResourceID)
gh := m.getGuestThresholds(guestContext, guestResourceID)
m.mu.RUnlock()
if gh.Disabled {
continue
@@ -6576,6 +6671,12 @@ func (m *Manager) CheckBackups(
"lastBackupTime": record.lastTime,
"ageDays": ageDays,
"thresholdDays": threshold,
"guestName": displayName,
"guestType": record.lookup.Type,
"guestInstance": instance,
"guestNode": node,
"guestVmid": metadataIntValue(record.vmID),
"orphaned": record.vmID != "" && guestResourceID == "",
}
specResourceID := canonicalBackupSubjectResourceID(alertKey, *record)
specResourceType := canonicalBackupSubjectResourceType(*record)
@@ -7865,6 +7966,17 @@ func metadataIntValue(value interface{}) int {
return 0
}
func metadataFloatValue(metadata map[string]interface{}, key string) (float64, bool) {
if metadata == nil {
return 0, false
}
value, ok := metadata[key]
if !ok {
return 0, false
}
return numericConditionValue(value)
}
func metadataStringValue(metadata map[string]interface{}, key string) string {
if metadata == nil {
return ""
@@ -9223,7 +9335,7 @@ func (m *Manager) clearStorageOfflineAlert(storage models.Storage) {
// checkGuestPoweredOff creates an alert for powered-off guests
func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestType string, monitorOnly bool) {
m.mu.RLock()
thresholds := m.resolveGuestThresholdOverride(cloneThresholdConfig(m.config.GuestDefaults), nil, guestID)
thresholds := m.getGuestThresholds(guestSnapshotFromIdentity(guestID, name, node, instanceName, guestType, "stopped"), guestID)
m.mu.RUnlock()
m.checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType, thresholds, monitorOnly)
}
+117 -1
View File
@@ -1,6 +1,10 @@
package alerts
import "github.com/rcourtman/pulse-go-rewrite/internal/models"
import (
"strings"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
type guestKind uint8
@@ -126,12 +130,124 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot {
}.normalizeCollections()
}
func guestKindFromType(guestType string) guestKind {
switch strings.ToLower(strings.TrimSpace(guestType)) {
case "qemu", "vm":
return guestKindVM
case "lxc", "container", "system-container":
return guestKindContainer
default:
return guestKindUnknown
}
}
func guestSnapshotFromIdentity(resourceID, name, node, instance, guestType, status string) guestSnapshot {
snapshot := guestSnapshot{
Kind: guestKindFromType(guestType),
ID: strings.TrimSpace(resourceID),
Name: strings.TrimSpace(name),
Node: strings.TrimSpace(node),
Instance: strings.TrimSpace(instance),
Status: strings.TrimSpace(status),
}
if ident, ok := guestOverrideIdentityFromGuestOrID(nil, resourceID); ok {
if snapshot.VMID <= 0 {
snapshot.VMID = ident.vmid
}
if snapshot.Instance == "" {
snapshot.Instance = ident.instance
}
if snapshot.Node == "" {
snapshot.Node = ident.node
}
}
if snapshot.Instance == "" {
snapshot.Instance = snapshot.Node
}
return snapshot.normalizeCollections()
}
func guestSnapshotFromLookup(lookup GuestLookup, fallbackName string) guestSnapshot {
resourceID := strings.TrimSpace(lookup.ResourceID)
if resourceID == "" && lookup.Instance != "" && lookup.Node != "" && lookup.VMID > 0 {
resourceID = BuildGuestKey(lookup.Instance, lookup.Node, lookup.VMID)
}
name := strings.TrimSpace(lookup.Name)
if name == "" {
name = strings.TrimSpace(fallbackName)
}
snapshot := guestSnapshotFromIdentity(resourceID, name, lookup.Node, lookup.Instance, lookup.Type, "")
if snapshot.VMID <= 0 && lookup.VMID > 0 {
snapshot.VMID = lookup.VMID
}
return snapshot.normalizeCollections()
}
func guestSnapshotFromAlert(alert *Alert, resourceID string) guestSnapshot {
if alert == nil {
return guestSnapshotFromIdentity(resourceID, "", "", "", "", "")
}
name := metadataStringValue(alert.Metadata, "guestName")
if name == "" {
name = alert.ResourceName
}
node := metadataStringValue(alert.Metadata, "guestNode")
if node == "" {
node = alert.Node
}
instance := metadataStringValue(alert.Metadata, "guestInstance")
if instance == "" {
instance = alert.Instance
}
guestType := metadataStringValue(alert.Metadata, "guestType")
if guestType == "" {
guestType = metadataStringValue(alert.Metadata, "resourceType")
}
status := metadataStringValue(alert.Metadata, "guestStatus")
if status == "" {
status = metadataStringValue(alert.Metadata, "status")
}
snapshot := guestSnapshotFromIdentity(resourceID, name, node, instance, guestType, status)
if snapshot.VMID <= 0 {
snapshot.VMID = metadataIntValue(alert.Metadata["guestVmid"])
}
return snapshot.normalizeCollections()
}
func extractGuestSnapshot(guest any) (guestSnapshot, bool) {
switch g := guest.(type) {
case models.VM:
return guestSnapshotFromVM(g), true
case *models.VM:
if g == nil {
return emptyGuestSnapshot(), false
}
return guestSnapshotFromVM(*g), true
case models.Container:
return guestSnapshotFromContainer(g), true
case *models.Container:
if g == nil {
return emptyGuestSnapshot(), false
}
return guestSnapshotFromContainer(*g), true
case guestSnapshot:
return g.normalizeCollections(), true
case *guestSnapshot:
if g == nil {
return emptyGuestSnapshot(), false
}
return g.normalizeCollections(), true
default:
return emptyGuestSnapshot(), false
}
@@ -2,9 +2,11 @@ package alerts
import (
"testing"
"time"
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
)
func boolPtr(v bool) *bool {
@@ -318,3 +320,333 @@ func TestCheckStorageOfflineUsesSharedThresholdResolution(t *testing.T) {
t.Fatalf("expected storage offline confirmations to clear when shared thresholds disable connectivity")
}
}
func TestCheckSnapshotsUsesGuestContextForCustomRules(t *testing.T) {
m := newTestManager(t)
m.ClearActiveAlerts()
cfg := AlertConfig{
Enabled: true,
SnapshotDefaults: SnapshotAlertConfig{
Enabled: true,
WarningDays: 7,
CriticalDays: 14,
},
CustomRules: []CustomAlertRule{
{
Name: "db-snapshots",
Enabled: true,
Priority: 10,
FilterConditions: FilterStack{
LogicalOperator: "AND",
Filters: []FilterCondition{
{Type: "text", Field: "name", Value: "db"},
},
},
Thresholds: ThresholdConfig{
Snapshot: &SnapshotAlertConfig{
Enabled: true,
WarningDays: 15,
CriticalDays: 20,
},
},
},
},
}
m.UpdateConfig(cfg)
m.mu.Lock()
m.config.TimeThresholds = map[string]int{}
m.mu.Unlock()
now := time.Now()
snapshots := []models.GuestSnapshot{
{
ID: "inst-node-100-weekly",
Name: "weekly",
Node: "node",
Instance: "inst",
Type: "qemu",
VMID: 100,
Time: now.Add(-10 * 24 * time.Hour),
},
{
ID: "inst-node-101-weekly",
Name: "weekly",
Node: "node",
Instance: "inst",
Type: "qemu",
VMID: 101,
Time: now.Add(-10 * 24 * time.Hour),
},
}
guestNames := map[string]string{
BuildGuestKey("inst", "node", 100): "db-server",
BuildGuestKey("inst", "node", 101): "web-server",
}
m.CheckSnapshotsForInstance("inst", snapshots, guestNames)
m.mu.RLock()
_, dbExists := testLookupActiveAlert(t, m, "snapshot-age-inst-node-100-weekly")
_, webExists := testLookupActiveAlert(t, m, "snapshot-age-inst-node-101-weekly")
m.mu.RUnlock()
if dbExists {
t.Fatalf("expected db snapshot alert to be suppressed by custom rule thresholds")
}
if !webExists {
t.Fatalf("expected non-matching snapshot alert to use default thresholds")
}
}
func TestCheckBackupsUsesGuestContextForCustomRules(t *testing.T) {
m := newTestManager(t)
m.ClearActiveAlerts()
cfg := AlertConfig{
Enabled: true,
BackupDefaults: BackupAlertConfig{
Enabled: true,
WarningDays: 7,
CriticalDays: 14,
},
CustomRules: []CustomAlertRule{
{
Name: "db-backups",
Enabled: true,
Priority: 10,
FilterConditions: FilterStack{
LogicalOperator: "AND",
Filters: []FilterCondition{
{Type: "text", Field: "name", Value: "db"},
},
},
Thresholds: ThresholdConfig{
Backup: &BackupAlertConfig{
Enabled: true,
WarningDays: 15,
CriticalDays: 20,
},
},
},
},
}
m.UpdateConfig(cfg)
now := time.Now()
rollups := []recovery.ProtectionRollup{
{
RollupID: "db-rollup",
SubjectRef: &recovery.ExternalRef{
Type: "proxmox-vm",
Namespace: "inst",
Name: "db-server",
ID: BuildGuestKey("inst", "node", 100),
Class: "node",
},
LastSuccessAt: ptrTime(now.Add(-10 * 24 * time.Hour)),
LastOutcome: recovery.OutcomeSuccess,
Providers: []recovery.Provider{recovery.ProviderProxmoxPVE},
},
{
RollupID: "web-rollup",
SubjectRef: &recovery.ExternalRef{
Type: "proxmox-vm",
Namespace: "inst",
Name: "web-server",
ID: BuildGuestKey("inst", "node", 101),
Class: "node",
},
LastSuccessAt: ptrTime(now.Add(-10 * 24 * time.Hour)),
LastOutcome: recovery.OutcomeSuccess,
Providers: []recovery.Provider{recovery.ProviderProxmoxPVE},
},
}
guest100 := GuestLookup{
ResourceID: BuildGuestKey("inst", "node", 100),
Name: "db-server",
Instance: "inst",
Node: "node",
Type: "qemu",
VMID: 100,
}
guest101 := GuestLookup{
ResourceID: BuildGuestKey("inst", "node", 101),
Name: "web-server",
Instance: "inst",
Node: "node",
Type: "qemu",
VMID: 101,
}
guestsByKey := map[string]GuestLookup{
guest100.ResourceID: guest100,
guest101.ResourceID: guest101,
}
guestsByVMID := map[string][]GuestLookup{
"100": {guest100},
"101": {guest101},
}
m.CheckBackups(rollups, guestsByKey, guestsByVMID)
m.mu.RLock()
_, dbExists := testLookupActiveAlert(t, m, "backup-age-"+sanitizeAlertKey(guest100.ResourceID))
_, webExists := testLookupActiveAlert(t, m, "backup-age-"+sanitizeAlertKey(guest101.ResourceID))
m.mu.RUnlock()
if dbExists {
t.Fatalf("expected db backup alert to be suppressed by custom rule thresholds")
}
if !webExists {
t.Fatalf("expected non-matching backup alert to use default thresholds")
}
}
func TestReevaluateActiveAlertsUsesGuestContextForMetricCustomRules(t *testing.T) {
m := newTestManager(t)
resourceID := BuildGuestKey("pve1", "node1", 100)
m.mu.Lock()
m.config.Enabled = true
m.config.GuestDefaults = ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
}
m.config.CustomRules = []CustomAlertRule{
{
Name: "db-metrics",
Enabled: true,
Priority: 10,
FilterConditions: FilterStack{
LogicalOperator: "AND",
Filters: []FilterCondition{
{Type: "text", Field: "name", Value: "db"},
},
},
Thresholds: ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 95, Clear: 90},
},
},
}
state, alert := testNewCanonicalAlert(resourceID, canonicalMetricSpecID(resourceID, "cpu"), string(alertspecs.AlertSpecKindMetricThreshold), "cpu")
alert.Value = 90
alert.Threshold = 80
alert.ResourceName = "db-server"
alert.Node = "node1"
alert.Instance = "pve1"
alert.Metadata = map[string]interface{}{
"resourceType": "vm",
}
m.setActiveAlertNoLock(state, alert)
m.reevaluateActiveAlertsLocked()
m.mu.Unlock()
m.mu.RLock()
_, exists := m.activeAlerts[state]
m.mu.RUnlock()
if exists {
t.Fatalf("expected guest metric alert to resolve when custom rule raises the trigger")
}
}
func TestReevaluateActiveAlertsUsesGuestContextForBackupCustomRules(t *testing.T) {
m := newTestManager(t)
resourceID := BuildGuestKey("pve1", "node1", 100)
m.mu.Lock()
m.config.Enabled = true
m.config.BackupDefaults = BackupAlertConfig{
Enabled: true,
WarningDays: 7,
CriticalDays: 14,
}
m.config.CustomRules = []CustomAlertRule{
{
Name: "db-backup-reeval",
Enabled: true,
Priority: 10,
FilterConditions: FilterStack{
LogicalOperator: "AND",
Filters: []FilterCondition{
{Type: "text", Field: "name", Value: "db"},
},
},
Thresholds: ThresholdConfig{
Backup: &BackupAlertConfig{
Enabled: true,
WarningDays: 15,
CriticalDays: 20,
},
},
},
}
state, alert := testNewCanonicalAlert(resourceID, resourceID+"-backup-age", string(alertspecs.AlertSpecKindPostureThreshold), "backup-age")
alert.Value = 10
alert.Threshold = 7
alert.ResourceName = "db-server backup"
alert.Node = "node1"
alert.Instance = "pve1"
alert.Metadata = map[string]interface{}{
"ageDays": 10.0,
"guestName": "db-server",
"guestType": "qemu",
"guestInstance": "pve1",
"guestNode": "node1",
"guestVmid": 100,
"orphaned": false,
}
m.setActiveAlertNoLock(state, alert)
m.reevaluateActiveAlertsLocked()
m.mu.Unlock()
m.mu.RLock()
_, exists := m.activeAlerts[state]
m.mu.RUnlock()
if exists {
t.Fatalf("expected guest backup alert to resolve when custom rule raises backup thresholds")
}
}
func TestReevaluateActiveAlertsUsesGuestContextForPoweredOffCustomRules(t *testing.T) {
m := newTestManager(t)
resourceID := BuildGuestKey("pve1", "node1", 100)
m.mu.Lock()
m.config.Enabled = true
m.config.CustomRules = []CustomAlertRule{
{
Name: "db-no-powered-off",
Enabled: true,
Priority: 10,
FilterConditions: FilterStack{
LogicalOperator: "AND",
Filters: []FilterCondition{
{Type: "text", Field: "name", Value: "db"},
},
},
Thresholds: ThresholdConfig{
DisableConnectivity: true,
},
},
}
state, alert := testNewCanonicalAlert(resourceID, canonicalPoweredStateSpecID(resourceID), string(alertspecs.AlertSpecKindPoweredState), "powered-off")
alert.ResourceName = "db-server"
alert.Node = "node1"
alert.Instance = "pve1"
alert.Metadata = map[string]interface{}{
"resourceType": "vm",
}
m.setActiveAlertNoLock(state, alert)
m.reevaluateActiveAlertsLocked()
m.mu.Unlock()
m.mu.RLock()
_, exists := m.activeAlerts[state]
m.mu.RUnlock()
if exists {
t.Fatalf("expected guest powered-off alert to resolve when custom rule disables connectivity")
}
}