mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
Resolve guest custom-rule alert paths
Fix snapshot, backup, powered-off, and config-reevaluation guest threshold resolution by routing them through shared guest alert context instead of override-only fallback paths.\n\nFixes #1418
This commit is contained in:
@@ -82,6 +82,14 @@ runtime must migrate the active alert, history entry, acknowledgment record,
|
||||
suppression/rate-limit/flapping tracking, and guest per-disk metric identity
|
||||
to the current canonical state instead of reopening a duplicate alert or
|
||||
resolving only the stale node-scoped identity.
|
||||
That same guest-threshold owner also governs guest-derived lifecycle and
|
||||
posture alerts. Snapshot age, backup age, powered-off state, and
|
||||
configuration-change reevaluation must all construct a canonical lightweight
|
||||
guest snapshot and route threshold resolution through the shared
|
||||
guest-defaults → filter-driven custom rules → guest-override chain.
|
||||
Passing `nil` guest context or resolving only overrides/defaults is forbidden
|
||||
because it silently bypasses custom guest rules and makes guest lifecycle
|
||||
alerting diverge from running-guest metric truth.
|
||||
That same guest-alert owner also has to retire per-disk guest alerts when the
|
||||
guest stops, disk alerting is disabled, or the reported disk set changes.
|
||||
Canonical guest disk identity is only valid while the guest still exposes that
|
||||
|
||||
+128
-16
@@ -2162,22 +2162,34 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
|
||||
alertsToResolve = append(alertsToResolve, alertID)
|
||||
continue
|
||||
}
|
||||
// We need to evaluate custom rules, but we don't have the guest object here.
|
||||
// For now, we'll mark these alerts for re-evaluation by the monitor.
|
||||
// The next poll cycle will properly evaluate them with custom rules.
|
||||
|
||||
thresholds := m.resolveGuestThresholdOverride(cloneThresholdConfig(m.config.GuestDefaults), nil, resourceID)
|
||||
if thresholds.Disabled {
|
||||
guestThresholds := m.getGuestThresholds(guestSnapshotFromAlert(alert, resourceID), resourceID)
|
||||
if guestThresholds.Disabled {
|
||||
alertsToResolve = append(alertsToResolve, alertID)
|
||||
continue
|
||||
}
|
||||
|
||||
// If no custom rule context is available, reevaluation still uses the
|
||||
// shared default+override resolution path and waits for the next poll
|
||||
// to apply filter-driven guest rules.
|
||||
// Note: This doesn't consider custom rules - those will be evaluated
|
||||
// on the next poll cycle when we have the full guest object
|
||||
threshold = getThresholdForMetric(thresholds, metricType)
|
||||
switch alert.Type {
|
||||
case "snapshot-age":
|
||||
if !snapshotAlertStillTriggered(alert, m.resolvedSnapshotAlertConfigNoLock(guestThresholds)) {
|
||||
alertsToResolve = append(alertsToResolve, alertID)
|
||||
}
|
||||
continue
|
||||
case "backup-age":
|
||||
if !backupAlertStillTriggered(alert, m.resolvedBackupAlertConfigNoLock(guestThresholds)) {
|
||||
alertsToResolve = append(alertsToResolve, alertID)
|
||||
}
|
||||
continue
|
||||
case "powered-off":
|
||||
if guestThresholds.DisableConnectivity {
|
||||
alertsToResolve = append(alertsToResolve, alertID)
|
||||
continue
|
||||
}
|
||||
alert.Level = normalizePoweredOffSeverity(guestThresholds.PoweredOffSeverity)
|
||||
continue
|
||||
}
|
||||
|
||||
threshold = getThresholdForMetric(guestThresholds, metricType)
|
||||
}
|
||||
|
||||
// If no threshold found or threshold is disabled (trigger <= 0), resolve the alert
|
||||
@@ -2260,6 +2272,83 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) resolvedSnapshotAlertConfigNoLock(thresholds ThresholdConfig) SnapshotAlertConfig {
|
||||
cfg := m.config.SnapshotDefaults
|
||||
if thresholds.Snapshot != nil {
|
||||
cfg = *thresholds.Snapshot
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func (m *Manager) resolvedBackupAlertConfigNoLock(thresholds ThresholdConfig) BackupAlertConfig {
|
||||
cfg := m.config.BackupDefaults
|
||||
if thresholds.Backup != nil {
|
||||
cfg = *thresholds.Backup
|
||||
}
|
||||
if cfg.AlertOrphaned == nil {
|
||||
alertOrphaned := true
|
||||
cfg.AlertOrphaned = &alertOrphaned
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func snapshotAlertStillTriggered(alert *Alert, cfg SnapshotAlertConfig) bool {
|
||||
if alert == nil || !cfg.Enabled {
|
||||
return false
|
||||
}
|
||||
|
||||
ageValue, _ := metadataFloatValue(alert.Metadata, "snapshotAgeDays")
|
||||
sizeValue, _ := metadataFloatValue(alert.Metadata, "snapshotSizeGiB")
|
||||
|
||||
if cfg.CriticalDays > 0 && ageValue >= float64(cfg.CriticalDays) {
|
||||
return true
|
||||
}
|
||||
if cfg.WarningDays > 0 && ageValue >= float64(cfg.WarningDays) {
|
||||
return true
|
||||
}
|
||||
if cfg.CriticalSizeGiB > 0 && sizeValue >= cfg.CriticalSizeGiB {
|
||||
return true
|
||||
}
|
||||
if cfg.WarningSizeGiB > 0 && sizeValue >= cfg.WarningSizeGiB {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func backupAlertStillTriggered(alert *Alert, cfg BackupAlertConfig) bool {
|
||||
if alert == nil || !cfg.Enabled {
|
||||
return false
|
||||
}
|
||||
|
||||
vmid := metadataStringValue(alert.Metadata, "guestVmid")
|
||||
if vmid == "" {
|
||||
if parsed := metadataIntValue(alert.Metadata["guestVmid"]); parsed > 0 {
|
||||
vmid = strconv.Itoa(parsed)
|
||||
}
|
||||
}
|
||||
if backupIgnoreVMID(vmid, cfg.IgnoreVMIDs) {
|
||||
return false
|
||||
}
|
||||
if metadataBoolValue(alert.Metadata, "orphaned") && cfg.AlertOrphaned != nil && !*cfg.AlertOrphaned {
|
||||
return false
|
||||
}
|
||||
|
||||
ageValue, ok := metadataFloatValue(alert.Metadata, "ageDays")
|
||||
if !ok {
|
||||
ageValue = alert.Value
|
||||
}
|
||||
|
||||
if cfg.CriticalDays > 0 && ageValue >= float64(cfg.CriticalDays) {
|
||||
return true
|
||||
}
|
||||
if cfg.WarningDays > 0 && ageValue >= float64(cfg.WarningDays) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// ReevaluateGuestAlert reevaluates a specific guest's alerts with full threshold resolution including custom rules
|
||||
// This should be called by the monitor with the current guest state
|
||||
func (m *Manager) ReevaluateGuestAlert(guest any, guestID string) {
|
||||
@@ -6094,8 +6183,10 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
|
||||
|
||||
// Determine thresholds for this snapshot
|
||||
resourceID := fmt.Sprintf("%s:%s:%d", snapshot.Instance, snapshot.Node, snapshot.VMID)
|
||||
guestName := strings.TrimSpace(guestNames[BuildGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID)])
|
||||
guestContext := guestSnapshotFromIdentity(resourceID, guestName, snapshot.Node, snapshot.Instance, snapshot.Type, "")
|
||||
m.mu.RLock()
|
||||
gh := m.getGuestThresholds(nil, resourceID)
|
||||
gh := m.getGuestThresholds(guestContext, resourceID)
|
||||
m.mu.RUnlock()
|
||||
|
||||
if gh.Disabled {
|
||||
@@ -6153,7 +6244,6 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
|
||||
alertID := fmt.Sprintf("snapshot-age-%s", snapshot.ID)
|
||||
|
||||
guestKey := BuildGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID)
|
||||
guestName := strings.TrimSpace(guestNames[guestKey])
|
||||
|
||||
guestType := "VM"
|
||||
if strings.EqualFold(snapshot.Type, "lxc") {
|
||||
@@ -6484,9 +6574,14 @@ func (m *Manager) CheckBackups(
|
||||
|
||||
// Determine thresholds for this backup
|
||||
currentBackupCfg := backupCfg
|
||||
if record.lookup.ResourceID != "" {
|
||||
guestContext := guestSnapshotFromLookup(record.lookup, record.fallbackName)
|
||||
guestResourceID := strings.TrimSpace(record.lookup.ResourceID)
|
||||
if guestResourceID == "" {
|
||||
guestResourceID = guestContext.ID
|
||||
}
|
||||
if guestResourceID != "" {
|
||||
m.mu.RLock()
|
||||
gh := m.getGuestThresholds(nil, record.lookup.ResourceID)
|
||||
gh := m.getGuestThresholds(guestContext, guestResourceID)
|
||||
m.mu.RUnlock()
|
||||
if gh.Disabled {
|
||||
continue
|
||||
@@ -6576,6 +6671,12 @@ func (m *Manager) CheckBackups(
|
||||
"lastBackupTime": record.lastTime,
|
||||
"ageDays": ageDays,
|
||||
"thresholdDays": threshold,
|
||||
"guestName": displayName,
|
||||
"guestType": record.lookup.Type,
|
||||
"guestInstance": instance,
|
||||
"guestNode": node,
|
||||
"guestVmid": metadataIntValue(record.vmID),
|
||||
"orphaned": record.vmID != "" && guestResourceID == "",
|
||||
}
|
||||
specResourceID := canonicalBackupSubjectResourceID(alertKey, *record)
|
||||
specResourceType := canonicalBackupSubjectResourceType(*record)
|
||||
@@ -7865,6 +7966,17 @@ func metadataIntValue(value interface{}) int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func metadataFloatValue(metadata map[string]interface{}, key string) (float64, bool) {
|
||||
if metadata == nil {
|
||||
return 0, false
|
||||
}
|
||||
value, ok := metadata[key]
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
return numericConditionValue(value)
|
||||
}
|
||||
|
||||
func metadataStringValue(metadata map[string]interface{}, key string) string {
|
||||
if metadata == nil {
|
||||
return ""
|
||||
@@ -9223,7 +9335,7 @@ func (m *Manager) clearStorageOfflineAlert(storage models.Storage) {
|
||||
// checkGuestPoweredOff creates an alert for powered-off guests
|
||||
func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestType string, monitorOnly bool) {
|
||||
m.mu.RLock()
|
||||
thresholds := m.resolveGuestThresholdOverride(cloneThresholdConfig(m.config.GuestDefaults), nil, guestID)
|
||||
thresholds := m.getGuestThresholds(guestSnapshotFromIdentity(guestID, name, node, instanceName, guestType, "stopped"), guestID)
|
||||
m.mu.RUnlock()
|
||||
m.checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType, thresholds, monitorOnly)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
package alerts
|
||||
|
||||
import "github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
)
|
||||
|
||||
type guestKind uint8
|
||||
|
||||
@@ -126,12 +130,124 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot {
|
||||
}.normalizeCollections()
|
||||
}
|
||||
|
||||
func guestKindFromType(guestType string) guestKind {
|
||||
switch strings.ToLower(strings.TrimSpace(guestType)) {
|
||||
case "qemu", "vm":
|
||||
return guestKindVM
|
||||
case "lxc", "container", "system-container":
|
||||
return guestKindContainer
|
||||
default:
|
||||
return guestKindUnknown
|
||||
}
|
||||
}
|
||||
|
||||
func guestSnapshotFromIdentity(resourceID, name, node, instance, guestType, status string) guestSnapshot {
|
||||
snapshot := guestSnapshot{
|
||||
Kind: guestKindFromType(guestType),
|
||||
ID: strings.TrimSpace(resourceID),
|
||||
Name: strings.TrimSpace(name),
|
||||
Node: strings.TrimSpace(node),
|
||||
Instance: strings.TrimSpace(instance),
|
||||
Status: strings.TrimSpace(status),
|
||||
}
|
||||
|
||||
if ident, ok := guestOverrideIdentityFromGuestOrID(nil, resourceID); ok {
|
||||
if snapshot.VMID <= 0 {
|
||||
snapshot.VMID = ident.vmid
|
||||
}
|
||||
if snapshot.Instance == "" {
|
||||
snapshot.Instance = ident.instance
|
||||
}
|
||||
if snapshot.Node == "" {
|
||||
snapshot.Node = ident.node
|
||||
}
|
||||
}
|
||||
|
||||
if snapshot.Instance == "" {
|
||||
snapshot.Instance = snapshot.Node
|
||||
}
|
||||
|
||||
return snapshot.normalizeCollections()
|
||||
}
|
||||
|
||||
func guestSnapshotFromLookup(lookup GuestLookup, fallbackName string) guestSnapshot {
|
||||
resourceID := strings.TrimSpace(lookup.ResourceID)
|
||||
if resourceID == "" && lookup.Instance != "" && lookup.Node != "" && lookup.VMID > 0 {
|
||||
resourceID = BuildGuestKey(lookup.Instance, lookup.Node, lookup.VMID)
|
||||
}
|
||||
|
||||
name := strings.TrimSpace(lookup.Name)
|
||||
if name == "" {
|
||||
name = strings.TrimSpace(fallbackName)
|
||||
}
|
||||
|
||||
snapshot := guestSnapshotFromIdentity(resourceID, name, lookup.Node, lookup.Instance, lookup.Type, "")
|
||||
if snapshot.VMID <= 0 && lookup.VMID > 0 {
|
||||
snapshot.VMID = lookup.VMID
|
||||
}
|
||||
return snapshot.normalizeCollections()
|
||||
}
|
||||
|
||||
func guestSnapshotFromAlert(alert *Alert, resourceID string) guestSnapshot {
|
||||
if alert == nil {
|
||||
return guestSnapshotFromIdentity(resourceID, "", "", "", "", "")
|
||||
}
|
||||
|
||||
name := metadataStringValue(alert.Metadata, "guestName")
|
||||
if name == "" {
|
||||
name = alert.ResourceName
|
||||
}
|
||||
|
||||
node := metadataStringValue(alert.Metadata, "guestNode")
|
||||
if node == "" {
|
||||
node = alert.Node
|
||||
}
|
||||
|
||||
instance := metadataStringValue(alert.Metadata, "guestInstance")
|
||||
if instance == "" {
|
||||
instance = alert.Instance
|
||||
}
|
||||
|
||||
guestType := metadataStringValue(alert.Metadata, "guestType")
|
||||
if guestType == "" {
|
||||
guestType = metadataStringValue(alert.Metadata, "resourceType")
|
||||
}
|
||||
|
||||
status := metadataStringValue(alert.Metadata, "guestStatus")
|
||||
if status == "" {
|
||||
status = metadataStringValue(alert.Metadata, "status")
|
||||
}
|
||||
|
||||
snapshot := guestSnapshotFromIdentity(resourceID, name, node, instance, guestType, status)
|
||||
if snapshot.VMID <= 0 {
|
||||
snapshot.VMID = metadataIntValue(alert.Metadata["guestVmid"])
|
||||
}
|
||||
return snapshot.normalizeCollections()
|
||||
}
|
||||
|
||||
func extractGuestSnapshot(guest any) (guestSnapshot, bool) {
|
||||
switch g := guest.(type) {
|
||||
case models.VM:
|
||||
return guestSnapshotFromVM(g), true
|
||||
case *models.VM:
|
||||
if g == nil {
|
||||
return emptyGuestSnapshot(), false
|
||||
}
|
||||
return guestSnapshotFromVM(*g), true
|
||||
case models.Container:
|
||||
return guestSnapshotFromContainer(g), true
|
||||
case *models.Container:
|
||||
if g == nil {
|
||||
return emptyGuestSnapshot(), false
|
||||
}
|
||||
return guestSnapshotFromContainer(*g), true
|
||||
case guestSnapshot:
|
||||
return g.normalizeCollections(), true
|
||||
case *guestSnapshot:
|
||||
if g == nil {
|
||||
return emptyGuestSnapshot(), false
|
||||
}
|
||||
return g.normalizeCollections(), true
|
||||
default:
|
||||
return emptyGuestSnapshot(), false
|
||||
}
|
||||
|
||||
@@ -2,9 +2,11 @@ package alerts
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
|
||||
)
|
||||
|
||||
func boolPtr(v bool) *bool {
|
||||
@@ -318,3 +320,333 @@ func TestCheckStorageOfflineUsesSharedThresholdResolution(t *testing.T) {
|
||||
t.Fatalf("expected storage offline confirmations to clear when shared thresholds disable connectivity")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckSnapshotsUsesGuestContextForCustomRules(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
m.ClearActiveAlerts()
|
||||
|
||||
cfg := AlertConfig{
|
||||
Enabled: true,
|
||||
SnapshotDefaults: SnapshotAlertConfig{
|
||||
Enabled: true,
|
||||
WarningDays: 7,
|
||||
CriticalDays: 14,
|
||||
},
|
||||
CustomRules: []CustomAlertRule{
|
||||
{
|
||||
Name: "db-snapshots",
|
||||
Enabled: true,
|
||||
Priority: 10,
|
||||
FilterConditions: FilterStack{
|
||||
LogicalOperator: "AND",
|
||||
Filters: []FilterCondition{
|
||||
{Type: "text", Field: "name", Value: "db"},
|
||||
},
|
||||
},
|
||||
Thresholds: ThresholdConfig{
|
||||
Snapshot: &SnapshotAlertConfig{
|
||||
Enabled: true,
|
||||
WarningDays: 15,
|
||||
CriticalDays: 20,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
m.UpdateConfig(cfg)
|
||||
m.mu.Lock()
|
||||
m.config.TimeThresholds = map[string]int{}
|
||||
m.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
snapshots := []models.GuestSnapshot{
|
||||
{
|
||||
ID: "inst-node-100-weekly",
|
||||
Name: "weekly",
|
||||
Node: "node",
|
||||
Instance: "inst",
|
||||
Type: "qemu",
|
||||
VMID: 100,
|
||||
Time: now.Add(-10 * 24 * time.Hour),
|
||||
},
|
||||
{
|
||||
ID: "inst-node-101-weekly",
|
||||
Name: "weekly",
|
||||
Node: "node",
|
||||
Instance: "inst",
|
||||
Type: "qemu",
|
||||
VMID: 101,
|
||||
Time: now.Add(-10 * 24 * time.Hour),
|
||||
},
|
||||
}
|
||||
guestNames := map[string]string{
|
||||
BuildGuestKey("inst", "node", 100): "db-server",
|
||||
BuildGuestKey("inst", "node", 101): "web-server",
|
||||
}
|
||||
|
||||
m.CheckSnapshotsForInstance("inst", snapshots, guestNames)
|
||||
|
||||
m.mu.RLock()
|
||||
_, dbExists := testLookupActiveAlert(t, m, "snapshot-age-inst-node-100-weekly")
|
||||
_, webExists := testLookupActiveAlert(t, m, "snapshot-age-inst-node-101-weekly")
|
||||
m.mu.RUnlock()
|
||||
|
||||
if dbExists {
|
||||
t.Fatalf("expected db snapshot alert to be suppressed by custom rule thresholds")
|
||||
}
|
||||
if !webExists {
|
||||
t.Fatalf("expected non-matching snapshot alert to use default thresholds")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckBackupsUsesGuestContextForCustomRules(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
m.ClearActiveAlerts()
|
||||
|
||||
cfg := AlertConfig{
|
||||
Enabled: true,
|
||||
BackupDefaults: BackupAlertConfig{
|
||||
Enabled: true,
|
||||
WarningDays: 7,
|
||||
CriticalDays: 14,
|
||||
},
|
||||
CustomRules: []CustomAlertRule{
|
||||
{
|
||||
Name: "db-backups",
|
||||
Enabled: true,
|
||||
Priority: 10,
|
||||
FilterConditions: FilterStack{
|
||||
LogicalOperator: "AND",
|
||||
Filters: []FilterCondition{
|
||||
{Type: "text", Field: "name", Value: "db"},
|
||||
},
|
||||
},
|
||||
Thresholds: ThresholdConfig{
|
||||
Backup: &BackupAlertConfig{
|
||||
Enabled: true,
|
||||
WarningDays: 15,
|
||||
CriticalDays: 20,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
m.UpdateConfig(cfg)
|
||||
|
||||
now := time.Now()
|
||||
rollups := []recovery.ProtectionRollup{
|
||||
{
|
||||
RollupID: "db-rollup",
|
||||
SubjectRef: &recovery.ExternalRef{
|
||||
Type: "proxmox-vm",
|
||||
Namespace: "inst",
|
||||
Name: "db-server",
|
||||
ID: BuildGuestKey("inst", "node", 100),
|
||||
Class: "node",
|
||||
},
|
||||
LastSuccessAt: ptrTime(now.Add(-10 * 24 * time.Hour)),
|
||||
LastOutcome: recovery.OutcomeSuccess,
|
||||
Providers: []recovery.Provider{recovery.ProviderProxmoxPVE},
|
||||
},
|
||||
{
|
||||
RollupID: "web-rollup",
|
||||
SubjectRef: &recovery.ExternalRef{
|
||||
Type: "proxmox-vm",
|
||||
Namespace: "inst",
|
||||
Name: "web-server",
|
||||
ID: BuildGuestKey("inst", "node", 101),
|
||||
Class: "node",
|
||||
},
|
||||
LastSuccessAt: ptrTime(now.Add(-10 * 24 * time.Hour)),
|
||||
LastOutcome: recovery.OutcomeSuccess,
|
||||
Providers: []recovery.Provider{recovery.ProviderProxmoxPVE},
|
||||
},
|
||||
}
|
||||
|
||||
guest100 := GuestLookup{
|
||||
ResourceID: BuildGuestKey("inst", "node", 100),
|
||||
Name: "db-server",
|
||||
Instance: "inst",
|
||||
Node: "node",
|
||||
Type: "qemu",
|
||||
VMID: 100,
|
||||
}
|
||||
guest101 := GuestLookup{
|
||||
ResourceID: BuildGuestKey("inst", "node", 101),
|
||||
Name: "web-server",
|
||||
Instance: "inst",
|
||||
Node: "node",
|
||||
Type: "qemu",
|
||||
VMID: 101,
|
||||
}
|
||||
guestsByKey := map[string]GuestLookup{
|
||||
guest100.ResourceID: guest100,
|
||||
guest101.ResourceID: guest101,
|
||||
}
|
||||
guestsByVMID := map[string][]GuestLookup{
|
||||
"100": {guest100},
|
||||
"101": {guest101},
|
||||
}
|
||||
|
||||
m.CheckBackups(rollups, guestsByKey, guestsByVMID)
|
||||
|
||||
m.mu.RLock()
|
||||
_, dbExists := testLookupActiveAlert(t, m, "backup-age-"+sanitizeAlertKey(guest100.ResourceID))
|
||||
_, webExists := testLookupActiveAlert(t, m, "backup-age-"+sanitizeAlertKey(guest101.ResourceID))
|
||||
m.mu.RUnlock()
|
||||
|
||||
if dbExists {
|
||||
t.Fatalf("expected db backup alert to be suppressed by custom rule thresholds")
|
||||
}
|
||||
if !webExists {
|
||||
t.Fatalf("expected non-matching backup alert to use default thresholds")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReevaluateActiveAlertsUsesGuestContextForMetricCustomRules(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
resourceID := BuildGuestKey("pve1", "node1", 100)
|
||||
|
||||
m.mu.Lock()
|
||||
m.config.Enabled = true
|
||||
m.config.GuestDefaults = ThresholdConfig{
|
||||
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
||||
}
|
||||
m.config.CustomRules = []CustomAlertRule{
|
||||
{
|
||||
Name: "db-metrics",
|
||||
Enabled: true,
|
||||
Priority: 10,
|
||||
FilterConditions: FilterStack{
|
||||
LogicalOperator: "AND",
|
||||
Filters: []FilterCondition{
|
||||
{Type: "text", Field: "name", Value: "db"},
|
||||
},
|
||||
},
|
||||
Thresholds: ThresholdConfig{
|
||||
CPU: &HysteresisThreshold{Trigger: 95, Clear: 90},
|
||||
},
|
||||
},
|
||||
}
|
||||
state, alert := testNewCanonicalAlert(resourceID, canonicalMetricSpecID(resourceID, "cpu"), string(alertspecs.AlertSpecKindMetricThreshold), "cpu")
|
||||
alert.Value = 90
|
||||
alert.Threshold = 80
|
||||
alert.ResourceName = "db-server"
|
||||
alert.Node = "node1"
|
||||
alert.Instance = "pve1"
|
||||
alert.Metadata = map[string]interface{}{
|
||||
"resourceType": "vm",
|
||||
}
|
||||
m.setActiveAlertNoLock(state, alert)
|
||||
m.reevaluateActiveAlertsLocked()
|
||||
m.mu.Unlock()
|
||||
|
||||
m.mu.RLock()
|
||||
_, exists := m.activeAlerts[state]
|
||||
m.mu.RUnlock()
|
||||
|
||||
if exists {
|
||||
t.Fatalf("expected guest metric alert to resolve when custom rule raises the trigger")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReevaluateActiveAlertsUsesGuestContextForBackupCustomRules(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
resourceID := BuildGuestKey("pve1", "node1", 100)
|
||||
|
||||
m.mu.Lock()
|
||||
m.config.Enabled = true
|
||||
m.config.BackupDefaults = BackupAlertConfig{
|
||||
Enabled: true,
|
||||
WarningDays: 7,
|
||||
CriticalDays: 14,
|
||||
}
|
||||
m.config.CustomRules = []CustomAlertRule{
|
||||
{
|
||||
Name: "db-backup-reeval",
|
||||
Enabled: true,
|
||||
Priority: 10,
|
||||
FilterConditions: FilterStack{
|
||||
LogicalOperator: "AND",
|
||||
Filters: []FilterCondition{
|
||||
{Type: "text", Field: "name", Value: "db"},
|
||||
},
|
||||
},
|
||||
Thresholds: ThresholdConfig{
|
||||
Backup: &BackupAlertConfig{
|
||||
Enabled: true,
|
||||
WarningDays: 15,
|
||||
CriticalDays: 20,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
state, alert := testNewCanonicalAlert(resourceID, resourceID+"-backup-age", string(alertspecs.AlertSpecKindPostureThreshold), "backup-age")
|
||||
alert.Value = 10
|
||||
alert.Threshold = 7
|
||||
alert.ResourceName = "db-server backup"
|
||||
alert.Node = "node1"
|
||||
alert.Instance = "pve1"
|
||||
alert.Metadata = map[string]interface{}{
|
||||
"ageDays": 10.0,
|
||||
"guestName": "db-server",
|
||||
"guestType": "qemu",
|
||||
"guestInstance": "pve1",
|
||||
"guestNode": "node1",
|
||||
"guestVmid": 100,
|
||||
"orphaned": false,
|
||||
}
|
||||
m.setActiveAlertNoLock(state, alert)
|
||||
m.reevaluateActiveAlertsLocked()
|
||||
m.mu.Unlock()
|
||||
|
||||
m.mu.RLock()
|
||||
_, exists := m.activeAlerts[state]
|
||||
m.mu.RUnlock()
|
||||
|
||||
if exists {
|
||||
t.Fatalf("expected guest backup alert to resolve when custom rule raises backup thresholds")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReevaluateActiveAlertsUsesGuestContextForPoweredOffCustomRules(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
resourceID := BuildGuestKey("pve1", "node1", 100)
|
||||
|
||||
m.mu.Lock()
|
||||
m.config.Enabled = true
|
||||
m.config.CustomRules = []CustomAlertRule{
|
||||
{
|
||||
Name: "db-no-powered-off",
|
||||
Enabled: true,
|
||||
Priority: 10,
|
||||
FilterConditions: FilterStack{
|
||||
LogicalOperator: "AND",
|
||||
Filters: []FilterCondition{
|
||||
{Type: "text", Field: "name", Value: "db"},
|
||||
},
|
||||
},
|
||||
Thresholds: ThresholdConfig{
|
||||
DisableConnectivity: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
state, alert := testNewCanonicalAlert(resourceID, canonicalPoweredStateSpecID(resourceID), string(alertspecs.AlertSpecKindPoweredState), "powered-off")
|
||||
alert.ResourceName = "db-server"
|
||||
alert.Node = "node1"
|
||||
alert.Instance = "pve1"
|
||||
alert.Metadata = map[string]interface{}{
|
||||
"resourceType": "vm",
|
||||
}
|
||||
m.setActiveAlertNoLock(state, alert)
|
||||
m.reevaluateActiveAlertsLocked()
|
||||
m.mu.Unlock()
|
||||
|
||||
m.mu.RLock()
|
||||
_, exists := m.activeAlerts[state]
|
||||
m.mu.RUnlock()
|
||||
|
||||
if exists {
|
||||
t.Fatalf("expected guest powered-off alert to resolve when custom rule disables connectivity")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user