mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-27 05:57:27 +00:00
rcourtman
019963b60e
Host-agent alerts about something other than the machine itself (filesystems, disk temperatures, SMART disks, RAID and Unraid arrays, custom sensors) cloned the host's alert metadata and carried resourceType "agent". The alert card's Monitoring menu writes policy to their child resource id, yet it named the Pulse agent as the inventory owner, the copy meant for retiring the agent's own machine. CheckHost now stamps platformType "agent" on every host alert and gives child alerts agent-disk, agent-storage or agent-sensor, so the card still links to Machines and uses the neutral source-system copy. Webhook payloads carry the same resourceType values. Each child type keeps "agent" among its CanonicalResourceTypeKeys, and configuration re-evaluation now treats agent policy as final for agent-keyed alerts. Before, a SMART, RAID or sensor alert fell through to the guest switch, so disabling guest alerts resolved it on save and the next agent report raised and notified it again. A metric alert held between its clear and trigger thresholds returned before merging metadata, so one restored from a checkpoint written before platformType or these child types existed kept linking to the wrong page until it re-fired or cleared. The hold now refreshes resourceType and platformType only; value, message and last-seen time stay at the last breach.
1062 lines
38 KiB
Go
1062 lines
38 KiB
Go
package alerts
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
)
|
|
|
|
func characterizationBaseConfig() AlertConfig {
|
|
return AlertConfig{
|
|
Enabled: true,
|
|
ActivationState: ActivationActive,
|
|
GuestDefaults: ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
|
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
|
|
},
|
|
NodeDefaults: ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
|
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
|
|
},
|
|
AgentDefaults: ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
|
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
|
|
},
|
|
PBSDefaults: ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
|
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
},
|
|
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
Overrides: map[string]ThresholdConfig{},
|
|
TimeThresholds: map[string]int{},
|
|
}
|
|
}
|
|
|
|
func newCharacterizationManager(t *testing.T, cfg AlertConfig) *Manager {
|
|
t.Helper()
|
|
|
|
m := newTestManager(t)
|
|
m.UpdateConfig(cfg)
|
|
|
|
// Force deterministic immediate evaluation for characterization tests.
|
|
m.mu.Lock()
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.config.MetricTimeThresholds = nil
|
|
m.mu.Unlock()
|
|
|
|
m.ClearActiveAlerts()
|
|
return m
|
|
}
|
|
|
|
func activeAlert(t *testing.T, m *Manager, alertID string) *Alert {
|
|
t.Helper()
|
|
|
|
m.mu.RLock()
|
|
alert := testRequireActiveAlert(t, m, alertID)
|
|
m.mu.RUnlock()
|
|
if alert != nil {
|
|
return alert.Clone()
|
|
}
|
|
|
|
t.Fatalf("expected active alert %q, got active alerts %v", alertID, alertKeys(m))
|
|
return nil
|
|
}
|
|
|
|
func testVM(resourceID string, vmID int, name, node, instance, status string, cpu float64, tags ...string) models.VM {
|
|
return models.VM{
|
|
ID: resourceID,
|
|
VMID: vmID,
|
|
Name: name,
|
|
Node: node,
|
|
Instance: instance,
|
|
Status: status,
|
|
CPU: cpu,
|
|
Tags: tags,
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationCanonicalGuestIdentityAcrossTypedAndUnifiedChecks(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
specID := canonicalMetricSpecID(resourceID, "cpu")
|
|
alertID := buildCanonicalStateID(resourceID, specID)
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.85), "pve1")
|
|
typedAlert := activeAlert(t, m, alertID)
|
|
|
|
m.ClearActiveAlerts()
|
|
m.CheckUnifiedResource(&UnifiedResourceInput{
|
|
ID: resourceID,
|
|
Type: "vm",
|
|
Name: "app01",
|
|
Node: "node1",
|
|
Instance: "pve1",
|
|
CPU: &UnifiedResourceMetric{Percent: 85},
|
|
})
|
|
unifiedAlert := activeAlert(t, m, alertID)
|
|
|
|
if typedAlert.ResourceID != resourceID {
|
|
t.Fatalf("typed ResourceID = %q, want %q", typedAlert.ResourceID, resourceID)
|
|
}
|
|
if unifiedAlert.ResourceID != resourceID {
|
|
t.Fatalf("unified ResourceID = %q, want %q", unifiedAlert.ResourceID, resourceID)
|
|
}
|
|
if typedAlert.ID != alertID || unifiedAlert.ID != alertID {
|
|
t.Fatalf("expected stable alert ID %q, got typed=%q unified=%q", alertID, typedAlert.ID, unifiedAlert.ID)
|
|
}
|
|
if typedAlert.CanonicalSpecID != specID || unifiedAlert.CanonicalSpecID != specID {
|
|
t.Fatalf("expected canonical spec id %q, got typed=%q unified=%q", specID, typedAlert.CanonicalSpecID, unifiedAlert.CanonicalSpecID)
|
|
}
|
|
if typedAlert.CanonicalState != alertID || unifiedAlert.CanonicalState != alertID {
|
|
t.Fatalf("expected canonical state %q, got typed=%q unified=%q", alertID, typedAlert.CanonicalState, unifiedAlert.CanonicalState)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGetActiveAlertsExportsCanonicalIdentity(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
canonicalState := canonicalMetricStateID(resourceID, "cpu")
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.85), "pve1")
|
|
|
|
alerts := m.GetActiveAlerts()
|
|
if len(alerts) != 1 {
|
|
t.Fatalf("expected 1 active alert, got %d", len(alerts))
|
|
}
|
|
if alerts[0].ID != canonicalState {
|
|
t.Fatalf("GetActiveAlerts() ID = %q, want canonical ID %q", alerts[0].ID, canonicalState)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationAcknowledgmentSurvivesAlertIDChangeForSameCanonicalState(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
oldAlertID := "legacy-" + resourceID + "-cpu"
|
|
newAlertID := resourceID + "-cpu"
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
|
|
m.mu.Lock()
|
|
oldAlert := &Alert{
|
|
ID: oldAlertID,
|
|
Type: "cpu",
|
|
Level: AlertLevelWarning,
|
|
ResourceID: resourceID,
|
|
Message: "legacy alert",
|
|
StartTime: time.Now().Add(-5 * time.Minute),
|
|
LastSeen: time.Now().Add(-1 * time.Minute),
|
|
}
|
|
applyCanonicalIdentity(oldAlert, newAlertID, "metric-threshold")
|
|
m.activeAlerts[oldAlertID] = oldAlert
|
|
m.mu.Unlock()
|
|
|
|
if err := m.AcknowledgeAlert(oldAlertID, "alice"); err != nil {
|
|
t.Fatalf("AcknowledgeAlert() error = %v", err)
|
|
}
|
|
|
|
m.mu.Lock()
|
|
m.removeActiveAlertNoLock(oldAlertID)
|
|
replacement := &Alert{
|
|
ID: newAlertID,
|
|
Type: "cpu",
|
|
Level: AlertLevelWarning,
|
|
ResourceID: resourceID,
|
|
Message: "canonical alert",
|
|
StartTime: time.Now(),
|
|
LastSeen: time.Now(),
|
|
}
|
|
applyCanonicalIdentity(replacement, newAlertID, "metric-threshold")
|
|
m.preserveAlertState(newAlertID, replacement)
|
|
m.activeAlerts[newAlertID] = replacement
|
|
m.mu.Unlock()
|
|
|
|
alert := activeAlert(t, m, newAlertID)
|
|
if !alert.Acknowledged {
|
|
t.Fatal("expected acknowledgment to survive alert ID change")
|
|
}
|
|
if alert.AckUser != "alice" {
|
|
t.Fatalf("AckUser = %q, want alice", alert.AckUser)
|
|
}
|
|
if alert.CanonicalState != resourceID+"::"+newAlertID {
|
|
t.Fatalf("CanonicalState = %q, want %q", alert.CanonicalState, resourceID+"::"+newAlertID)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationHistoryUpdateUsesCanonicalStateAcrossAlertIDChange(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
oldAlertID := "legacy-" + resourceID + "-cpu"
|
|
newAlertID := resourceID + "-cpu"
|
|
lastSeen := time.Now()
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
|
|
oldHistoryAlert := Alert{
|
|
ID: oldAlertID,
|
|
Type: "cpu",
|
|
ResourceID: resourceID,
|
|
StartTime: time.Now().Add(-10 * time.Minute),
|
|
LastSeen: time.Now().Add(-5 * time.Minute),
|
|
}
|
|
applyCanonicalIdentity(&oldHistoryAlert, newAlertID, "metric-threshold")
|
|
m.historyManager.AddAlert(oldHistoryAlert)
|
|
|
|
current := &Alert{
|
|
ID: newAlertID,
|
|
Type: "cpu",
|
|
ResourceID: resourceID,
|
|
StartTime: time.Now().Add(-2 * time.Minute),
|
|
LastSeen: lastSeen,
|
|
}
|
|
applyCanonicalIdentity(current, newAlertID, "metric-threshold")
|
|
|
|
m.historyManager.UpdateAlertLastSeenForAlert(current, lastSeen)
|
|
|
|
history := m.GetAlertHistory(10)
|
|
if len(history) == 0 {
|
|
t.Fatalf("expected history entry")
|
|
}
|
|
if !history[0].LastSeen.Equal(lastSeen) {
|
|
t.Fatalf("LastSeen = %v, want %v", history[0].LastSeen, lastSeen)
|
|
}
|
|
expectedCanonicalState := buildCanonicalStateID(resourceID, newAlertID)
|
|
if history[0].ID != expectedCanonicalState {
|
|
t.Fatalf("history alert ID = %q, want canonical ID %q", history[0].ID, expectedCanonicalState)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationRecentSuppressionSurvivesAlertIDChangeForSameCanonicalState(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
oldAlertID := "legacy-" + resourceID + "-cpu"
|
|
newAlertID := resourceID + "-cpu"
|
|
clear := 75.0
|
|
critical := 90.0
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.mu.Lock()
|
|
m.config.MinimumDelta = 5
|
|
m.config.SuppressionWindow = 10
|
|
recent := &Alert{
|
|
ID: oldAlertID,
|
|
Type: "cpu",
|
|
Level: AlertLevelWarning,
|
|
ResourceID: resourceID,
|
|
Value: 90,
|
|
StartTime: time.Now().Add(-1 * time.Minute),
|
|
LastSeen: time.Now().Add(-30 * time.Second),
|
|
}
|
|
applyCanonicalIdentity(recent, newAlertID, "metric-threshold")
|
|
m.recentAlerts[recent.CanonicalState] = recent
|
|
m.mu.Unlock()
|
|
|
|
spec := alertspecs.ResourceAlertSpec{
|
|
ID: newAlertID,
|
|
ResourceID: resourceID,
|
|
ResourceType: unifiedresources.ResourceTypeVM,
|
|
Kind: alertspecs.AlertSpecKindMetricThreshold,
|
|
Severity: alertspecs.AlertSeverityWarning,
|
|
MetricThreshold: &alertspecs.MetricThresholdSpec{
|
|
Metric: "cpu",
|
|
Direction: alertspecs.ThresholdDirectionAbove,
|
|
Trigger: 80,
|
|
Recovery: &clear,
|
|
Critical: &critical,
|
|
},
|
|
}
|
|
|
|
m.evaluateCanonicalMetricAlert(spec, "app01", "node1", "pve1", "cpu", 91, &HysteresisThreshold{Trigger: 80, Clear: 75}, nil)
|
|
|
|
assertAlertMissing(t, m, newAlertID)
|
|
|
|
m.mu.RLock()
|
|
suppressedUntil, ok := m.suppressedUntil[resourceID+"::"+newAlertID]
|
|
m.mu.RUnlock()
|
|
if !ok {
|
|
t.Fatal("expected canonical suppression window to be recorded")
|
|
}
|
|
if time.Until(suppressedUntil) <= 0 {
|
|
t.Fatal("expected canonical suppression window to be in the future")
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationAcknowledgeByCanonicalStateAlias(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
alertID := canonicalMetricStateID(resourceID, "cpu")
|
|
canonicalState := alertID
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
|
|
|
|
if err := m.AcknowledgeAlert(canonicalState, "alice"); err != nil {
|
|
t.Fatalf("AcknowledgeAlert(%q) error = %v", canonicalState, err)
|
|
}
|
|
|
|
alert := activeAlert(t, m, alertID)
|
|
if !alert.Acknowledged {
|
|
t.Fatal("expected alert to be acknowledged through canonical state alias")
|
|
}
|
|
if alert.AckUser != "alice" {
|
|
t.Fatalf("AckUser = %q, want alice", alert.AckUser)
|
|
}
|
|
|
|
m.mu.RLock()
|
|
_, legacyAck := m.ackState[alertID]
|
|
record, canonicalAck := m.ackStateByCanonical[canonicalState]
|
|
m.mu.RUnlock()
|
|
if legacyAck {
|
|
t.Fatalf("expected canonical alert acknowledgment to be keyed by canonical state, not legacy alert ID")
|
|
}
|
|
if !canonicalAck || !record.acknowledged || record.user != "alice" {
|
|
t.Fatalf("expected canonical ack record for %q, got %+v exists=%t", canonicalState, record, canonicalAck)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestThresholdPrecedence(t *testing.T) {
|
|
overrideGuestID := BuildGuestKey("pve1", "node1", 101)
|
|
ruleGuestID := BuildGuestKey("pve1", "node1", 102)
|
|
|
|
cfg := characterizationBaseConfig()
|
|
cfg.GuestDefaults.CPU = &HysteresisThreshold{Trigger: 90, Clear: 85}
|
|
cfg.CustomRules = []CustomAlertRule{
|
|
{
|
|
Name: "named-apps",
|
|
Enabled: true,
|
|
Priority: 10,
|
|
FilterConditions: FilterStack{
|
|
LogicalOperator: "AND",
|
|
Filters: []FilterCondition{
|
|
{Type: "text", Field: "name", Value: "app"},
|
|
},
|
|
},
|
|
Thresholds: ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 70, Clear: 65},
|
|
},
|
|
},
|
|
}
|
|
cfg.Overrides[overrideGuestID] = ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 95, Clear: 90},
|
|
}
|
|
|
|
m := newCharacterizationManager(t, cfg)
|
|
m.CheckGuest(testVM(ruleGuestID, 102, "app-rule", "node1", "pve1", "running", 0.80), "pve1")
|
|
m.CheckGuest(testVM(overrideGuestID, 101, "app-override", "node1", "pve1", "running", 0.80), "pve1")
|
|
|
|
assertAlertPresent(t, m, canonicalMetricStateID(ruleGuestID, "cpu"))
|
|
assertAlertMissing(t, m, canonicalMetricStateID(overrideGuestID, "cpu"))
|
|
}
|
|
|
|
func TestAlertCharacterizationDisableConnectivitySuppressesPoweredOffButNotMetrics(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
cfg := characterizationBaseConfig()
|
|
cfg.Overrides[resourceID] = ThresholdConfig{DisableConnectivity: true}
|
|
|
|
m := newCharacterizationManager(t, cfg)
|
|
stopped := testVM(resourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
|
|
assertAlertMissing(t, m, "guest-powered-off-"+resourceID)
|
|
|
|
m.mu.RLock()
|
|
hasConfirmations := testCoreHasIncident(m, resourceID, canonicalPoweredStateSpecID(resourceID))
|
|
m.mu.RUnlock()
|
|
if hasConfirmations {
|
|
t.Fatalf("expected powered-off tracking to stay clear when connectivity is disabled")
|
|
}
|
|
|
|
m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
|
|
assertAlertPresent(t, m, canonicalMetricStateID(resourceID, "cpu"))
|
|
}
|
|
|
|
func TestAlertCharacterizationOnBootFalseSuppressesPoweredOffAlert(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
cfg := characterizationBaseConfig()
|
|
m := newCharacterizationManager(t, cfg)
|
|
|
|
off := false
|
|
stopped := testVM(resourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
stopped.OnBoot = &off
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
|
|
assertAlertMissing(t, m, "guest-powered-off-"+resourceID)
|
|
|
|
m.mu.RLock()
|
|
hasConfirmations := testCoreHasIncident(m, resourceID, canonicalPoweredStateSpecID(resourceID))
|
|
m.mu.RUnlock()
|
|
if hasConfirmations {
|
|
t.Fatalf("expected no powered-off tracking for guest with onboot=false")
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationOnBootTrueStillGeneratesPoweredOffAlert(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
cfg := characterizationBaseConfig()
|
|
m := newCharacterizationManager(t, cfg)
|
|
|
|
on := true
|
|
stopped := testVM(resourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
stopped.OnBoot = &on
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
|
|
assertAlertPresent(t, m, "guest-powered-off-"+resourceID)
|
|
}
|
|
|
|
func TestAlertCharacterizationOnBootNilPreservesCurrentBehavior(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
cfg := characterizationBaseConfig()
|
|
m := newCharacterizationManager(t, cfg)
|
|
|
|
stopped := testVM(resourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
|
|
assertAlertPresent(t, m, "guest-powered-off-"+resourceID)
|
|
}
|
|
|
|
func TestAlertCharacterizationReevaluatesAlertsWhenConfigChanges(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
alertID := canonicalMetricStateID(resourceID, "cpu")
|
|
cfg := characterizationBaseConfig()
|
|
|
|
m := newCharacterizationManager(t, cfg)
|
|
resolved := make(chan string, 1)
|
|
m.SetResolvedCallback(func(id string) {
|
|
select {
|
|
case resolved <- id:
|
|
default:
|
|
}
|
|
})
|
|
|
|
m.CheckUnifiedResource(&UnifiedResourceInput{
|
|
ID: resourceID,
|
|
Type: "vm",
|
|
Name: "app01",
|
|
Node: "node1",
|
|
Instance: "pve1",
|
|
CPU: &UnifiedResourceMetric{Percent: 87},
|
|
})
|
|
assertAlertPresent(t, m, alertID)
|
|
|
|
updated := cfg
|
|
updated.GuestDefaults.CPU = &HysteresisThreshold{Trigger: 90, Clear: 85}
|
|
m.UpdateConfig(updated)
|
|
|
|
select {
|
|
case got := <-resolved:
|
|
expectedResolvedID := alertID
|
|
if got != expectedResolvedID {
|
|
t.Fatalf("resolved callback = %q, want %q", got, expectedResolvedID)
|
|
}
|
|
case <-time.After(250 * time.Millisecond):
|
|
t.Fatalf("expected alert %q to resolve after config change", alertID)
|
|
}
|
|
|
|
assertAlertMissing(t, m, alertID)
|
|
|
|
_, wasResolved := testLookupResolvedAlert(t, m, alertID)
|
|
if !wasResolved {
|
|
t.Fatalf("expected %q in recently resolved after config change", alertID)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationPoweredOffLifecycle(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
alertID := "guest-powered-off-" + resourceID
|
|
cfg := characterizationBaseConfig()
|
|
m := newCharacterizationManager(t, cfg)
|
|
|
|
running := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.95)
|
|
stopped := testVM(resourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
paused := testVM(resourceID, 101, "app01", "node1", "pve1", "paused", 0)
|
|
|
|
m.CheckGuest(running, "pve1")
|
|
assertAlertPresent(t, m, canonicalMetricStateID(resourceID, "cpu"))
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
|
|
assertAlertMissing(t, m, canonicalMetricStateID(resourceID, "cpu"))
|
|
assertAlertPresent(t, m, alertID)
|
|
|
|
m.CheckGuest(paused, "pve1")
|
|
assertAlertMissing(t, m, alertID)
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
assertAlertPresent(t, m, alertID)
|
|
|
|
m.CheckGuest(running, "pve1")
|
|
assertAlertMissing(t, m, alertID)
|
|
}
|
|
|
|
func TestAlertCharacterizationAcknowledgmentSurvivesPoweredOffReappearance(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
alertID := canonicalPoweredStateStateID(resourceID)
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
|
|
stopped := testVM(resourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
running := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.10)
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
if err := m.AcknowledgeAlert(alertID, "alice"); err != nil {
|
|
t.Fatalf("AcknowledgeAlert(%q) failed: %v", alertID, err)
|
|
}
|
|
|
|
acknowledged := activeAlert(t, m, alertID)
|
|
if acknowledged.AckTime == nil {
|
|
t.Fatalf("expected acknowledged alert to record ack time")
|
|
}
|
|
|
|
m.CheckGuest(running, "pve1")
|
|
assertAlertMissing(t, m, alertID)
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
|
|
reappeared := activeAlert(t, m, alertID)
|
|
if !reappeared.Acknowledged {
|
|
t.Fatalf("expected acknowledgment to be restored when the same powered-off identity reappears")
|
|
}
|
|
if reappeared.AckUser != "alice" {
|
|
t.Fatalf("AckUser = %q, want %q", reappeared.AckUser, "alice")
|
|
}
|
|
if reappeared.AckTime == nil || !reappeared.AckTime.Equal(*acknowledged.AckTime) {
|
|
t.Fatalf("expected AckTime to be preserved across clear/recreate, got %v want %v", reappeared.AckTime, acknowledged.AckTime)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationSuppressTagClearsMetricSuppressionIdentity(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
alertID := canonicalMetricStateID(resourceID, "cpu")
|
|
trackingKey := alertID
|
|
cfg := characterizationBaseConfig()
|
|
cfg.SuppressionWindow = 30
|
|
cfg.MinimumDelta = 5
|
|
|
|
m := newCharacterizationManager(t, cfg)
|
|
running := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.90)
|
|
cleared := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.70)
|
|
similarSpike := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.91)
|
|
suppressedByTag := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.91, "pulse-no-alerts")
|
|
|
|
m.CheckGuest(running, "pve1")
|
|
assertAlertPresent(t, m, alertID)
|
|
|
|
m.CheckGuest(cleared, "pve1")
|
|
assertAlertMissing(t, m, alertID)
|
|
|
|
m.CheckGuest(similarSpike, "pve1")
|
|
assertAlertMissing(t, m, alertID)
|
|
|
|
m.mu.RLock()
|
|
_, isSuppressed := m.suppressedUntil[trackingKey]
|
|
m.mu.RUnlock()
|
|
if !isSuppressed {
|
|
t.Fatalf("expected similar retrigger to be suppressed for canonical tracking key %q", trackingKey)
|
|
}
|
|
|
|
m.CheckGuest(suppressedByTag, "pve1")
|
|
|
|
m.mu.RLock()
|
|
_, isSuppressed = m.suppressedUntil[trackingKey]
|
|
m.mu.RUnlock()
|
|
if isSuppressed {
|
|
t.Fatalf("expected pulse-no-alerts suppression to clear stale suppression state for %q", trackingKey)
|
|
}
|
|
|
|
m.CheckGuest(similarSpike, "pve1")
|
|
assertAlertPresent(t, m, alertID)
|
|
}
|
|
|
|
func TestAlertCharacterizationResolvedLookupByCanonicalStateAlias(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
alertID := canonicalMetricStateID(resourceID, "cpu")
|
|
canonicalState := alertID
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
|
|
m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.70), "pve1")
|
|
|
|
resolved := m.GetResolvedAlert(canonicalState)
|
|
if resolved == nil || resolved.Alert == nil {
|
|
t.Fatalf("expected resolved alert lookup by canonical state %q", canonicalState)
|
|
}
|
|
if resolved.Alert.ID != canonicalState {
|
|
t.Fatalf("resolved alert ID = %q, want canonical ID %q", resolved.Alert.ID, canonicalState)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestMetricAlertMigratesAcrossNodeMove(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
oldState := canonicalMetricStateID(oldResourceID, "cpu")
|
|
newState := canonicalMetricStateID(newResourceID, "cpu")
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(oldResourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
|
|
|
|
if err := m.AcknowledgeAlert(oldState, "alice"); err != nil {
|
|
t.Fatalf("AcknowledgeAlert(%q) error = %v", oldState, err)
|
|
}
|
|
|
|
m.mu.Lock()
|
|
m.suppressedUntil[oldState] = time.Now().Add(time.Hour)
|
|
m.alertRateLimit[oldState] = []time.Time{time.Now()}
|
|
m.flappingHistory[oldState] = []time.Time{time.Now()}
|
|
m.flappingActive[oldState] = true
|
|
m.mu.Unlock()
|
|
|
|
m.CheckGuest(testVM(newResourceID, 101, "app01", "node2", "pve1", "running", 0.92), "pve1")
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
alert := activeAlert(t, m, newState)
|
|
if alert.ResourceID != newResourceID {
|
|
t.Fatalf("ResourceID = %q, want %q", alert.ResourceID, newResourceID)
|
|
}
|
|
if alert.Node != "node2" {
|
|
t.Fatalf("Node = %q, want node2", alert.Node)
|
|
}
|
|
if !alert.Acknowledged || alert.AckUser != "alice" {
|
|
t.Fatalf("expected acknowledgment to follow migrated alert, got acknowledged=%t user=%q", alert.Acknowledged, alert.AckUser)
|
|
}
|
|
|
|
m.mu.RLock()
|
|
_, oldAck := m.ackStateByCanonical[oldState]
|
|
record, newAck := m.ackStateByCanonical[newState]
|
|
_, oldSuppression := m.suppressedUntil[oldState]
|
|
_, newSuppression := m.suppressedUntil[newState]
|
|
_, oldRate := m.alertRateLimit[oldState]
|
|
_, newRate := m.alertRateLimit[newState]
|
|
_, oldFlapping := m.flappingHistory[oldState]
|
|
_, newFlapping := m.flappingHistory[newState]
|
|
m.mu.RUnlock()
|
|
|
|
if oldAck || !newAck || !record.acknowledged || record.user != "alice" {
|
|
t.Fatalf("expected canonical ack record to migrate, old=%t new=%t record=%+v", oldAck, newAck, record)
|
|
}
|
|
if oldSuppression || !newSuppression || oldRate || !newRate || oldFlapping || !newFlapping {
|
|
t.Fatalf(
|
|
"expected tracking state to migrate, suppression old=%t new=%t rate old=%t new=%t flapping old=%t new=%t",
|
|
oldSuppression,
|
|
newSuppression,
|
|
oldRate,
|
|
newRate,
|
|
oldFlapping,
|
|
newFlapping,
|
|
)
|
|
}
|
|
|
|
history := m.GetAlertHistory(5)
|
|
if len(history) != 1 || history[0].ID != newState {
|
|
t.Fatalf("expected migrated history entry under %q, got %#v", newState, history)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestMetricResolutionUsesCurrentNodeIdentityAfterMove(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
newState := canonicalMetricStateID(newResourceID, "cpu")
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(oldResourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
|
|
m.CheckGuest(testVM(newResourceID, 101, "app01", "node2", "pve1", "running", 0.70), "pve1")
|
|
|
|
assertAlertMissing(t, m, canonicalMetricStateID(oldResourceID, "cpu"))
|
|
assertAlertMissing(t, m, newState)
|
|
|
|
resolved := m.GetResolvedAlert(newState)
|
|
if resolved == nil || resolved.Alert == nil {
|
|
t.Fatalf("expected resolved alert lookup by migrated canonical state %q", newState)
|
|
}
|
|
if resolved.Alert.ResourceID != newResourceID {
|
|
t.Fatalf("resolved alert resource ID = %q, want %q", resolved.Alert.ResourceID, newResourceID)
|
|
}
|
|
// Identity migration must not preserve the old firing observation in
|
|
// the snapshot used by recovery callbacks and recent-resolution reads.
|
|
if resolved.Value != 70 || !strings.HasPrefix(resolved.Message, "Resolved: VM cpu ") || !strings.Contains(resolved.Message, "70.0%") || strings.Contains(resolved.Message, "95.0%") {
|
|
t.Fatalf("resolved snapshot retained firing content: value=%v message=%q", resolved.Value, resolved.Message)
|
|
}
|
|
if !resolved.LastSeen.After(resolved.StartTime) {
|
|
t.Fatalf("resolved observation time %v did not advance past incident start %v", resolved.LastSeen, resolved.StartTime)
|
|
}
|
|
|
|
history := m.GetAlertHistory(5)
|
|
if len(history) != 1 || history[0].ID != newState {
|
|
t.Fatalf("expected resolved history entry under %q, got %#v", newState, history)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestPerDiskMetricAlertMigratesAcrossNodeMove(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101) + "-disk-root"
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101) + "-disk-root"
|
|
|
|
threshold := &HysteresisThreshold{Trigger: 90, Clear: 85}
|
|
oldSpec, err := buildCanonicalMetricSpec(oldResourceID, "app01", unifiedresources.ResourceTypeVM, "disk", threshold)
|
|
if err != nil {
|
|
t.Fatalf("buildCanonicalMetricSpec(oldResourceID) error = %v", err)
|
|
}
|
|
newSpec, err := buildCanonicalMetricSpec(newResourceID, "app01", unifiedresources.ResourceTypeVM, "disk", threshold)
|
|
if err != nil {
|
|
t.Fatalf("buildCanonicalMetricSpec(newResourceID) error = %v", err)
|
|
}
|
|
|
|
oldState := buildCanonicalStateID(oldResourceID, oldSpec.ID)
|
|
newState := buildCanonicalStateID(newResourceID, newSpec.ID)
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.checkMetricWithCanonicalSpec(oldSpec, "app01", "node1", "pve1", "VM", 95, threshold, &metricOptions{Message: "VM disk (/root) at 95%"})
|
|
m.checkMetricWithCanonicalSpec(newSpec, "app01", "node2", "pve1", "VM", 95, threshold, &metricOptions{Message: "VM disk (/root) at 95%"})
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
alert := activeAlert(t, m, newState)
|
|
if alert.ResourceID != newResourceID {
|
|
t.Fatalf("ResourceID = %q, want %q", alert.ResourceID, newResourceID)
|
|
}
|
|
|
|
history := m.GetAlertHistory(5)
|
|
if len(history) != 1 || history[0].ID != newState {
|
|
t.Fatalf("expected per-disk history entry under %q, got %#v", newState, history)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestPoweredOffAlertMigratesAcrossNodeMove(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
oldState := canonicalPoweredStateStateID(oldResourceID)
|
|
newState := canonicalPoweredStateStateID(newResourceID)
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
stoppedOld := testVM(oldResourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
m.CheckGuest(stoppedOld, "pve1")
|
|
m.CheckGuest(stoppedOld, "pve1")
|
|
original := activeAlert(t, m, oldState)
|
|
|
|
if err := m.AcknowledgeAlert(oldState, "alice"); err != nil {
|
|
t.Fatalf("AcknowledgeAlert(%q) error = %v", oldState, err)
|
|
}
|
|
|
|
m.CheckGuest(testVM(newResourceID, 101, "app01", "node2", "pve1", "stopped", 0), "pve1")
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
alert := activeAlert(t, m, newState)
|
|
if alert.ResourceID != newResourceID {
|
|
t.Fatalf("ResourceID = %q, want %q", alert.ResourceID, newResourceID)
|
|
}
|
|
if alert.Node != "node2" {
|
|
t.Fatalf("Node = %q, want node2", alert.Node)
|
|
}
|
|
if !alert.StartTime.Equal(original.StartTime) {
|
|
t.Fatalf("StartTime = %v, want original %v preserved across node move", alert.StartTime, original.StartTime)
|
|
}
|
|
if !alert.Acknowledged || alert.AckUser != "alice" {
|
|
t.Fatalf("expected acknowledgment to follow migrated powered-off alert, got acknowledged=%t user=%q", alert.Acknowledged, alert.AckUser)
|
|
}
|
|
|
|
history := m.GetAlertHistory(5)
|
|
if len(history) != 1 || history[0].ID != newState {
|
|
t.Fatalf("expected single migrated history entry under %q, got %#v", newState, history)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestPoweredOffAlertResolvesAfterNodeMove(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
oldState := canonicalPoweredStateStateID(oldResourceID)
|
|
newState := canonicalPoweredStateStateID(newResourceID)
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
stoppedOld := testVM(oldResourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
m.CheckGuest(stoppedOld, "pve1")
|
|
m.CheckGuest(stoppedOld, "pve1")
|
|
assertAlertPresent(t, m, oldState)
|
|
|
|
m.CheckGuest(testVM(newResourceID, 101, "app01", "node2", "pve1", "running", 0.10), "pve1")
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
assertAlertMissing(t, m, newState)
|
|
|
|
resolved := m.GetResolvedAlert(newState)
|
|
if resolved == nil || resolved.Alert == nil {
|
|
t.Fatalf("expected powered-off alert to resolve under migrated canonical state %q", newState)
|
|
}
|
|
if resolved.Alert.ResourceID != newResourceID {
|
|
t.Fatalf("resolved alert resource ID = %q, want %q", resolved.Alert.ResourceID, newResourceID)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestMetricAlertClearsAfterNodeMoveWhenThresholdDisabled(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
oldState := canonicalMetricStateID(oldResourceID, "cpu")
|
|
newState := canonicalMetricStateID(newResourceID, "cpu")
|
|
|
|
threshold := &HysteresisThreshold{Trigger: 80, Clear: 75}
|
|
oldSpec, err := buildCanonicalMetricSpec(oldResourceID, "app01", unifiedresources.ResourceTypeVM, "cpu", threshold)
|
|
if err != nil {
|
|
t.Fatalf("buildCanonicalMetricSpec(oldResourceID) error = %v", err)
|
|
}
|
|
disabledSpec, err := buildCanonicalMetricSpec(newResourceID, "app01", unifiedresources.ResourceTypeVM, "cpu", nil)
|
|
if err != nil {
|
|
t.Fatalf("buildCanonicalMetricSpec(newResourceID, nil threshold) error = %v", err)
|
|
}
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.checkMetricWithCanonicalSpec(oldSpec, "app01", "node1", "pve1", "vm", 95, threshold, nil)
|
|
assertAlertPresent(t, m, oldState)
|
|
|
|
m.checkMetricWithCanonicalSpec(disabledSpec, "app01", "node2", "pve1", "vm", 95, nil, nil)
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
assertAlertMissing(t, m, newState)
|
|
resolved := m.GetResolvedAlert(newState)
|
|
if resolved == nil || resolved.Alert == nil {
|
|
t.Fatalf("expected disabled-threshold clear to resolve migrated alert under %q", newState)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationLegacyCheckMetricClearsAfterNodeMoveWhenThresholdDisabled(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
oldState := canonicalMetricStateID(oldResourceID, "cpu")
|
|
newState := canonicalMetricStateID(newResourceID, "cpu")
|
|
|
|
threshold := &HysteresisThreshold{Trigger: 80, Clear: 75}
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.checkMetric(oldResourceID, "app01", "node1", "pve1", "guest", "cpu", 95, threshold, nil)
|
|
assertAlertPresent(t, m, oldState)
|
|
|
|
m.checkMetric(newResourceID, "app01", "node2", "pve1", "guest", "cpu", 95, nil, nil)
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
assertAlertMissing(t, m, newState)
|
|
resolved := m.GetResolvedAlert(newState)
|
|
if resolved == nil || resolved.Alert == nil {
|
|
t.Fatalf("expected disabled-threshold clear to resolve migrated alert under %q", newState)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestMetricAlertClearsWhenStoppedAfterNodeMove(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
oldState := canonicalMetricStateID(oldResourceID, "cpu")
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(oldResourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
|
|
assertAlertPresent(t, m, oldState)
|
|
|
|
m.CheckGuest(testVM(newResourceID, 101, "app01", "node2", "pve1", "stopped", 0), "pve1")
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestDiskAlertCleanupAfterNodeMove(t *testing.T) {
|
|
oldGuestID := BuildGuestKey("pve1", "node1", 101)
|
|
newGuestID := BuildGuestKey("pve1", "node2", 101)
|
|
oldResourceID := oldGuestID + "-disk-root"
|
|
|
|
threshold := &HysteresisThreshold{Trigger: 90, Clear: 85}
|
|
oldSpec, err := buildCanonicalMetricSpec(oldResourceID, "app01", unifiedresources.ResourceTypeVM, "disk", threshold)
|
|
if err != nil {
|
|
t.Fatalf("buildCanonicalMetricSpec(oldResourceID) error = %v", err)
|
|
}
|
|
oldState := buildCanonicalStateID(oldResourceID, oldSpec.ID)
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.checkMetricWithCanonicalSpec(oldSpec, "app01", "node1", "pve1", "VM", 95, threshold, &metricOptions{Message: "VM disk (/root) at 95%"})
|
|
assertAlertPresent(t, m, oldState)
|
|
|
|
// Guest reappears on the new node without that disk in its report.
|
|
m.CheckGuest(testVM(newGuestID, 101, "app01", "node2", "pve1", "running", 0.10), "pve1")
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
}
|
|
|
|
func TestAlertCharacterizationSuppressTagClearsAlertsAfterNodeMove(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
oldState := canonicalMetricStateID(oldResourceID, "cpu")
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(oldResourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
|
|
assertAlertPresent(t, m, oldState)
|
|
|
|
m.CheckGuest(testVM(newResourceID, 101, "app01", "node2", "pve1", "running", 0.95, "pulse-no-alerts"), "pve1")
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
}
|
|
|
|
func TestAlertCharacterizationManualClearRemovesCanonicalTrackingState(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
specID := canonicalMetricSpecID(resourceID, "cpu")
|
|
alertID := buildCanonicalStateID(resourceID, specID)
|
|
trackingKey := alertID
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.85), "pve1")
|
|
assertAlertPresent(t, m, alertID)
|
|
|
|
m.mu.Lock()
|
|
m.recentAlerts[trackingKey] = &Alert{ID: alertID, ResourceID: resourceID, CanonicalState: trackingKey, CanonicalSpecID: specID, StartTime: time.Now(), LastSeen: time.Now()}
|
|
m.suppressedUntil[trackingKey] = time.Now().Add(time.Hour)
|
|
m.alertRateLimit[trackingKey] = []time.Time{time.Now()}
|
|
m.mu.Unlock()
|
|
|
|
if !m.ClearAlert(alertID) {
|
|
t.Fatalf("expected ClearAlert(%q) to succeed", alertID)
|
|
}
|
|
|
|
m.mu.RLock()
|
|
_, recentExists := m.recentAlerts[trackingKey]
|
|
_, suppressedExists := m.suppressedUntil[trackingKey]
|
|
_, rateExists := m.alertRateLimit[trackingKey]
|
|
m.mu.RUnlock()
|
|
if recentExists || suppressedExists || rateExists {
|
|
t.Fatalf("expected manual clear to remove canonical tracking entries, got recent=%t suppressed=%t rate=%t", recentExists, suppressedExists, rateExists)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationResolvedCallbackUsesCanonicalIDForCanonicalAliasClear(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
alertID := canonicalMetricSpecID(resourceID, "cpu")
|
|
canonicalState := canonicalMetricStateID(resourceID, "cpu")
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
resolved := make(chan string, 1)
|
|
m.SetResolvedCallback(func(id string) {
|
|
select {
|
|
case resolved <- id:
|
|
default:
|
|
}
|
|
})
|
|
|
|
alert := &Alert{
|
|
ID: alertID,
|
|
Type: "cpu",
|
|
ResourceID: resourceID,
|
|
ResourceName: "app01",
|
|
CanonicalSpecID: canonicalMetricSpecID(resourceID, "cpu"),
|
|
CanonicalKind: "metric_threshold",
|
|
CanonicalState: canonicalState,
|
|
StartTime: time.Now().Add(-time.Minute),
|
|
LastSeen: time.Now(),
|
|
}
|
|
|
|
m.mu.Lock()
|
|
m.setActiveAlertNoLock(canonicalState, alert)
|
|
m.mu.Unlock()
|
|
|
|
m.clearAlert(canonicalState)
|
|
|
|
select {
|
|
case got := <-resolved:
|
|
if got != canonicalState {
|
|
t.Fatalf("resolved callback = %q, want %q", got, canonicalState)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatalf("expected resolved callback for %q", alertID)
|
|
}
|
|
}
|
|
|
|
// A held threshold alert keeps the value and message of its last breach, but
|
|
// what the alert is about can be newer than the alert: one restored from a
|
|
// checkpoint written before its producer named a child type or stamped
|
|
// platformType would otherwise keep linking to the wrong page and naming the
|
|
// wrong policy owner until it re-fires or clears.
|
|
func TestAlertCharacterizationHeldMetricAlertRefreshesItsClassification(t *testing.T) {
|
|
type staleAlert struct {
|
|
resourceType string
|
|
platformType string
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
alertID string
|
|
fire func(*Manager)
|
|
hold func(*Manager)
|
|
stale staleAlert
|
|
wantResourceType string
|
|
wantPlatform string
|
|
wantValue float64
|
|
}{
|
|
{
|
|
name: "agent filesystem",
|
|
alertID: canonicalMetricStateID("agent:held-host/disk:srv", "disk"),
|
|
fire: func(m *Manager) {
|
|
m.CheckHost(models.Host{ID: "held-host", Hostname: "held", Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sdc1", Usage: 92}}})
|
|
},
|
|
hold: func(m *Manager) {
|
|
m.CheckHost(models.Host{ID: "held-host", Hostname: "held", Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sdc1", Usage: 75}}})
|
|
},
|
|
stale: staleAlert{resourceType: "agent"},
|
|
wantResourceType: "agent-disk",
|
|
wantPlatform: "agent",
|
|
wantValue: 92,
|
|
},
|
|
{
|
|
name: "vSphere VM",
|
|
alertID: canonicalMetricStateID("vmware:vc-1:vm:vm-201", "disk"),
|
|
fire: func(m *Manager) {
|
|
m.CheckUnifiedResource(&UnifiedResourceInput{ID: "vmware:vc-1:vm:vm-201", Type: "vmware-vm", Name: "app-01", Node: "esxi-01", Instance: "Lab vCenter", Disk: &UnifiedResourceMetric{Percent: 94}})
|
|
},
|
|
hold: func(m *Manager) {
|
|
m.CheckUnifiedResource(&UnifiedResourceInput{ID: "vmware:vc-1:vm:vm-201", Type: "vmware-vm", Name: "app-01", Node: "esxi-01", Instance: "Lab vCenter", Disk: &UnifiedResourceMetric{Percent: 87}})
|
|
},
|
|
stale: staleAlert{resourceType: "vSphere VM"},
|
|
wantResourceType: "vSphere VM",
|
|
wantPlatform: "vmware-vsphere",
|
|
wantValue: 94,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
m := newTestManager(t)
|
|
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
|
|
m.mu.Lock()
|
|
m.config.AgentDefaults.Disk = &HysteresisThreshold{Trigger: 80, Clear: 70}
|
|
m.mu.Unlock()
|
|
|
|
tt.fire(m)
|
|
m.mu.Lock()
|
|
alert, ok := m.getActiveAlertNoLock(tt.alertID)
|
|
if !ok {
|
|
m.mu.Unlock()
|
|
t.Fatalf("expected active alert %q", tt.alertID)
|
|
}
|
|
alert.Metadata["resourceType"] = tt.stale.resourceType
|
|
delete(alert.Metadata, alertPlatformTypeKey)
|
|
m.mu.Unlock()
|
|
|
|
tt.hold(m)
|
|
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
held, ok := m.getActiveAlertNoLock(tt.alertID)
|
|
if !ok {
|
|
t.Fatal("held alert resolved; the reading should sit between clear and trigger")
|
|
}
|
|
if held.Value != tt.wantValue {
|
|
t.Fatalf("held alert value = %v, want the last breach %v", held.Value, tt.wantValue)
|
|
}
|
|
if got := held.Metadata["resourceType"]; got != tt.wantResourceType {
|
|
t.Fatalf("resourceType = %v, want %s", got, tt.wantResourceType)
|
|
}
|
|
if got := held.Metadata[alertPlatformTypeKey]; got != tt.wantPlatform {
|
|
t.Fatalf("platformType = %v, want %s", got, tt.wantPlatform)
|
|
}
|
|
})
|
|
}
|
|
}
|