mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-15 08:33:03 +00:00
rcourtman
c471d8d201
Guest alerts are keyed by node-scoped resource IDs, and only metric threshold alerts were re-homed after a live migration. Lifecycle alerts (powered-state and future guest kinds) stayed keyed to the old node forever because their resolvers only look up the new-node key, node existence GC never fires while the source node is still online, and nothing else consumes the move. Generalize the migration helper to match lifecycle spec IDs (which embed the node-scoped resource ID) as well as node-independent metric spec IDs, and call it from the canonical lifecycle and stateful evaluators, the powered-off resolver, and the disabled-threshold early returns in both metric paths. Guest-wide sweep clears (metric clear on stop, per-disk cleanup, tag suppression) now match the stable instance+vmid identity across nodes too. Reported by Johannes Strasser: a VM live-migrated between Proxmox nodes stranded its pre-existing active alert with no way to clear it.
965 lines
34 KiB
Go
965 lines
34 KiB
Go
package alerts
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
)
|
|
|
|
func characterizationBaseConfig() AlertConfig {
|
|
return AlertConfig{
|
|
Enabled: true,
|
|
ActivationState: ActivationActive,
|
|
GuestDefaults: ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
|
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
|
|
},
|
|
NodeDefaults: ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
|
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
|
|
},
|
|
AgentDefaults: ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
|
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
|
|
},
|
|
PBSDefaults: ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
|
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
},
|
|
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
Overrides: map[string]ThresholdConfig{},
|
|
TimeThresholds: map[string]int{},
|
|
}
|
|
}
|
|
|
|
func newCharacterizationManager(t *testing.T, cfg AlertConfig) *Manager {
|
|
t.Helper()
|
|
|
|
m := newTestManager(t)
|
|
m.UpdateConfig(cfg)
|
|
|
|
// Force deterministic immediate evaluation for characterization tests.
|
|
m.mu.Lock()
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.config.MetricTimeThresholds = nil
|
|
m.mu.Unlock()
|
|
|
|
m.ClearActiveAlerts()
|
|
return m
|
|
}
|
|
|
|
func activeAlert(t *testing.T, m *Manager, alertID string) *Alert {
|
|
t.Helper()
|
|
|
|
m.mu.RLock()
|
|
alert := testRequireActiveAlert(t, m, alertID)
|
|
m.mu.RUnlock()
|
|
if alert != nil {
|
|
return alert.Clone()
|
|
}
|
|
|
|
t.Fatalf("expected active alert %q, got active alerts %v", alertID, alertKeys(m))
|
|
return nil
|
|
}
|
|
|
|
func testVM(resourceID string, vmID int, name, node, instance, status string, cpu float64, tags ...string) models.VM {
|
|
return models.VM{
|
|
ID: resourceID,
|
|
VMID: vmID,
|
|
Name: name,
|
|
Node: node,
|
|
Instance: instance,
|
|
Status: status,
|
|
CPU: cpu,
|
|
Tags: tags,
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationCanonicalGuestIdentityAcrossTypedAndUnifiedChecks(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
specID := canonicalMetricSpecID(resourceID, "cpu")
|
|
alertID := buildCanonicalStateID(resourceID, specID)
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.85), "pve1")
|
|
typedAlert := activeAlert(t, m, alertID)
|
|
|
|
m.ClearActiveAlerts()
|
|
m.CheckUnifiedResource(&UnifiedResourceInput{
|
|
ID: resourceID,
|
|
Type: "vm",
|
|
Name: "app01",
|
|
Node: "node1",
|
|
Instance: "pve1",
|
|
CPU: &UnifiedResourceMetric{Percent: 85},
|
|
})
|
|
unifiedAlert := activeAlert(t, m, alertID)
|
|
|
|
if typedAlert.ResourceID != resourceID {
|
|
t.Fatalf("typed ResourceID = %q, want %q", typedAlert.ResourceID, resourceID)
|
|
}
|
|
if unifiedAlert.ResourceID != resourceID {
|
|
t.Fatalf("unified ResourceID = %q, want %q", unifiedAlert.ResourceID, resourceID)
|
|
}
|
|
if typedAlert.ID != alertID || unifiedAlert.ID != alertID {
|
|
t.Fatalf("expected stable alert ID %q, got typed=%q unified=%q", alertID, typedAlert.ID, unifiedAlert.ID)
|
|
}
|
|
if typedAlert.CanonicalSpecID != specID || unifiedAlert.CanonicalSpecID != specID {
|
|
t.Fatalf("expected canonical spec id %q, got typed=%q unified=%q", specID, typedAlert.CanonicalSpecID, unifiedAlert.CanonicalSpecID)
|
|
}
|
|
if typedAlert.CanonicalState != alertID || unifiedAlert.CanonicalState != alertID {
|
|
t.Fatalf("expected canonical state %q, got typed=%q unified=%q", alertID, typedAlert.CanonicalState, unifiedAlert.CanonicalState)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGetActiveAlertsExportsCanonicalIdentity(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
canonicalState := canonicalMetricStateID(resourceID, "cpu")
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.85), "pve1")
|
|
|
|
alerts := m.GetActiveAlerts()
|
|
if len(alerts) != 1 {
|
|
t.Fatalf("expected 1 active alert, got %d", len(alerts))
|
|
}
|
|
if alerts[0].ID != canonicalState {
|
|
t.Fatalf("GetActiveAlerts() ID = %q, want canonical ID %q", alerts[0].ID, canonicalState)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationAcknowledgmentSurvivesAlertIDChangeForSameCanonicalState(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
oldAlertID := "legacy-" + resourceID + "-cpu"
|
|
newAlertID := resourceID + "-cpu"
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
|
|
m.mu.Lock()
|
|
oldAlert := &Alert{
|
|
ID: oldAlertID,
|
|
Type: "cpu",
|
|
Level: AlertLevelWarning,
|
|
ResourceID: resourceID,
|
|
Message: "legacy alert",
|
|
StartTime: time.Now().Add(-5 * time.Minute),
|
|
LastSeen: time.Now().Add(-1 * time.Minute),
|
|
}
|
|
applyCanonicalIdentity(oldAlert, newAlertID, "metric-threshold")
|
|
m.activeAlerts[oldAlertID] = oldAlert
|
|
m.mu.Unlock()
|
|
|
|
if err := m.AcknowledgeAlert(oldAlertID, "alice"); err != nil {
|
|
t.Fatalf("AcknowledgeAlert() error = %v", err)
|
|
}
|
|
|
|
m.mu.Lock()
|
|
m.removeActiveAlertNoLock(oldAlertID)
|
|
replacement := &Alert{
|
|
ID: newAlertID,
|
|
Type: "cpu",
|
|
Level: AlertLevelWarning,
|
|
ResourceID: resourceID,
|
|
Message: "canonical alert",
|
|
StartTime: time.Now(),
|
|
LastSeen: time.Now(),
|
|
}
|
|
applyCanonicalIdentity(replacement, newAlertID, "metric-threshold")
|
|
m.preserveAlertState(newAlertID, replacement)
|
|
m.activeAlerts[newAlertID] = replacement
|
|
m.mu.Unlock()
|
|
|
|
alert := activeAlert(t, m, newAlertID)
|
|
if !alert.Acknowledged {
|
|
t.Fatal("expected acknowledgment to survive alert ID change")
|
|
}
|
|
if alert.AckUser != "alice" {
|
|
t.Fatalf("AckUser = %q, want alice", alert.AckUser)
|
|
}
|
|
if alert.CanonicalState != resourceID+"::"+newAlertID {
|
|
t.Fatalf("CanonicalState = %q, want %q", alert.CanonicalState, resourceID+"::"+newAlertID)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationHistoryUpdateUsesCanonicalStateAcrossAlertIDChange(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
oldAlertID := "legacy-" + resourceID + "-cpu"
|
|
newAlertID := resourceID + "-cpu"
|
|
lastSeen := time.Now()
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
|
|
oldHistoryAlert := Alert{
|
|
ID: oldAlertID,
|
|
Type: "cpu",
|
|
ResourceID: resourceID,
|
|
StartTime: time.Now().Add(-10 * time.Minute),
|
|
LastSeen: time.Now().Add(-5 * time.Minute),
|
|
}
|
|
applyCanonicalIdentity(&oldHistoryAlert, newAlertID, "metric-threshold")
|
|
m.historyManager.AddAlert(oldHistoryAlert)
|
|
|
|
current := &Alert{
|
|
ID: newAlertID,
|
|
Type: "cpu",
|
|
ResourceID: resourceID,
|
|
StartTime: time.Now().Add(-2 * time.Minute),
|
|
LastSeen: lastSeen,
|
|
}
|
|
applyCanonicalIdentity(current, newAlertID, "metric-threshold")
|
|
|
|
m.historyManager.UpdateAlertLastSeenForAlert(current, lastSeen)
|
|
|
|
history := m.GetAlertHistory(10)
|
|
if len(history) == 0 {
|
|
t.Fatalf("expected history entry")
|
|
}
|
|
if !history[0].LastSeen.Equal(lastSeen) {
|
|
t.Fatalf("LastSeen = %v, want %v", history[0].LastSeen, lastSeen)
|
|
}
|
|
expectedCanonicalState := buildCanonicalStateID(resourceID, newAlertID)
|
|
if history[0].ID != expectedCanonicalState {
|
|
t.Fatalf("history alert ID = %q, want canonical ID %q", history[0].ID, expectedCanonicalState)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationRecentSuppressionSurvivesAlertIDChangeForSameCanonicalState(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
oldAlertID := "legacy-" + resourceID + "-cpu"
|
|
newAlertID := resourceID + "-cpu"
|
|
clear := 75.0
|
|
critical := 90.0
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.mu.Lock()
|
|
m.config.MinimumDelta = 5
|
|
m.config.SuppressionWindow = 10
|
|
recent := &Alert{
|
|
ID: oldAlertID,
|
|
Type: "cpu",
|
|
Level: AlertLevelWarning,
|
|
ResourceID: resourceID,
|
|
Value: 90,
|
|
StartTime: time.Now().Add(-1 * time.Minute),
|
|
LastSeen: time.Now().Add(-30 * time.Second),
|
|
}
|
|
applyCanonicalIdentity(recent, newAlertID, "metric-threshold")
|
|
m.recentAlerts[recent.CanonicalState] = recent
|
|
m.mu.Unlock()
|
|
|
|
spec := alertspecs.ResourceAlertSpec{
|
|
ID: newAlertID,
|
|
ResourceID: resourceID,
|
|
ResourceType: unifiedresources.ResourceTypeVM,
|
|
Kind: alertspecs.AlertSpecKindMetricThreshold,
|
|
Severity: alertspecs.AlertSeverityWarning,
|
|
MetricThreshold: &alertspecs.MetricThresholdSpec{
|
|
Metric: "cpu",
|
|
Direction: alertspecs.ThresholdDirectionAbove,
|
|
Trigger: 80,
|
|
Recovery: &clear,
|
|
Critical: &critical,
|
|
},
|
|
}
|
|
|
|
m.evaluateCanonicalMetricAlert(spec, "app01", "node1", "pve1", "cpu", 91, &HysteresisThreshold{Trigger: 80, Clear: 75}, nil)
|
|
|
|
assertAlertMissing(t, m, newAlertID)
|
|
|
|
m.mu.RLock()
|
|
suppressedUntil, ok := m.suppressedUntil[resourceID+"::"+newAlertID]
|
|
m.mu.RUnlock()
|
|
if !ok {
|
|
t.Fatal("expected canonical suppression window to be recorded")
|
|
}
|
|
if time.Until(suppressedUntil) <= 0 {
|
|
t.Fatal("expected canonical suppression window to be in the future")
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationAcknowledgeByCanonicalStateAlias(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
alertID := canonicalMetricStateID(resourceID, "cpu")
|
|
canonicalState := alertID
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
|
|
|
|
if err := m.AcknowledgeAlert(canonicalState, "alice"); err != nil {
|
|
t.Fatalf("AcknowledgeAlert(%q) error = %v", canonicalState, err)
|
|
}
|
|
|
|
alert := activeAlert(t, m, alertID)
|
|
if !alert.Acknowledged {
|
|
t.Fatal("expected alert to be acknowledged through canonical state alias")
|
|
}
|
|
if alert.AckUser != "alice" {
|
|
t.Fatalf("AckUser = %q, want alice", alert.AckUser)
|
|
}
|
|
|
|
m.mu.RLock()
|
|
_, legacyAck := m.ackState[alertID]
|
|
record, canonicalAck := m.ackStateByCanonical[canonicalState]
|
|
m.mu.RUnlock()
|
|
if legacyAck {
|
|
t.Fatalf("expected canonical alert acknowledgment to be keyed by canonical state, not legacy alert ID")
|
|
}
|
|
if !canonicalAck || !record.acknowledged || record.user != "alice" {
|
|
t.Fatalf("expected canonical ack record for %q, got %+v exists=%t", canonicalState, record, canonicalAck)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestThresholdPrecedence(t *testing.T) {
|
|
overrideGuestID := BuildGuestKey("pve1", "node1", 101)
|
|
ruleGuestID := BuildGuestKey("pve1", "node1", 102)
|
|
|
|
cfg := characterizationBaseConfig()
|
|
cfg.GuestDefaults.CPU = &HysteresisThreshold{Trigger: 90, Clear: 85}
|
|
cfg.CustomRules = []CustomAlertRule{
|
|
{
|
|
Name: "named-apps",
|
|
Enabled: true,
|
|
Priority: 10,
|
|
FilterConditions: FilterStack{
|
|
LogicalOperator: "AND",
|
|
Filters: []FilterCondition{
|
|
{Type: "text", Field: "name", Value: "app"},
|
|
},
|
|
},
|
|
Thresholds: ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 70, Clear: 65},
|
|
},
|
|
},
|
|
}
|
|
cfg.Overrides[overrideGuestID] = ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 95, Clear: 90},
|
|
}
|
|
|
|
m := newCharacterizationManager(t, cfg)
|
|
m.CheckGuest(testVM(ruleGuestID, 102, "app-rule", "node1", "pve1", "running", 0.80), "pve1")
|
|
m.CheckGuest(testVM(overrideGuestID, 101, "app-override", "node1", "pve1", "running", 0.80), "pve1")
|
|
|
|
assertAlertPresent(t, m, canonicalMetricStateID(ruleGuestID, "cpu"))
|
|
assertAlertMissing(t, m, canonicalMetricStateID(overrideGuestID, "cpu"))
|
|
}
|
|
|
|
func TestAlertCharacterizationDisableConnectivitySuppressesPoweredOffButNotMetrics(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
cfg := characterizationBaseConfig()
|
|
cfg.Overrides[resourceID] = ThresholdConfig{DisableConnectivity: true}
|
|
|
|
m := newCharacterizationManager(t, cfg)
|
|
stopped := testVM(resourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
|
|
assertAlertMissing(t, m, "guest-powered-off-"+resourceID)
|
|
|
|
m.mu.RLock()
|
|
_, hasConfirmations := m.offlineConfirmations[resourceID]
|
|
m.mu.RUnlock()
|
|
if hasConfirmations {
|
|
t.Fatalf("expected powered-off tracking to stay clear when connectivity is disabled")
|
|
}
|
|
|
|
m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
|
|
assertAlertPresent(t, m, canonicalMetricStateID(resourceID, "cpu"))
|
|
}
|
|
|
|
func TestAlertCharacterizationOnBootFalseSuppressesPoweredOffAlert(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
cfg := characterizationBaseConfig()
|
|
m := newCharacterizationManager(t, cfg)
|
|
|
|
off := false
|
|
stopped := testVM(resourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
stopped.OnBoot = &off
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
|
|
assertAlertMissing(t, m, "guest-powered-off-"+resourceID)
|
|
|
|
m.mu.RLock()
|
|
_, hasConfirmations := m.offlineConfirmations[resourceID]
|
|
m.mu.RUnlock()
|
|
if hasConfirmations {
|
|
t.Fatalf("expected no powered-off tracking for guest with onboot=false")
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationOnBootTrueStillGeneratesPoweredOffAlert(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
cfg := characterizationBaseConfig()
|
|
m := newCharacterizationManager(t, cfg)
|
|
|
|
on := true
|
|
stopped := testVM(resourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
stopped.OnBoot = &on
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
|
|
assertAlertPresent(t, m, "guest-powered-off-"+resourceID)
|
|
}
|
|
|
|
func TestAlertCharacterizationOnBootNilPreservesCurrentBehavior(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
cfg := characterizationBaseConfig()
|
|
m := newCharacterizationManager(t, cfg)
|
|
|
|
stopped := testVM(resourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
|
|
assertAlertPresent(t, m, "guest-powered-off-"+resourceID)
|
|
}
|
|
|
|
func TestAlertCharacterizationReevaluatesAlertsWhenConfigChanges(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
alertID := canonicalMetricStateID(resourceID, "cpu")
|
|
cfg := characterizationBaseConfig()
|
|
|
|
m := newCharacterizationManager(t, cfg)
|
|
resolved := make(chan string, 1)
|
|
m.SetResolvedCallback(func(id string) {
|
|
select {
|
|
case resolved <- id:
|
|
default:
|
|
}
|
|
})
|
|
|
|
m.CheckUnifiedResource(&UnifiedResourceInput{
|
|
ID: resourceID,
|
|
Type: "vm",
|
|
Name: "app01",
|
|
Node: "node1",
|
|
Instance: "pve1",
|
|
CPU: &UnifiedResourceMetric{Percent: 87},
|
|
})
|
|
assertAlertPresent(t, m, alertID)
|
|
|
|
updated := cfg
|
|
updated.GuestDefaults.CPU = &HysteresisThreshold{Trigger: 90, Clear: 85}
|
|
m.UpdateConfig(updated)
|
|
|
|
select {
|
|
case got := <-resolved:
|
|
expectedResolvedID := alertID
|
|
if got != expectedResolvedID {
|
|
t.Fatalf("resolved callback = %q, want %q", got, expectedResolvedID)
|
|
}
|
|
case <-time.After(250 * time.Millisecond):
|
|
t.Fatalf("expected alert %q to resolve after config change", alertID)
|
|
}
|
|
|
|
assertAlertMissing(t, m, alertID)
|
|
|
|
m.resolvedMutex.RLock()
|
|
_, wasResolved := testLookupResolvedAlert(t, m, alertID)
|
|
m.resolvedMutex.RUnlock()
|
|
if !wasResolved {
|
|
t.Fatalf("expected %q in recently resolved after config change", alertID)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationPoweredOffLifecycle(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
alertID := "guest-powered-off-" + resourceID
|
|
cfg := characterizationBaseConfig()
|
|
m := newCharacterizationManager(t, cfg)
|
|
|
|
running := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.95)
|
|
stopped := testVM(resourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
paused := testVM(resourceID, 101, "app01", "node1", "pve1", "paused", 0)
|
|
|
|
m.CheckGuest(running, "pve1")
|
|
assertAlertPresent(t, m, canonicalMetricStateID(resourceID, "cpu"))
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
|
|
assertAlertMissing(t, m, canonicalMetricStateID(resourceID, "cpu"))
|
|
assertAlertPresent(t, m, alertID)
|
|
|
|
m.CheckGuest(paused, "pve1")
|
|
assertAlertMissing(t, m, alertID)
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
assertAlertPresent(t, m, alertID)
|
|
|
|
m.CheckGuest(running, "pve1")
|
|
assertAlertMissing(t, m, alertID)
|
|
}
|
|
|
|
func TestAlertCharacterizationAcknowledgmentSurvivesPoweredOffReappearance(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
alertID := canonicalPoweredStateStateID(resourceID)
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
|
|
stopped := testVM(resourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
running := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.10)
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
if err := m.AcknowledgeAlert(alertID, "alice"); err != nil {
|
|
t.Fatalf("AcknowledgeAlert(%q) failed: %v", alertID, err)
|
|
}
|
|
|
|
acknowledged := activeAlert(t, m, alertID)
|
|
if acknowledged.AckTime == nil {
|
|
t.Fatalf("expected acknowledged alert to record ack time")
|
|
}
|
|
|
|
m.CheckGuest(running, "pve1")
|
|
assertAlertMissing(t, m, alertID)
|
|
|
|
m.CheckGuest(stopped, "pve1")
|
|
m.CheckGuest(stopped, "pve1")
|
|
|
|
reappeared := activeAlert(t, m, alertID)
|
|
if !reappeared.Acknowledged {
|
|
t.Fatalf("expected acknowledgment to be restored when the same powered-off identity reappears")
|
|
}
|
|
if reappeared.AckUser != "alice" {
|
|
t.Fatalf("AckUser = %q, want %q", reappeared.AckUser, "alice")
|
|
}
|
|
if reappeared.AckTime == nil || !reappeared.AckTime.Equal(*acknowledged.AckTime) {
|
|
t.Fatalf("expected AckTime to be preserved across clear/recreate, got %v want %v", reappeared.AckTime, acknowledged.AckTime)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationSuppressTagClearsMetricSuppressionIdentity(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
alertID := canonicalMetricStateID(resourceID, "cpu")
|
|
trackingKey := alertID
|
|
cfg := characterizationBaseConfig()
|
|
cfg.SuppressionWindow = 30
|
|
cfg.MinimumDelta = 5
|
|
|
|
m := newCharacterizationManager(t, cfg)
|
|
running := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.90)
|
|
cleared := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.70)
|
|
similarSpike := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.91)
|
|
suppressedByTag := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.91, "pulse-no-alerts")
|
|
|
|
m.CheckGuest(running, "pve1")
|
|
assertAlertPresent(t, m, alertID)
|
|
|
|
m.CheckGuest(cleared, "pve1")
|
|
assertAlertMissing(t, m, alertID)
|
|
|
|
m.CheckGuest(similarSpike, "pve1")
|
|
assertAlertMissing(t, m, alertID)
|
|
|
|
m.mu.RLock()
|
|
_, isSuppressed := m.suppressedUntil[trackingKey]
|
|
m.mu.RUnlock()
|
|
if !isSuppressed {
|
|
t.Fatalf("expected similar retrigger to be suppressed for canonical tracking key %q", trackingKey)
|
|
}
|
|
|
|
m.CheckGuest(suppressedByTag, "pve1")
|
|
|
|
m.mu.RLock()
|
|
_, isSuppressed = m.suppressedUntil[trackingKey]
|
|
m.mu.RUnlock()
|
|
if isSuppressed {
|
|
t.Fatalf("expected pulse-no-alerts suppression to clear stale suppression state for %q", trackingKey)
|
|
}
|
|
|
|
m.CheckGuest(similarSpike, "pve1")
|
|
assertAlertPresent(t, m, alertID)
|
|
}
|
|
|
|
func TestAlertCharacterizationResolvedLookupByCanonicalStateAlias(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
alertID := canonicalMetricStateID(resourceID, "cpu")
|
|
canonicalState := alertID
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
|
|
m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.70), "pve1")
|
|
|
|
resolved := m.GetResolvedAlert(canonicalState)
|
|
if resolved == nil || resolved.Alert == nil {
|
|
t.Fatalf("expected resolved alert lookup by canonical state %q", canonicalState)
|
|
}
|
|
if resolved.Alert.ID != canonicalState {
|
|
t.Fatalf("resolved alert ID = %q, want canonical ID %q", resolved.Alert.ID, canonicalState)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestMetricAlertMigratesAcrossNodeMove(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
oldState := canonicalMetricStateID(oldResourceID, "cpu")
|
|
newState := canonicalMetricStateID(newResourceID, "cpu")
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(oldResourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
|
|
|
|
if err := m.AcknowledgeAlert(oldState, "alice"); err != nil {
|
|
t.Fatalf("AcknowledgeAlert(%q) error = %v", oldState, err)
|
|
}
|
|
|
|
m.mu.Lock()
|
|
m.suppressedUntil[oldState] = time.Now().Add(time.Hour)
|
|
m.alertRateLimit[oldState] = []time.Time{time.Now()}
|
|
m.flappingHistory[oldState] = []time.Time{time.Now()}
|
|
m.flappingActive[oldState] = true
|
|
m.mu.Unlock()
|
|
|
|
m.CheckGuest(testVM(newResourceID, 101, "app01", "node2", "pve1", "running", 0.92), "pve1")
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
alert := activeAlert(t, m, newState)
|
|
if alert.ResourceID != newResourceID {
|
|
t.Fatalf("ResourceID = %q, want %q", alert.ResourceID, newResourceID)
|
|
}
|
|
if alert.Node != "node2" {
|
|
t.Fatalf("Node = %q, want node2", alert.Node)
|
|
}
|
|
if !alert.Acknowledged || alert.AckUser != "alice" {
|
|
t.Fatalf("expected acknowledgment to follow migrated alert, got acknowledged=%t user=%q", alert.Acknowledged, alert.AckUser)
|
|
}
|
|
|
|
m.mu.RLock()
|
|
_, oldAck := m.ackStateByCanonical[oldState]
|
|
record, newAck := m.ackStateByCanonical[newState]
|
|
_, oldSuppression := m.suppressedUntil[oldState]
|
|
_, newSuppression := m.suppressedUntil[newState]
|
|
_, oldRate := m.alertRateLimit[oldState]
|
|
_, newRate := m.alertRateLimit[newState]
|
|
_, oldFlapping := m.flappingHistory[oldState]
|
|
_, newFlapping := m.flappingHistory[newState]
|
|
m.mu.RUnlock()
|
|
|
|
if oldAck || !newAck || !record.acknowledged || record.user != "alice" {
|
|
t.Fatalf("expected canonical ack record to migrate, old=%t new=%t record=%+v", oldAck, newAck, record)
|
|
}
|
|
if oldSuppression || !newSuppression || oldRate || !newRate || oldFlapping || !newFlapping {
|
|
t.Fatalf(
|
|
"expected tracking state to migrate, suppression old=%t new=%t rate old=%t new=%t flapping old=%t new=%t",
|
|
oldSuppression,
|
|
newSuppression,
|
|
oldRate,
|
|
newRate,
|
|
oldFlapping,
|
|
newFlapping,
|
|
)
|
|
}
|
|
|
|
history := m.GetAlertHistory(5)
|
|
if len(history) != 1 || history[0].ID != newState {
|
|
t.Fatalf("expected migrated history entry under %q, got %#v", newState, history)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestMetricResolutionUsesCurrentNodeIdentityAfterMove(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
newState := canonicalMetricStateID(newResourceID, "cpu")
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(oldResourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
|
|
m.CheckGuest(testVM(newResourceID, 101, "app01", "node2", "pve1", "running", 0.70), "pve1")
|
|
|
|
assertAlertMissing(t, m, canonicalMetricStateID(oldResourceID, "cpu"))
|
|
assertAlertMissing(t, m, newState)
|
|
|
|
resolved := m.GetResolvedAlert(newState)
|
|
if resolved == nil || resolved.Alert == nil {
|
|
t.Fatalf("expected resolved alert lookup by migrated canonical state %q", newState)
|
|
}
|
|
if resolved.Alert.ResourceID != newResourceID {
|
|
t.Fatalf("resolved alert resource ID = %q, want %q", resolved.Alert.ResourceID, newResourceID)
|
|
}
|
|
|
|
history := m.GetAlertHistory(5)
|
|
if len(history) != 1 || history[0].ID != newState {
|
|
t.Fatalf("expected resolved history entry under %q, got %#v", newState, history)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestPerDiskMetricAlertMigratesAcrossNodeMove(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101) + "-disk-root"
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101) + "-disk-root"
|
|
|
|
threshold := &HysteresisThreshold{Trigger: 90, Clear: 85}
|
|
oldSpec, err := buildCanonicalMetricSpec(oldResourceID, "app01", unifiedresources.ResourceTypeVM, "disk", threshold)
|
|
if err != nil {
|
|
t.Fatalf("buildCanonicalMetricSpec(oldResourceID) error = %v", err)
|
|
}
|
|
newSpec, err := buildCanonicalMetricSpec(newResourceID, "app01", unifiedresources.ResourceTypeVM, "disk", threshold)
|
|
if err != nil {
|
|
t.Fatalf("buildCanonicalMetricSpec(newResourceID) error = %v", err)
|
|
}
|
|
|
|
oldState := buildCanonicalStateID(oldResourceID, oldSpec.ID)
|
|
newState := buildCanonicalStateID(newResourceID, newSpec.ID)
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.checkMetricWithCanonicalSpec(oldSpec, "app01", "node1", "pve1", "VM", 95, threshold, &metricOptions{Message: "VM disk (/root) at 95%"})
|
|
m.checkMetricWithCanonicalSpec(newSpec, "app01", "node2", "pve1", "VM", 95, threshold, &metricOptions{Message: "VM disk (/root) at 95%"})
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
alert := activeAlert(t, m, newState)
|
|
if alert.ResourceID != newResourceID {
|
|
t.Fatalf("ResourceID = %q, want %q", alert.ResourceID, newResourceID)
|
|
}
|
|
|
|
history := m.GetAlertHistory(5)
|
|
if len(history) != 1 || history[0].ID != newState {
|
|
t.Fatalf("expected per-disk history entry under %q, got %#v", newState, history)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestPoweredOffAlertMigratesAcrossNodeMove(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
oldState := canonicalPoweredStateStateID(oldResourceID)
|
|
newState := canonicalPoweredStateStateID(newResourceID)
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
stoppedOld := testVM(oldResourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
m.CheckGuest(stoppedOld, "pve1")
|
|
m.CheckGuest(stoppedOld, "pve1")
|
|
original := activeAlert(t, m, oldState)
|
|
|
|
if err := m.AcknowledgeAlert(oldState, "alice"); err != nil {
|
|
t.Fatalf("AcknowledgeAlert(%q) error = %v", oldState, err)
|
|
}
|
|
|
|
m.CheckGuest(testVM(newResourceID, 101, "app01", "node2", "pve1", "stopped", 0), "pve1")
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
alert := activeAlert(t, m, newState)
|
|
if alert.ResourceID != newResourceID {
|
|
t.Fatalf("ResourceID = %q, want %q", alert.ResourceID, newResourceID)
|
|
}
|
|
if alert.Node != "node2" {
|
|
t.Fatalf("Node = %q, want node2", alert.Node)
|
|
}
|
|
if !alert.StartTime.Equal(original.StartTime) {
|
|
t.Fatalf("StartTime = %v, want original %v preserved across node move", alert.StartTime, original.StartTime)
|
|
}
|
|
if !alert.Acknowledged || alert.AckUser != "alice" {
|
|
t.Fatalf("expected acknowledgment to follow migrated powered-off alert, got acknowledged=%t user=%q", alert.Acknowledged, alert.AckUser)
|
|
}
|
|
|
|
history := m.GetAlertHistory(5)
|
|
if len(history) != 1 || history[0].ID != newState {
|
|
t.Fatalf("expected single migrated history entry under %q, got %#v", newState, history)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestPoweredOffAlertResolvesAfterNodeMove(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
oldState := canonicalPoweredStateStateID(oldResourceID)
|
|
newState := canonicalPoweredStateStateID(newResourceID)
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
stoppedOld := testVM(oldResourceID, 101, "app01", "node1", "pve1", "stopped", 0)
|
|
m.CheckGuest(stoppedOld, "pve1")
|
|
m.CheckGuest(stoppedOld, "pve1")
|
|
assertAlertPresent(t, m, oldState)
|
|
|
|
m.CheckGuest(testVM(newResourceID, 101, "app01", "node2", "pve1", "running", 0.10), "pve1")
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
assertAlertMissing(t, m, newState)
|
|
|
|
resolved := m.GetResolvedAlert(newState)
|
|
if resolved == nil || resolved.Alert == nil {
|
|
t.Fatalf("expected powered-off alert to resolve under migrated canonical state %q", newState)
|
|
}
|
|
if resolved.Alert.ResourceID != newResourceID {
|
|
t.Fatalf("resolved alert resource ID = %q, want %q", resolved.Alert.ResourceID, newResourceID)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestMetricAlertClearsAfterNodeMoveWhenThresholdDisabled(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
oldState := canonicalMetricStateID(oldResourceID, "cpu")
|
|
newState := canonicalMetricStateID(newResourceID, "cpu")
|
|
|
|
threshold := &HysteresisThreshold{Trigger: 80, Clear: 75}
|
|
oldSpec, err := buildCanonicalMetricSpec(oldResourceID, "app01", unifiedresources.ResourceTypeVM, "cpu", threshold)
|
|
if err != nil {
|
|
t.Fatalf("buildCanonicalMetricSpec(oldResourceID) error = %v", err)
|
|
}
|
|
disabledSpec, err := buildCanonicalMetricSpec(newResourceID, "app01", unifiedresources.ResourceTypeVM, "cpu", nil)
|
|
if err != nil {
|
|
t.Fatalf("buildCanonicalMetricSpec(newResourceID, nil threshold) error = %v", err)
|
|
}
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.checkMetricWithCanonicalSpec(oldSpec, "app01", "node1", "pve1", "vm", 95, threshold, nil)
|
|
assertAlertPresent(t, m, oldState)
|
|
|
|
m.checkMetricWithCanonicalSpec(disabledSpec, "app01", "node2", "pve1", "vm", 95, nil, nil)
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
assertAlertMissing(t, m, newState)
|
|
resolved := m.GetResolvedAlert(newState)
|
|
if resolved == nil || resolved.Alert == nil {
|
|
t.Fatalf("expected disabled-threshold clear to resolve migrated alert under %q", newState)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationLegacyCheckMetricClearsAfterNodeMoveWhenThresholdDisabled(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
oldState := canonicalMetricStateID(oldResourceID, "cpu")
|
|
newState := canonicalMetricStateID(newResourceID, "cpu")
|
|
|
|
threshold := &HysteresisThreshold{Trigger: 80, Clear: 75}
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.checkMetric(oldResourceID, "app01", "node1", "pve1", "guest", "cpu", 95, threshold, nil)
|
|
assertAlertPresent(t, m, oldState)
|
|
|
|
m.checkMetric(newResourceID, "app01", "node2", "pve1", "guest", "cpu", 95, nil, nil)
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
assertAlertMissing(t, m, newState)
|
|
resolved := m.GetResolvedAlert(newState)
|
|
if resolved == nil || resolved.Alert == nil {
|
|
t.Fatalf("expected disabled-threshold clear to resolve migrated alert under %q", newState)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestMetricAlertClearsWhenStoppedAfterNodeMove(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
oldState := canonicalMetricStateID(oldResourceID, "cpu")
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(oldResourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
|
|
assertAlertPresent(t, m, oldState)
|
|
|
|
m.CheckGuest(testVM(newResourceID, 101, "app01", "node2", "pve1", "stopped", 0), "pve1")
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
}
|
|
|
|
func TestAlertCharacterizationGuestDiskAlertCleanupAfterNodeMove(t *testing.T) {
|
|
oldGuestID := BuildGuestKey("pve1", "node1", 101)
|
|
newGuestID := BuildGuestKey("pve1", "node2", 101)
|
|
oldResourceID := oldGuestID + "-disk-root"
|
|
|
|
threshold := &HysteresisThreshold{Trigger: 90, Clear: 85}
|
|
oldSpec, err := buildCanonicalMetricSpec(oldResourceID, "app01", unifiedresources.ResourceTypeVM, "disk", threshold)
|
|
if err != nil {
|
|
t.Fatalf("buildCanonicalMetricSpec(oldResourceID) error = %v", err)
|
|
}
|
|
oldState := buildCanonicalStateID(oldResourceID, oldSpec.ID)
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.checkMetricWithCanonicalSpec(oldSpec, "app01", "node1", "pve1", "VM", 95, threshold, &metricOptions{Message: "VM disk (/root) at 95%"})
|
|
assertAlertPresent(t, m, oldState)
|
|
|
|
// Guest reappears on the new node without that disk in its report.
|
|
m.CheckGuest(testVM(newGuestID, 101, "app01", "node2", "pve1", "running", 0.10), "pve1")
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
}
|
|
|
|
func TestAlertCharacterizationSuppressTagClearsAlertsAfterNodeMove(t *testing.T) {
|
|
oldResourceID := BuildGuestKey("pve1", "node1", 101)
|
|
newResourceID := BuildGuestKey("pve1", "node2", 101)
|
|
oldState := canonicalMetricStateID(oldResourceID, "cpu")
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(oldResourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
|
|
assertAlertPresent(t, m, oldState)
|
|
|
|
m.CheckGuest(testVM(newResourceID, 101, "app01", "node2", "pve1", "running", 0.95, "pulse-no-alerts"), "pve1")
|
|
|
|
assertAlertMissing(t, m, oldState)
|
|
}
|
|
|
|
func TestAlertCharacterizationManualClearRemovesCanonicalTrackingState(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
specID := canonicalMetricSpecID(resourceID, "cpu")
|
|
alertID := buildCanonicalStateID(resourceID, specID)
|
|
trackingKey := alertID
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.85), "pve1")
|
|
assertAlertPresent(t, m, alertID)
|
|
|
|
m.mu.Lock()
|
|
m.recentAlerts[trackingKey] = &Alert{ID: alertID, ResourceID: resourceID, CanonicalState: trackingKey, CanonicalSpecID: specID, StartTime: time.Now(), LastSeen: time.Now()}
|
|
m.suppressedUntil[trackingKey] = time.Now().Add(time.Hour)
|
|
m.alertRateLimit[trackingKey] = []time.Time{time.Now()}
|
|
m.mu.Unlock()
|
|
|
|
if !m.ClearAlert(alertID) {
|
|
t.Fatalf("expected ClearAlert(%q) to succeed", alertID)
|
|
}
|
|
|
|
m.mu.RLock()
|
|
_, recentExists := m.recentAlerts[trackingKey]
|
|
_, suppressedExists := m.suppressedUntil[trackingKey]
|
|
_, rateExists := m.alertRateLimit[trackingKey]
|
|
m.mu.RUnlock()
|
|
if recentExists || suppressedExists || rateExists {
|
|
t.Fatalf("expected manual clear to remove canonical tracking entries, got recent=%t suppressed=%t rate=%t", recentExists, suppressedExists, rateExists)
|
|
}
|
|
}
|
|
|
|
func TestAlertCharacterizationResolvedCallbackUsesCanonicalIDForCanonicalAliasClear(t *testing.T) {
|
|
resourceID := BuildGuestKey("pve1", "node1", 101)
|
|
alertID := canonicalMetricSpecID(resourceID, "cpu")
|
|
canonicalState := canonicalMetricStateID(resourceID, "cpu")
|
|
|
|
m := newCharacterizationManager(t, characterizationBaseConfig())
|
|
resolved := make(chan string, 1)
|
|
m.SetResolvedCallback(func(id string) {
|
|
select {
|
|
case resolved <- id:
|
|
default:
|
|
}
|
|
})
|
|
|
|
alert := &Alert{
|
|
ID: alertID,
|
|
Type: "cpu",
|
|
ResourceID: resourceID,
|
|
ResourceName: "app01",
|
|
CanonicalSpecID: canonicalMetricSpecID(resourceID, "cpu"),
|
|
CanonicalKind: "metric_threshold",
|
|
CanonicalState: canonicalState,
|
|
StartTime: time.Now().Add(-time.Minute),
|
|
LastSeen: time.Now(),
|
|
}
|
|
|
|
m.mu.Lock()
|
|
m.setActiveAlertNoLock(canonicalState, alert)
|
|
m.mu.Unlock()
|
|
|
|
m.clearAlert(canonicalState)
|
|
|
|
select {
|
|
case got := <-resolved:
|
|
if got != canonicalState {
|
|
t.Fatalf("resolved callback = %q, want %q", got, canonicalState)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatalf("expected resolved callback for %q", alertID)
|
|
}
|
|
}
|