mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 10:35:51 +00:00
6eee5a1641
The reducer core became the authoritative transition state for every per-observation family in the Phase 2 cutovers; the manager's tracking maps (offlineConfirmations, offlineRecoveryConfirmations, nodeOfflineCount, connectionDegradedCount, dockerOfflineCount, dockerStateConfirm, pendingAlerts) had been reduced to write-only mirrors. This deletes them, per the plan's retirement list (docs/ALERT_ENGINE_EVOLUTION.md). Hygiene the maps' cleanup loops used to provide moves into the core: - reducer.PruneStalePending reaps pending runs whose resource stopped being observed (Cleanup at 10 minutes, cleanupStaleMaps at the stale threshold) — previously the pendingAlerts age sweep. - Docker container cleanup drops core pending runs for containers no longer in the seen set (reducer.PendingResourceIDs + DropPendingForResource) — previously the dockerStateConfirm loops. - HandleHostOnline / HandleDockerHostOnline apply a healthy observation to the core so an in-flight offline confirmation run ends — previously a map delete. Two real gaps surfaced by the test conversion, fixed at the root: - Config-change auto-resolution removed alerts without mirroring the forget into the core, leaving the incident firing after a policy disabled it. - Intent pending state created by evaluateIntentNoLock carried no ResourceID/ResourceType/TrackingKey, so per-resource clears (guest suppression) could not match it. Guest node-move migration no longer re-keys pending runs: a move restarts an in-flight pending run (firing continuity still comes from alert adoption). Deliberate simplification, noted in the helper. Tests convert their map seeds and asserts to the core seams (testCoreConfirmations / testCoreRecoveryCount / testCoreHasIncident / testCoreIsPending, direct ApplyDiscrete seeding); assertions that only tested the deleted maps' bookkeeping are removed.
154 lines
5.0 KiB
Go
154 lines
5.0 KiB
Go
package alerts
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
)
|
|
|
|
func TestDisableAllNodesOfflinePreventsOfflineAlert(t *testing.T) {
|
|
manager := NewManager()
|
|
|
|
// Reset state to avoid interference from persisted alerts.
|
|
manager.mu.Lock()
|
|
manager.activeAlerts = make(map[string]*Alert)
|
|
manager.core.Reset()
|
|
manager.mu.Unlock()
|
|
|
|
config := manager.GetConfig()
|
|
config.DisableAllNodesOffline = true
|
|
manager.UpdateConfig(config)
|
|
|
|
node := models.Node{
|
|
ID: "node-1",
|
|
Name: "node-1",
|
|
Status: "offline",
|
|
ConnectionHealth: "error",
|
|
}
|
|
|
|
manager.CheckNode(node)
|
|
|
|
manager.mu.RLock()
|
|
_, alertExists := manager.activeAlerts["node-offline-node-1"]
|
|
counterExists := testCoreHasIncident(manager, "node-1", canonicalConnectivitySpecID("node-1"))
|
|
manager.mu.RUnlock()
|
|
|
|
if alertExists {
|
|
t.Fatalf("expected no node offline alert when DisableAllNodesOffline is true")
|
|
}
|
|
if counterExists {
|
|
t.Fatalf("expected node offline counter to be cleared when DisableAllNodesOffline is true")
|
|
}
|
|
}
|
|
|
|
func TestUpdateConfigClearsExistingNodeOfflineAlerts(t *testing.T) {
|
|
manager := NewManager()
|
|
|
|
manager.mu.Lock()
|
|
manager.activeAlerts = make(map[string]*Alert)
|
|
manager.core.Reset()
|
|
manager.activeAlerts["node-offline-node-1"] = &Alert{
|
|
ID: "node-offline-node-1",
|
|
Type: "connectivity",
|
|
ResourceID: "node-1",
|
|
ResourceName: "node-1",
|
|
Node: "node-1",
|
|
StartTime: time.Now().Add(-5 * time.Minute),
|
|
LastSeen: time.Now(),
|
|
}
|
|
manager.mu.Unlock()
|
|
|
|
config := manager.GetConfig()
|
|
config.DisableAllNodesOffline = true
|
|
manager.UpdateConfig(config)
|
|
|
|
// Allow asynchronous resolution callbacks, if any, to run.
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
manager.mu.RLock()
|
|
_, alertExists := manager.activeAlerts["node-offline-node-1"]
|
|
counterExists := testCoreHasIncident(manager, "node-1", canonicalConnectivitySpecID("node-1"))
|
|
manager.mu.RUnlock()
|
|
|
|
if alertExists {
|
|
t.Fatalf("expected node offline alert to be cleared when DisableAllNodesOffline is enabled")
|
|
}
|
|
if counterExists {
|
|
t.Fatalf("expected node offline counter to be reset when DisableAllNodesOffline is enabled")
|
|
}
|
|
}
|
|
|
|
func TestUpdateConfigClearsDockerContainerAlertsWhenDisabled(t *testing.T) {
|
|
manager := NewManager()
|
|
|
|
containerResourceID := "docker:host-1/container-1"
|
|
containerAlertIDs := []string{
|
|
"docker-container-state-" + containerResourceID,
|
|
"docker-container-health-" + containerResourceID,
|
|
"docker-container-restart-loop-" + containerResourceID,
|
|
"docker-container-oom-" + containerResourceID,
|
|
"docker-container-memory-limit-" + containerResourceID,
|
|
}
|
|
|
|
// Canonical stateful alerts (e.g. the image-update alert) are stored under
|
|
// "<resourceID>::<specID>" state IDs rather than legacy prefixed IDs.
|
|
canonicalUpdateAlertID := buildCanonicalStateID(containerResourceID, containerResourceID+"-image-update")
|
|
containerAlertIDs = append(containerAlertIDs, canonicalUpdateAlertID)
|
|
|
|
manager.mu.Lock()
|
|
for _, id := range containerAlertIDs {
|
|
manager.activeAlerts[id] = &Alert{ID: id, ResourceID: containerResourceID}
|
|
}
|
|
manager.dockerRestartTracking[containerResourceID] = &dockerRestartRecord{}
|
|
manager.mu.Unlock()
|
|
|
|
config := manager.GetConfig()
|
|
config.DisableAllDockerContainers = true
|
|
manager.UpdateConfig(config)
|
|
|
|
time.Sleep(10 * time.Millisecond)
|
|
|
|
manager.mu.RLock()
|
|
defer manager.mu.RUnlock()
|
|
for _, id := range containerAlertIDs {
|
|
if _, exists := manager.activeAlerts[id]; exists {
|
|
t.Fatalf("expected docker container alert %s to be cleared when DisableAllDockerContainers is enabled", id)
|
|
}
|
|
}
|
|
if testCoreHasIncident(manager, containerResourceID, canonicalDiscreteStateSpecID(containerResourceID, "runtime-state")) {
|
|
t.Fatalf("expected container state confirmation tracking to be cleared when DisableAllDockerContainers is enabled")
|
|
}
|
|
if len(manager.dockerRestartTracking) != 0 {
|
|
t.Fatalf("expected dockerRestartTracking map to be cleared when DisableAllDockerContainers is enabled")
|
|
}
|
|
}
|
|
|
|
func TestUpdateConfigClearsDockerServiceAlertsWhenDisabled(t *testing.T) {
|
|
manager := NewManager()
|
|
|
|
serviceResourceID := "docker:host-2/service/frontend"
|
|
serviceAlertID := "docker-service-health-" + serviceResourceID
|
|
canonicalServiceAlertID := buildCanonicalStateID(serviceResourceID, serviceResourceID+"-replica-gap")
|
|
|
|
manager.mu.Lock()
|
|
manager.activeAlerts[serviceAlertID] = &Alert{ID: serviceAlertID, ResourceID: serviceResourceID}
|
|
manager.activeAlerts[canonicalServiceAlertID] = &Alert{ID: canonicalServiceAlertID, ResourceID: serviceResourceID}
|
|
manager.mu.Unlock()
|
|
|
|
config := manager.GetConfig()
|
|
config.DisableAllDockerServices = true
|
|
manager.UpdateConfig(config)
|
|
|
|
time.Sleep(10 * time.Millisecond)
|
|
|
|
manager.mu.RLock()
|
|
defer manager.mu.RUnlock()
|
|
if _, exists := manager.activeAlerts[serviceAlertID]; exists {
|
|
t.Fatalf("expected docker service alert to be cleared when DisableAllDockerServices is enabled")
|
|
}
|
|
if _, exists := manager.activeAlerts[canonicalServiceAlertID]; exists {
|
|
t.Fatalf("expected canonical docker service alert to be cleared when DisableAllDockerServices is enabled")
|
|
}
|
|
}
|