diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 9b5c0836e..45efd93fb 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -265,12 +265,16 @@ type AlertConfig struct { CustomRules []CustomAlertRule `json:"customRules,omitempty"` Schedule ScheduleConfig `json:"schedule"` // Global disable flags per resource type - DisableAllNodes bool `json:"disableAllNodes"` // Disable all alerts for Proxmox nodes - DisableAllGuests bool `json:"disableAllGuests"` // Disable all alerts for VMs/containers - DisableAllStorage bool `json:"disableAllStorage"` // Disable all alerts for storage - DisableAllPBS bool `json:"disableAllPBS"` // Disable all alerts for PBS servers - DisableAllDockerHosts bool `json:"disableAllDockerHosts"` // Disable all alerts for Docker hosts - DisableAllDockerContainers bool `json:"disableAllDockerContainers"` // Disable all alerts for Docker containers + DisableAllNodes bool `json:"disableAllNodes"` // Disable all alerts for Proxmox nodes + DisableAllGuests bool `json:"disableAllGuests"` // Disable all alerts for VMs/containers + DisableAllStorage bool `json:"disableAllStorage"` // Disable all alerts for storage + DisableAllPBS bool `json:"disableAllPBS"` // Disable all alerts for PBS servers + DisableAllDockerHosts bool `json:"disableAllDockerHosts"` // Disable all alerts for Docker hosts + DisableAllDockerContainers bool `json:"disableAllDockerContainers"` // Disable all alerts for Docker containers + DisableAllNodesOffline bool `json:"disableAllNodesOffline"` // Disable node offline/connectivity alerts globally + DisableAllGuestsOffline bool `json:"disableAllGuestsOffline"` // Disable guest powered-off alerts globally + DisableAllPBSOffline bool `json:"disableAllPBSOffline"` // Disable PBS offline alerts globally + DisableAllDockerHostsOffline bool `json:"disableAllDockerHostsOffline"` // Disable Docker host offline alerts globally // New configuration options MinimumDelta float64 `json:"minimumDelta"` // Minimum % change to trigger new alert SuppressionWindow int `json:"suppressionWindow"` // Minutes to suppress duplicate alerts @@ -358,13 +362,13 @@ func NewManager() *Manager { Enabled: true, GuestDefaults: ThresholdConfig{ PoweredOffSeverity: AlertLevelWarning, - CPU: &HysteresisThreshold{Trigger: 80, Clear: 75}, - Memory: &HysteresisThreshold{Trigger: 85, Clear: 80}, - Disk: &HysteresisThreshold{Trigger: 90, Clear: 85}, - DiskRead: &HysteresisThreshold{Trigger: 0, Clear: 0}, // Off by default - DiskWrite: &HysteresisThreshold{Trigger: 0, Clear: 0}, // Off by default - NetworkIn: &HysteresisThreshold{Trigger: 0, Clear: 0}, // Off by default - NetworkOut: &HysteresisThreshold{Trigger: 0, Clear: 0}, // Off by default + CPU: &HysteresisThreshold{Trigger: 80, Clear: 75}, + Memory: &HysteresisThreshold{Trigger: 85, Clear: 80}, + Disk: &HysteresisThreshold{Trigger: 90, Clear: 85}, + DiskRead: &HysteresisThreshold{Trigger: 0, Clear: 0}, // Off by default + DiskWrite: &HysteresisThreshold{Trigger: 0, Clear: 0}, // Off by default + NetworkIn: &HysteresisThreshold{Trigger: 0, Clear: 0}, // Off by default + NetworkOut: &HysteresisThreshold{Trigger: 0, Clear: 0}, // Off by default }, NodeDefaults: ThresholdConfig{ CPU: &HysteresisThreshold{Trigger: 80, Clear: 75}, @@ -538,7 +542,6 @@ func (m *Manager) UpdateConfig(config AlertConfig) { config.GuestDefaults.PoweredOffSeverity = normalizePoweredOffSeverity(config.GuestDefaults.PoweredOffSeverity) config.NodeDefaults.PoweredOffSeverity = normalizePoweredOffSeverity(config.NodeDefaults.PoweredOffSeverity) - m.config = config for id, override := range m.config.Overrides { override.PoweredOffSeverity = normalizePoweredOffSeverity(override.PoweredOffSeverity) @@ -548,6 +551,9 @@ func (m *Manager) UpdateConfig(config AlertConfig) { } m.config.Overrides[id] = override } + + m.applyGlobalOfflineSettingsLocked() + log.Info(). Bool("enabled", config.Enabled). Interface("guestDefaults", config.GuestDefaults). @@ -557,6 +563,62 @@ func (m *Manager) UpdateConfig(config AlertConfig) { m.reevaluateActiveAlertsLocked() } +// applyGlobalOfflineSettingsLocked clears tracking and active alerts for globally disabled offline detectors. +// Caller must hold m.mu. +func (m *Manager) applyGlobalOfflineSettingsLocked() { + if m.config.DisableAllNodesOffline { + var nodeAlerts []string + for alertID := range m.activeAlerts { + if strings.HasPrefix(alertID, "node-offline-") { + nodeAlerts = append(nodeAlerts, alertID) + } + } + for _, alertID := range nodeAlerts { + m.clearAlertNoLock(alertID) + } + m.nodeOfflineCount = make(map[string]int) + } + + if m.config.DisableAllPBSOffline { + var pbsAlerts []string + for alertID, alert := range m.activeAlerts { + if strings.HasPrefix(alertID, "pbs-offline-") { + pbsAlerts = append(pbsAlerts, alertID) + delete(m.offlineConfirmations, alert.ResourceID) + } + } + for _, alertID := range pbsAlerts { + m.clearAlertNoLock(alertID) + } + } + + if m.config.DisableAllGuestsOffline { + var guestAlerts []string + for alertID, alert := range m.activeAlerts { + if strings.HasPrefix(alertID, "guest-powered-off-") { + guestAlerts = append(guestAlerts, alertID) + delete(m.offlineConfirmations, alert.ResourceID) + } + } + for _, alertID := range guestAlerts { + m.clearAlertNoLock(alertID) + } + } + + if m.config.DisableAllDockerHostsOffline { + var hostAlerts []string + for alertID := range m.activeAlerts { + if strings.HasPrefix(alertID, "docker-host-offline-") { + hostAlerts = append(hostAlerts, alertID) + } + } + for _, alertID := range hostAlerts { + m.clearAlertNoLock(alertID) + } + m.dockerOfflineCount = make(map[string]int) + } +} + // reevaluateActiveAlertsLocked re-evaluates all active alerts against the current configuration // This should only be called with m.mu already locked func (m *Manager) reevaluateActiveAlertsLocked() { @@ -963,17 +1025,19 @@ func (m *Manager) GetConfig() AlertConfig { // CheckGuest checks a guest (VM or container) against thresholds func (m *Manager) CheckGuest(guest interface{}, instanceName string) { m.mu.RLock() - if !m.config.Enabled { - m.mu.RUnlock() + enabled := m.config.Enabled + disableAllGuests := m.config.DisableAllGuests + disableAllGuestsOffline := m.config.DisableAllGuestsOffline + m.mu.RUnlock() + + if !enabled { log.Debug().Msg("CheckGuest: alerts disabled globally") return } - if m.config.DisableAllGuests { - m.mu.RUnlock() + if disableAllGuests { log.Debug().Msg("CheckGuest: all guest alerts disabled") return } - m.mu.RUnlock() var guestID, name, node, guestType, status string var cpu, memUsage, diskUsage float64 @@ -1031,7 +1095,15 @@ func (m *Manager) CheckGuest(guest interface{}, instanceName string) { if status != "running" { // Check for powered-off state and generate alert if configured if status == "stopped" { - m.checkGuestPoweredOff(guestID, name, node, instanceName, guestType) + if disableAllGuestsOffline { + // Clear any pending powered-off tracking and alerts when globally disabled + m.mu.Lock() + delete(m.offlineConfirmations, guestID) + m.mu.Unlock() + m.clearAlert(fmt.Sprintf("guest-powered-off-%s", guestID)) + } else { + m.checkGuestPoweredOff(guestID, name, node, instanceName, guestType) + } } else { // For paused/suspended, clear powered-off alert m.clearGuestPoweredOffAlert(guestID, name) @@ -1190,18 +1262,27 @@ func (m *Manager) CheckNode(node models.Node) { m.mu.RUnlock() return } + disableNodesOffline := m.config.DisableAllNodesOffline thresholds := m.config.NodeDefaults if override, exists := m.config.Overrides[node.ID]; exists { thresholds = m.applyThresholdOverride(thresholds, override) } m.mu.RUnlock() - // CRITICAL: Check if node is offline first - if node.Status == "offline" || node.ConnectionHealth == "error" || node.ConnectionHealth == "failed" { - m.checkNodeOffline(node) + if disableNodesOffline { + // Clear tracking and any existing offline alerts when globally disabled + m.mu.Lock() + delete(m.nodeOfflineCount, node.ID) + m.mu.Unlock() + m.clearAlert(fmt.Sprintf("node-offline-%s", node.ID)) } else { - // Clear any existing offline alert if node is back online - m.clearNodeOfflineAlert(node) + // CRITICAL: Check if node is offline first + if node.Status == "offline" || node.ConnectionHealth == "error" || node.ConnectionHealth == "failed" { + m.checkNodeOffline(node) + } else { + // Clear any existing offline alert if node is back online + m.clearNodeOfflineAlert(node) + } } // Check each metric (only if node is online) - checkMetric will skip if threshold is nil or <= 0 @@ -1240,14 +1321,23 @@ func (m *Manager) CheckPBS(pbs models.PBSInstance) { // Use node defaults for PBS (same as nodes: CPU, Memory) cpuThreshold := m.config.NodeDefaults.CPU memoryThreshold := m.config.NodeDefaults.Memory + disablePBSOffline := m.config.DisableAllPBSOffline m.mu.RUnlock() - // Check if PBS is offline first (similar to nodes) - if pbs.Status == "offline" || pbs.ConnectionHealth == "error" || pbs.ConnectionHealth == "unhealthy" { - m.checkPBSOffline(pbs) + if disablePBSOffline { + // Clear tracking and any existing offline alerts when globally disabled + m.mu.Lock() + delete(m.offlineConfirmations, pbs.ID) + m.mu.Unlock() + m.clearAlert(fmt.Sprintf("pbs-offline-%s", pbs.ID)) } else { - // Clear any existing offline alert if PBS is back online - m.clearPBSOfflineAlert(pbs) + // Check if PBS is offline first (similar to nodes) + if pbs.Status == "offline" || pbs.ConnectionHealth == "error" || pbs.ConnectionHealth == "unhealthy" { + m.checkPBSOffline(pbs) + } else { + // Clear any existing offline alert if PBS is back online + m.clearPBSOfflineAlert(pbs) + } } // If alerts are disabled for this PBS instance, clear any existing alerts and return @@ -1511,6 +1601,7 @@ func (m *Manager) HandleDockerHostOffline(host models.DockerHost) { m.mu.RUnlock() return } + disableDockerHostsOffline := m.config.DisableAllDockerHostsOffline m.mu.RUnlock() alertID := fmt.Sprintf("docker-host-offline-%s", host.ID) @@ -1518,6 +1609,14 @@ func (m *Manager) HandleDockerHostOffline(host models.DockerHost) { instanceName := dockerInstanceName(host) nodeName := strings.TrimSpace(host.Hostname) + if disableDockerHostsOffline { + m.mu.Lock() + delete(m.dockerOfflineCount, host.ID) + m.mu.Unlock() + m.clearAlert(alertID) + return + } + var disableConnectivity bool m.mu.RLock() if override, exists := m.config.Overrides[host.ID]; exists { @@ -3330,15 +3429,15 @@ func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestT thresholds = m.config.GuestDefaults } - severity := normalizePoweredOffSeverity(thresholds.PoweredOffSeverity) + severity := normalizePoweredOffSeverity(thresholds.PoweredOffSeverity) - // Check if powered-off alerts are disabled for this guest - if thresholds.Disabled || thresholds.DisableConnectivity { - // Powered-off alerts are disabled, clear any existing alert and return - if _, alertExists := m.activeAlerts[alertID]; alertExists { - m.clearAlertNoLock(alertID) - log.Debug(). - Str("guest", name). + // Check if powered-off alerts are disabled for this guest + if thresholds.Disabled || thresholds.DisableConnectivity { + // Powered-off alerts are disabled, clear any existing alert and return + if _, alertExists := m.activeAlerts[alertID]; alertExists { + m.clearAlertNoLock(alertID) + log.Debug(). + Str("guest", name). Msg("Guest powered-off alert cleared (alerts disabled)") } delete(m.offlineConfirmations, guestID) @@ -3346,12 +3445,12 @@ func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestT } // Check if alert already exists - if alert, exists := m.activeAlerts[alertID]; exists { - // Alert already exists, just update LastSeen - alert.LastSeen = time.Now() - alert.Level = severity - return - } + if alert, exists := m.activeAlerts[alertID]; exists { + // Alert already exists, just update LastSeen + alert.LastSeen = time.Now() + alert.Level = severity + return + } // Increment confirmation count m.offlineConfirmations[guestID]++ @@ -3376,10 +3475,10 @@ func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestT } // Create new powered-off alert after confirmation - alert := &Alert{ - ID: alertID, - Type: "powered-off", - Level: severity, + alert := &Alert{ + ID: alertID, + Type: "powered-off", + Level: severity, ResourceID: guestID, ResourceName: name, Node: node, diff --git a/internal/alerts/offline_toggle_test.go b/internal/alerts/offline_toggle_test.go new file mode 100644 index 000000000..2c5bd670e --- /dev/null +++ b/internal/alerts/offline_toggle_test.go @@ -0,0 +1,81 @@ +package alerts + +import ( + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/models" +) + +func TestDisableAllNodesOfflinePreventsOfflineAlert(t *testing.T) { + manager := NewManager() + + // Reset state to avoid interference from persisted alerts. + manager.mu.Lock() + manager.activeAlerts = make(map[string]*Alert) + manager.nodeOfflineCount = make(map[string]int) + manager.mu.Unlock() + + config := manager.GetConfig() + config.DisableAllNodesOffline = true + manager.UpdateConfig(config) + + node := models.Node{ + ID: "node-1", + Name: "node-1", + Status: "offline", + ConnectionHealth: "error", + } + + manager.CheckNode(node) + + manager.mu.RLock() + _, alertExists := manager.activeAlerts["node-offline-node-1"] + _, counterExists := manager.nodeOfflineCount["node-1"] + manager.mu.RUnlock() + + if alertExists { + t.Fatalf("expected no node offline alert when DisableAllNodesOffline is true") + } + if counterExists { + t.Fatalf("expected node offline counter to be cleared when DisableAllNodesOffline is true") + } +} + +func TestUpdateConfigClearsExistingNodeOfflineAlerts(t *testing.T) { + manager := NewManager() + + manager.mu.Lock() + manager.activeAlerts = make(map[string]*Alert) + manager.nodeOfflineCount = make(map[string]int) + manager.activeAlerts["node-offline-node-1"] = &Alert{ + ID: "node-offline-node-1", + Type: "connectivity", + ResourceID: "node-1", + ResourceName: "node-1", + Node: "node-1", + StartTime: time.Now().Add(-5 * time.Minute), + LastSeen: time.Now(), + } + manager.nodeOfflineCount["node-1"] = 3 + manager.mu.Unlock() + + config := manager.GetConfig() + config.DisableAllNodesOffline = true + manager.UpdateConfig(config) + + // Allow asynchronous resolution callbacks, if any, to run. + time.Sleep(50 * time.Millisecond) + + manager.mu.RLock() + _, alertExists := manager.activeAlerts["node-offline-node-1"] + _, counterExists := manager.nodeOfflineCount["node-1"] + manager.mu.RUnlock() + + if alertExists { + t.Fatalf("expected node offline alert to be cleared when DisableAllNodesOffline is enabled") + } + if counterExists { + t.Fatalf("expected node offline counter to be reset when DisableAllNodesOffline is enabled") + } +}