diff --git a/frontend-modern/src/components/Alerts/ThresholdsTable.tsx b/frontend-modern/src/components/Alerts/ThresholdsTable.tsx index f96572126..ebcc24357 100644 --- a/frontend-modern/src/components/Alerts/ThresholdsTable.tsx +++ b/frontend-modern/src/components/Alerts/ThresholdsTable.tsx @@ -15,6 +15,7 @@ import type { } from '@/types/api'; import type { RawOverrideConfig } from '@/types/alerts'; import { ResourceTable, Resource, GroupHeaderMeta } from './ResourceTable'; +import { TimeThresholdSettings } from './TimeThresholdSettings'; import { Card } from '@/components/shared/Card'; import { SectionHeader } from '@/components/shared/SectionHeader'; @@ -1190,6 +1191,13 @@ const dockerContainersGroupedByHost = createMemo>((pr return (
+ {/* Time Threshold Settings */} + + {/* Search Bar */}
{ guest: number; node: number; storage: number; pbs: number }; + setTimeThresholds: (value: { guest: number; node: number; storage: number; pbs: number }) => void; + setHasUnsavedChanges: (value: boolean) => void; +} + +export function TimeThresholdSettings(props: TimeThresholdSettingsProps) { + const thresholdConfigs = [ + { + key: 'guest' as const, + label: 'VMs & Containers', + description: 'Delay before triggering alerts for virtual machines and containers', + min: 0, + max: 300, + step: 5, + }, + { + key: 'node' as const, + label: 'Proxmox Nodes', + description: 'Delay before triggering alerts for Proxmox nodes', + min: 0, + max: 300, + step: 5, + }, + { + key: 'storage' as const, + label: 'Storage Devices', + description: 'Delay before triggering alerts for storage devices', + min: 0, + max: 300, + step: 5, + }, + { + key: 'pbs' as const, + label: 'PBS Servers', + description: 'Delay before triggering alerts for Proxmox Backup Server instances', + min: 0, + max: 300, + step: 5, + }, + ]; + + return ( + +
+
+ +

+ Configure how long a metric must remain above threshold before triggering an alert. + This prevents alerts from being triggered by brief spikes. +

+
+ +
+ {thresholdConfigs.map((config) => ( +
+ +
+ { + const value = parseInt(e.currentTarget.value); + if (!isNaN(value) && value >= config.min && value <= config.max) { + props.setTimeThresholds({ + ...props.timeThresholds(), + [config.key]: value, + }); + props.setHasUnsavedChanges(true); + } + }} + class={controlClass('pr-20')} + /> + + seconds + +
+

+ {config.description} + {props.timeThresholds()[config.key] === 0 && ( + + (Alerts trigger immediately) + + )} +

+
+ ))} +
+ +
+
+ + + +
+

How Alert Delays Work

+

+ When a metric exceeds its threshold, Pulse waits for the configured delay period before triggering an alert. + If the metric drops below the threshold during this waiting period, no alert is created. + This helps reduce false alarms from temporary spikes. +

+
+
+
+
+ + ); +} diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index ac48b1e4a..d776af48d 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -2529,6 +2529,13 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource alert.Level = AlertLevelCritical } + log.Debug(). + Str("alertID", alertID). + Time("alertStartTime", alertStartTime). + Time("now", time.Now()). + Dur("initialDuration", time.Since(alertStartTime)). + Msg("Creating new alert with start time") + m.preserveAlertState(alertID, alert) m.activeAlerts[alertID] = alert @@ -2834,6 +2841,8 @@ func (m *Manager) preserveAlertState(alertID string, updated *Alert) { existing, exists := m.activeAlerts[alertID] if exists && existing != nil { + // Preserve the original start time so duration calculations are correct + updated.StartTime = existing.StartTime updated.Acknowledged = existing.Acknowledged updated.AckUser = existing.AckUser if existing.AckTime != nil { @@ -2848,6 +2857,12 @@ func (m *Manager) preserveAlertState(alertID string, updated *Alert) { } else { updated.EscalationTimes = nil } + + log.Debug(). + Str("alertID", alertID). + Time("originalStartTime", existing.StartTime). + Dur("currentDuration", time.Since(existing.StartTime)). + Msg("Preserving alert state including StartTime") return } diff --git a/internal/notifications/notifications.go b/internal/notifications/notifications.go index 0331f69fd..65325d775 100644 --- a/internal/notifications/notifications.go +++ b/internal/notifications/notifications.go @@ -332,12 +332,24 @@ func (n *NotificationManager) SendAlert(alert *alerts.Alert) { if exists && record.alertStart.Equal(alert.StartTime) && time.Since(record.lastSent) < n.cooldown { log.Info(). Str("alertID", alert.ID). + Str("resourceName", alert.ResourceName). + Str("type", alert.Type). Dur("timeSince", time.Since(record.lastSent)). Dur("cooldown", n.cooldown). - Msg("Alert notification in cooldown for active alert") + Dur("remainingCooldown", n.cooldown-time.Since(record.lastSent)). + Msg("Alert notification in cooldown for active alert - notification suppressed") return } + log.Info(). + Str("alertID", alert.ID). + Str("resourceName", alert.ResourceName). + Str("type", alert.Type). + Float64("value", alert.Value). + Float64("threshold", alert.Threshold). + Bool("inCooldown", exists). + Msg("Alert passed cooldown check - adding to pending notifications") + // Add to pending alerts for grouping n.pendingAlerts = append(n.pendingAlerts, alert) @@ -425,7 +437,18 @@ func (n *NotificationManager) sendGroupedAlerts() { // Send notifications using the captured snapshots outside the lock to avoid blocking writers if emailConfig.Enabled { + log.Info(). + Int("alertCount", len(alertsToSend)). + Str("smtpHost", emailConfig.SMTPHost). + Int("smtpPort", emailConfig.SMTPPort). + Strs("recipients", emailConfig.To). + Bool("hasAuth", emailConfig.Username != "" && emailConfig.Password != ""). + Msg("Email notifications enabled - sending grouped email") go n.sendGroupedEmail(emailConfig, alertsToSend) + } else { + log.Debug(). + Int("alertCount", len(alertsToSend)). + Msg("Email notifications disabled - skipping email delivery") } for _, webhook := range webhooks {