Improve alert duration tracking and add time threshold UI

This commit addresses issues reported in #470 related to alert duration tracking, time threshold configuration, and email notification debugging. Backend Changes: - Preserve alert StartTime in preserveAlertState() to maintain accurate duration calculations across monitoring cycles - Add debug logging to track alert creation times and duration preservation - Add comprehensive logging to notification pipeline for email delivery tracking including SMTP config, cooldown status, and delivery attempts Frontend Changes: - Add TimeThresholdSettings component to display and configure per-resource-type alert delays (VMs/Containers, Nodes, Storage, PBS) - Integrate time threshold UI into Thresholds tab with clear labels explaining "seconds above threshold before triggering" - Add informational help text about how alert delays work Related to #470
2026-09-22 11:13:26 +00:00 · 2025-10-08 08:51:02 +00:00
parent a4569875b8
commit df7bc21caf
4 changed files with 178 additions and 1 deletions
@@ -15,6 +15,7 @@ import type {
 } from '@/types/api';
 import type { RawOverrideConfig } from '@/types/alerts';
 import { ResourceTable, Resource, GroupHeaderMeta } from './ResourceTable';
+import { TimeThresholdSettings } from './TimeThresholdSettings';
 import { Card } from '@/components/shared/Card';
 import { SectionHeader } from '@/components/shared/SectionHeader';

@@ -1190,6 +1191,13 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr

  return (
    <div class="space-y-6">
+      {/* Time Threshold Settings */}
+      <TimeThresholdSettings
+        timeThresholds={props.timeThresholds}
+        setTimeThresholds={props.setTimeThresholds}
+        setHasUnsavedChanges={props.setHasUnsavedChanges}
+      />
+
      {/* Search Bar */}
      <div class="relative">
        <input
@@ -0,0 +1,131 @@
+import { Card } from '@/components/shared/Card';
+import { SectionHeader } from '@/components/shared/SectionHeader';
+import { formField, labelClass, controlClass, formHelpText } from '@/components/shared/Form';
+
+interface TimeThresholdSettingsProps {
+  timeThresholds: () => { guest: number; node: number; storage: number; pbs: number };
+  setTimeThresholds: (value: { guest: number; node: number; storage: number; pbs: number }) => void;
+  setHasUnsavedChanges: (value: boolean) => void;
+}
+
+export function TimeThresholdSettings(props: TimeThresholdSettingsProps) {
+  const thresholdConfigs = [
+    {
+      key: 'guest' as const,
+      label: 'VMs & Containers',
+      description: 'Delay before triggering alerts for virtual machines and containers',
+      min: 0,
+      max: 300,
+      step: 5,
+    },
+    {
+      key: 'node' as const,
+      label: 'Proxmox Nodes',
+      description: 'Delay before triggering alerts for Proxmox nodes',
+      min: 0,
+      max: 300,
+      step: 5,
+    },
+    {
+      key: 'storage' as const,
+      label: 'Storage Devices',
+      description: 'Delay before triggering alerts for storage devices',
+      min: 0,
+      max: 300,
+      step: 5,
+    },
+    {
+      key: 'pbs' as const,
+      label: 'PBS Servers',
+      description: 'Delay before triggering alerts for Proxmox Backup Server instances',
+      min: 0,
+      max: 300,
+      step: 5,
+    },
+  ];
+
+  return (
+    <Card>
+      <div class="space-y-4">
+        <div>
+          <SectionHeader
+            title="Alert Delay Thresholds"
+            size="md"
+            class="mb-2"
+          />
+          <p class="text-sm text-gray-600 dark:text-gray-400">
+            Configure how long a metric must remain above threshold before triggering an alert.
+            This prevents alerts from being triggered by brief spikes.
+          </p>
+        </div>
+
+        <div class="grid gap-4 md:grid-cols-2">
+          {thresholdConfigs.map((config) => (
+            <div class={formField}>
+              <label class={labelClass()}>
+                {config.label}
+              </label>
+              <div class="relative">
+                <input
+                  type="number"
+                  min={config.min}
+                  max={config.max}
+                  step={config.step}
+                  value={props.timeThresholds()[config.key]}
+                  onInput={(e) => {
+                    const value = parseInt(e.currentTarget.value);
+                    if (!isNaN(value) && value >= config.min && value <= config.max) {
+                      props.setTimeThresholds({
+                        ...props.timeThresholds(),
+                        [config.key]: value,
+                      });
+                      props.setHasUnsavedChanges(true);
+                    }
+                  }}
+                  class={controlClass('pr-20')}
+                />
+                <span class="pointer-events-none absolute inset-y-0 right-3 flex items-center text-sm text-gray-500 dark:text-gray-400">
+                  seconds
+                </span>
+              </div>
+              <p class={formHelpText}>
+                {config.description}
+                {props.timeThresholds()[config.key] === 0 && (
+                  <span class="ml-1 font-medium text-amber-600 dark:text-amber-400">
+                    (Alerts trigger immediately)
+                  </span>
+                )}
+              </p>
+            </div>
+          ))}
+        </div>
+
+        <div class="rounded-md bg-blue-50 dark:bg-blue-900/20 border border-blue-200 dark:border-blue-800 p-3">
+          <div class="flex gap-2">
+            <svg
+              class="h-5 w-5 text-blue-600 dark:text-blue-400 flex-shrink-0 mt-0.5"
+              fill="none"
+              stroke="currentColor"
+              viewBox="0 0 24 24"
+            >
+              <path
+                stroke-linecap="round"
+                stroke-linejoin="round"
+                stroke-width="2"
+                d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z"
+              />
+            </svg>
+            <div class="text-sm text-blue-800 dark:text-blue-200">
+              <p class="font-medium mb-1">How Alert Delays Work</p>
+              <p>
+                When a metric exceeds its threshold, Pulse waits for the configured delay period before triggering an alert.
+                If the metric drops below the threshold during this waiting period, no alert is created.
+                This helps reduce false alarms from temporary spikes.
+              </p>
+            </div>
+          </div>
+        </div>
+      </div>
+    </Card>
+  );
+}
@@ -2529,6 +2529,13 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
 				alert.Level = AlertLevelCritical
 			}

+			log.Debug().
+				Str("alertID", alertID).
+				Time("alertStartTime", alertStartTime).
+				Time("now", time.Now()).
+				Dur("initialDuration", time.Since(alertStartTime)).
+				Msg("Creating new alert with start time")
+
 			m.preserveAlertState(alertID, alert)

 			m.activeAlerts[alertID] = alert
@@ -2834,6 +2841,8 @@ func (m *Manager) preserveAlertState(alertID string, updated *Alert) {

 	existing, exists := m.activeAlerts[alertID]
 	if exists && existing != nil {
+		// Preserve the original start time so duration calculations are correct
+		updated.StartTime = existing.StartTime
 		updated.Acknowledged = existing.Acknowledged
 		updated.AckUser = existing.AckUser
 		if existing.AckTime != nil {
@@ -2848,6 +2857,12 @@ func (m *Manager) preserveAlertState(alertID string, updated *Alert) {
 		} else {
 			updated.EscalationTimes = nil
 		}
+
+		log.Debug().
+			Str("alertID", alertID).
+			Time("originalStartTime", existing.StartTime).
+			Dur("currentDuration", time.Since(existing.StartTime)).
+			Msg("Preserving alert state including StartTime")
 		return
 	}

@@ -332,12 +332,24 @@ func (n *NotificationManager) SendAlert(alert *alerts.Alert) {
 	if exists && record.alertStart.Equal(alert.StartTime) && time.Since(record.lastSent) < n.cooldown {
 		log.Info().
 			Str("alertID", alert.ID).
+			Str("resourceName", alert.ResourceName).
+			Str("type", alert.Type).
 			Dur("timeSince", time.Since(record.lastSent)).
 			Dur("cooldown", n.cooldown).
-			Msg("Alert notification in cooldown for active alert")
+			Dur("remainingCooldown", n.cooldown-time.Since(record.lastSent)).
+			Msg("Alert notification in cooldown for active alert - notification suppressed")
 		return
 	}

+	log.Info().
+		Str("alertID", alert.ID).
+		Str("resourceName", alert.ResourceName).
+		Str("type", alert.Type).
+		Float64("value", alert.Value).
+		Float64("threshold", alert.Threshold).
+		Bool("inCooldown", exists).
+		Msg("Alert passed cooldown check - adding to pending notifications")
+
 	// Add to pending alerts for grouping
 	n.pendingAlerts = append(n.pendingAlerts, alert)

@@ -425,7 +437,18 @@ func (n *NotificationManager) sendGroupedAlerts() {

 	// Send notifications using the captured snapshots outside the lock to avoid blocking writers
 	if emailConfig.Enabled {
+		log.Info().
+			Int("alertCount", len(alertsToSend)).
+			Str("smtpHost", emailConfig.SMTPHost).
+			Int("smtpPort", emailConfig.SMTPPort).
+			Strs("recipients", emailConfig.To).
+			Bool("hasAuth", emailConfig.Username != "" && emailConfig.Password != "").
+			Msg("Email notifications enabled - sending grouped email")
 		go n.sendGroupedEmail(emailConfig, alertsToSend)
+	} else {
+		log.Debug().
+			Int("alertCount", len(alertsToSend)).
+			Msg("Email notifications disabled - skipping email delivery")
 	}

 	for _, webhook := range webhooks {