Files
pulse/internal/monitoring/system_alerts_test.go
T
rcourtman 6f9a01a72a Keep the delivery-health system alert standing and quiet across count drift
The Proxmox node sweep removes any non-preserved alert whose Node is
empty, and system alerts have no node, so every sweep silently deleted
the notification-delivery alert. Its five-minute evaluation then
re-raised it as a brand-new alert, firing a fresh notification each
cycle with no recovery in between, which reads as an alert appearing,
vanishing without a recovery, and paging again minutes later. System
alerts are now preserved outside node cleanup.

Raises also carried the delivery counts inside the message, and a
message change re-notifies, so each newly retained failure re-paged
even while the condition was unchanged. System alerts now take an
optional fingerprint: while level and fingerprint hold, a re-raise
refreshes the message and metadata silently. The delivery alert
fingerprints on status and failure classes, keeping its counter text
current without paging on drift.

Refs #1721

Contract-Neutral: behavioral fix: delivery-health system alert survived node sweep and stops re-paging on count drift, no public contract delta (Refs #1721)
2026-08-19 16:54:39 +01:00

133 lines
4.1 KiB
Go

package monitoring
import (
"strings"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
)
func TestNotificationDeliveryAlertMessageNamesTheOutcome(t *testing.T) {
cases := []struct {
name string
health notifications.DeliveryHealth
want []string
}{
{
name: "failed only",
health: notifications.DeliveryHealth{
Status: notifications.DeliveryDegraded,
Failed: 4,
},
want: []string{"4 failed deliveries", "not reaching their destinations"},
},
{
name: "dead lettered only",
health: notifications.DeliveryHealth{
Status: notifications.DeliveryDegraded,
DeadLetter: 1,
},
want: []string{"1 dead-lettered delivery", "gave up after repeated failures"},
},
{
name: "both",
health: notifications.DeliveryHealth{
Status: notifications.DeliveryDegraded,
Failed: 2,
DeadLetter: 3,
},
want: []string{"2 failed deliveries", "3 dead-lettered deliveries"},
},
{
name: "unavailable",
health: notifications.DeliveryHealth{Status: notifications.DeliveryUnavailable},
want: []string{"cannot read the notification queue"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
message := notificationDeliveryAlertMessage(tc.health)
for _, want := range tc.want {
if !strings.Contains(message, want) {
t.Errorf("message %q does not contain %q", message, want)
}
}
// The operator reading this has not received a notification about
// it, so the message must say where to go.
if tc.health.Status == notifications.DeliveryDegraded &&
!strings.Contains(message, "Alerts, Notifications") {
t.Errorf("message %q does not point at the destinations surface", message)
}
})
}
}
func TestNotificationDeliveryAlertMessageSingularises(t *testing.T) {
message := notificationDeliveryAlertMessage(notifications.DeliveryHealth{
Status: notifications.DeliveryDegraded,
Failed: 1,
})
if !strings.Contains(message, "1 failed delivery ") {
t.Errorf("expected a singular delivery in %q", message)
}
}
func TestDeliveryHealthFingerprintIgnoresCountDrift(t *testing.T) {
base := notifications.ClassifyQueueHealth(map[string]int{
string(notifications.QueueStatusDLQ): 11,
})
drifted := notifications.ClassifyQueueHealth(map[string]int{
string(notifications.QueueStatusDLQ): 14,
})
if deliveryHealthFingerprint(base) != deliveryHealthFingerprint(drifted) {
t.Error("expected count drift within one failure class to keep the fingerprint stable")
}
withFailed := notifications.ClassifyQueueHealth(map[string]int{
string(notifications.QueueStatusDLQ): 14,
string(notifications.QueueStatusFailed): 1,
})
if deliveryHealthFingerprint(base) == deliveryHealthFingerprint(withFailed) {
t.Error("expected a new failure class to change the fingerprint")
}
unavailable := notifications.UnavailableDeliveryHealth()
if deliveryHealthFingerprint(base) == deliveryHealthFingerprint(unavailable) {
t.Error("expected an unavailable queue to change the fingerprint")
}
}
func TestEvaluateNotificationDeliveryThrottles(t *testing.T) {
// The poll ticker runs on the polling cadence, which can be seconds, and
// reading queue health costs a SQLite query. A nil notification manager
// means the evaluation returns early, but the throttle stamp must still be
// taken so the interval is honoured.
m := &Monitor{}
start := time.Now()
m.evaluateNotificationDelivery(start)
firstStamp := m.lastDeliveryHealthCheck
if firstStamp.IsZero() {
t.Fatal("expected the first evaluation to record a check time")
}
m.evaluateNotificationDelivery(start.Add(notificationDeliveryCheckInterval / 2))
if !m.lastDeliveryHealthCheck.Equal(firstStamp) {
t.Error("expected an evaluation inside the interval to be skipped")
}
due := start.Add(notificationDeliveryCheckInterval + time.Second)
m.evaluateNotificationDelivery(due)
if !m.lastDeliveryHealthCheck.Equal(due) {
t.Error("expected an evaluation past the interval to run")
}
}
func TestEvaluateNotificationDeliveryIsSafeWithoutAMonitor(t *testing.T) {
var m *Monitor
m.evaluateNotificationDelivery(time.Now())
}