mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-11 14:00:29 +00:00
6f9a01a72a
The Proxmox node sweep removes any non-preserved alert whose Node is empty, and system alerts have no node, so every sweep silently deleted the notification-delivery alert. Its five-minute evaluation then re-raised it as a brand-new alert, firing a fresh notification each cycle with no recovery in between, which reads as an alert appearing, vanishing without a recovery, and paging again minutes later. System alerts are now preserved outside node cleanup. Raises also carried the delivery counts inside the message, and a message change re-notifies, so each newly retained failure re-paged even while the condition was unchanged. System alerts now take an optional fingerprint: while level and fingerprint hold, a re-raise refreshes the message and metadata silently. The delivery alert fingerprints on status and failure classes, keeping its counter text current without paging on drift. Refs #1721 Contract-Neutral: behavioral fix: delivery-health system alert survived node sweep and stops re-paging on count drift, no public contract delta (Refs #1721)
133 lines
4.1 KiB
Go
133 lines
4.1 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
|
|
)
|
|
|
|
func TestNotificationDeliveryAlertMessageNamesTheOutcome(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
health notifications.DeliveryHealth
|
|
want []string
|
|
}{
|
|
{
|
|
name: "failed only",
|
|
health: notifications.DeliveryHealth{
|
|
Status: notifications.DeliveryDegraded,
|
|
Failed: 4,
|
|
},
|
|
want: []string{"4 failed deliveries", "not reaching their destinations"},
|
|
},
|
|
{
|
|
name: "dead lettered only",
|
|
health: notifications.DeliveryHealth{
|
|
Status: notifications.DeliveryDegraded,
|
|
DeadLetter: 1,
|
|
},
|
|
want: []string{"1 dead-lettered delivery", "gave up after repeated failures"},
|
|
},
|
|
{
|
|
name: "both",
|
|
health: notifications.DeliveryHealth{
|
|
Status: notifications.DeliveryDegraded,
|
|
Failed: 2,
|
|
DeadLetter: 3,
|
|
},
|
|
want: []string{"2 failed deliveries", "3 dead-lettered deliveries"},
|
|
},
|
|
{
|
|
name: "unavailable",
|
|
health: notifications.DeliveryHealth{Status: notifications.DeliveryUnavailable},
|
|
want: []string{"cannot read the notification queue"},
|
|
},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
message := notificationDeliveryAlertMessage(tc.health)
|
|
for _, want := range tc.want {
|
|
if !strings.Contains(message, want) {
|
|
t.Errorf("message %q does not contain %q", message, want)
|
|
}
|
|
}
|
|
// The operator reading this has not received a notification about
|
|
// it, so the message must say where to go.
|
|
if tc.health.Status == notifications.DeliveryDegraded &&
|
|
!strings.Contains(message, "Alerts, Notifications") {
|
|
t.Errorf("message %q does not point at the destinations surface", message)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNotificationDeliveryAlertMessageSingularises(t *testing.T) {
|
|
message := notificationDeliveryAlertMessage(notifications.DeliveryHealth{
|
|
Status: notifications.DeliveryDegraded,
|
|
Failed: 1,
|
|
})
|
|
|
|
if !strings.Contains(message, "1 failed delivery ") {
|
|
t.Errorf("expected a singular delivery in %q", message)
|
|
}
|
|
}
|
|
|
|
func TestDeliveryHealthFingerprintIgnoresCountDrift(t *testing.T) {
|
|
base := notifications.ClassifyQueueHealth(map[string]int{
|
|
string(notifications.QueueStatusDLQ): 11,
|
|
})
|
|
drifted := notifications.ClassifyQueueHealth(map[string]int{
|
|
string(notifications.QueueStatusDLQ): 14,
|
|
})
|
|
if deliveryHealthFingerprint(base) != deliveryHealthFingerprint(drifted) {
|
|
t.Error("expected count drift within one failure class to keep the fingerprint stable")
|
|
}
|
|
|
|
withFailed := notifications.ClassifyQueueHealth(map[string]int{
|
|
string(notifications.QueueStatusDLQ): 14,
|
|
string(notifications.QueueStatusFailed): 1,
|
|
})
|
|
if deliveryHealthFingerprint(base) == deliveryHealthFingerprint(withFailed) {
|
|
t.Error("expected a new failure class to change the fingerprint")
|
|
}
|
|
|
|
unavailable := notifications.UnavailableDeliveryHealth()
|
|
if deliveryHealthFingerprint(base) == deliveryHealthFingerprint(unavailable) {
|
|
t.Error("expected an unavailable queue to change the fingerprint")
|
|
}
|
|
}
|
|
|
|
func TestEvaluateNotificationDeliveryThrottles(t *testing.T) {
|
|
// The poll ticker runs on the polling cadence, which can be seconds, and
|
|
// reading queue health costs a SQLite query. A nil notification manager
|
|
// means the evaluation returns early, but the throttle stamp must still be
|
|
// taken so the interval is honoured.
|
|
m := &Monitor{}
|
|
|
|
start := time.Now()
|
|
m.evaluateNotificationDelivery(start)
|
|
firstStamp := m.lastDeliveryHealthCheck
|
|
if firstStamp.IsZero() {
|
|
t.Fatal("expected the first evaluation to record a check time")
|
|
}
|
|
|
|
m.evaluateNotificationDelivery(start.Add(notificationDeliveryCheckInterval / 2))
|
|
if !m.lastDeliveryHealthCheck.Equal(firstStamp) {
|
|
t.Error("expected an evaluation inside the interval to be skipped")
|
|
}
|
|
|
|
due := start.Add(notificationDeliveryCheckInterval + time.Second)
|
|
m.evaluateNotificationDelivery(due)
|
|
if !m.lastDeliveryHealthCheck.Equal(due) {
|
|
t.Error("expected an evaluation past the interval to run")
|
|
}
|
|
}
|
|
|
|
func TestEvaluateNotificationDeliveryIsSafeWithoutAMonitor(t *testing.T) {
|
|
var m *Monitor
|
|
m.evaluateNotificationDelivery(time.Now())
|
|
}
|