mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 18:45:53 +00:00
2160 lines
65 KiB
Go
2160 lines
65 KiB
Go
package notifications
|
|
|
|
import (
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/utils"
|
|
)
|
|
|
|
func TestCalculateBackoff(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
attempt int
|
|
expected time.Duration
|
|
}{
|
|
{
|
|
name: "negative attempt defaults to first backoff",
|
|
attempt: -1,
|
|
expected: 1 * time.Second,
|
|
},
|
|
{
|
|
name: "attempt 0 (first retry)",
|
|
attempt: 0,
|
|
expected: 1 * time.Second,
|
|
},
|
|
{
|
|
name: "attempt 1",
|
|
attempt: 1,
|
|
expected: 2 * time.Second,
|
|
},
|
|
{
|
|
name: "attempt 2",
|
|
attempt: 2,
|
|
expected: 4 * time.Second,
|
|
},
|
|
{
|
|
name: "attempt 3",
|
|
attempt: 3,
|
|
expected: 8 * time.Second,
|
|
},
|
|
{
|
|
name: "attempt 4",
|
|
attempt: 4,
|
|
expected: 16 * time.Second,
|
|
},
|
|
{
|
|
name: "attempt 5",
|
|
attempt: 5,
|
|
expected: 32 * time.Second,
|
|
},
|
|
{
|
|
name: "attempt 6 (capped at 60s)",
|
|
attempt: 6,
|
|
expected: 60 * time.Second,
|
|
},
|
|
{
|
|
name: "attempt 7 (stays at cap)",
|
|
attempt: 7,
|
|
expected: 60 * time.Second,
|
|
},
|
|
{
|
|
name: "attempt 10 (stays at cap)",
|
|
attempt: 10,
|
|
expected: 60 * time.Second,
|
|
},
|
|
{
|
|
name: "very large attempts stay capped",
|
|
attempt: 60,
|
|
expected: 60 * time.Second,
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
result := calculateBackoff(tc.attempt)
|
|
if result != tc.expected {
|
|
t.Errorf("calculateBackoff(%d) = %v, want %v", tc.attempt, result, tc.expected)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNewNotificationQueue_WhitespaceDataDirUsesDefault(t *testing.T) {
|
|
dataDir := t.TempDir()
|
|
t.Setenv("PULSE_DATA_DIR", dataDir)
|
|
|
|
nq, err := NewNotificationQueue(" \t ")
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue with whitespace data dir: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
expectedDBPath := filepath.Join(utils.GetDataDir(), notificationQueueDirName, notificationQueueFileName)
|
|
if nq.dbPath != expectedDBPath {
|
|
t.Fatalf("expected db path %q, got %q", expectedDBPath, nq.dbPath)
|
|
}
|
|
}
|
|
|
|
func TestResolveNotificationQueuePathCanonicalizesExplicitDataDir(t *testing.T) {
|
|
rawDataDir := filepath.Join(t.TempDir(), "queue", "..", "queue", ".")
|
|
|
|
resolvedDir, dbPath, err := resolveNotificationQueuePath(rawDataDir)
|
|
if err != nil {
|
|
t.Fatalf("resolveNotificationQueuePath() error = %v", err)
|
|
}
|
|
|
|
expectedDir := filepath.Clean(rawDataDir)
|
|
if resolvedDir != expectedDir {
|
|
t.Fatalf("resolved dir = %q, want %q", resolvedDir, expectedDir)
|
|
}
|
|
|
|
expectedDBPath := filepath.Join(expectedDir, notificationQueueFileName)
|
|
if dbPath != expectedDBPath {
|
|
t.Fatalf("db path = %q, want %q", dbPath, expectedDBPath)
|
|
}
|
|
}
|
|
|
|
func TestResolveNotificationQueuePathWhitespaceUsesDefaultNotificationsDir(t *testing.T) {
|
|
dataDir := t.TempDir()
|
|
t.Setenv("PULSE_DATA_DIR", dataDir)
|
|
|
|
resolvedDir, dbPath, err := resolveNotificationQueuePath(" \t ")
|
|
if err != nil {
|
|
t.Fatalf("resolveNotificationQueuePath() error = %v", err)
|
|
}
|
|
|
|
expectedDir := filepath.Join(utils.GetDataDir(), notificationQueueDirName)
|
|
if resolvedDir != expectedDir {
|
|
t.Fatalf("resolved dir = %q, want %q", resolvedDir, expectedDir)
|
|
}
|
|
|
|
expectedDBPath := filepath.Join(expectedDir, notificationQueueFileName)
|
|
if dbPath != expectedDBPath {
|
|
t.Fatalf("db path = %q, want %q", dbPath, expectedDBPath)
|
|
}
|
|
}
|
|
|
|
func TestEnqueue_ValidatesAndNormalizesInput(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
t.Run("rejects nil notification", func(t *testing.T) {
|
|
err := nq.Enqueue(nil)
|
|
if err == nil {
|
|
t.Fatalf("expected error for nil notification")
|
|
}
|
|
})
|
|
|
|
t.Run("rejects empty type", func(t *testing.T) {
|
|
err := nq.Enqueue(&QueuedNotification{
|
|
Config: []byte(`{}`),
|
|
})
|
|
if err == nil {
|
|
t.Fatalf("expected error for empty notification type")
|
|
}
|
|
})
|
|
|
|
t.Run("rejects empty config", func(t *testing.T) {
|
|
err := nq.Enqueue(&QueuedNotification{
|
|
Type: "email",
|
|
})
|
|
if err == nil {
|
|
t.Fatalf("expected error for empty notification config")
|
|
}
|
|
})
|
|
|
|
t.Run("normalizes attempts and type", func(t *testing.T) {
|
|
futureRetry := time.Now().Add(1 * time.Hour)
|
|
notif := &QueuedNotification{
|
|
ID: "normalize-test",
|
|
Type: " email ",
|
|
Status: QueueStatusPending,
|
|
Attempts: -10,
|
|
MaxAttempts: -2,
|
|
Config: []byte(`{}`),
|
|
NextRetryAt: &futureRetry, // keep background worker from picking it up
|
|
Alerts: []*alerts.Alert{{ID: "a-1"}},
|
|
}
|
|
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("enqueue failed: %v", err)
|
|
}
|
|
|
|
var dbType string
|
|
var attempts int
|
|
var maxAttempts int
|
|
err := nq.db.QueryRow(`SELECT type, attempts, max_attempts FROM notification_queue WHERE id = ?`, notif.ID).Scan(&dbType, &attempts, &maxAttempts)
|
|
if err != nil {
|
|
t.Fatalf("failed to query normalized notification: %v", err)
|
|
}
|
|
|
|
if dbType != "email" {
|
|
t.Fatalf("expected trimmed type 'email', got %q", dbType)
|
|
}
|
|
if attempts != 0 {
|
|
t.Fatalf("expected attempts to normalize to 0, got %d", attempts)
|
|
}
|
|
if maxAttempts != defaultQueueMaxAttempts {
|
|
t.Fatalf("expected max attempts to normalize to %d, got %d", defaultQueueMaxAttempts, maxAttempts)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestCalculateBackoff_ExponentialGrowth(t *testing.T) {
|
|
// Verify backoff grows exponentially until cap
|
|
prev := calculateBackoff(0)
|
|
for attempt := 1; attempt <= 5; attempt++ {
|
|
curr := calculateBackoff(attempt)
|
|
if curr != prev*2 {
|
|
t.Errorf("calculateBackoff(%d) = %v, expected %v (2x previous)", attempt, curr, prev*2)
|
|
}
|
|
prev = curr
|
|
}
|
|
}
|
|
|
|
func TestCalculateBackoff_NeverExceedsCap(t *testing.T) {
|
|
cap := 60 * time.Second
|
|
// Test a range of practical attempt values (0-20 is realistic range)
|
|
for attempt := 0; attempt <= 20; attempt++ {
|
|
result := calculateBackoff(attempt)
|
|
if result > cap {
|
|
t.Errorf("calculateBackoff(%d) = %v, exceeds cap of %v", attempt, result, cap)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCalculateBackoff_LargeAttemptStillCapped(t *testing.T) {
|
|
for _, attempt := range []int{60, 1000} {
|
|
if got := calculateBackoff(attempt); got != 60*time.Second {
|
|
t.Errorf("calculateBackoff(%d) = %v, want %v", attempt, got, 60*time.Second)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNotificationQueueStatus_Values(t *testing.T) {
|
|
// Verify status constants have expected string values
|
|
tests := []struct {
|
|
status NotificationQueueStatus
|
|
expected string
|
|
}{
|
|
{QueueStatusPending, "pending"},
|
|
{QueueStatusSending, "sending"},
|
|
{QueueStatusSent, "sent"},
|
|
{QueueStatusFailed, "failed"},
|
|
{QueueStatusDLQ, "dlq"},
|
|
{QueueStatusCancelled, "cancelled"},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(string(tc.status), func(t *testing.T) {
|
|
if string(tc.status) != tc.expected {
|
|
t.Errorf("status = %q, want %q", tc.status, tc.expected)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestQueuedNotification_Fields(t *testing.T) {
|
|
now := time.Now()
|
|
lastAttempt := now.Add(-1 * time.Minute)
|
|
nextRetry := now.Add(5 * time.Minute)
|
|
errorMsg := "connection refused"
|
|
|
|
notif := QueuedNotification{
|
|
ID: "test-123",
|
|
Type: "email",
|
|
Method: "smtp",
|
|
Status: QueueStatusPending,
|
|
Alerts: nil,
|
|
Config: []byte(`{"host":"smtp.example.com"}`),
|
|
Attempts: 2,
|
|
MaxAttempts: 5,
|
|
LastAttempt: &lastAttempt,
|
|
LastError: &errorMsg,
|
|
CreatedAt: now,
|
|
NextRetryAt: &nextRetry,
|
|
}
|
|
|
|
if notif.ID != "test-123" {
|
|
t.Errorf("ID = %q, want 'test-123'", notif.ID)
|
|
}
|
|
if notif.Type != "email" {
|
|
t.Errorf("Type = %q, want 'email'", notif.Type)
|
|
}
|
|
if notif.Method != "smtp" {
|
|
t.Errorf("Method = %q, want 'smtp'", notif.Method)
|
|
}
|
|
if notif.Status != QueueStatusPending {
|
|
t.Errorf("Status = %q, want 'pending'", notif.Status)
|
|
}
|
|
if notif.Attempts != 2 {
|
|
t.Errorf("Attempts = %d, want 2", notif.Attempts)
|
|
}
|
|
if notif.MaxAttempts != 5 {
|
|
t.Errorf("MaxAttempts = %d, want 5", notif.MaxAttempts)
|
|
}
|
|
if notif.LastAttempt == nil {
|
|
t.Error("LastAttempt should not be nil")
|
|
}
|
|
if notif.LastError == nil || *notif.LastError != "connection refused" {
|
|
t.Errorf("LastError = %v, want 'connection refused'", notif.LastError)
|
|
}
|
|
if notif.NextRetryAt == nil {
|
|
t.Error("NextRetryAt should not be nil")
|
|
}
|
|
}
|
|
|
|
func TestQueuedNotification_ZeroValues(t *testing.T) {
|
|
notif := QueuedNotification{}
|
|
|
|
if notif.ID != "" {
|
|
t.Error("ID should be empty by default")
|
|
}
|
|
if notif.Type != "" {
|
|
t.Error("Type should be empty by default")
|
|
}
|
|
if notif.Status != "" {
|
|
t.Error("Status should be empty by default")
|
|
}
|
|
if notif.Attempts != 0 {
|
|
t.Error("Attempts should be 0 by default")
|
|
}
|
|
if notif.MaxAttempts != 0 {
|
|
t.Error("MaxAttempts should be 0 by default")
|
|
}
|
|
if notif.LastAttempt != nil {
|
|
t.Error("LastAttempt should be nil by default")
|
|
}
|
|
if notif.LastError != nil {
|
|
t.Error("LastError should be nil by default")
|
|
}
|
|
if !notif.CreatedAt.IsZero() {
|
|
t.Error("CreatedAt should be zero by default")
|
|
}
|
|
if notif.NextRetryAt != nil {
|
|
t.Error("NextRetryAt should be nil by default")
|
|
}
|
|
if notif.CompletedAt != nil {
|
|
t.Error("CompletedAt should be nil by default")
|
|
}
|
|
}
|
|
|
|
func TestCancelByAlertIdentifiers_EmptyInput(t *testing.T) {
|
|
// Create a temporary queue for testing
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
|
|
// Empty slice should return nil without error
|
|
count, err := nq.CancelByAlertIdentifiers([]string{})
|
|
if err != nil {
|
|
t.Errorf("CancelByAlertIdentifiers with empty slice returned error: %v", err)
|
|
}
|
|
if count != 0 {
|
|
t.Errorf("CancelByAlertIdentifiers with empty slice returned count %d, want 0", count)
|
|
}
|
|
|
|
// Nil slice should also return nil without error
|
|
count, err = nq.CancelByAlertIdentifiers(nil)
|
|
if err != nil {
|
|
t.Errorf("CancelByAlertIdentifiers with nil slice returned error: %v", err)
|
|
}
|
|
if count != 0 {
|
|
t.Errorf("CancelByAlertIdentifiers with nil slice returned count %d, want 0", count)
|
|
}
|
|
}
|
|
|
|
func TestCancelByAlertIdentifiers_NoMatchingNotifications(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
// Enqueue a notification with alert-1 (far future NextRetryAt so background processor doesn't pick it up)
|
|
futureRetry := time.Now().Add(1 * time.Hour)
|
|
notif := &QueuedNotification{
|
|
ID: "notif-1",
|
|
Type: "email",
|
|
Status: QueueStatusPending,
|
|
MaxAttempts: 3,
|
|
Config: []byte(`{}`),
|
|
NextRetryAt: &futureRetry,
|
|
Alerts: []*alerts.Alert{{ID: "alert-1"}},
|
|
}
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Failed to enqueue: %v", err)
|
|
}
|
|
|
|
// Cancel with non-matching alert ID
|
|
count, err := nq.CancelByAlertIdentifiers([]string{"alert-2"})
|
|
if err != nil {
|
|
t.Errorf("CancelByAlertIdentifiers returned error: %v", err)
|
|
}
|
|
if count != 0 {
|
|
t.Errorf("CancelByAlertIdentifiers returned count %d, want 0", count)
|
|
}
|
|
|
|
// Verify the notification is still pending using GetQueueStats
|
|
stats, err := nq.GetQueueStats()
|
|
if err != nil {
|
|
t.Fatalf("GetQueueStats failed: %v", err)
|
|
}
|
|
if stats["pending"] != 1 {
|
|
t.Errorf("Expected 1 pending notification, got %d (stats: %v)", stats["pending"], stats)
|
|
}
|
|
if stats["cancelled"] != 0 {
|
|
t.Errorf("Expected 0 cancelled notifications, got %d", stats["cancelled"])
|
|
}
|
|
}
|
|
|
|
func TestNewNotificationQueueCreatesSecureDirectory(t *testing.T) {
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("directory permission bits are not reliably enforced on windows")
|
|
}
|
|
|
|
baseDir := t.TempDir()
|
|
queueDir := baseDir + "/notification-queue"
|
|
|
|
nq, err := NewNotificationQueue(queueDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
info, err := os.Stat(queueDir)
|
|
if err != nil {
|
|
t.Fatalf("failed to stat queue directory: %v", err)
|
|
}
|
|
|
|
if perms := info.Mode().Perm(); perms != 0700 {
|
|
t.Fatalf("expected queue directory permissions 0700, got %#o", perms)
|
|
}
|
|
}
|
|
|
|
func TestNewNotificationQueue_MigratesAuditAlertIdentifiersColumn(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
dbPath := filepath.Join(tempDir, "notification_queue.db")
|
|
|
|
db, err := sql.Open("sqlite", dbPath)
|
|
if err != nil {
|
|
t.Fatalf("open legacy db: %v", err)
|
|
}
|
|
|
|
legacySchema := `
|
|
CREATE TABLE notification_queue (
|
|
id TEXT PRIMARY KEY,
|
|
type TEXT NOT NULL,
|
|
method TEXT,
|
|
status TEXT NOT NULL,
|
|
alerts TEXT NOT NULL,
|
|
config TEXT NOT NULL,
|
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
max_attempts INTEGER NOT NULL DEFAULT 3,
|
|
last_attempt INTEGER,
|
|
last_error TEXT,
|
|
created_at INTEGER NOT NULL,
|
|
next_retry_at INTEGER,
|
|
completed_at INTEGER,
|
|
payload_bytes INTEGER
|
|
);
|
|
|
|
CREATE TABLE notification_audit (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
notification_id TEXT NOT NULL,
|
|
type TEXT NOT NULL,
|
|
method TEXT,
|
|
status TEXT NOT NULL,
|
|
alert_ids TEXT,
|
|
alert_count INTEGER,
|
|
attempts INTEGER,
|
|
success BOOLEAN,
|
|
error_message TEXT,
|
|
payload_size INTEGER,
|
|
timestamp INTEGER NOT NULL
|
|
);
|
|
`
|
|
if _, err := db.Exec(legacySchema); err != nil {
|
|
_ = db.Close()
|
|
t.Fatalf("create legacy schema: %v", err)
|
|
}
|
|
if err := db.Close(); err != nil {
|
|
t.Fatalf("close legacy db: %v", err)
|
|
}
|
|
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("NewNotificationQueue failed: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
columns, err := nq.tableColumns("notification_audit")
|
|
if err != nil {
|
|
t.Fatalf("tableColumns(notification_audit) failed: %v", err)
|
|
}
|
|
if !columns[notificationAuditAlertIdentifiersColumn] {
|
|
t.Fatalf("expected migrated alert_identifiers column, got %#v", columns)
|
|
}
|
|
if columns[legacyNotificationAuditAlertIdentifiersColumn] {
|
|
t.Fatalf(
|
|
"did not expect legacy %s column after migration, got %#v",
|
|
legacyNotificationAuditAlertIdentifiersColumn,
|
|
columns,
|
|
)
|
|
}
|
|
queueColumns, err := nq.tableColumns("notification_queue")
|
|
if err != nil {
|
|
t.Fatalf("tableColumns(notification_queue) failed: %v", err)
|
|
}
|
|
if !queueColumns[notificationOperationalLinksColumn] ||
|
|
!columns[notificationOperationalLinksColumn] ||
|
|
!columns[notificationFailureClassColumn] {
|
|
t.Fatalf(
|
|
"expected migrated operational link and failure class columns, queue=%#v audit=%#v",
|
|
queueColumns,
|
|
columns,
|
|
)
|
|
}
|
|
}
|
|
|
|
func TestCancelByAlertIdentifiers_MatchingNotificationCancelled(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
// Enqueue a notification with alert-1 (far future NextRetryAt so background processor doesn't pick it up)
|
|
futureRetry := time.Now().Add(1 * time.Hour)
|
|
notif := &QueuedNotification{
|
|
ID: "notif-1",
|
|
Type: "email",
|
|
Status: QueueStatusPending,
|
|
MaxAttempts: 3,
|
|
Config: []byte(`{}`),
|
|
NextRetryAt: &futureRetry,
|
|
Alerts: []*alerts.Alert{{ID: "alert-1"}},
|
|
}
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Failed to enqueue: %v", err)
|
|
}
|
|
|
|
// Cancel with matching alert ID
|
|
count, err := nq.CancelByAlertIdentifiers([]string{"alert-1"})
|
|
if err != nil {
|
|
t.Errorf("CancelByAlertIdentifiers returned error: %v", err)
|
|
}
|
|
if count != 1 {
|
|
t.Errorf("CancelByAlertIdentifiers returned count %d, want 1", count)
|
|
}
|
|
|
|
// Verify the notification is now cancelled using GetQueueStats
|
|
stats, err := nq.GetQueueStats()
|
|
if err != nil {
|
|
t.Fatalf("GetQueueStats failed: %v", err)
|
|
}
|
|
if stats["pending"] != 0 {
|
|
t.Errorf("Expected 0 pending notifications, got %d", stats["pending"])
|
|
}
|
|
if stats["cancelled"] != 1 {
|
|
t.Errorf("Expected 1 cancelled notification, got %d (stats: %v)", stats["cancelled"], stats)
|
|
}
|
|
|
|
var completedAt sql.NullInt64
|
|
err = nq.db.QueryRow(`SELECT completed_at FROM notification_queue WHERE id = ?`, "notif-1").Scan(&completedAt)
|
|
if err != nil {
|
|
t.Fatalf("Failed to query completed_at: %v", err)
|
|
}
|
|
if !completedAt.Valid || completedAt.Int64 <= 0 {
|
|
t.Errorf("Expected completed_at to be set for cancelled notification, got %+v", completedAt)
|
|
}
|
|
}
|
|
|
|
func TestCancelByAlertIdentifiers_PreservesResolvedNotifications(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
futureRetry := time.Now().Add(1 * time.Hour)
|
|
firing := &QueuedNotification{
|
|
ID: "notif-firing",
|
|
Type: "webhook",
|
|
Status: QueueStatusPending,
|
|
MaxAttempts: 3,
|
|
Config: []byte(`{}`),
|
|
NextRetryAt: &futureRetry,
|
|
Alerts: []*alerts.Alert{{ID: "alert-1"}},
|
|
}
|
|
resolved := &QueuedNotification{
|
|
ID: "notif-resolved",
|
|
Type: "webhook" + queueTypeSuffixResolved,
|
|
Status: QueueStatusPending,
|
|
MaxAttempts: 3,
|
|
Config: []byte(`{}`),
|
|
NextRetryAt: &futureRetry,
|
|
Alerts: []*alerts.Alert{{ID: "alert-1"}},
|
|
}
|
|
|
|
for _, notif := range []*QueuedNotification{firing, resolved} {
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Failed to enqueue %s: %v", notif.ID, err)
|
|
}
|
|
}
|
|
|
|
count, err := nq.CancelByAlertIdentifiers([]string{"alert-1"})
|
|
if err != nil {
|
|
t.Fatalf("CancelByAlertIdentifiers returned error: %v", err)
|
|
}
|
|
if count != 1 {
|
|
t.Fatalf("CancelByAlertIdentifiers returned count %d, want 1 (only the firing entry counts)", count)
|
|
}
|
|
|
|
var statusFiring NotificationQueueStatus
|
|
if err := nq.db.QueryRow(`SELECT status FROM notification_queue WHERE id = ?`, firing.ID).Scan(&statusFiring); err != nil {
|
|
t.Fatalf("failed to read firing notification status: %v", err)
|
|
}
|
|
if statusFiring != QueueStatusCancelled {
|
|
t.Fatalf("expected firing notification to be cancelled, got %s", statusFiring)
|
|
}
|
|
|
|
var statusResolved NotificationQueueStatus
|
|
if err := nq.db.QueryRow(`SELECT status FROM notification_queue WHERE id = ?`, resolved.ID).Scan(&statusResolved); err != nil {
|
|
t.Fatalf("failed to read resolved notification status: %v", err)
|
|
}
|
|
if statusResolved != QueueStatusPending {
|
|
t.Fatalf("expected resolved notification to remain pending, got %s", statusResolved)
|
|
}
|
|
}
|
|
|
|
func TestCancelByAlertIdentifiers_MultipleAlertsPartialMatch(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
// Enqueue a notification with multiple alerts (far future NextRetryAt so background processor doesn't pick it up)
|
|
futureRetry := time.Now().Add(1 * time.Hour)
|
|
notif := &QueuedNotification{
|
|
ID: "notif-multi",
|
|
Type: "webhook",
|
|
DestinationID: "webhook:primary",
|
|
Status: QueueStatusPending,
|
|
MaxAttempts: 3,
|
|
Config: []byte(`{"url":"https://hooks.example.test"}`),
|
|
NextRetryAt: &futureRetry,
|
|
Alerts: []*alerts.Alert{
|
|
{
|
|
ID: "alert-1",
|
|
OperationalRecord: &operationaltrust.OperationalRecord{ID: "record-1"},
|
|
LatestTransition: &operationaltrust.LifecycleTransition{
|
|
ID: "transition-1", To: operationaltrust.OperationalOpen, CauseKey: "cause-1",
|
|
},
|
|
},
|
|
{
|
|
ID: "alert-2",
|
|
OperationalRecord: &operationaltrust.OperationalRecord{ID: "record-2"},
|
|
LatestTransition: &operationaltrust.LifecycleTransition{
|
|
ID: "transition-2", To: operationaltrust.OperationalOpen, CauseKey: "cause-2",
|
|
},
|
|
},
|
|
{
|
|
ID: "alert-3",
|
|
OperationalRecord: &operationaltrust.OperationalRecord{ID: "record-3"},
|
|
LatestTransition: &operationaltrust.LifecycleTransition{
|
|
ID: "transition-3", To: operationaltrust.OperationalOpen, CauseKey: "cause-3",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Failed to enqueue: %v", err)
|
|
}
|
|
|
|
count, err := nq.CancelByAlertIdentifiers([]string{"alert-1", "alert-3"})
|
|
if err != nil {
|
|
t.Errorf("CancelByAlertIdentifiers returned error: %v", err)
|
|
}
|
|
if count != 2 {
|
|
t.Errorf("CancelByAlertIdentifiers returned count %d, want 2", count)
|
|
}
|
|
|
|
stats, err := nq.GetQueueStats()
|
|
if err != nil {
|
|
t.Fatalf("GetQueueStats failed: %v", err)
|
|
}
|
|
if stats["pending"] != 1 {
|
|
t.Errorf("Expected 1 pending notification after partial match rewrite, got %d", stats["pending"])
|
|
}
|
|
if stats["cancelled"] != 0 {
|
|
t.Errorf("Expected 0 cancelled notifications after partial match rewrite, got %d (stats: %v)", stats["cancelled"], stats)
|
|
}
|
|
|
|
var notifType string
|
|
var alertsJSON string
|
|
var linksJSON string
|
|
var configJSON string
|
|
var nextRetryAt sql.NullInt64
|
|
if err := nq.db.QueryRow(
|
|
`SELECT type, alerts, operational_links, config, next_retry_at FROM notification_queue WHERE id = ?`,
|
|
notif.ID,
|
|
).Scan(¬ifType, &alertsJSON, &linksJSON, &configJSON, &nextRetryAt); err != nil {
|
|
t.Fatalf("Failed to query rewritten notification row: %v", err)
|
|
}
|
|
if notifType != notif.Type {
|
|
t.Fatalf("type = %q, want %q", notifType, notif.Type)
|
|
}
|
|
if configJSON != string(notif.Config) {
|
|
t.Fatalf("config = %q, want %q", configJSON, string(notif.Config))
|
|
}
|
|
if !nextRetryAt.Valid || nextRetryAt.Int64 != futureRetry.Unix() {
|
|
t.Fatalf("next_retry_at = %+v, want %d", nextRetryAt, futureRetry.Unix())
|
|
}
|
|
|
|
var remaining []*alerts.Alert
|
|
if err := json.Unmarshal([]byte(alertsJSON), &remaining); err != nil {
|
|
t.Fatalf("failed to unmarshal rewritten alerts: %v", err)
|
|
}
|
|
if len(remaining) != 1 || remaining[0].ID != "alert-2" {
|
|
t.Fatalf("remaining alerts = %#v, want only alert-2", remaining)
|
|
}
|
|
var remainingLinks []operationaltrust.NotificationLink
|
|
if err := json.Unmarshal([]byte(linksJSON), &remainingLinks); err != nil {
|
|
t.Fatalf("failed to unmarshal rewritten operational links: %v", err)
|
|
}
|
|
if len(remainingLinks) != 1 ||
|
|
remainingLinks[0].OperationalRecordID != "record-2" ||
|
|
remainingLinks[0].TransitionID != "transition-2" {
|
|
t.Fatalf("remaining operational links = %#v, want only alert-2 link", remainingLinks)
|
|
}
|
|
}
|
|
|
|
func TestProcessNotificationReloadsAlertsAfterCancellationRewrite(t *testing.T) {
|
|
nq, err := NewNotificationQueue(t.TempDir())
|
|
if err != nil {
|
|
t.Fatalf("NewNotificationQueue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
futureRetry := time.Now().Add(time.Hour)
|
|
queued := &QueuedNotification{
|
|
ID: "notif-rewritten-before-claim", Type: "webhook", Status: QueueStatusPending,
|
|
MaxAttempts: 3, Config: []byte(`{}`), NextRetryAt: &futureRetry,
|
|
Alerts: []*alerts.Alert{{ID: "alert-1"}, {ID: "alert-2"}},
|
|
}
|
|
if err := nq.Enqueue(queued); err != nil {
|
|
t.Fatalf("Enqueue: %v", err)
|
|
}
|
|
|
|
// Model a worker snapshot returned by GetPending before resolution rewrites
|
|
// the durable grouped row.
|
|
workerSnapshot := *queued
|
|
workerSnapshot.Alerts = append([]*alerts.Alert(nil), queued.Alerts...)
|
|
if count, err := nq.CancelByAlertIdentifiers([]string{"alert-1"}); err != nil {
|
|
t.Fatalf("CancelByAlertIdentifiers: %v", err)
|
|
} else if count != 1 {
|
|
t.Fatalf("cancelled alert count = %d, want 1", count)
|
|
}
|
|
|
|
var delivered []string
|
|
nq.SetProcessor(func(notif *QueuedNotification) error {
|
|
for _, alert := range notif.Alerts {
|
|
if alert != nil {
|
|
delivered = append(delivered, alert.ID)
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
nq.processNotification(&workerSnapshot)
|
|
|
|
if len(delivered) != 1 || delivered[0] != "alert-2" {
|
|
t.Fatalf("delivered alerts = %v, want only alert-2", delivered)
|
|
}
|
|
}
|
|
|
|
func TestCancelByAlertIdentifiersWaitsForInFlightDelivery(t *testing.T) {
|
|
nq, err := NewNotificationQueue(t.TempDir())
|
|
if err != nil {
|
|
t.Fatalf("NewNotificationQueue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
futureRetry := time.Now().Add(time.Hour)
|
|
queued := &QueuedNotification{
|
|
ID: "notif-in-flight", Type: "webhook", Status: QueueStatusPending,
|
|
MaxAttempts: 3, Config: []byte(`{}`), NextRetryAt: &futureRetry,
|
|
Alerts: []*alerts.Alert{{ID: "alert-1"}},
|
|
}
|
|
if err := nq.Enqueue(queued); err != nil {
|
|
t.Fatalf("Enqueue: %v", err)
|
|
}
|
|
|
|
started := make(chan struct{})
|
|
release := make(chan struct{})
|
|
nq.SetProcessor(func(*QueuedNotification) error {
|
|
close(started)
|
|
<-release
|
|
return nil
|
|
})
|
|
processed := make(chan struct{})
|
|
go func() {
|
|
nq.processNotification(queued)
|
|
close(processed)
|
|
}()
|
|
|
|
select {
|
|
case <-started:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("delivery did not start")
|
|
}
|
|
|
|
cancelled := make(chan int, 1)
|
|
cancelErr := make(chan error, 1)
|
|
go func() {
|
|
count, err := nq.CancelByAlertIdentifiers([]string{"alert-1"})
|
|
cancelled <- count
|
|
cancelErr <- err
|
|
}()
|
|
select {
|
|
case count := <-cancelled:
|
|
close(release)
|
|
t.Fatalf("cancellation returned before in-flight delivery completed: %d", count)
|
|
case <-time.After(50 * time.Millisecond):
|
|
}
|
|
|
|
close(release)
|
|
select {
|
|
case <-processed:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("delivery did not finish")
|
|
}
|
|
select {
|
|
case err := <-cancelErr:
|
|
if err != nil {
|
|
t.Fatalf("CancelByAlertIdentifiers: %v", err)
|
|
}
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("cancellation did not finish")
|
|
}
|
|
if count := <-cancelled; count != 0 {
|
|
t.Fatalf("cancelled alert count = %d, want 0 after delivery", count)
|
|
}
|
|
|
|
var status string
|
|
if err := nq.db.QueryRow(`SELECT status FROM notification_queue WHERE id = ?`, queued.ID).Scan(&status); err != nil {
|
|
t.Fatalf("read notification status: %v", err)
|
|
}
|
|
if status != string(QueueStatusSent) {
|
|
t.Fatalf("status = %q, want %q", status, QueueStatusSent)
|
|
}
|
|
}
|
|
|
|
func TestCancelByAlertIdentifiers_SendingRowsCancelledButNotCounted(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
futureRetry := time.Now().Add(1 * time.Hour)
|
|
notif := &QueuedNotification{
|
|
ID: "notif-sending",
|
|
Type: "webhook",
|
|
Status: QueueStatusPending,
|
|
MaxAttempts: 3,
|
|
Config: []byte(`{}`),
|
|
NextRetryAt: &futureRetry,
|
|
Alerts: []*alerts.Alert{{ID: "alert-1"}},
|
|
}
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Failed to enqueue: %v", err)
|
|
}
|
|
if _, err := nq.db.Exec(`UPDATE notification_queue SET status = 'sending' WHERE id = ?`, notif.ID); err != nil {
|
|
t.Fatalf("Failed to mark notification as sending: %v", err)
|
|
}
|
|
|
|
// A mid-send row may still complete delivery, so it must not count as an
|
|
// undelivered firing entry even though the cancellation is still applied.
|
|
count, err := nq.CancelByAlertIdentifiers([]string{"alert-1"})
|
|
if err != nil {
|
|
t.Fatalf("CancelByAlertIdentifiers returned error: %v", err)
|
|
}
|
|
if count != 0 {
|
|
t.Fatalf("CancelByAlertIdentifiers returned count %d, want 0 for a sending row", count)
|
|
}
|
|
|
|
var status string
|
|
if err := nq.db.QueryRow(`SELECT status FROM notification_queue WHERE id = ?`, notif.ID).Scan(&status); err != nil {
|
|
t.Fatalf("Failed to query notification status: %v", err)
|
|
}
|
|
if status != string(QueueStatusCancelled) {
|
|
t.Fatalf("status = %q, want %q", status, QueueStatusCancelled)
|
|
}
|
|
}
|
|
|
|
func TestCancelByAlertIdentifiers_SetsCompletedAtAndClearsNextRetry(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
futureRetry := time.Now().Add(1 * time.Hour)
|
|
notif := &QueuedNotification{
|
|
ID: "notif-cancelled-metadata",
|
|
Type: "email",
|
|
Status: QueueStatusPending,
|
|
MaxAttempts: 3,
|
|
Config: []byte(`{}`),
|
|
NextRetryAt: &futureRetry,
|
|
Alerts: []*alerts.Alert{{ID: "alert-1"}},
|
|
}
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Failed to enqueue: %v", err)
|
|
}
|
|
|
|
if _, err := nq.CancelByAlertIdentifiers([]string{"alert-1"}); err != nil {
|
|
t.Fatalf("CancelByAlertIdentifiers returned error: %v", err)
|
|
}
|
|
|
|
var status string
|
|
var completedAt sql.NullInt64
|
|
var nextRetryAt sql.NullInt64
|
|
if err := nq.db.QueryRow(
|
|
`SELECT status, completed_at, next_retry_at FROM notification_queue WHERE id = ?`,
|
|
notif.ID,
|
|
).Scan(&status, &completedAt, &nextRetryAt); err != nil {
|
|
t.Fatalf("Failed to query cancelled notification row: %v", err)
|
|
}
|
|
|
|
if status != string(QueueStatusCancelled) {
|
|
t.Fatalf("status = %q, want %q", status, QueueStatusCancelled)
|
|
}
|
|
if !completedAt.Valid || completedAt.Int64 <= 0 {
|
|
t.Fatalf("completed_at should be set for cancelled notifications, got %v", completedAt)
|
|
}
|
|
if nextRetryAt.Valid {
|
|
t.Fatalf("next_retry_at should be NULL after cancellation, got %v", nextRetryAt)
|
|
}
|
|
}
|
|
|
|
func TestCancelByTypes_CancelsOnlyMatchingNotificationTypes(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
futureRetry := time.Now().Add(1 * time.Hour)
|
|
for _, notif := range []*QueuedNotification{
|
|
{
|
|
ID: "notif-email",
|
|
Type: "email",
|
|
Status: QueueStatusPending,
|
|
MaxAttempts: 3,
|
|
Config: []byte(`{}`),
|
|
NextRetryAt: &futureRetry,
|
|
Alerts: []*alerts.Alert{{ID: "alert-email"}},
|
|
},
|
|
{
|
|
ID: "notif-email-resolved",
|
|
Type: "email_resolved",
|
|
Status: QueueStatusPending,
|
|
MaxAttempts: 3,
|
|
Config: []byte(`{}`),
|
|
NextRetryAt: &futureRetry,
|
|
Alerts: []*alerts.Alert{{ID: "alert-email-resolved"}},
|
|
},
|
|
{
|
|
ID: "notif-webhook",
|
|
Type: "webhook",
|
|
Status: QueueStatusPending,
|
|
MaxAttempts: 3,
|
|
Config: []byte(`{}`),
|
|
NextRetryAt: &futureRetry,
|
|
Alerts: []*alerts.Alert{{ID: "alert-webhook"}},
|
|
},
|
|
} {
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Failed to enqueue %s: %v", notif.ID, err)
|
|
}
|
|
}
|
|
|
|
if err := nq.CancelByTypes([]string{"email", "email_resolved"}, "Email notifications disabled"); err != nil {
|
|
t.Fatalf("CancelByTypes returned error: %v", err)
|
|
}
|
|
|
|
stats, err := nq.GetQueueStats()
|
|
if err != nil {
|
|
t.Fatalf("GetQueueStats failed: %v", err)
|
|
}
|
|
if stats["cancelled"] != 2 {
|
|
t.Fatalf("expected 2 cancelled notifications, got %d (stats: %v)", stats["cancelled"], stats)
|
|
}
|
|
if stats["pending"] != 1 {
|
|
t.Fatalf("expected 1 pending notification remaining, got %d (stats: %v)", stats["pending"], stats)
|
|
}
|
|
}
|
|
|
|
func TestProcessNotification_CancelledNotification(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
// Create a cancelled notification
|
|
notif := &QueuedNotification{
|
|
ID: "test-cancelled",
|
|
Type: "email",
|
|
Status: QueueStatusCancelled,
|
|
}
|
|
|
|
// processNotification should return early without processing
|
|
// No panic or error expected
|
|
nq.processNotification(notif)
|
|
|
|
// Verify the notification wasn't modified (no attempts incremented)
|
|
// Since it's cancelled, it should just return
|
|
}
|
|
|
|
func TestProcessNotification_NoProcessor(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
futureRetry := time.Now().Add(1 * time.Hour)
|
|
|
|
// Enqueue a notification first so IncrementAttemptAndSetStatus works
|
|
notif := &QueuedNotification{
|
|
ID: "test-no-processor",
|
|
Type: "email",
|
|
Status: QueueStatusPending,
|
|
MaxAttempts: 3,
|
|
Config: []byte(`{}`),
|
|
NextRetryAt: &futureRetry,
|
|
}
|
|
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Failed to enqueue: %v", err)
|
|
}
|
|
|
|
// Don't set a processor - processNotification should handle this
|
|
nq.processNotification(notif)
|
|
|
|
// The notification should be scheduled for retry or moved to DLQ
|
|
// since no processor means failure
|
|
}
|
|
|
|
func TestProcessNotification_ProcessorSuccess(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
futureRetry := time.Now().Add(1 * time.Hour)
|
|
|
|
// Enqueue a notification
|
|
notif := &QueuedNotification{
|
|
ID: "test-success",
|
|
Type: "email",
|
|
Status: QueueStatusPending,
|
|
MaxAttempts: 3,
|
|
Config: []byte(`{}`),
|
|
NextRetryAt: &futureRetry,
|
|
}
|
|
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Failed to enqueue: %v", err)
|
|
}
|
|
|
|
// Set a processor that succeeds
|
|
processorCalled := make(chan struct{}, 1)
|
|
nq.SetProcessor(func(n *QueuedNotification) error {
|
|
select {
|
|
case processorCalled <- struct{}{}:
|
|
default:
|
|
}
|
|
return nil
|
|
})
|
|
|
|
nq.processNotification(notif)
|
|
|
|
select {
|
|
case <-processorCalled:
|
|
default:
|
|
t.Error("Processor was not called")
|
|
}
|
|
}
|
|
|
|
func TestProcessNotification_ProcessorFailure(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
futureRetry := time.Now().Add(1 * time.Hour)
|
|
|
|
// Enqueue a notification with low max attempts
|
|
notif := &QueuedNotification{
|
|
ID: "test-failure",
|
|
Type: "email",
|
|
Status: QueueStatusPending,
|
|
MaxAttempts: 1, // Only 1 attempt, so failure goes to DLQ
|
|
Config: []byte(`{}`),
|
|
NextRetryAt: &futureRetry,
|
|
}
|
|
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Failed to enqueue: %v", err)
|
|
}
|
|
|
|
// Set a processor that fails
|
|
nq.SetProcessor(func(n *QueuedNotification) error {
|
|
return fmt.Errorf("simulated failure")
|
|
})
|
|
|
|
nq.processNotification(notif)
|
|
|
|
// Notification should be in DLQ since max attempts reached
|
|
}
|
|
|
|
func TestScanNotification_DLQWithTimestamps(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
futureRetry := time.Now().Add(1 * time.Hour)
|
|
|
|
// Enqueue a notification with max 1 attempt
|
|
notif := &QueuedNotification{
|
|
ID: "test-dlq-timestamps",
|
|
Type: "webhook",
|
|
Status: QueueStatusPending,
|
|
MaxAttempts: 1, // Will go to DLQ on first failure
|
|
Config: []byte(`{"url":"http://example.com"}`),
|
|
NextRetryAt: &futureRetry,
|
|
}
|
|
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Failed to enqueue: %v", err)
|
|
}
|
|
|
|
// Set a failing processor to trigger DLQ
|
|
nq.SetProcessor(func(n *QueuedNotification) error {
|
|
return fmt.Errorf("simulated failure")
|
|
})
|
|
|
|
nq.processNotification(notif)
|
|
|
|
// Get DLQ notifications - this exercises scanNotification with timestamps
|
|
dlq, err := nq.GetDLQ(10)
|
|
if err != nil {
|
|
t.Fatalf("GetDLQ failed: %v", err)
|
|
}
|
|
|
|
if len(dlq) != 1 {
|
|
t.Fatalf("Expected 1 DLQ notification, got %d", len(dlq))
|
|
}
|
|
|
|
// DLQ notification should have CompletedAt set (when it was moved to DLQ)
|
|
if dlq[0].CompletedAt == nil {
|
|
t.Error("Expected CompletedAt to be set for DLQ notification")
|
|
}
|
|
|
|
// Also verify LastAttempt is set
|
|
if dlq[0].LastAttempt == nil {
|
|
t.Error("Expected LastAttempt to be set for DLQ notification")
|
|
}
|
|
}
|
|
|
|
func TestIncrementAttempt(t *testing.T) {
|
|
t.Run("increments attempt counter", func(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
// Set next_retry_at far in the future so background processor doesn't pick it up
|
|
futureRetry := time.Now().Add(1 * time.Hour)
|
|
|
|
// Enqueue a notification
|
|
notif := &QueuedNotification{
|
|
ID: "test-increment",
|
|
Type: "email",
|
|
Status: QueueStatusPending,
|
|
MaxAttempts: 3,
|
|
Config: []byte(`{}`),
|
|
NextRetryAt: &futureRetry,
|
|
}
|
|
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Failed to enqueue: %v", err)
|
|
}
|
|
|
|
// Increment the attempt counter multiple times
|
|
for i := 0; i < 3; i++ {
|
|
if err := nq.IncrementAttempt("test-increment"); err != nil {
|
|
t.Fatalf("IncrementAttempt failed on iteration %d: %v", i, err)
|
|
}
|
|
}
|
|
|
|
// Verify via DLQ - first move to DLQ to query it
|
|
// (This exercises the function; actual count verification would require db access)
|
|
if err := nq.UpdateStatus("test-increment", QueueStatusDLQ, "test"); err != nil {
|
|
t.Fatalf("UpdateStatus to DLQ failed: %v", err)
|
|
}
|
|
|
|
dlq, err := nq.GetDLQ(10)
|
|
if err != nil {
|
|
t.Fatalf("GetDLQ failed: %v", err)
|
|
}
|
|
if len(dlq) != 1 {
|
|
t.Fatalf("Expected 1 DLQ notification, got %d", len(dlq))
|
|
}
|
|
if dlq[0].Attempts != 3 {
|
|
t.Errorf("After 3 increments, attempts = %d, want 3", dlq[0].Attempts)
|
|
}
|
|
})
|
|
|
|
t.Run("non-existent ID does not error", func(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
// Calling IncrementAttempt on non-existent ID should not error
|
|
// (the SQL UPDATE just affects 0 rows)
|
|
err = nq.IncrementAttempt("non-existent-id")
|
|
if err != nil {
|
|
t.Errorf("IncrementAttempt with non-existent ID returned error: %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestGetQueueStats(t *testing.T) {
|
|
t.Run("empty queue", func(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
stats, err := nq.GetQueueStats()
|
|
if err != nil {
|
|
t.Fatalf("GetQueueStats failed: %v", err)
|
|
}
|
|
|
|
// Empty queue should return empty map
|
|
if len(stats) != 0 {
|
|
t.Errorf("Expected empty stats map, got %v", stats)
|
|
}
|
|
})
|
|
|
|
t.Run("with notifications in various statuses", func(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
// Enqueue notifications with different statuses
|
|
notifications := []*QueuedNotification{
|
|
{ID: "pending-1", Type: "email", Status: QueueStatusPending, MaxAttempts: 3, Config: []byte(`{}`)},
|
|
{ID: "pending-2", Type: "email", Status: QueueStatusPending, MaxAttempts: 3, Config: []byte(`{}`)},
|
|
{ID: "sending-1", Type: "webhook", Status: QueueStatusSending, MaxAttempts: 3, Config: []byte(`{}`)},
|
|
}
|
|
|
|
for _, notif := range notifications {
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Failed to enqueue %s: %v", notif.ID, err)
|
|
}
|
|
}
|
|
|
|
// Mark one as sent (completed)
|
|
if err := nq.UpdateStatus("pending-1", QueueStatusSent, ""); err != nil {
|
|
t.Fatalf("Failed to update status: %v", err)
|
|
}
|
|
|
|
// Mark one as failed
|
|
if err := nq.UpdateStatus("sending-1", QueueStatusFailed, "connection refused"); err != nil {
|
|
t.Fatalf("Failed to update status with error: %v", err)
|
|
}
|
|
|
|
stats, err := nq.GetQueueStats()
|
|
if err != nil {
|
|
t.Fatalf("GetQueueStats failed: %v", err)
|
|
}
|
|
|
|
// Verify counts
|
|
if stats["pending"] != 1 {
|
|
t.Errorf("pending count = %d, want 1", stats["pending"])
|
|
}
|
|
if stats["sent"] != 1 {
|
|
t.Errorf("sent count = %d, want 1", stats["sent"])
|
|
}
|
|
if stats["failed"] != 1 {
|
|
t.Errorf("failed count = %d, want 1", stats["failed"])
|
|
}
|
|
})
|
|
|
|
t.Run("includes terminal rows for their full retention window", func(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
for _, notif := range []*QueuedNotification{
|
|
{ID: "failed-retained", Type: "email", Status: QueueStatusPending, MaxAttempts: 3, Config: []byte(`{}`)},
|
|
{ID: "dlq-retained", Type: "webhook", Status: QueueStatusPending, MaxAttempts: 3, Config: []byte(`{}`)},
|
|
} {
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Failed to enqueue %s: %v", notif.ID, err)
|
|
}
|
|
}
|
|
if err := nq.UpdateStatus("failed-retained", QueueStatusFailed, "terminal"); err != nil {
|
|
t.Fatalf("mark failed: %v", err)
|
|
}
|
|
if err := nq.UpdateStatus("dlq-retained", QueueStatusDLQ, "terminal"); err != nil {
|
|
t.Fatalf("mark dlq: %v", err)
|
|
}
|
|
retainedAt := time.Now().Add(-48 * time.Hour).Unix()
|
|
if _, err := nq.db.Exec(
|
|
`UPDATE notification_queue SET completed_at = ? WHERE id IN (?, ?)`,
|
|
retainedAt,
|
|
"failed-retained",
|
|
"dlq-retained",
|
|
); err != nil {
|
|
t.Fatalf("age retained terminal rows: %v", err)
|
|
}
|
|
|
|
stats, err := nq.GetQueueStats()
|
|
if err != nil {
|
|
t.Fatalf("GetQueueStats failed: %v", err)
|
|
}
|
|
if stats["failed"] != 1 || stats["dlq"] != 1 {
|
|
t.Fatalf("retained terminal stats = %#v, want failed=1 dlq=1", stats)
|
|
}
|
|
})
|
|
|
|
t.Run("operator recovery retries or dismisses every terminal row without deleting history", func(t *testing.T) {
|
|
nq, err := NewNotificationQueue(t.TempDir())
|
|
if err != nil {
|
|
t.Fatalf("NewNotificationQueue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
observedHealth := make([]DeliveryHealth, 0, 6)
|
|
nq.SetDeliveryHealthChangedCallback(func() {
|
|
stats, statsErr := nq.GetQueueStats()
|
|
if statsErr != nil {
|
|
t.Fatalf("read queue health from committed callback: %v", statsErr)
|
|
}
|
|
observedHealth = append(observedHealth, ClassifyQueueHealth(stats))
|
|
})
|
|
|
|
for _, notif := range []*QueuedNotification{
|
|
{ID: "failed-recovery", Type: "email", Status: QueueStatusPending, Attempts: 8, MaxAttempts: 8, Config: []byte(`{}`)},
|
|
{ID: "dlq-recovery", Type: "webhook", Status: QueueStatusPending, Attempts: 8, MaxAttempts: 8, Config: []byte(`{}`)},
|
|
{ID: "pending-unrelated", Type: "email", Status: QueueStatusPending, MaxAttempts: 8, Config: []byte(`{}`)},
|
|
} {
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("enqueue %s: %v", notif.ID, err)
|
|
}
|
|
}
|
|
if err := nq.UpdateStatus("failed-recovery", QueueStatusFailed, "authentication failed"); err != nil {
|
|
t.Fatalf("mark failed: %v", err)
|
|
}
|
|
if err := nq.UpdateStatus("dlq-recovery", QueueStatusDLQ, "retries exhausted"); err != nil {
|
|
t.Fatalf("mark dlq: %v", err)
|
|
}
|
|
|
|
affected, err := nq.RetryTerminalFailures()
|
|
if err != nil {
|
|
t.Fatalf("RetryTerminalFailures: %v", err)
|
|
}
|
|
if affected != 2 {
|
|
t.Fatalf("retry affected = %d, want 2", affected)
|
|
}
|
|
if len(observedHealth) == 0 || !observedHealth[len(observedHealth)-1].Healthy {
|
|
t.Fatalf("health after retry = %#v, want healthy callback after commit", observedHealth)
|
|
}
|
|
for _, id := range []string{"failed-recovery", "dlq-recovery"} {
|
|
var status string
|
|
var attempts int
|
|
var lastAttempt, lastError, completedAt *int64
|
|
if err := nq.db.QueryRow(`
|
|
SELECT status, attempts, last_attempt, last_error, completed_at
|
|
FROM notification_queue WHERE id = ?
|
|
`, id).Scan(&status, &attempts, &lastAttempt, &lastError, &completedAt); err != nil {
|
|
t.Fatalf("read retried %s: %v", id, err)
|
|
}
|
|
if status != string(QueueStatusPending) || attempts != 0 || lastAttempt != nil || lastError != nil || completedAt != nil {
|
|
t.Fatalf("retried %s = status %q attempts %d lastAttempt %v lastError %v completedAt %v", id, status, attempts, lastAttempt, lastError, completedAt)
|
|
}
|
|
}
|
|
|
|
if err := nq.UpdateStatus("failed-recovery", QueueStatusFailed, "still invalid"); err != nil {
|
|
t.Fatalf("mark failed again: %v", err)
|
|
}
|
|
if err := nq.UpdateStatus("dlq-recovery", QueueStatusDLQ, "still unavailable"); err != nil {
|
|
t.Fatalf("mark dlq again: %v", err)
|
|
}
|
|
affected, err = nq.DismissTerminalFailures()
|
|
if err != nil {
|
|
t.Fatalf("DismissTerminalFailures: %v", err)
|
|
}
|
|
if affected != 2 {
|
|
t.Fatalf("dismiss affected = %d, want 2", affected)
|
|
}
|
|
if len(observedHealth) == 0 || !observedHealth[len(observedHealth)-1].Healthy {
|
|
t.Fatalf("health after dismiss = %#v, want healthy callback after commit", observedHealth)
|
|
}
|
|
stats, err := nq.GetQueueStats()
|
|
if err != nil {
|
|
t.Fatalf("GetQueueStats: %v", err)
|
|
}
|
|
if stats[string(QueueStatusFailed)] != 0 || stats[string(QueueStatusDLQ)] != 0 || stats[string(QueueStatusCancelled)] != 2 || stats[string(QueueStatusPending)] != 1 {
|
|
t.Fatalf("stats after dismissal = %#v", stats)
|
|
}
|
|
})
|
|
|
|
t.Run("UpdateStatus returns error for non-existent notification", func(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
err = nq.UpdateStatus("non-existent-id", QueueStatusSent, "")
|
|
if err == nil {
|
|
t.Error("expected error when updating non-existent notification, got nil")
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestGetTelemetryStatsReturnsOnlyWindowedOutcomeCounts(t *testing.T) {
|
|
nq, err := NewNotificationQueue(t.TempDir())
|
|
if err != nil {
|
|
t.Fatalf("NewNotificationQueue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
now := time.Now().UTC()
|
|
entries := []*QueuedNotification{
|
|
{ID: "sent", Type: "email", Status: QueueStatusPending, MaxAttempts: 3, Config: []byte(`{}`), CreatedAt: now.Add(-time.Hour)},
|
|
{ID: "failed", Type: "webhook", Status: QueueStatusPending, MaxAttempts: 3, Config: []byte(`{}`), CreatedAt: now.Add(-2 * time.Hour)},
|
|
{ID: "old", Type: "email", Status: QueueStatusPending, MaxAttempts: 3, Config: []byte(`{}`), CreatedAt: now.Add(-8 * 24 * time.Hour)},
|
|
}
|
|
for _, entry := range entries {
|
|
if err := nq.Enqueue(entry); err != nil {
|
|
t.Fatalf("enqueue %s: %v", entry.ID, err)
|
|
}
|
|
}
|
|
entries[0].Attempts = 1
|
|
entries[0].Status = QueueStatusPending
|
|
if err := nq.RecordAudit(entries[0], false, "first attempt failed"); err != nil {
|
|
t.Fatalf("record sent retry: %v", err)
|
|
}
|
|
entries[0].Attempts = 2
|
|
entries[0].Status = QueueStatusSent
|
|
if err := nq.RecordAudit(entries[0], true, ""); err != nil {
|
|
t.Fatalf("record sent delivery: %v", err)
|
|
}
|
|
entries[1].Attempts = 1
|
|
entries[1].Status = QueueStatusDLQ
|
|
if err := nq.RecordAudit(entries[1], false, "final attempt failed"); err != nil {
|
|
t.Fatalf("record failed delivery: %v", err)
|
|
}
|
|
entries[2].Attempts = 1
|
|
entries[2].Status = QueueStatusSent
|
|
if err := nq.RecordAudit(entries[2], true, ""); err != nil {
|
|
t.Fatalf("record old delivery: %v", err)
|
|
}
|
|
if _, err := nq.db.Exec(
|
|
`UPDATE notification_audit SET timestamp = ? WHERE notification_id = ?`,
|
|
now.Add(-8*24*time.Hour).Unix(),
|
|
"old",
|
|
); err != nil {
|
|
t.Fatalf("age old audit row: %v", err)
|
|
}
|
|
|
|
stats, err := nq.GetTelemetryStats(now.Add(-7 * 24 * time.Hour))
|
|
if err != nil {
|
|
t.Fatalf("GetTelemetryStats: %v", err)
|
|
}
|
|
if stats != (TelemetryStats{
|
|
Attempts: 3,
|
|
Deliveries: 1,
|
|
Failures: 1,
|
|
FailureClasses: NotificationFailureClassCounts{
|
|
Unknown: 1,
|
|
},
|
|
}) {
|
|
t.Fatalf("telemetry stats = %#v", stats)
|
|
}
|
|
}
|
|
|
|
func TestClassifyNotificationFailureUsesBoundedPrivacySafeBuckets(t *testing.T) {
|
|
tests := []struct {
|
|
errorMessage string
|
|
want NotificationFailureClass
|
|
}{
|
|
{"webhook returned status 401: Unauthorized", NotificationFailureAuthentication},
|
|
{"SMTP auth failed: 535 invalid credentials", NotificationFailureAuthentication},
|
|
{"webhook returned HTTP 429: Too Many Requests", NotificationFailureRateLimited},
|
|
{"x509: certificate signed by unknown authority", NotificationFailureTLS},
|
|
{"dial tcp 10.0.0.1:443: i/o timeout", NotificationFailureConnectivity},
|
|
{"apprise server URL is not configured", NotificationFailureConfiguration},
|
|
{"webhook returned status 413: Payload Too Large", NotificationFailureRejected},
|
|
{"webhook returned status 500: Internal Server Error", NotificationFailureServerError},
|
|
{"webhook returned HTTP 504: Gateway Timeout", NotificationFailureServerError},
|
|
{"provider-specific secret detail", NotificationFailureUnknown},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
if got := ClassifyNotificationFailure(test.errorMessage); got != test.want {
|
|
t.Errorf("ClassifyNotificationFailure(%q) = %q, want %q", test.errorMessage, got, test.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGetTelemetryStatsClassifiesTerminalFailuresOnly(t *testing.T) {
|
|
nq, err := NewNotificationQueue(t.TempDir())
|
|
if err != nil {
|
|
t.Fatalf("NewNotificationQueue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
now := time.Now().UTC()
|
|
entries := []*QueuedNotification{
|
|
{ID: "auth-dlq", Type: "webhook", Status: QueueStatusDLQ, Attempts: 3, Config: []byte(`{}`), CreatedAt: now},
|
|
{ID: "rate-retry", Type: "webhook", Status: QueueStatusPending, Attempts: 1, Config: []byte(`{}`), CreatedAt: now},
|
|
{ID: "tls-failed", Type: "email", Status: QueueStatusFailed, Attempts: 3, Config: []byte(`{}`), CreatedAt: now},
|
|
{ID: "server-error-dlq", Type: "webhook", Status: QueueStatusDLQ, Attempts: 3, Config: []byte(`{}`), CreatedAt: now},
|
|
}
|
|
for _, entry := range entries {
|
|
status := entry.Status
|
|
entry.Status = QueueStatusPending
|
|
if err := nq.Enqueue(entry); err != nil {
|
|
t.Fatalf("enqueue %s: %v", entry.ID, err)
|
|
}
|
|
entry.Status = status
|
|
}
|
|
if err := nq.RecordAudit(entries[0], false, "HTTP 401 Unauthorized"); err != nil {
|
|
t.Fatalf("record auth audit: %v", err)
|
|
}
|
|
if err := nq.RecordAudit(entries[1], false, "HTTP 429 Too Many Requests"); err != nil {
|
|
t.Fatalf("record retry audit: %v", err)
|
|
}
|
|
if err := nq.RecordAudit(entries[2], false, "x509 certificate failure"); err != nil {
|
|
t.Fatalf("record TLS audit: %v", err)
|
|
}
|
|
if err := nq.RecordAudit(entries[3], false, "HTTP 503 Service Unavailable"); err != nil {
|
|
t.Fatalf("record server-error audit: %v", err)
|
|
}
|
|
|
|
stats, err := nq.GetTelemetryStats(now.Add(-time.Hour))
|
|
if err != nil {
|
|
t.Fatalf("GetTelemetryStats: %v", err)
|
|
}
|
|
if stats.Attempts != 4 || stats.Failures != 3 {
|
|
t.Fatalf("telemetry totals = %#v, want 4 attempts and 3 terminal failures", stats)
|
|
}
|
|
if stats.FailureClasses.Authentication != 1 || stats.FailureClasses.TLS != 1 ||
|
|
stats.FailureClasses.ServerError != 1 || stats.FailureClasses.Unknown != 0 {
|
|
t.Fatalf("failure classes = %#v, want authentication=1 tls=1 server_error=1 unknown=0", stats.FailureClasses)
|
|
}
|
|
if stats.FailureClasses.RateLimited != 0 {
|
|
t.Fatalf("retry failure leaked into terminal class counts: %#v", stats.FailureClasses)
|
|
}
|
|
}
|
|
|
|
func TestPerformCleanup(t *testing.T) {
|
|
t.Run("cleanup removes old completed entries", func(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
// Insert a notification directly with old completed_at timestamp
|
|
oldTime := time.Now().Add(-10 * 24 * time.Hour).Unix() // 10 days ago
|
|
|
|
_, err = nq.db.Exec(`
|
|
INSERT INTO notification_queue
|
|
(id, type, status, config, alerts, attempts, max_attempts, created_at, completed_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
"old-sent-1", "email", "sent", "{}", "[]", 1, 3, oldTime, oldTime)
|
|
if err != nil {
|
|
t.Fatalf("Failed to insert old notification: %v", err)
|
|
}
|
|
|
|
// Insert a recent completed notification (should NOT be cleaned)
|
|
recentTime := time.Now().Add(-1 * 24 * time.Hour).Unix() // 1 day ago
|
|
_, err = nq.db.Exec(`
|
|
INSERT INTO notification_queue
|
|
(id, type, status, config, alerts, attempts, max_attempts, created_at, completed_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
"recent-sent-1", "email", "sent", "{}", "[]", 1, 3, recentTime, recentTime)
|
|
if err != nil {
|
|
t.Fatalf("Failed to insert recent notification: %v", err)
|
|
}
|
|
|
|
// Run cleanup
|
|
nq.performCleanup()
|
|
|
|
// Verify old entry was removed
|
|
var count int
|
|
err = nq.db.QueryRow(`SELECT COUNT(*) FROM notification_queue WHERE id = ?`, "old-sent-1").Scan(&count)
|
|
if err != nil {
|
|
t.Fatalf("Failed to query: %v", err)
|
|
}
|
|
if count != 0 {
|
|
t.Error("old completed notification should have been cleaned up")
|
|
}
|
|
|
|
// Verify recent entry still exists
|
|
err = nq.db.QueryRow(`SELECT COUNT(*) FROM notification_queue WHERE id = ?`, "recent-sent-1").Scan(&count)
|
|
if err != nil {
|
|
t.Fatalf("Failed to query: %v", err)
|
|
}
|
|
if count != 1 {
|
|
t.Error("recent completed notification should NOT have been cleaned up")
|
|
}
|
|
})
|
|
|
|
t.Run("cleanup removes old completed entries with audit rows", func(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
oldTime := time.Now().Add(-10 * 24 * time.Hour).Unix() // 10 days ago
|
|
recentAuditTime := time.Now().Add(-1 * time.Hour).Unix()
|
|
|
|
_, err = nq.db.Exec(`
|
|
INSERT INTO notification_queue
|
|
(id, type, status, config, alerts, attempts, max_attempts, created_at, completed_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
"old-sent-with-audit", "email", "sent", "{}", "[]", 1, 3, oldTime, oldTime)
|
|
if err != nil {
|
|
t.Fatalf("Failed to insert old notification: %v", err)
|
|
}
|
|
|
|
_, err = nq.db.Exec(`
|
|
INSERT INTO notification_audit (notification_id, type, status, timestamp)
|
|
VALUES (?, ?, ?, ?)`,
|
|
"old-sent-with-audit", "email", "sent", recentAuditTime)
|
|
if err != nil {
|
|
t.Fatalf("Failed to insert audit row: %v", err)
|
|
}
|
|
|
|
nq.performCleanup()
|
|
|
|
var count int
|
|
err = nq.db.QueryRow(`SELECT COUNT(*) FROM notification_queue WHERE id = ?`, "old-sent-with-audit").Scan(&count)
|
|
if err != nil {
|
|
t.Fatalf("Failed to query queue row: %v", err)
|
|
}
|
|
if count != 0 {
|
|
t.Error("old completed notification with audit row should have been cleaned up")
|
|
}
|
|
|
|
err = nq.db.QueryRow(`SELECT COUNT(*) FROM notification_audit WHERE notification_id = ?`, "old-sent-with-audit").Scan(&count)
|
|
if err != nil {
|
|
t.Fatalf("Failed to query audit rows: %v", err)
|
|
}
|
|
if count != 0 {
|
|
t.Error("audit rows for deleted notification should have been cleaned up")
|
|
}
|
|
})
|
|
|
|
t.Run("cleanup removes old DLQ entries", func(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
// Insert old DLQ entry (> 30 days)
|
|
oldTime := time.Now().Add(-35 * 24 * time.Hour).Unix()
|
|
_, err = nq.db.Exec(`
|
|
INSERT INTO notification_queue
|
|
(id, type, status, config, alerts, attempts, max_attempts, created_at, completed_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
"old-dlq-1", "webhook", "dlq", "{}", "[]", 5, 3, oldTime, oldTime)
|
|
if err != nil {
|
|
t.Fatalf("Failed to insert old DLQ entry: %v", err)
|
|
}
|
|
|
|
// Insert recent DLQ entry (< 30 days)
|
|
recentTime := time.Now().Add(-20 * 24 * time.Hour).Unix()
|
|
_, err = nq.db.Exec(`
|
|
INSERT INTO notification_queue
|
|
(id, type, status, config, alerts, attempts, max_attempts, created_at, completed_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
"recent-dlq-1", "webhook", "dlq", "{}", "[]", 5, 3, recentTime, recentTime)
|
|
if err != nil {
|
|
t.Fatalf("Failed to insert recent DLQ entry: %v", err)
|
|
}
|
|
|
|
// Run cleanup
|
|
nq.performCleanup()
|
|
|
|
// Verify old DLQ was removed
|
|
var count int
|
|
err = nq.db.QueryRow(`SELECT COUNT(*) FROM notification_queue WHERE id = ?`, "old-dlq-1").Scan(&count)
|
|
if err != nil {
|
|
t.Fatalf("Failed to query: %v", err)
|
|
}
|
|
if count != 0 {
|
|
t.Error("old DLQ entry should have been cleaned up")
|
|
}
|
|
|
|
// Verify recent DLQ still exists
|
|
err = nq.db.QueryRow(`SELECT COUNT(*) FROM notification_queue WHERE id = ?`, "recent-dlq-1").Scan(&count)
|
|
if err != nil {
|
|
t.Fatalf("Failed to query: %v", err)
|
|
}
|
|
if count != 1 {
|
|
t.Error("recent DLQ entry should NOT have been cleaned up")
|
|
}
|
|
})
|
|
|
|
t.Run("cleanup removes old audit logs", func(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
// Insert parent notifications first (foreign key constraint)
|
|
oldTime := time.Now().Add(-35 * 24 * time.Hour).Unix()
|
|
recentTime := time.Now().Add(-5 * 24 * time.Hour).Unix()
|
|
|
|
_, err = nq.db.Exec(`
|
|
INSERT INTO notification_queue
|
|
(id, type, status, config, alerts, attempts, max_attempts, created_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
"test-1", "email", "sent", "{}", "[]", 1, 3, oldTime)
|
|
if err != nil {
|
|
t.Fatalf("Failed to insert parent notification 1: %v", err)
|
|
}
|
|
|
|
_, err = nq.db.Exec(`
|
|
INSERT INTO notification_queue
|
|
(id, type, status, config, alerts, attempts, max_attempts, created_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
"test-2", "email", "sent", "{}", "[]", 1, 3, recentTime)
|
|
if err != nil {
|
|
t.Fatalf("Failed to insert parent notification 2: %v", err)
|
|
}
|
|
|
|
// Insert old audit log (> 30 days)
|
|
_, err = nq.db.Exec(`
|
|
INSERT INTO notification_audit (notification_id, type, status, timestamp)
|
|
VALUES (?, ?, ?, ?)`,
|
|
"test-1", "email", "created", oldTime)
|
|
if err != nil {
|
|
t.Fatalf("Failed to insert old audit: %v", err)
|
|
}
|
|
|
|
// Insert recent audit log (< 30 days)
|
|
_, err = nq.db.Exec(`
|
|
INSERT INTO notification_audit (notification_id, type, status, timestamp)
|
|
VALUES (?, ?, ?, ?)`,
|
|
"test-2", "email", "sent", recentTime)
|
|
if err != nil {
|
|
t.Fatalf("Failed to insert recent audit: %v", err)
|
|
}
|
|
|
|
// Run cleanup
|
|
nq.performCleanup()
|
|
|
|
// Verify old audit was removed
|
|
var count int
|
|
err = nq.db.QueryRow(`SELECT COUNT(*) FROM notification_audit WHERE timestamp = ?`, oldTime).Scan(&count)
|
|
if err != nil {
|
|
t.Fatalf("Failed to query: %v", err)
|
|
}
|
|
if count != 0 {
|
|
t.Error("old audit log should have been cleaned up")
|
|
}
|
|
|
|
// Verify recent audit still exists
|
|
err = nq.db.QueryRow(`SELECT COUNT(*) FROM notification_audit WHERE timestamp = ?`, recentTime).Scan(&count)
|
|
if err != nil {
|
|
t.Fatalf("Failed to query: %v", err)
|
|
}
|
|
if count != 1 {
|
|
t.Error("recent audit log should NOT have been cleaned up")
|
|
}
|
|
})
|
|
|
|
t.Run("cleanup with empty database", func(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(tempDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
// Should not panic or error
|
|
nq.performCleanup()
|
|
})
|
|
}
|
|
|
|
func TestNewNotificationQueue_InvalidPath(t *testing.T) {
|
|
// Test with a path that cannot be created (file exists where directory expected)
|
|
tempDir := t.TempDir()
|
|
|
|
// Create a file at the path where we'd want to create a directory
|
|
blockingFile := tempDir + "/blocked"
|
|
if err := os.WriteFile(blockingFile, []byte("blocking"), 0644); err != nil {
|
|
t.Fatalf("failed to create blocking file: %v", err)
|
|
}
|
|
|
|
// Try to create queue at a path nested under the blocking file
|
|
invalidPath := blockingFile + "/subdir"
|
|
_, err := NewNotificationQueue(invalidPath)
|
|
if err == nil {
|
|
t.Error("expected error when creating notification queue with invalid path, got nil")
|
|
}
|
|
}
|
|
|
|
func TestNotificationQueueStopIsIdempotent(t *testing.T) {
|
|
nq, err := NewNotificationQueue(t.TempDir())
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
|
|
const callers = 16
|
|
var wg sync.WaitGroup
|
|
wg.Add(callers)
|
|
|
|
panicCh := make(chan interface{}, callers)
|
|
errCh := make(chan error, callers)
|
|
|
|
for i := 0; i < callers; i++ {
|
|
go func() {
|
|
defer wg.Done()
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
panicCh <- r
|
|
}
|
|
}()
|
|
errCh <- nq.Stop()
|
|
}()
|
|
}
|
|
|
|
wg.Wait()
|
|
close(panicCh)
|
|
close(errCh)
|
|
|
|
if len(panicCh) > 0 {
|
|
t.Fatalf("Stop panicked under concurrent calls: %v", <-panicCh)
|
|
}
|
|
|
|
for stopErr := range errCh {
|
|
if stopErr != nil {
|
|
t.Fatalf("Stop returned error under concurrent calls: %v", stopErr)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProcessBatchWithoutProcessorLeavesNotificationPending(t *testing.T) {
|
|
nq, err := NewNotificationQueue(t.TempDir())
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
notif := &QueuedNotification{
|
|
ID: "pending-without-processor",
|
|
Type: "email",
|
|
Status: QueueStatusPending,
|
|
Config: []byte(`{}`),
|
|
Alerts: []*alerts.Alert{{ID: "alert-1"}},
|
|
MaxAttempts: 3,
|
|
}
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Enqueue() error = %v", err)
|
|
}
|
|
|
|
nq.processBatch()
|
|
|
|
var status string
|
|
var attempts int
|
|
if err := nq.db.QueryRow(`SELECT status, attempts FROM notification_queue WHERE id = ?`, notif.ID).Scan(&status, &attempts); err != nil {
|
|
t.Fatalf("query notification: %v", err)
|
|
}
|
|
if status != string(QueueStatusPending) {
|
|
t.Fatalf("status = %q, want %q", status, QueueStatusPending)
|
|
}
|
|
if attempts != 0 {
|
|
t.Fatalf("attempts = %d, want 0", attempts)
|
|
}
|
|
}
|
|
|
|
func TestSetProcessorTriggersPendingDelivery(t *testing.T) {
|
|
nq, err := NewNotificationQueue(t.TempDir())
|
|
if err != nil {
|
|
t.Fatalf("Failed to create notification queue: %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
notif := &QueuedNotification{
|
|
ID: "process-after-set-processor",
|
|
Type: "email",
|
|
Status: QueueStatusPending,
|
|
Config: []byte(`{}`),
|
|
Alerts: []*alerts.Alert{{ID: "alert-1"}},
|
|
MaxAttempts: 3,
|
|
}
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Enqueue() error = %v", err)
|
|
}
|
|
|
|
processed := make(chan string, 1)
|
|
nq.SetProcessor(func(notif *QueuedNotification) error {
|
|
processed <- notif.ID
|
|
return nil
|
|
})
|
|
|
|
select {
|
|
case got := <-processed:
|
|
if got != notif.ID {
|
|
t.Fatalf("processed notification id = %q, want %q", got, notif.ID)
|
|
}
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("SetProcessor did not trigger pending notification delivery")
|
|
}
|
|
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for {
|
|
var status string
|
|
var attempts int
|
|
if err := nq.db.QueryRow(`SELECT status, attempts FROM notification_queue WHERE id = ?`, notif.ID).Scan(&status, &attempts); err != nil {
|
|
t.Fatalf("query notification: %v", err)
|
|
}
|
|
if status == string(QueueStatusSent) {
|
|
if attempts != 1 {
|
|
t.Fatalf("attempts = %d, want 1 after successful delivery", attempts)
|
|
}
|
|
return
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("notification status = %q after processor wakeup, want %q", status, QueueStatusSent)
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func TestNotificationQueuePersistsGroupedOperationalLinksAcrossDeliveryStates(t *testing.T) {
|
|
dataDir := t.TempDir()
|
|
nq, err := NewNotificationQueue(dataDir)
|
|
if err != nil {
|
|
t.Fatalf("NewNotificationQueue() error = %v", err)
|
|
}
|
|
defer func() { _ = nq.Stop() }()
|
|
|
|
makeAlert := func(id, recordID, transitionID string) *alerts.Alert {
|
|
return &alerts.Alert{
|
|
ID: id,
|
|
OperationalRecord: &operationaltrust.OperationalRecord{
|
|
ID: recordID,
|
|
},
|
|
LatestTransition: &operationaltrust.LifecycleTransition{
|
|
ID: transitionID,
|
|
OperationalRecordID: recordID,
|
|
To: operationaltrust.OperationalOpen,
|
|
CauseKey: id + "-cause",
|
|
},
|
|
}
|
|
}
|
|
createdAt := time.Date(2026, 7, 18, 20, 0, 0, 123, time.UTC)
|
|
notif := &QueuedNotification{
|
|
Type: "email",
|
|
DestinationID: "destination-primary-email",
|
|
Alerts: []*alerts.Alert{
|
|
makeAlert("alert-1", "record-1", "transition-1"),
|
|
makeAlert("alert-2", "record-2", "transition-2"),
|
|
},
|
|
Config: json.RawMessage(`{}`),
|
|
CreatedAt: createdAt,
|
|
MaxAttempts: 3,
|
|
}
|
|
if err := nq.Enqueue(notif); err != nil {
|
|
t.Fatalf("Enqueue() error = %v", err)
|
|
}
|
|
if notif.ID == "" {
|
|
t.Fatal("notification id is empty")
|
|
}
|
|
if len(notif.Links) != 2 {
|
|
t.Fatalf("queued link count = %d, want 2", len(notif.Links))
|
|
}
|
|
for _, link := range notif.Links {
|
|
if link.NotificationID != notif.ID ||
|
|
link.DestinationID != notif.DestinationID ||
|
|
link.DeliveryState != operationaltrust.NotificationQueued {
|
|
t.Fatalf("queued link = %+v", link)
|
|
}
|
|
if err := link.Validate(); err != nil {
|
|
t.Fatalf("queued link Validate() error = %v", err)
|
|
}
|
|
}
|
|
|
|
if err := nq.IncrementAttemptAndSetStatus(
|
|
notif.ID,
|
|
QueueStatusSending,
|
|
); err != nil {
|
|
t.Fatalf("IncrementAttemptAndSetStatus() error = %v", err)
|
|
}
|
|
delivering, err := nq.getNotificationLinks(notif.ID)
|
|
if err != nil {
|
|
t.Fatalf("get delivering links error = %v", err)
|
|
}
|
|
for _, link := range delivering {
|
|
if link.DeliveryState != operationaltrust.NotificationDelivering ||
|
|
link.AttemptedAt == nil ||
|
|
link.CompletedAt != nil {
|
|
t.Fatalf("delivering link = %+v", link)
|
|
}
|
|
}
|
|
|
|
if err := nq.ScheduleRetry(notif.ID, 1); err != nil {
|
|
t.Fatalf("ScheduleRetry() error = %v", err)
|
|
}
|
|
retrying, err := nq.getNotificationLinks(notif.ID)
|
|
if err != nil {
|
|
t.Fatalf("get retrying links error = %v", err)
|
|
}
|
|
for index, link := range retrying {
|
|
if link.NotificationID != notif.ID ||
|
|
link.TransitionID != delivering[index].TransitionID ||
|
|
link.DeliveryState != operationaltrust.NotificationRetrying ||
|
|
link.AttemptedAt == nil ||
|
|
link.CompletedAt != nil {
|
|
t.Fatalf("retrying link = %+v", link)
|
|
}
|
|
}
|
|
|
|
if err := nq.Stop(); err != nil {
|
|
t.Fatalf("Stop() before restart error = %v", err)
|
|
}
|
|
nq, err = NewNotificationQueue(dataDir)
|
|
if err != nil {
|
|
t.Fatalf("NewNotificationQueue() after restart error = %v", err)
|
|
}
|
|
restarted, err := nq.getNotificationLinks(notif.ID)
|
|
if err != nil {
|
|
t.Fatalf("get restarted links error = %v", err)
|
|
}
|
|
for index, link := range restarted {
|
|
if link.NotificationID != notif.ID ||
|
|
link.TransitionID != retrying[index].TransitionID ||
|
|
link.DeliveryState != operationaltrust.NotificationRetrying ||
|
|
link.AttemptedAt == nil ||
|
|
link.CompletedAt != nil {
|
|
t.Fatalf("restarted link = %+v", link)
|
|
}
|
|
}
|
|
|
|
if err := nq.UpdateStatus(notif.ID, QueueStatusSent, ""); err != nil {
|
|
t.Fatalf("UpdateStatus(sent) error = %v", err)
|
|
}
|
|
delivered, err := nq.getNotificationLinks(notif.ID)
|
|
if err != nil {
|
|
t.Fatalf("get delivered links error = %v", err)
|
|
}
|
|
for _, link := range delivered {
|
|
if link.NotificationID != notif.ID ||
|
|
link.DeliveryState != operationaltrust.NotificationDelivered ||
|
|
link.AttemptedAt == nil ||
|
|
link.CompletedAt == nil {
|
|
t.Fatalf("delivered link = %+v", link)
|
|
}
|
|
if err := link.Validate(); err != nil {
|
|
t.Fatalf("delivered link Validate() error = %v", err)
|
|
}
|
|
}
|
|
|
|
notif.Status = QueueStatusSent
|
|
notif.Attempts = 1
|
|
notif.Links = delivered
|
|
if err := nq.RecordAudit(notif, true, ""); err != nil {
|
|
t.Fatalf("RecordAudit() error = %v", err)
|
|
}
|
|
var auditLinksJSON string
|
|
if err := nq.db.QueryRow(
|
|
`SELECT operational_links FROM notification_audit WHERE notification_id = ?`,
|
|
notif.ID,
|
|
).Scan(&auditLinksJSON); err != nil {
|
|
t.Fatalf("query audit links: %v", err)
|
|
}
|
|
var auditLinks []operationaltrust.NotificationLink
|
|
if err := json.Unmarshal([]byte(auditLinksJSON), &auditLinks); err != nil {
|
|
t.Fatalf("unmarshal audit links: %v", err)
|
|
}
|
|
if len(auditLinks) != 2 {
|
|
t.Fatalf("audit link count = %d, want 2", len(auditLinks))
|
|
}
|
|
for index := range auditLinks {
|
|
if auditLinks[index].NotificationID != delivered[index].NotificationID ||
|
|
auditLinks[index].TransitionID != delivered[index].TransitionID ||
|
|
auditLinks[index].DeliveryState != operationaltrust.NotificationDelivered {
|
|
t.Fatalf("audit link[%d] = %+v, want %+v", index, auditLinks[index], delivered[index])
|
|
}
|
|
}
|
|
}
|