mirror of
https://github.com/shankar0123/certctl.git
synced 2026-06-07 17:51:29 +00:00
675b87ba63
Critical alerts can no longer be silently dropped by a transient
notifier failure. Failed notification attempts now ride an exponential
backoff retry loop, with a 5-attempt budget before promotion to the
dead-letter queue for operator intervention.
Schema (migration 000016, idempotent):
- retry_count INTEGER NOT NULL DEFAULT 0
- next_retry_at TIMESTAMPTZ
- last_error TEXT
- idx_notification_events_retry_sweep partial index
(next_retry_at) WHERE status='failed' AND next_retry_at IS NOT NULL
Dead rows clear next_retry_at so the index stops matching them.
Service contract:
- NotificationService.RetryFailedNotifications drives 2^n-minute
exponential backoff capped at 1h (notifRetryBackoffCap) with
5-attempt budget (notifRetryMaxAttempts).
- Exhaustion (RetryCount >= notifRetryMaxAttempts-1) promotes to
status='dead' via MarkAsDead.
- Non-terminal failures record via RecordFailedAttempt.
- Success path promotes to 'sent' without touching retry_count
(audit preserves "delivered on attempt N").
- Missing-notifier branch defensively promotes to 'sent' to avoid
wedging a row on a deleted channel.
- RequeueNotification operator escape hatch atomically resets
retry_count -> 0, next_retry_at -> NULL, last_error -> NULL,
status -> pending via notifRepo.Requeue.
Scheduler:
- New always-on notificationRetryLoop wired into the base loop set at
CERTCTL_NOTIFICATION_RETRY_INTERVAL (default 2m).
- sync/atomic.Bool idempotency guard.
- sync.WaitGroup shutdown drain via WaitForCompletion.
StatsService:
- SetNotifRepo setter pattern preserves 9 pre-existing
NewStatsService call sites (main.go + stats_test.go + 8 digest
tests) without touching the constructor signature.
- DashboardSummary.NotificationsDead populated via
notifRepo.CountByStatus(ctx, "dead") — nil-safe when unwired
(reports zero on systems without a notification repository).
- CountByStatus error is non-fatal (dashboard summary is
best-effort for this field).
- Prometheus certctl_notification_dead_total counter emitted from
the same snapshot.
Handler:
- New POST /api/v1/notifications/{id}/requeue endpoint.
- dead status surfaces to MCP + CLI.
Frontend:
- NotificationsPage gains two-tab toolbar ("All" / "Dead letter")
with queryKey: ['notifications', activeTab] so switching tabs
doesn't serve stale data until the 30s refetch.
- Dead rows surface "Retry {n}/5" + truncated last_error with
full-text title tooltip.
- Requeue mutation wrapped as
mutationFn: (id: string) => requeueNotification(id)
to prevent react-query v5's positional context argument from
leaking into the API client — pinned against future refactors
by strict-match toHaveBeenCalledWith('notif-dead-001') in
NotificationsPage.test.tsx:181.
Closes I-005.
128 lines
4.3 KiB
Go
128 lines
4.3 KiB
Go
package domain
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestNotificationType_Constants(t *testing.T) {
|
|
tests := map[string]NotificationType{
|
|
"ExpirationWarning": NotificationTypeExpirationWarning,
|
|
"RenewalSuccess": NotificationTypeRenewalSuccess,
|
|
"RenewalFailure": NotificationTypeRenewalFailure,
|
|
"DeploymentSuccess": NotificationTypeDeploymentSuccess,
|
|
"DeploymentFailure": NotificationTypeDeploymentFailure,
|
|
"PolicyViolation": NotificationTypePolicyViolation,
|
|
"Revocation": NotificationTypeRevocation,
|
|
}
|
|
for expected, got := range tests {
|
|
if string(got) != expected {
|
|
t.Errorf("expected %q, got %q", expected, string(got))
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNotificationChannel_Constants(t *testing.T) {
|
|
tests := map[string]NotificationChannel{
|
|
"Email": NotificationChannelEmail,
|
|
"Webhook": NotificationChannelWebhook,
|
|
"Slack": NotificationChannelSlack,
|
|
"Teams": NotificationChannelTeams,
|
|
"PagerDuty": NotificationChannelPagerDuty,
|
|
"OpsGenie": NotificationChannelOpsGenie,
|
|
}
|
|
for expected, got := range tests {
|
|
if string(got) != expected {
|
|
t.Errorf("expected %q, got %q", expected, string(got))
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNotificationEvent_Fields(t *testing.T) {
|
|
// This test verifies the NotificationEvent struct can be instantiated
|
|
// with all expected fields.
|
|
certID := "mc-123"
|
|
errorMsg := "failed to send"
|
|
event := &NotificationEvent{
|
|
ID: "notif-1",
|
|
Type: NotificationTypeExpirationWarning,
|
|
CertificateID: &certID,
|
|
Channel: NotificationChannelSlack,
|
|
Recipient: "alerts@example.com",
|
|
Message: "Certificate expiring in 30 days",
|
|
Status: "sent",
|
|
Error: &errorMsg,
|
|
}
|
|
|
|
if event.ID != "notif-1" {
|
|
t.Errorf("expected ID 'notif-1', got %s", event.ID)
|
|
}
|
|
|
|
if event.Type != NotificationTypeExpirationWarning {
|
|
t.Errorf("expected type ExpirationWarning, got %s", string(event.Type))
|
|
}
|
|
|
|
if event.Channel != NotificationChannelSlack {
|
|
t.Errorf("expected channel Slack, got %s", string(event.Channel))
|
|
}
|
|
|
|
if event.CertificateID == nil || *event.CertificateID != "mc-123" {
|
|
t.Errorf("expected CertificateID mc-123, got %v", event.CertificateID)
|
|
}
|
|
|
|
if event.Error == nil || *event.Error != "failed to send" {
|
|
t.Errorf("expected error 'failed to send', got %v", event.Error)
|
|
}
|
|
}
|
|
|
|
// TestNotificationStatus_Constants verifies that I-005 introduces a typed
|
|
// NotificationStatus alongside canonical lowercase string constants covering
|
|
// the pending → sent, pending → failed → dead, and pending → read transitions.
|
|
// The Red signal here is a compile error: the type and the NotificationStatusDead
|
|
// constant do not exist before Phase 2 Green.
|
|
func TestNotificationStatus_Constants(t *testing.T) {
|
|
tests := map[string]NotificationStatus{
|
|
"pending": NotificationStatusPending,
|
|
"sent": NotificationStatusSent,
|
|
"failed": NotificationStatusFailed,
|
|
"dead": NotificationStatusDead,
|
|
"read": NotificationStatusRead,
|
|
}
|
|
for expected, got := range tests {
|
|
if string(got) != expected {
|
|
t.Errorf("expected %q, got %q", expected, string(got))
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestNotificationEvent_RetryFields verifies the I-005 retry/DLQ columns are
|
|
// surfaced on the domain model: a RetryCount counter, a nullable NextRetryAt
|
|
// timestamp used by the retry-sweep partial index, and a nullable LastError
|
|
// string preserving the most recent transient failure for operator triage.
|
|
// The Red signal is a compile error — these fields do not exist yet.
|
|
func TestNotificationEvent_RetryFields(t *testing.T) {
|
|
next := time.Now().Add(2 * time.Minute)
|
|
lastErr := "connection refused"
|
|
event := &NotificationEvent{
|
|
ID: "notif-retry-001",
|
|
Type: NotificationTypeExpirationWarning,
|
|
Channel: NotificationChannelWebhook,
|
|
Recipient: "https://hooks.example.com/certs",
|
|
Message: "retry me",
|
|
Status: string(NotificationStatusFailed),
|
|
RetryCount: 3,
|
|
NextRetryAt: &next,
|
|
LastError: &lastErr,
|
|
}
|
|
|
|
if event.RetryCount != 3 {
|
|
t.Errorf("expected RetryCount 3, got %d", event.RetryCount)
|
|
}
|
|
if event.NextRetryAt == nil || !event.NextRetryAt.Equal(next) {
|
|
t.Errorf("expected NextRetryAt %v, got %v", next, event.NextRetryAt)
|
|
}
|
|
if event.LastError == nil || *event.LastError != "connection refused" {
|
|
t.Errorf("expected LastError 'connection refused', got %v", event.LastError)
|
|
}
|
|
}
|