I-005: notification retry loop + dead-letter queue

Critical alerts can no longer be silently dropped by a transient notifier failure. Failed notification attempts now ride an exponential backoff retry loop, with a 5-attempt budget before promotion to the dead-letter queue for operator intervention. Schema (migration 000016, idempotent): - retry_count INTEGER NOT NULL DEFAULT 0 - next_retry_at TIMESTAMPTZ - last_error TEXT - idx_notification_events_retry_sweep partial index (next_retry_at) WHERE status='failed' AND next_retry_at IS NOT NULL Dead rows clear next_retry_at so the index stops matching them. Service contract: - NotificationService.RetryFailedNotifications drives 2^n-minute exponential backoff capped at 1h (notifRetryBackoffCap) with 5-attempt budget (notifRetryMaxAttempts). - Exhaustion (RetryCount >= notifRetryMaxAttempts-1) promotes to status='dead' via MarkAsDead. - Non-terminal failures record via RecordFailedAttempt. - Success path promotes to 'sent' without touching retry_count (audit preserves "delivered on attempt N"). - Missing-notifier branch defensively promotes to 'sent' to avoid wedging a row on a deleted channel. - RequeueNotification operator escape hatch atomically resets retry_count -> 0, next_retry_at -> NULL, last_error -> NULL, status -> pending via notifRepo.Requeue. Scheduler: - New always-on notificationRetryLoop wired into the base loop set at CERTCTL_NOTIFICATION_RETRY_INTERVAL (default 2m). - sync/atomic.Bool idempotency guard. - sync.WaitGroup shutdown drain via WaitForCompletion. StatsService: - SetNotifRepo setter pattern preserves 9 pre-existing NewStatsService call sites (main.go + stats_test.go + 8 digest tests) without touching the constructor signature. - DashboardSummary.NotificationsDead populated via notifRepo.CountByStatus(ctx, "dead") — nil-safe when unwired (reports zero on systems without a notification repository). - CountByStatus error is non-fatal (dashboard summary is best-effort for this field). - Prometheus certctl_notification_dead_total counter emitted from the same snapshot. Handler: - New POST /api/v1/notifications/{id}/requeue endpoint. - dead status surfaces to MCP + CLI. Frontend: - NotificationsPage gains two-tab toolbar ("All" / "Dead letter") with queryKey: ['notifications', activeTab] so switching tabs doesn't serve stale data until the 30s refetch. - Dead rows surface "Retry {n}/5" + truncated last_error with full-text title tooltip. - Requeue mutation wrapped as mutationFn: (id: string) => requeueNotification(id) to prevent react-query v5's positional context argument from leaking into the API client — pinned against future refactors by strict-match toHaveBeenCalledWith('notif-dead-001') in NotificationsPage.test.tsx:181. Closes I-005.
2026-06-07 19:21:29 +00:00 · 2026-04-19 15:17:27 +00:00
parent 707d8de4fb
commit 675b87ba63
33 changed files with 3758 additions and 228 deletions
@@ -13,9 +13,11 @@ import (

 // MockNotificationService is a mock implementation of NotificationService interface.
 type MockNotificationService struct {
-	ListNotificationsFn func(page, perPage int) ([]domain.NotificationEvent, int64, error)
-	GetNotificationFn   func(id string) (*domain.NotificationEvent, error)
-	MarkAsReadFn        func(id string) error
+	ListNotificationsFn         func(page, perPage int) ([]domain.NotificationEvent, int64, error)
+	ListNotificationsByStatusFn func(status string, page, perPage int) ([]domain.NotificationEvent, int64, error)
+	GetNotificationFn           func(id string) (*domain.NotificationEvent, error)
+	MarkAsReadFn                func(id string) error
+	RequeueFn                   func(id string) error
 }

 func (m *MockNotificationService) ListNotifications(_ context.Context, page, perPage int) ([]domain.NotificationEvent, int64, error) {
@@ -25,6 +27,13 @@ func (m *MockNotificationService) ListNotifications(_ context.Context, page, per
 	return nil, 0, nil
 }

+func (m *MockNotificationService) ListNotificationsByStatus(_ context.Context, status string, page, perPage int) ([]domain.NotificationEvent, int64, error) {
+	if m.ListNotificationsByStatusFn != nil {
+		return m.ListNotificationsByStatusFn(status, page, perPage)
+	}
+	return nil, 0, nil
+}
+
 func (m *MockNotificationService) GetNotification(_ context.Context, id string) (*domain.NotificationEvent, error) {
 	if m.GetNotificationFn != nil {
 		return m.GetNotificationFn(id)
@@ -39,6 +48,13 @@ func (m *MockNotificationService) MarkAsRead(_ context.Context, id string) error
 	return nil
 }

+func (m *MockNotificationService) RequeueNotification(_ context.Context, id string) error {
+	if m.RequeueFn != nil {
+		return m.RequeueFn(id)
+	}
+	return nil
+}
+
 func TestListNotifications_Success(t *testing.T) {
 	now := time.Now()
 	certID := "mc-prod-001"
@@ -282,3 +298,224 @@ func TestMarkAsRead_EmptyID(t *testing.T) {
 		t.Fatalf("expected status 400, got %d", w.Code)
 	}
 }
+
+// ---------------------------------------------------------------------------
+// I-005: Notification Retry + Dead-Letter Queue handler contract (Phase 1 Red)
+//
+// These tests pin the HTTP surface Phase 2 Green must implement:
+//
+//   1. POST /api/v1/notifications/{id}/requeue — flips a dead notification
+//      back to 'pending' so the retry loop can pick it up again. The handler
+//      method does not exist yet (NotificationHandler has no RequeueNotification
+//      method) and the NotificationService interface does not declare
+//      RequeueNotification — both are compile-time Red halts.
+//
+//   2. GET /api/v1/notifications?status=dead — routes dead-letter list requests
+//      through ListNotificationsByStatus instead of ListNotifications. The
+//      status-filter routing does not exist yet, so ListNotificationsByStatusFn
+//      never fires — a runtime Red halt.
+// ---------------------------------------------------------------------------
+
+func TestRequeueNotification_Success(t *testing.T) {
+	var requeuedID string
+	mock := &MockNotificationService{
+		RequeueFn: func(id string) error {
+			requeuedID = id
+			return nil
+		},
+	}
+
+	handler := NewNotificationHandler(mock)
+	req := httptest.NewRequest(http.MethodPost, "/api/v1/notifications/notif-dead-001/requeue", nil)
+	req = req.WithContext(contextWithRequestID())
+	w := httptest.NewRecorder()
+
+	handler.RequeueNotification(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected status 200, got %d", w.Code)
+	}
+	if requeuedID != "notif-dead-001" {
+		t.Errorf("expected requeued ID 'notif-dead-001', got '%s'", requeuedID)
+	}
+
+	var resp map[string]string
+	if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
+		t.Fatalf("failed to decode response: %v", err)
+	}
+	if resp["status"] != "requeued" {
+		t.Errorf("expected status 'requeued', got '%s'", resp["status"])
+	}
+}
+
+func TestRequeueNotification_NotFound(t *testing.T) {
+	mock := &MockNotificationService{
+		RequeueFn: func(id string) error {
+			return ErrMockNotFound
+		},
+	}
+
+	handler := NewNotificationHandler(mock)
+	req := httptest.NewRequest(http.MethodPost, "/api/v1/notifications/nonexistent/requeue", nil)
+	req = req.WithContext(contextWithRequestID())
+	w := httptest.NewRecorder()
+
+	handler.RequeueNotification(w, req)
+
+	if w.Code != http.StatusNotFound {
+		t.Fatalf("expected status 404, got %d", w.Code)
+	}
+}
+
+func TestRequeueNotification_ServiceError(t *testing.T) {
+	mock := &MockNotificationService{
+		RequeueFn: func(id string) error {
+			return ErrMockServiceFailed
+		},
+	}
+
+	handler := NewNotificationHandler(mock)
+	req := httptest.NewRequest(http.MethodPost, "/api/v1/notifications/notif-dead-001/requeue", nil)
+	req = req.WithContext(contextWithRequestID())
+	w := httptest.NewRecorder()
+
+	handler.RequeueNotification(w, req)
+
+	if w.Code != http.StatusInternalServerError {
+		t.Fatalf("expected status 500, got %d", w.Code)
+	}
+}
+
+func TestRequeueNotification_MethodNotAllowed(t *testing.T) {
+	handler := NewNotificationHandler(&MockNotificationService{})
+	req := httptest.NewRequest(http.MethodGet, "/api/v1/notifications/notif-dead-001/requeue", nil)
+	w := httptest.NewRecorder()
+
+	handler.RequeueNotification(w, req)
+
+	if w.Code != http.StatusMethodNotAllowed {
+		t.Fatalf("expected status 405, got %d", w.Code)
+	}
+}
+
+func TestRequeueNotification_EmptyID(t *testing.T) {
+	handler := NewNotificationHandler(&MockNotificationService{})
+	req := httptest.NewRequest(http.MethodPost, "/api/v1/notifications//requeue", nil)
+	req = req.WithContext(contextWithRequestID())
+	w := httptest.NewRecorder()
+
+	handler.RequeueNotification(w, req)
+
+	if w.Code != http.StatusBadRequest {
+		t.Fatalf("expected status 400, got %d", w.Code)
+	}
+}
+
+func TestListNotifications_StatusFilter_Dead(t *testing.T) {
+	now := time.Now()
+	certID := "mc-prod-001"
+	lastErr := "SMTP connection refused"
+	nextRetry := now.Add(1 * time.Minute)
+	dead := domain.NotificationEvent{
+		ID:            "notif-dead-001",
+		Type:          domain.NotificationTypeExpirationWarning,
+		CertificateID: &certID,
+		Channel:       domain.NotificationChannelEmail,
+		Recipient:     "admin@example.com",
+		Message:       "Certificate expiring in 7 days",
+		Status:        "dead",
+		CreatedAt:     now,
+		RetryCount:    5,
+		NextRetryAt:   &nextRetry,
+		LastError:     &lastErr,
+	}
+
+	var capturedStatus string
+	var capturedPage, capturedPerPage int
+	byStatusCalled := false
+	listCalled := false
+
+	mock := &MockNotificationService{
+		ListNotificationsFn: func(page, perPage int) ([]domain.NotificationEvent, int64, error) {
+			listCalled = true
+			return nil, 0, nil
+		},
+		ListNotificationsByStatusFn: func(status string, page, perPage int) ([]domain.NotificationEvent, int64, error) {
+			byStatusCalled = true
+			capturedStatus = status
+			capturedPage = page
+			capturedPerPage = perPage
+			return []domain.NotificationEvent{dead}, 1, nil
+		},
+	}
+
+	handler := NewNotificationHandler(mock)
+	req := httptest.NewRequest(http.MethodGet, "/api/v1/notifications?status=dead&page=1&per_page=50", nil)
+	req = req.WithContext(contextWithRequestID())
+	w := httptest.NewRecorder()
+
+	handler.ListNotifications(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected status 200, got %d", w.Code)
+	}
+	if !byStatusCalled {
+		t.Fatalf("expected ListNotificationsByStatus to be called for ?status=dead, but it was not")
+	}
+	if listCalled {
+		t.Errorf("ListNotifications should not be called when status filter is present")
+	}
+	if capturedStatus != "dead" {
+		t.Errorf("expected status='dead', got '%s'", capturedStatus)
+	}
+	if capturedPage != 1 {
+		t.Errorf("expected page=1, got %d", capturedPage)
+	}
+	if capturedPerPage != 50 {
+		t.Errorf("expected per_page=50, got %d", capturedPerPage)
+	}
+
+	var resp PagedResponse
+	if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
+		t.Fatalf("failed to decode response: %v", err)
+	}
+	if resp.Total != 1 {
+		t.Errorf("expected total=1 dead notification, got %d", resp.Total)
+	}
+}
+
+func TestListNotifications_NoStatusFilter_CallsDefault(t *testing.T) {
+	// Pin the inverse: when no ?status= is provided, the handler must call the
+	// existing ListNotifications path (not ListNotificationsByStatus). Phase 2
+	// Green must not break the default listing behavior for the plain tab.
+	listCalled := false
+	byStatusCalled := false
+
+	mock := &MockNotificationService{
+		ListNotificationsFn: func(page, perPage int) ([]domain.NotificationEvent, int64, error) {
+			listCalled = true
+			return []domain.NotificationEvent{}, 0, nil
+		},
+		ListNotificationsByStatusFn: func(status string, page, perPage int) ([]domain.NotificationEvent, int64, error) {
+			byStatusCalled = true
+			return nil, 0, nil
+		},
+	}
+
+	handler := NewNotificationHandler(mock)
+	req := httptest.NewRequest(http.MethodGet, "/api/v1/notifications", nil)
+	req = req.WithContext(contextWithRequestID())
+	w := httptest.NewRecorder()
+
+	handler.ListNotifications(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected status 200, got %d", w.Code)
+	}
+	if !listCalled {
+		t.Errorf("expected ListNotifications to be called when no status filter is present")
+	}
+	if byStatusCalled {
+		t.Errorf("ListNotificationsByStatus should not be called when no status filter is present")
+	}
+}