I-005: notification retry loop + dead-letter queue

Critical alerts can no longer be silently dropped by a transient
notifier failure. Failed notification attempts now ride an exponential
backoff retry loop, with a 5-attempt budget before promotion to the
dead-letter queue for operator intervention.

Schema (migration 000016, idempotent):
- retry_count INTEGER NOT NULL DEFAULT 0
- next_retry_at TIMESTAMPTZ
- last_error TEXT
- idx_notification_events_retry_sweep partial index
  (next_retry_at) WHERE status='failed' AND next_retry_at IS NOT NULL
  Dead rows clear next_retry_at so the index stops matching them.

Service contract:
- NotificationService.RetryFailedNotifications drives 2^n-minute
  exponential backoff capped at 1h (notifRetryBackoffCap) with
  5-attempt budget (notifRetryMaxAttempts).
- Exhaustion (RetryCount >= notifRetryMaxAttempts-1) promotes to
  status='dead' via MarkAsDead.
- Non-terminal failures record via RecordFailedAttempt.
- Success path promotes to 'sent' without touching retry_count
  (audit preserves "delivered on attempt N").
- Missing-notifier branch defensively promotes to 'sent' to avoid
  wedging a row on a deleted channel.
- RequeueNotification operator escape hatch atomically resets
  retry_count -> 0, next_retry_at -> NULL, last_error -> NULL,
  status -> pending via notifRepo.Requeue.

Scheduler:
- New always-on notificationRetryLoop wired into the base loop set at
  CERTCTL_NOTIFICATION_RETRY_INTERVAL (default 2m).
- sync/atomic.Bool idempotency guard.
- sync.WaitGroup shutdown drain via WaitForCompletion.

StatsService:
- SetNotifRepo setter pattern preserves 9 pre-existing
  NewStatsService call sites (main.go + stats_test.go + 8 digest
  tests) without touching the constructor signature.
- DashboardSummary.NotificationsDead populated via
  notifRepo.CountByStatus(ctx, "dead") — nil-safe when unwired
  (reports zero on systems without a notification repository).
- CountByStatus error is non-fatal (dashboard summary is
  best-effort for this field).
- Prometheus certctl_notification_dead_total counter emitted from
  the same snapshot.

Handler:
- New POST /api/v1/notifications/{id}/requeue endpoint.
- dead status surfaces to MCP + CLI.

Frontend:
- NotificationsPage gains two-tab toolbar ("All" / "Dead letter")
  with queryKey: ['notifications', activeTab] so switching tabs
  doesn't serve stale data until the 30s refetch.
- Dead rows surface "Retry {n}/5" + truncated last_error with
  full-text title tooltip.
- Requeue mutation wrapped as
    mutationFn: (id: string) => requeueNotification(id)
  to prevent react-query v5's positional context argument from
  leaking into the API client — pinned against future refactors
  by strict-match toHaveBeenCalledWith('notif-dead-001') in
  NotificationsPage.test.tsx:181.

Closes I-005.

internal/api/handler/metrics.go

@@ -41,20 +41,26 @@ type MetricsResponse struct {
 
 // MetricsGauge represents gauge metrics (point-in-time values).
 type MetricsGauge struct {
-	CertificateTotal       int64   `json:"certificate_total"`
-	CertificateActive      int64   `json:"certificate_active"`
-	CertificateExpiringSoon int64  `json:"certificate_expiring_soon"` // Within 30d
-	CertificateExpired     int64   `json:"certificate_expired"`
-	CertificateRevoked     int64   `json:"certificate_revoked"`
-	AgentTotal             int64   `json:"agent_total"`
-	AgentOnline            int64   `json:"agent_online"`
-	JobPending             int64   `json:"job_pending"`
+	CertificateTotal        int64 `json:"certificate_total"`
+	CertificateActive       int64 `json:"certificate_active"`
+	CertificateExpiringSoon int64 `json:"certificate_expiring_soon"` // Within 30d
+	CertificateExpired      int64 `json:"certificate_expired"`
+	CertificateRevoked      int64 `json:"certificate_revoked"`
+	AgentTotal              int64 `json:"agent_total"`
+	AgentOnline             int64 `json:"agent_online"`
+	JobPending              int64 `json:"job_pending"`
 }
 
 // MetricsCounter represents counter metrics (cumulative values).
 type MetricsCounter struct {
 	JobCompletedTotal int64 `json:"job_completed_total"`
 	JobFailedTotal    int64 `json:"job_failed_total"`
+	// NotificationsDeadTotal is a point-in-time count of notifications in the
+	// dead-letter queue (status="dead"), exposed here with the _total suffix
+	// to match Prometheus DB-snapshot counter convention (same semantics as
+	// JobFailedTotal and JobCompletedTotal — see metrics.md). I-005 DLQ
+	// observability gate.
+	NotificationsDeadTotal int64 `json:"notifications_dead_total"`
 }
 
 // UptimeMetric represents server uptime information.
@@ -95,18 +101,19 @@ func (h MetricsHandler) GetMetrics(w http.ResponseWriter, r *http.Request) {
 	// Build metrics response
 	metricsResp := MetricsResponse{
 		Gauge: MetricsGauge{
-			CertificateTotal:      dashboardSummary.TotalCertificates,
-			CertificateActive:     dashboardSummary.TotalCertificates - dashboardSummary.ExpiringCertificates - dashboardSummary.ExpiredCertificates - dashboardSummary.RevokedCertificates,
+			CertificateTotal:        dashboardSummary.TotalCertificates,
+			CertificateActive:       dashboardSummary.TotalCertificates - dashboardSummary.ExpiringCertificates - dashboardSummary.ExpiredCertificates - dashboardSummary.RevokedCertificates,
 			CertificateExpiringSoon: dashboardSummary.ExpiringCertificates,
-			CertificateExpired:    dashboardSummary.ExpiredCertificates,
-			CertificateRevoked:    dashboardSummary.RevokedCertificates,
-			AgentTotal:            dashboardSummary.TotalAgents,
-			AgentOnline:           dashboardSummary.ActiveAgents,
-			JobPending:            dashboardSummary.PendingJobs,
+			CertificateExpired:      dashboardSummary.ExpiredCertificates,
+			CertificateRevoked:      dashboardSummary.RevokedCertificates,
+			AgentTotal:              dashboardSummary.TotalAgents,
+			AgentOnline:             dashboardSummary.ActiveAgents,
+			JobPending:              dashboardSummary.PendingJobs,
 		},
 		Counter: MetricsCounter{
-			JobCompletedTotal: dashboardSummary.CompleteJobs,
-			JobFailedTotal:    dashboardSummary.FailedJobs,
+			JobCompletedTotal:      dashboardSummary.CompleteJobs,
+			JobFailedTotal:         dashboardSummary.FailedJobs,
+			NotificationsDeadTotal: dashboardSummary.NotificationsDead,
 		},
 		Uptime: UptimeMetric{
 			UptimeSeconds: int64(time.Since(h.serverStarted).Seconds()),
@@ -200,6 +207,17 @@ func (h MetricsHandler) GetPrometheusMetrics(w http.ResponseWriter, r *http.Requ
 	fmt.Fprintf(w, "# TYPE certctl_job_failed_total counter\n")
 	fmt.Fprintf(w, "certctl_job_failed_total %d\n\n", dashboardSummary.FailedJobs)
 
+	// I-005: notification dead-letter queue depth. Emitted with the _total
+	// suffix to match the existing certctl_job_completed_total /
+	// certctl_job_failed_total convention for DB-snapshot counters — the
+	// value is a point-in-time COUNT(*) of notification_events rows where
+	// status='dead', not a monotonically increasing process-lifetime counter.
+	// Operators alert on this as "dead-letter depth" (thresholds in the
+	// I-005 spec: > 0 → warning, > 10 → critical).
+	fmt.Fprintf(w, "# HELP certctl_notification_dead_total Number of notifications in the dead-letter queue.\n")
+	fmt.Fprintf(w, "# TYPE certctl_notification_dead_total counter\n")
+	fmt.Fprintf(w, "certctl_notification_dead_total %d\n\n", dashboardSummary.NotificationsDead)
+
 	// Info — server uptime
 	fmt.Fprintf(w, "# HELP certctl_uptime_seconds Server uptime in seconds.\n")
 	fmt.Fprintf(w, "# TYPE certctl_uptime_seconds gauge\n")
@@ -209,15 +227,21 @@ func (h MetricsHandler) GetPrometheusMetrics(w http.ResponseWriter, r *http.Requ
 // DashboardSummary mirrors the service.DashboardSummary for JSON unmarshaling.
 // JSON tags must match the service-layer struct exactly.
 type DashboardSummary struct {
-	TotalCertificates    int64     `json:"total_certificates"`
-	ExpiringCertificates int64     `json:"expiring_certificates"`
-	ExpiredCertificates  int64     `json:"expired_certificates"`
-	RevokedCertificates  int64     `json:"revoked_certificates"`
-	ActiveAgents         int64     `json:"active_agents"`
-	OfflineAgents        int64     `json:"offline_agents"`
-	TotalAgents          int64     `json:"total_agents"`
-	PendingJobs          int64     `json:"pending_jobs"`
-	FailedJobs           int64     `json:"failed_jobs"`
-	CompleteJobs         int64     `json:"complete_jobs"`
-	CompletedAt          time.Time `json:"completed_at"`
+	TotalCertificates    int64 `json:"total_certificates"`
+	ExpiringCertificates int64 `json:"expiring_certificates"`
+	ExpiredCertificates  int64 `json:"expired_certificates"`
+	RevokedCertificates  int64 `json:"revoked_certificates"`
+	ActiveAgents         int64 `json:"active_agents"`
+	OfflineAgents        int64 `json:"offline_agents"`
+	TotalAgents          int64 `json:"total_agents"`
+	PendingJobs          int64 `json:"pending_jobs"`
+	FailedJobs           int64 `json:"failed_jobs"`
+	CompleteJobs         int64 `json:"complete_jobs"`
+	// NotificationsDead mirrors service.DashboardSummary.NotificationsDead.
+	// JSON tag "notifications_dead" must match the service-layer struct
+	// exactly — this cross-package mirror avoids a direct import cycle and
+	// is driven by the I-005 Prometheus counter emission path. See
+	// GetPrometheusMetrics and MetricsCounter.NotificationsDeadTotal.
+	NotificationsDead int64     `json:"notifications_dead"`
+	CompletedAt       time.Time `json:"completed_at"`
 }

internal/api/handler/notification_handler_test.go

@@ -13,9 +13,11 @@ import (
 
 // MockNotificationService is a mock implementation of NotificationService interface.
 type MockNotificationService struct {
-	ListNotificationsFn func(page, perPage int) ([]domain.NotificationEvent, int64, error)
-	GetNotificationFn   func(id string) (*domain.NotificationEvent, error)
-	MarkAsReadFn        func(id string) error
+	ListNotificationsFn         func(page, perPage int) ([]domain.NotificationEvent, int64, error)
+	ListNotificationsByStatusFn func(status string, page, perPage int) ([]domain.NotificationEvent, int64, error)
+	GetNotificationFn           func(id string) (*domain.NotificationEvent, error)
+	MarkAsReadFn                func(id string) error
+	RequeueFn                   func(id string) error
 }
 
 func (m *MockNotificationService) ListNotifications(_ context.Context, page, perPage int) ([]domain.NotificationEvent, int64, error) {
@@ -25,6 +27,13 @@ func (m *MockNotificationService) ListNotifications(_ context.Context, page, per
 	return nil, 0, nil
 }
 
+func (m *MockNotificationService) ListNotificationsByStatus(_ context.Context, status string, page, perPage int) ([]domain.NotificationEvent, int64, error) {
+	if m.ListNotificationsByStatusFn != nil {
+		return m.ListNotificationsByStatusFn(status, page, perPage)
+	}
+	return nil, 0, nil
+}
+
 func (m *MockNotificationService) GetNotification(_ context.Context, id string) (*domain.NotificationEvent, error) {
 	if m.GetNotificationFn != nil {
 		return m.GetNotificationFn(id)
@@ -39,6 +48,13 @@ func (m *MockNotificationService) MarkAsRead(_ context.Context, id string) error
 	return nil
 }
 
+func (m *MockNotificationService) RequeueNotification(_ context.Context, id string) error {
+	if m.RequeueFn != nil {
+		return m.RequeueFn(id)
+	}
+	return nil
+}
+
 func TestListNotifications_Success(t *testing.T) {
 	now := time.Now()
 	certID := "mc-prod-001"
@@ -282,3 +298,224 @@ func TestMarkAsRead_EmptyID(t *testing.T) {
 		t.Fatalf("expected status 400, got %d", w.Code)
 	}
 }
+
+// ---------------------------------------------------------------------------
+// I-005: Notification Retry + Dead-Letter Queue handler contract (Phase 1 Red)
+//
+// These tests pin the HTTP surface Phase 2 Green must implement:
+//
+//   1. POST /api/v1/notifications/{id}/requeue — flips a dead notification
+//      back to 'pending' so the retry loop can pick it up again. The handler
+//      method does not exist yet (NotificationHandler has no RequeueNotification
+//      method) and the NotificationService interface does not declare
+//      RequeueNotification — both are compile-time Red halts.
+//
+//   2. GET /api/v1/notifications?status=dead — routes dead-letter list requests
+//      through ListNotificationsByStatus instead of ListNotifications. The
+//      status-filter routing does not exist yet, so ListNotificationsByStatusFn
+//      never fires — a runtime Red halt.
+// ---------------------------------------------------------------------------
+
+func TestRequeueNotification_Success(t *testing.T) {
+	var requeuedID string
+	mock := &MockNotificationService{
+		RequeueFn: func(id string) error {
+			requeuedID = id
+			return nil
+		},
+	}
+
+	handler := NewNotificationHandler(mock)
+	req := httptest.NewRequest(http.MethodPost, "/api/v1/notifications/notif-dead-001/requeue", nil)
+	req = req.WithContext(contextWithRequestID())
+	w := httptest.NewRecorder()
+
+	handler.RequeueNotification(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected status 200, got %d", w.Code)
+	}
+	if requeuedID != "notif-dead-001" {
+		t.Errorf("expected requeued ID 'notif-dead-001', got '%s'", requeuedID)
+	}
+
+	var resp map[string]string
+	if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
+		t.Fatalf("failed to decode response: %v", err)
+	}
+	if resp["status"] != "requeued" {
+		t.Errorf("expected status 'requeued', got '%s'", resp["status"])
+	}
+}
+
+func TestRequeueNotification_NotFound(t *testing.T) {
+	mock := &MockNotificationService{
+		RequeueFn: func(id string) error {
+			return ErrMockNotFound
+		},
+	}
+
+	handler := NewNotificationHandler(mock)
+	req := httptest.NewRequest(http.MethodPost, "/api/v1/notifications/nonexistent/requeue", nil)
+	req = req.WithContext(contextWithRequestID())
+	w := httptest.NewRecorder()
+
+	handler.RequeueNotification(w, req)
+
+	if w.Code != http.StatusNotFound {
+		t.Fatalf("expected status 404, got %d", w.Code)
+	}
+}
+
+func TestRequeueNotification_ServiceError(t *testing.T) {
+	mock := &MockNotificationService{
+		RequeueFn: func(id string) error {
+			return ErrMockServiceFailed
+		},
+	}
+
+	handler := NewNotificationHandler(mock)
+	req := httptest.NewRequest(http.MethodPost, "/api/v1/notifications/notif-dead-001/requeue", nil)
+	req = req.WithContext(contextWithRequestID())
+	w := httptest.NewRecorder()
+
+	handler.RequeueNotification(w, req)
+
+	if w.Code != http.StatusInternalServerError {
+		t.Fatalf("expected status 500, got %d", w.Code)
+	}
+}
+
+func TestRequeueNotification_MethodNotAllowed(t *testing.T) {
+	handler := NewNotificationHandler(&MockNotificationService{})
+	req := httptest.NewRequest(http.MethodGet, "/api/v1/notifications/notif-dead-001/requeue", nil)
+	w := httptest.NewRecorder()
+
+	handler.RequeueNotification(w, req)
+
+	if w.Code != http.StatusMethodNotAllowed {
+		t.Fatalf("expected status 405, got %d", w.Code)
+	}
+}
+
+func TestRequeueNotification_EmptyID(t *testing.T) {
+	handler := NewNotificationHandler(&MockNotificationService{})
+	req := httptest.NewRequest(http.MethodPost, "/api/v1/notifications//requeue", nil)
+	req = req.WithContext(contextWithRequestID())
+	w := httptest.NewRecorder()
+
+	handler.RequeueNotification(w, req)
+
+	if w.Code != http.StatusBadRequest {
+		t.Fatalf("expected status 400, got %d", w.Code)
+	}
+}
+
+func TestListNotifications_StatusFilter_Dead(t *testing.T) {
+	now := time.Now()
+	certID := "mc-prod-001"
+	lastErr := "SMTP connection refused"
+	nextRetry := now.Add(1 * time.Minute)
+	dead := domain.NotificationEvent{
+		ID:            "notif-dead-001",
+		Type:          domain.NotificationTypeExpirationWarning,
+		CertificateID: &certID,
+		Channel:       domain.NotificationChannelEmail,
+		Recipient:     "admin@example.com",
+		Message:       "Certificate expiring in 7 days",
+		Status:        "dead",
+		CreatedAt:     now,
+		RetryCount:    5,
+		NextRetryAt:   &nextRetry,
+		LastError:     &lastErr,
+	}
+
+	var capturedStatus string
+	var capturedPage, capturedPerPage int
+	byStatusCalled := false
+	listCalled := false
+
+	mock := &MockNotificationService{
+		ListNotificationsFn: func(page, perPage int) ([]domain.NotificationEvent, int64, error) {
+			listCalled = true
+			return nil, 0, nil
+		},
+		ListNotificationsByStatusFn: func(status string, page, perPage int) ([]domain.NotificationEvent, int64, error) {
+			byStatusCalled = true
+			capturedStatus = status
+			capturedPage = page
+			capturedPerPage = perPage
+			return []domain.NotificationEvent{dead}, 1, nil
+		},
+	}
+
+	handler := NewNotificationHandler(mock)
+	req := httptest.NewRequest(http.MethodGet, "/api/v1/notifications?status=dead&page=1&per_page=50", nil)
+	req = req.WithContext(contextWithRequestID())
+	w := httptest.NewRecorder()
+
+	handler.ListNotifications(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected status 200, got %d", w.Code)
+	}
+	if !byStatusCalled {
+		t.Fatalf("expected ListNotificationsByStatus to be called for ?status=dead, but it was not")
+	}
+	if listCalled {
+		t.Errorf("ListNotifications should not be called when status filter is present")
+	}
+	if capturedStatus != "dead" {
+		t.Errorf("expected status='dead', got '%s'", capturedStatus)
+	}
+	if capturedPage != 1 {
+		t.Errorf("expected page=1, got %d", capturedPage)
+	}
+	if capturedPerPage != 50 {
+		t.Errorf("expected per_page=50, got %d", capturedPerPage)
+	}
+
+	var resp PagedResponse
+	if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
+		t.Fatalf("failed to decode response: %v", err)
+	}
+	if resp.Total != 1 {
+		t.Errorf("expected total=1 dead notification, got %d", resp.Total)
+	}
+}
+
+func TestListNotifications_NoStatusFilter_CallsDefault(t *testing.T) {
+	// Pin the inverse: when no ?status= is provided, the handler must call the
+	// existing ListNotifications path (not ListNotificationsByStatus). Phase 2
+	// Green must not break the default listing behavior for the plain tab.
+	listCalled := false
+	byStatusCalled := false
+
+	mock := &MockNotificationService{
+		ListNotificationsFn: func(page, perPage int) ([]domain.NotificationEvent, int64, error) {
+			listCalled = true
+			return []domain.NotificationEvent{}, 0, nil
+		},
+		ListNotificationsByStatusFn: func(status string, page, perPage int) ([]domain.NotificationEvent, int64, error) {
+			byStatusCalled = true
+			return nil, 0, nil
+		},
+	}
+
+	handler := NewNotificationHandler(mock)
+	req := httptest.NewRequest(http.MethodGet, "/api/v1/notifications", nil)
+	req = req.WithContext(contextWithRequestID())
+	w := httptest.NewRecorder()
+
+	handler.ListNotifications(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected status 200, got %d", w.Code)
+	}
+	if !listCalled {
+		t.Errorf("expected ListNotifications to be called when no status filter is present")
+	}
+	if byStatusCalled {
+		t.Errorf("ListNotificationsByStatus should not be called when no status filter is present")
+	}
+}

internal/api/handler/notifications.go

@@ -11,10 +11,17 @@ import (
 )
 
 // NotificationService defines the service interface for notification operations.
+//
+// ListNotificationsByStatus and RequeueNotification were added to close coverage
+// gap I-005: the Dead letter tab on the GUI (?status=dead) needs a scoped
+// listing path, and the Requeue action needs a dedicated endpoint that flips a
+// dead notification back to 'pending' so the retry sweep can pick it up again.
 type NotificationService interface {
 	ListNotifications(ctx context.Context, page, perPage int) ([]domain.NotificationEvent, int64, error)
+	ListNotificationsByStatus(ctx context.Context, status string, page, perPage int) ([]domain.NotificationEvent, int64, error)
 	GetNotification(ctx context.Context, id string) (*domain.NotificationEvent, error)
 	MarkAsRead(ctx context.Context, id string) error
+	RequeueNotification(ctx context.Context, id string) error
 }
 
 // NotificationHandler handles HTTP requests for notification operations.
@@ -51,7 +58,20 @@ func (h NotificationHandler) ListNotifications(w http.ResponseWriter, r *http.Re
 		}
 	}
 
-	notifications, total, err := h.svc.ListNotifications(r.Context(), page, perPage)
+	// I-005: branch to the status-scoped listing path when ?status= is present
+	// so the Dead letter tab on the GUI (?status=dead) can filter server-side.
+	// Empty status delegates to the original ListNotifications path to preserve
+	// the default tab's existing behavior.
+	var (
+		notifications []domain.NotificationEvent
+		total         int64
+		err           error
+	)
+	if status := query.Get("status"); status != "" {
+		notifications, total, err = h.svc.ListNotificationsByStatus(r.Context(), status, page, perPage)
+	} else {
+		notifications, total, err = h.svc.ListNotifications(r.Context(), page, perPage)
+	}
 	if err != nil {
 		ErrorWithRequestID(w, http.StatusInternalServerError, "Failed to list notifications", requestID)
 		return
@@ -124,3 +144,43 @@ func (h NotificationHandler) MarkAsRead(w http.ResponseWriter, r *http.Request)
 
 	JSON(w, http.StatusOK, response)
 }
+
+// RequeueNotification flips a dead notification back to 'pending' so the retry
+// sweep (coverage gap I-005) can pick it up again on its next tick. The handler
+// is strictly POST-only; GET/PUT/DELETE return 405. An empty id segment
+// (/api/v1/notifications//requeue) returns 400. Service errors that carry a
+// "not found" sentinel map to 404; all other service errors map to 500. This
+// 404-vs-500 split mirrors GetCertificateDeployments at certificates.go:644.
+// POST /api/v1/notifications/{id}/requeue
+func (h NotificationHandler) RequeueNotification(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		Error(w, http.StatusMethodNotAllowed, "Method not allowed")
+		return
+	}
+
+	requestID := middleware.GetRequestID(r.Context())
+
+	// Extract notification ID from path /api/v1/notifications/{id}/requeue
+	path := strings.TrimPrefix(r.URL.Path, "/api/v1/notifications/")
+	parts := strings.Split(path, "/")
+	if len(parts) < 2 || parts[0] == "" {
+		ErrorWithRequestID(w, http.StatusBadRequest, "Notification ID is required", requestID)
+		return
+	}
+	notificationID := parts[0]
+
+	if err := h.svc.RequeueNotification(r.Context(), notificationID); err != nil {
+		if strings.Contains(err.Error(), "not found") {
+			ErrorWithRequestID(w, http.StatusNotFound, "Notification not found", requestID)
+			return
+		}
+		ErrorWithRequestID(w, http.StatusInternalServerError, "Failed to requeue notification", requestID)
+		return
+	}
+
+	response := map[string]string{
+		"status": "requeued",
+	}
+
+	JSON(w, http.StatusOK, response)
+}