mirror of
https://github.com/shankar0123/certctl.git
synced 2026-06-07 20:41:30 +00:00
675b87ba63
Critical alerts can no longer be silently dropped by a transient
notifier failure. Failed notification attempts now ride an exponential
backoff retry loop, with a 5-attempt budget before promotion to the
dead-letter queue for operator intervention.
Schema (migration 000016, idempotent):
- retry_count INTEGER NOT NULL DEFAULT 0
- next_retry_at TIMESTAMPTZ
- last_error TEXT
- idx_notification_events_retry_sweep partial index
(next_retry_at) WHERE status='failed' AND next_retry_at IS NOT NULL
Dead rows clear next_retry_at so the index stops matching them.
Service contract:
- NotificationService.RetryFailedNotifications drives 2^n-minute
exponential backoff capped at 1h (notifRetryBackoffCap) with
5-attempt budget (notifRetryMaxAttempts).
- Exhaustion (RetryCount >= notifRetryMaxAttempts-1) promotes to
status='dead' via MarkAsDead.
- Non-terminal failures record via RecordFailedAttempt.
- Success path promotes to 'sent' without touching retry_count
(audit preserves "delivered on attempt N").
- Missing-notifier branch defensively promotes to 'sent' to avoid
wedging a row on a deleted channel.
- RequeueNotification operator escape hatch atomically resets
retry_count -> 0, next_retry_at -> NULL, last_error -> NULL,
status -> pending via notifRepo.Requeue.
Scheduler:
- New always-on notificationRetryLoop wired into the base loop set at
CERTCTL_NOTIFICATION_RETRY_INTERVAL (default 2m).
- sync/atomic.Bool idempotency guard.
- sync.WaitGroup shutdown drain via WaitForCompletion.
StatsService:
- SetNotifRepo setter pattern preserves 9 pre-existing
NewStatsService call sites (main.go + stats_test.go + 8 digest
tests) without touching the constructor signature.
- DashboardSummary.NotificationsDead populated via
notifRepo.CountByStatus(ctx, "dead") — nil-safe when unwired
(reports zero on systems without a notification repository).
- CountByStatus error is non-fatal (dashboard summary is
best-effort for this field).
- Prometheus certctl_notification_dead_total counter emitted from
the same snapshot.
Handler:
- New POST /api/v1/notifications/{id}/requeue endpoint.
- dead status surfaces to MCP + CLI.
Frontend:
- NotificationsPage gains two-tab toolbar ("All" / "Dead letter")
with queryKey: ['notifications', activeTab] so switching tabs
doesn't serve stale data until the 30s refetch.
- Dead rows surface "Retry {n}/5" + truncated last_error with
full-text title tooltip.
- Requeue mutation wrapped as
mutationFn: (id: string) => requeueNotification(id)
to prevent react-query v5's positional context argument from
leaking into the API client — pinned against future refactors
by strict-match toHaveBeenCalledWith('notif-dead-001') in
NotificationsPage.test.tsx:181.
Closes I-005.
187 lines
6.1 KiB
Go
187 lines
6.1 KiB
Go
package handler
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/shankar0123/certctl/internal/api/middleware"
|
|
"github.com/shankar0123/certctl/internal/domain"
|
|
)
|
|
|
|
// NotificationService defines the service interface for notification operations.
|
|
//
|
|
// ListNotificationsByStatus and RequeueNotification were added to close coverage
|
|
// gap I-005: the Dead letter tab on the GUI (?status=dead) needs a scoped
|
|
// listing path, and the Requeue action needs a dedicated endpoint that flips a
|
|
// dead notification back to 'pending' so the retry sweep can pick it up again.
|
|
type NotificationService interface {
|
|
ListNotifications(ctx context.Context, page, perPage int) ([]domain.NotificationEvent, int64, error)
|
|
ListNotificationsByStatus(ctx context.Context, status string, page, perPage int) ([]domain.NotificationEvent, int64, error)
|
|
GetNotification(ctx context.Context, id string) (*domain.NotificationEvent, error)
|
|
MarkAsRead(ctx context.Context, id string) error
|
|
RequeueNotification(ctx context.Context, id string) error
|
|
}
|
|
|
|
// NotificationHandler handles HTTP requests for notification operations.
|
|
type NotificationHandler struct {
|
|
svc NotificationService
|
|
}
|
|
|
|
// NewNotificationHandler creates a new NotificationHandler with a service dependency.
|
|
func NewNotificationHandler(svc NotificationService) NotificationHandler {
|
|
return NotificationHandler{svc: svc}
|
|
}
|
|
|
|
// ListNotifications lists notifications.
|
|
// GET /api/v1/notifications?page=1&per_page=50
|
|
func (h NotificationHandler) ListNotifications(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
Error(w, http.StatusMethodNotAllowed, "Method not allowed")
|
|
return
|
|
}
|
|
|
|
requestID := middleware.GetRequestID(r.Context())
|
|
|
|
page := 1
|
|
perPage := 50
|
|
query := r.URL.Query()
|
|
if p := query.Get("page"); p != "" {
|
|
if parsed, err := strconv.Atoi(p); err == nil && parsed > 0 {
|
|
page = parsed
|
|
}
|
|
}
|
|
if pp := query.Get("per_page"); pp != "" {
|
|
if parsed, err := strconv.Atoi(pp); err == nil && parsed > 0 && parsed <= 500 {
|
|
perPage = parsed
|
|
}
|
|
}
|
|
|
|
// I-005: branch to the status-scoped listing path when ?status= is present
|
|
// so the Dead letter tab on the GUI (?status=dead) can filter server-side.
|
|
// Empty status delegates to the original ListNotifications path to preserve
|
|
// the default tab's existing behavior.
|
|
var (
|
|
notifications []domain.NotificationEvent
|
|
total int64
|
|
err error
|
|
)
|
|
if status := query.Get("status"); status != "" {
|
|
notifications, total, err = h.svc.ListNotificationsByStatus(r.Context(), status, page, perPage)
|
|
} else {
|
|
notifications, total, err = h.svc.ListNotifications(r.Context(), page, perPage)
|
|
}
|
|
if err != nil {
|
|
ErrorWithRequestID(w, http.StatusInternalServerError, "Failed to list notifications", requestID)
|
|
return
|
|
}
|
|
|
|
response := PagedResponse{
|
|
Data: notifications,
|
|
Total: total,
|
|
Page: page,
|
|
PerPage: perPage,
|
|
}
|
|
|
|
JSON(w, http.StatusOK, response)
|
|
}
|
|
|
|
// GetNotification retrieves a single notification by ID.
|
|
// GET /api/v1/notifications/{id}
|
|
func (h NotificationHandler) GetNotification(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
Error(w, http.StatusMethodNotAllowed, "Method not allowed")
|
|
return
|
|
}
|
|
|
|
requestID := middleware.GetRequestID(r.Context())
|
|
|
|
id := strings.TrimPrefix(r.URL.Path, "/api/v1/notifications/")
|
|
parts := strings.Split(id, "/")
|
|
if len(parts) == 0 || parts[0] == "" {
|
|
ErrorWithRequestID(w, http.StatusBadRequest, "Notification ID is required", requestID)
|
|
return
|
|
}
|
|
id = parts[0]
|
|
|
|
notification, err := h.svc.GetNotification(r.Context(), id)
|
|
if err != nil {
|
|
ErrorWithRequestID(w, http.StatusNotFound, "Notification not found", requestID)
|
|
return
|
|
}
|
|
|
|
JSON(w, http.StatusOK, notification)
|
|
}
|
|
|
|
// MarkAsRead marks a notification as read.
|
|
// POST /api/v1/notifications/{id}/read
|
|
func (h NotificationHandler) MarkAsRead(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
Error(w, http.StatusMethodNotAllowed, "Method not allowed")
|
|
return
|
|
}
|
|
|
|
requestID := middleware.GetRequestID(r.Context())
|
|
|
|
// Extract notification ID from path /api/v1/notifications/{id}/read
|
|
path := strings.TrimPrefix(r.URL.Path, "/api/v1/notifications/")
|
|
parts := strings.Split(path, "/")
|
|
if len(parts) < 2 || parts[0] == "" {
|
|
ErrorWithRequestID(w, http.StatusBadRequest, "Notification ID is required", requestID)
|
|
return
|
|
}
|
|
notificationID := parts[0]
|
|
|
|
if err := h.svc.MarkAsRead(r.Context(), notificationID); err != nil {
|
|
ErrorWithRequestID(w, http.StatusInternalServerError, "Failed to mark notification as read", requestID)
|
|
return
|
|
}
|
|
|
|
response := map[string]string{
|
|
"status": "marked_as_read",
|
|
}
|
|
|
|
JSON(w, http.StatusOK, response)
|
|
}
|
|
|
|
// RequeueNotification flips a dead notification back to 'pending' so the retry
|
|
// sweep (coverage gap I-005) can pick it up again on its next tick. The handler
|
|
// is strictly POST-only; GET/PUT/DELETE return 405. An empty id segment
|
|
// (/api/v1/notifications//requeue) returns 400. Service errors that carry a
|
|
// "not found" sentinel map to 404; all other service errors map to 500. This
|
|
// 404-vs-500 split mirrors GetCertificateDeployments at certificates.go:644.
|
|
// POST /api/v1/notifications/{id}/requeue
|
|
func (h NotificationHandler) RequeueNotification(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
Error(w, http.StatusMethodNotAllowed, "Method not allowed")
|
|
return
|
|
}
|
|
|
|
requestID := middleware.GetRequestID(r.Context())
|
|
|
|
// Extract notification ID from path /api/v1/notifications/{id}/requeue
|
|
path := strings.TrimPrefix(r.URL.Path, "/api/v1/notifications/")
|
|
parts := strings.Split(path, "/")
|
|
if len(parts) < 2 || parts[0] == "" {
|
|
ErrorWithRequestID(w, http.StatusBadRequest, "Notification ID is required", requestID)
|
|
return
|
|
}
|
|
notificationID := parts[0]
|
|
|
|
if err := h.svc.RequeueNotification(r.Context(), notificationID); err != nil {
|
|
if strings.Contains(err.Error(), "not found") {
|
|
ErrorWithRequestID(w, http.StatusNotFound, "Notification not found", requestID)
|
|
return
|
|
}
|
|
ErrorWithRequestID(w, http.StatusInternalServerError, "Failed to requeue notification", requestID)
|
|
return
|
|
}
|
|
|
|
response := map[string]string{
|
|
"status": "requeued",
|
|
}
|
|
|
|
JSON(w, http.StatusOK, response)
|
|
}
|