mirror of
https://github.com/shankar0123/certctl.git
synced 2026-06-07 15:51:30 +00:00
I-005: notification retry loop + dead-letter queue
Critical alerts can no longer be silently dropped by a transient
notifier failure. Failed notification attempts now ride an exponential
backoff retry loop, with a 5-attempt budget before promotion to the
dead-letter queue for operator intervention.
Schema (migration 000016, idempotent):
- retry_count INTEGER NOT NULL DEFAULT 0
- next_retry_at TIMESTAMPTZ
- last_error TEXT
- idx_notification_events_retry_sweep partial index
(next_retry_at) WHERE status='failed' AND next_retry_at IS NOT NULL
Dead rows clear next_retry_at so the index stops matching them.
Service contract:
- NotificationService.RetryFailedNotifications drives 2^n-minute
exponential backoff capped at 1h (notifRetryBackoffCap) with
5-attempt budget (notifRetryMaxAttempts).
- Exhaustion (RetryCount >= notifRetryMaxAttempts-1) promotes to
status='dead' via MarkAsDead.
- Non-terminal failures record via RecordFailedAttempt.
- Success path promotes to 'sent' without touching retry_count
(audit preserves "delivered on attempt N").
- Missing-notifier branch defensively promotes to 'sent' to avoid
wedging a row on a deleted channel.
- RequeueNotification operator escape hatch atomically resets
retry_count -> 0, next_retry_at -> NULL, last_error -> NULL,
status -> pending via notifRepo.Requeue.
Scheduler:
- New always-on notificationRetryLoop wired into the base loop set at
CERTCTL_NOTIFICATION_RETRY_INTERVAL (default 2m).
- sync/atomic.Bool idempotency guard.
- sync.WaitGroup shutdown drain via WaitForCompletion.
StatsService:
- SetNotifRepo setter pattern preserves 9 pre-existing
NewStatsService call sites (main.go + stats_test.go + 8 digest
tests) without touching the constructor signature.
- DashboardSummary.NotificationsDead populated via
notifRepo.CountByStatus(ctx, "dead") — nil-safe when unwired
(reports zero on systems without a notification repository).
- CountByStatus error is non-fatal (dashboard summary is
best-effort for this field).
- Prometheus certctl_notification_dead_total counter emitted from
the same snapshot.
Handler:
- New POST /api/v1/notifications/{id}/requeue endpoint.
- dead status surfaces to MCP + CLI.
Frontend:
- NotificationsPage gains two-tab toolbar ("All" / "Dead letter")
with queryKey: ['notifications', activeTab] so switching tabs
doesn't serve stale data until the 30s refetch.
- Dead rows surface "Retry {n}/5" + truncated last_error with
full-text title tooltip.
- Requeue mutation wrapped as
mutationFn: (id: string) => requeueNotification(id)
to prevent react-query v5's positional context argument from
leaking into the API client — pinned against future refactors
by strict-match toHaveBeenCalledWith('notif-dead-001') in
NotificationsPage.test.tsx:181.
Closes I-005.
This commit is contained in:
@@ -103,25 +103,25 @@ func TestCertificateLifecycle(t *testing.T) {
|
||||
// Create router and register handlers
|
||||
r := router.New()
|
||||
r.RegisterHandlers(router.HandlerRegistry{
|
||||
Certificates: certificateHandler,
|
||||
Issuers: issuerHandler,
|
||||
Targets: targetHandler,
|
||||
Agents: agentHandler,
|
||||
Jobs: jobHandler,
|
||||
Policies: policyHandler,
|
||||
Profiles: profileHandler,
|
||||
Teams: teamHandler,
|
||||
Owners: ownerHandler,
|
||||
AgentGroups: agentGroupHandler,
|
||||
Audit: auditHandler,
|
||||
Notifications: notificationHandler,
|
||||
Stats: statsHandler,
|
||||
Metrics: metricsHandler,
|
||||
Health: healthHandler,
|
||||
Discovery: discoveryHandler,
|
||||
NetworkScan: networkScanHandler,
|
||||
Verification: verificationHandler,
|
||||
BulkRevocation: handler.BulkRevocationHandler{},
|
||||
Certificates: certificateHandler,
|
||||
Issuers: issuerHandler,
|
||||
Targets: targetHandler,
|
||||
Agents: agentHandler,
|
||||
Jobs: jobHandler,
|
||||
Policies: policyHandler,
|
||||
Profiles: profileHandler,
|
||||
Teams: teamHandler,
|
||||
Owners: ownerHandler,
|
||||
AgentGroups: agentGroupHandler,
|
||||
Audit: auditHandler,
|
||||
Notifications: notificationHandler,
|
||||
Stats: statsHandler,
|
||||
Metrics: metricsHandler,
|
||||
Health: healthHandler,
|
||||
Discovery: discoveryHandler,
|
||||
NetworkScan: networkScanHandler,
|
||||
Verification: verificationHandler,
|
||||
BulkRevocation: handler.BulkRevocationHandler{},
|
||||
})
|
||||
r.RegisterESTHandlers(estHandler)
|
||||
|
||||
@@ -1022,6 +1022,46 @@ func (m *mockNotificationRepository) UpdateStatus(ctx context.Context, id string
|
||||
return fmt.Errorf("notification not found")
|
||||
}
|
||||
|
||||
// I-005: retry/DLQ interface satisfiers. The integration tests in this package
|
||||
// drive the end-to-end lifecycle against a NotificationService which requires
|
||||
// the full repository.NotificationRepository interface, but none of the
|
||||
// lifecycle scenarios exercise the retry sweep or dead-letter transitions —
|
||||
// they're covered by unit tests in internal/service/notification_test.go. So
|
||||
// these are deliberate no-op / panic-free stubs whose only job is to satisfy
|
||||
// the compile-time interface contract. If a future integration test needs
|
||||
// real retry semantics, promote this mock to match internal/service's
|
||||
// mockNotifRepo (testutil_test.go:410) one-for-one.
|
||||
|
||||
func (m *mockNotificationRepository) ListRetryEligible(ctx context.Context, now time.Time, maxAttempts, limit int) ([]*domain.NotificationEvent, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockNotificationRepository) RecordFailedAttempt(ctx context.Context, id string, lastError string, nextRetryAt time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockNotificationRepository) MarkAsDead(ctx context.Context, id string, lastError string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockNotificationRepository) Requeue(ctx context.Context, id string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// CountByStatus satisfies the NotificationRepository interface contract added
|
||||
// by I-005 Phase 2 Green. Counts in-memory rows so StatsService wiring exercised
|
||||
// by the lifecycle integration tests gets a truthful count even though the
|
||||
// retry/DLQ surface isn't driven here.
|
||||
func (m *mockNotificationRepository) CountByStatus(ctx context.Context, status string) (int64, error) {
|
||||
var count int64
|
||||
for _, n := range m.notifications {
|
||||
if n.Status == status {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
type mockPolicyRepository struct {
|
||||
rules map[string]*domain.PolicyRule
|
||||
violations []*domain.PolicyViolation
|
||||
|
||||
Reference in New Issue
Block a user