mirror of
https://github.com/shankar0123/certctl.git
synced 2026-06-09 12:08:56 +00:00
15daf008aa
Critical alerts can no longer be silently dropped by a transient
notifier failure. Failed notification attempts now ride an exponential
backoff retry loop, with a 5-attempt budget before promotion to the
dead-letter queue for operator intervention.
Schema (migration 000016, idempotent):
- retry_count INTEGER NOT NULL DEFAULT 0
- next_retry_at TIMESTAMPTZ
- last_error TEXT
- idx_notification_events_retry_sweep partial index
(next_retry_at) WHERE status='failed' AND next_retry_at IS NOT NULL
Dead rows clear next_retry_at so the index stops matching them.
Service contract:
- NotificationService.RetryFailedNotifications drives 2^n-minute
exponential backoff capped at 1h (notifRetryBackoffCap) with
5-attempt budget (notifRetryMaxAttempts).
- Exhaustion (RetryCount >= notifRetryMaxAttempts-1) promotes to
status='dead' via MarkAsDead.
- Non-terminal failures record via RecordFailedAttempt.
- Success path promotes to 'sent' without touching retry_count
(audit preserves "delivered on attempt N").
- Missing-notifier branch defensively promotes to 'sent' to avoid
wedging a row on a deleted channel.
- RequeueNotification operator escape hatch atomically resets
retry_count -> 0, next_retry_at -> NULL, last_error -> NULL,
status -> pending via notifRepo.Requeue.
Scheduler:
- New always-on notificationRetryLoop wired into the base loop set at
CERTCTL_NOTIFICATION_RETRY_INTERVAL (default 2m).
- sync/atomic.Bool idempotency guard.
- sync.WaitGroup shutdown drain via WaitForCompletion.
StatsService:
- SetNotifRepo setter pattern preserves 9 pre-existing
NewStatsService call sites (main.go + stats_test.go + 8 digest
tests) without touching the constructor signature.
- DashboardSummary.NotificationsDead populated via
notifRepo.CountByStatus(ctx, "dead") — nil-safe when unwired
(reports zero on systems without a notification repository).
- CountByStatus error is non-fatal (dashboard summary is
best-effort for this field).
- Prometheus certctl_notification_dead_total counter emitted from
the same snapshot.
Handler:
- New POST /api/v1/notifications/{id}/requeue endpoint.
- dead status surfaces to MCP + CLI.
Frontend:
- NotificationsPage gains two-tab toolbar ("All" / "Dead letter")
with queryKey: ['notifications', activeTab] so switching tabs
doesn't serve stale data until the 30s refetch.
- Dead rows surface "Retry {n}/5" + truncated last_error with
full-text title tooltip.
- Requeue mutation wrapped as
mutationFn: (id: string) => requeueNotification(id)
to prevent react-query v5's positional context argument from
leaking into the API client — pinned against future refactors
by strict-match toHaveBeenCalledWith('notif-dead-001') in
NotificationsPage.test.tsx:181.
Closes I-005.
209 lines
7.1 KiB
TypeScript
209 lines
7.1 KiB
TypeScript
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
import { render, screen, waitFor, fireEvent, cleanup } from '@testing-library/react';
|
|
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
|
|
import { MemoryRouter } from 'react-router-dom';
|
|
import type { ReactNode } from 'react';
|
|
|
|
// -----------------------------------------------------------------------------
|
|
// I-005: NotificationsPage Phase 1 Red — Dead Letter tab + Requeue action
|
|
//
|
|
// This file pins the frontend contract Phase 2 Green must implement:
|
|
//
|
|
// 1. A "Dead letter" tab renders alongside the existing status filter, and
|
|
// selecting it causes the underlying query to fetch with { status: 'dead' }.
|
|
// The tab does not exist at HEAD — the tab-locator assertions are the Red.
|
|
//
|
|
// 2. Notifications in status='dead' render a "Requeue" action button. HEAD
|
|
// only renders "Mark read" for Pending rows and no action for anything
|
|
// else — the button-locator assertion is the Red.
|
|
//
|
|
// 3. Clicking "Requeue" invokes requeueNotification(id) from the API client
|
|
// and invalidates the notifications query. `requeueNotification` does not
|
|
// yet exist as an export from ../api/client — tsc --noEmit will fail with
|
|
// "Property 'requeueNotification' does not exist" when Phase 2 Green runs
|
|
// its verification gates, which is the compile-time Red halt. This file is
|
|
// structured so Phase 2 Green's single fix (add the client export + page
|
|
// wiring) flips the entire suite Green at once.
|
|
// -----------------------------------------------------------------------------
|
|
|
|
vi.mock('../api/client', () => ({
|
|
getNotifications: vi.fn(),
|
|
getNotification: vi.fn(),
|
|
markNotificationRead: vi.fn(),
|
|
requeueNotification: vi.fn(),
|
|
}));
|
|
|
|
// Imported after vi.mock so the mock replaces the real module.
|
|
import NotificationsPage from './NotificationsPage';
|
|
import * as client from '../api/client';
|
|
|
|
function renderWithQuery(ui: ReactNode) {
|
|
const qc = new QueryClient({
|
|
defaultOptions: {
|
|
queries: { retry: false, gcTime: 0, staleTime: 0 },
|
|
},
|
|
});
|
|
return render(
|
|
<QueryClientProvider client={qc}>
|
|
<MemoryRouter>{ui}</MemoryRouter>
|
|
</QueryClientProvider>,
|
|
);
|
|
}
|
|
|
|
const pendingNotif = {
|
|
id: 'notif-001',
|
|
type: 'ExpirationWarning',
|
|
channel: 'Email',
|
|
recipient: 'admin@example.com',
|
|
subject: 'Certificate expiring',
|
|
message: 'Certificate expiring in 7 days',
|
|
status: 'Pending',
|
|
certificate_id: 'mc-prod-001',
|
|
created_at: new Date().toISOString(),
|
|
};
|
|
|
|
const deadNotif = {
|
|
id: 'notif-dead-001',
|
|
type: 'ExpirationWarning',
|
|
channel: 'Email',
|
|
recipient: 'admin@example.com',
|
|
subject: 'Certificate expiring',
|
|
message: 'Certificate expiring in 7 days',
|
|
status: 'dead',
|
|
certificate_id: 'mc-prod-001',
|
|
created_at: new Date().toISOString(),
|
|
retry_count: 5,
|
|
last_error: 'SMTP connection refused',
|
|
};
|
|
|
|
describe('NotificationsPage — I-005 Dead Letter + Requeue (Phase 1 Red)', () => {
|
|
beforeEach(() => {
|
|
vi.clearAllMocks();
|
|
cleanup();
|
|
});
|
|
|
|
it('renders a Dead letter tab in the filter toolbar', async () => {
|
|
vi.mocked(client.getNotifications).mockResolvedValue({
|
|
data: [pendingNotif],
|
|
total: 1,
|
|
page: 1,
|
|
per_page: 100,
|
|
});
|
|
|
|
renderWithQuery(<NotificationsPage />);
|
|
|
|
await waitFor(() => {
|
|
expect(screen.queryByText(/Loading/i)).not.toBeInTheDocument();
|
|
});
|
|
|
|
// Red: no Dead letter tab exists at HEAD. Phase 2 Green adds a button/tab
|
|
// labeled "Dead letter" (matches docs/testing-guide UI label).
|
|
expect(screen.getByRole('button', { name: /Dead letter/i })).toBeInTheDocument();
|
|
});
|
|
|
|
it('clicking Dead letter tab fetches notifications with status=dead', async () => {
|
|
vi.mocked(client.getNotifications).mockResolvedValue({
|
|
data: [],
|
|
total: 0,
|
|
page: 1,
|
|
per_page: 100,
|
|
});
|
|
|
|
renderWithQuery(<NotificationsPage />);
|
|
|
|
await waitFor(() => {
|
|
expect(screen.queryByText(/Loading/i)).not.toBeInTheDocument();
|
|
});
|
|
|
|
const tab = screen.getByRole('button', { name: /Dead letter/i });
|
|
fireEvent.click(tab);
|
|
|
|
// Red: Phase 2 Green must route the Dead letter tab's query through
|
|
// getNotifications({ status: 'dead', per_page: '100' }). HEAD only ever
|
|
// calls getNotifications({ per_page: '100' }) — no status param is ever
|
|
// passed through.
|
|
await waitFor(() => {
|
|
const calls = vi.mocked(client.getNotifications).mock.calls;
|
|
const deadCall = calls.find(([params]) => (params as Record<string, string>)?.status === 'dead');
|
|
expect(deadCall, 'expected getNotifications to be called with status=dead').toBeTruthy();
|
|
});
|
|
});
|
|
|
|
it('renders a Requeue button on dead notifications', async () => {
|
|
vi.mocked(client.getNotifications).mockResolvedValue({
|
|
data: [deadNotif],
|
|
total: 1,
|
|
page: 1,
|
|
per_page: 100,
|
|
});
|
|
|
|
renderWithQuery(<NotificationsPage />);
|
|
|
|
await waitFor(() => {
|
|
expect(screen.queryByText(/Loading/i)).not.toBeInTheDocument();
|
|
});
|
|
|
|
// Switch to Dead letter tab so the mocked dead notification becomes visible.
|
|
const tab = screen.getByRole('button', { name: /Dead letter/i });
|
|
fireEvent.click(tab);
|
|
|
|
await waitFor(() => {
|
|
// Red: HEAD renders no action for status='dead'. Phase 2 Green adds a
|
|
// "Requeue" button next to each dead row.
|
|
expect(screen.getByRole('button', { name: /Requeue/i })).toBeInTheDocument();
|
|
});
|
|
});
|
|
|
|
it('clicking Requeue invokes requeueNotification(id) from the API client', async () => {
|
|
vi.mocked(client.getNotifications).mockResolvedValue({
|
|
data: [deadNotif],
|
|
total: 1,
|
|
page: 1,
|
|
per_page: 100,
|
|
});
|
|
vi.mocked(client.requeueNotification).mockResolvedValue({ status: 'requeued' });
|
|
|
|
renderWithQuery(<NotificationsPage />);
|
|
|
|
await waitFor(() => {
|
|
expect(screen.queryByText(/Loading/i)).not.toBeInTheDocument();
|
|
});
|
|
|
|
fireEvent.click(screen.getByRole('button', { name: /Dead letter/i }));
|
|
|
|
const requeueBtn = await screen.findByRole('button', { name: /Requeue/i });
|
|
fireEvent.click(requeueBtn);
|
|
|
|
// Red: client.requeueNotification is not an exported function at HEAD, and
|
|
// the page does not call it. Both the mock and the page wiring are added
|
|
// in Phase 2 Green.
|
|
await waitFor(() => {
|
|
expect(client.requeueNotification).toHaveBeenCalledWith('notif-dead-001');
|
|
});
|
|
});
|
|
|
|
it('dead notifications surface retry_count and last_error metadata', async () => {
|
|
vi.mocked(client.getNotifications).mockResolvedValue({
|
|
data: [deadNotif],
|
|
total: 1,
|
|
page: 1,
|
|
per_page: 100,
|
|
});
|
|
|
|
renderWithQuery(<NotificationsPage />);
|
|
|
|
await waitFor(() => {
|
|
expect(screen.queryByText(/Loading/i)).not.toBeInTheDocument();
|
|
});
|
|
|
|
fireEvent.click(screen.getByRole('button', { name: /Dead letter/i }));
|
|
|
|
await waitFor(() => {
|
|
// Red: HEAD does not display retry_count or last_error. Phase 2 Green
|
|
// must surface these so operators can see *why* a notification died.
|
|
expect(screen.getByText(/SMTP connection refused/i)).toBeInTheDocument();
|
|
expect(screen.getByText(/5/)).toBeInTheDocument();
|
|
});
|
|
});
|
|
});
|