feat: add configurable notification dispatch retries (#1655)

* feat: add configurable notification dispatch retries Allow each node to set 0-3 extra in-process delivery attempts with a fixed one-second delay for routes, agents, and Test dispatch. * fix: harden notification retry settings load/save and channel tests Guard Delivery retries against failed and out-of-order same-node settings responses, and cover Slack/webhook retry classification alongside Discord. * fix: clear Delivery retries saving state and correct screenshot alt Separate save-request ownership from value-generation invalidation so a successful PATCH cannot leave Save retries stuck on Saving, reset saving on node switch, and align the Channels screenshot alt with the committed image. * fix: surface invalid notification retry settings instead of false saved clamp Align Channels GET handling with the backend strict 0-3 parser so stored values like 9 or 1.5 show as error needing repair, matching runtime fallback to 0 instead of displaying a clamped saved policy.
2026-08-11 03:06:54 +00:00 · 2026-07-20 20:29:17 -04:00
parent 859839082c
commit 090a0d73ac
19 changed files with 1175 additions and 38 deletions
@@ -7,13 +7,17 @@ import { TogglePill } from '@/components/ui/toggle-pill';
 import { toast } from '@/components/ui/toast-store';
 import { apiFetch } from '@/lib/api';
 import { useNodes } from '@/context/NodeContext';
+import { useAuth } from '@/context/AuthContext';
 import { RefreshCw } from 'lucide-react';
 import type { Agent } from './types';
+import { DEFAULT_SETTINGS } from './types';
 import { SettingsSection } from './SettingsSection';
 import { SettingsField } from './SettingsField';
 import { SettingsActions, SettingsPrimaryButton } from './SettingsActions';
 import { useMastheadStats } from './MastheadStatsContext';
+import { NumberChip } from './SystemControls';
 import { classifyAppriseEndpoint, isKeyedAppriseEndpoint, isStatelessAppriseEndpoint } from '@/lib/appriseEndpoint';
+import { parseNotificationDispatchRetries } from '@/lib/notificationDispatchRetries';

 type ChannelType = 'discord' | 'slack' | 'webhook' | 'apprise';

@@ -38,8 +42,20 @@ function hasStoredAppriseAgent(agent: Agent): boolean {
    return Boolean(agent.config?.mode) || agent.url.includes('<redacted>');
 }

-export function NotificationsSection() {
+function clampRetryExtras(raw: string): string {
+    const n = Math.trunc(Number(raw));
+    if (!Number.isFinite(n)) return DEFAULT_SETTINGS.notification_dispatch_retries!;
+    return String(Math.max(0, Math.min(3, n)));
+}
+
+interface NotificationsSectionProps {
+    onDirtyChange?: (dirty: boolean) => void;
+}
+
+export function NotificationsSection({ onDirtyChange }: NotificationsSectionProps) {
    const { activeNode } = useNodes();
+    const { isAdmin } = useAuth();
+    const readOnly = !isAdmin;
    const activeNodeIdRef = useRef(activeNode?.id);
    useEffect(() => { activeNodeIdRef.current = activeNode?.id; }, [activeNode?.id]);

@@ -50,6 +66,22 @@ export function NotificationsSection() {
    const [appriseUrlDirty, setAppriseUrlDirty] = useState(false);
    const [appriseConfigDirty, setAppriseConfigDirty] = useState(false);

+    const [retries, setRetries] = useState(DEFAULT_SETTINGS.notification_dispatch_retries!);
+    const [savedRetries, setSavedRetries] = useState(DEFAULT_SETTINGS.notification_dispatch_retries!);
+    const [isSavingRetries, setIsSavingRetries] = useState(false);
+    // idle: node-switch reset before first successful load; ready: trusted saved value; error: GET failed.
+    const [retriesLoadState, setRetriesLoadState] = useState<'idle' | 'loading' | 'ready' | 'error'>('idle');
+    const [hasLoadedRetries, setHasLoadedRetries] = useState(false);
+    const [retriesNeedsRepair, setRetriesNeedsRepair] = useState(false);
+    const retriesFetchGenRef = useRef(0);
+    const retriesMutationGenRef = useRef(0);
+    const retriesSaveGenRef = useRef(0);
+    const retriesDirty = hasLoadedRetries && (retries !== savedRetries || retriesNeedsRepair);
+
+    useEffect(() => {
+        onDirtyChange?.(retriesDirty);
+    }, [retriesDirty, onDirtyChange]);
+
    const fetchAgents = async () => {
        const requestNodeId = activeNode?.id;
        try {
@@ -73,11 +105,98 @@ export function NotificationsSection() {
        }
    };

+    const fetchRetries = async () => {
+        const requestNodeId = activeNode?.id;
+        const fetchGen = ++retriesFetchGenRef.current;
+        const mutationAtStart = retriesMutationGenRef.current;
+        setRetriesLoadState('loading');
+        const isCurrent = () => (
+            activeNodeIdRef.current === requestNodeId
+            && fetchGen === retriesFetchGenRef.current
+            && retriesMutationGenRef.current === mutationAtStart
+        );
+        const restoreAfterStale = () => {
+            if (activeNodeIdRef.current !== requestNodeId) return;
+            // A newer fetch owns loading; do not fight it.
+            if (fetchGen !== retriesFetchGenRef.current) return;
+            // A newer edit/save owns the value; leave ready so edited UI stays interactive.
+            if (retriesMutationGenRef.current !== mutationAtStart) {
+                setRetriesLoadState('ready');
+            }
+        };
+        try {
+            const res = await apiFetch('/settings', {
+                nodeId: typeof requestNodeId === 'number' ? requestNodeId : undefined,
+            });
+            if (!isCurrent()) {
+                restoreAfterStale();
+                return;
+            }
+            if (!res.ok) {
+                setRetriesLoadState('error');
+                return;
+            }
+            const data: Record<string, string> = await res.json();
+            if (!isCurrent()) {
+                restoreAfterStale();
+                return;
+            }
+            // Missing key: same as seed/runtime default. Malformed stored values must NOT
+            // clamp into a false "saved" policy (backend falls back to 0 without accepting 1.5/9).
+            const raw = data.notification_dispatch_retries;
+            if (raw === undefined || raw === null || raw === '') {
+                const next = DEFAULT_SETTINGS.notification_dispatch_retries!;
+                setRetries(next);
+                setSavedRetries(next);
+                setRetriesNeedsRepair(false);
+                setRetriesLoadState('ready');
+                setHasLoadedRetries(true);
+                return;
+            }
+            const parsed = parseNotificationDispatchRetries(raw);
+            if (parsed === null) {
+                const next = DEFAULT_SETTINGS.notification_dispatch_retries!;
+                setRetries(next);
+                setSavedRetries(next);
+                setRetriesNeedsRepair(true);
+                setRetriesLoadState('error');
+                setHasLoadedRetries(true);
+                return;
+            }
+            const next = String(parsed);
+            setRetries(next);
+            setSavedRetries(next);
+            setRetriesNeedsRepair(false);
+            setRetriesLoadState('ready');
+            setHasLoadedRetries(true);
+        } catch (e) {
+            console.error('Failed to fetch notification retry setting', e);
+            if (!isCurrent()) {
+                restoreAfterStale();
+                return;
+            }
+            setRetriesLoadState('error');
+        }
+    };
+
    useEffect(() => {
+        // Reset local channel/retry state when the active node changes so a prior
+        // node's values cannot flash while the replacement fetches settle.
+        retriesFetchGenRef.current += 1;
+        retriesMutationGenRef.current += 1;
+        retriesSaveGenRef.current += 1;
+        // eslint-disable-next-line react-hooks/set-state-in-effect -- intentional node-switch reset
        setAgents(emptyAgents());
        setAppriseUrlDirty(false);
        setAppriseConfigDirty(false);
+        setRetries(DEFAULT_SETTINGS.notification_dispatch_retries!);
+        setSavedRetries(DEFAULT_SETTINGS.notification_dispatch_retries!);
+        setRetriesLoadState('idle');
+        setHasLoadedRetries(false);
+        setRetriesNeedsRepair(false);
+        setIsSavingRetries(false);
        void fetchAgents();
+        void fetchRetries();
    }, [activeNode?.id]); // eslint-disable-line react-hooks/exhaustive-deps

    const enabledCount = Object.values(agents).filter(a => a.enabled).length;
@@ -87,8 +206,63 @@ export function NotificationsSection() {
            value: `${enabledCount}/4`,
            tone: enabledCount > 0 ? 'value' : 'subtitle',
        },
+        ...(retriesDirty
+            ? [{ label: 'EDITED', value: 'retries', tone: 'warn' as const }]
+            : []),
    ]);

+    const saveRetries = async () => {
+        const requestNodeId = activeNode?.id;
+        const submitted = clampRetryExtras(retries);
+        const mutationAtStart = retriesMutationGenRef.current;
+        const saveGen = ++retriesSaveGenRef.current;
+        setIsSavingRetries(true);
+        try {
+            const res = await apiFetch('/settings', {
+                method: 'PATCH',
+                nodeId: typeof requestNodeId === 'number' ? requestNodeId : undefined,
+                body: JSON.stringify({ notification_dispatch_retries: submitted }),
+            });
+            if (activeNodeIdRef.current !== requestNodeId) return;
+            if (retriesMutationGenRef.current !== mutationAtStart) return;
+            if (!res.ok) {
+                const err = await res.json().catch(() => ({}));
+                toast.error(err?.error || err?.message || 'Something went wrong.');
+                return;
+            }
+            // Invalidate in-flight GETs so a late load cannot overwrite this save.
+            retriesFetchGenRef.current += 1;
+            retriesMutationGenRef.current += 1;
+            setRetries(submitted);
+            setSavedRetries(submitted);
+            setRetriesLoadState('ready');
+            setHasLoadedRetries(true);
+            setRetriesNeedsRepair(false);
+            toast.success('Delivery retries saved.');
+        } catch (e: unknown) {
+            if (activeNodeIdRef.current !== requestNodeId) return;
+            if (retriesMutationGenRef.current !== mutationAtStart) return;
+            toast.error((e as Error)?.message || 'Network error.');
+        } finally {
+            // Own the spinner by save generation, not value mutation (success bumps mutation).
+            if (activeNodeIdRef.current === requestNodeId && saveGen === retriesSaveGenRef.current) {
+                setIsSavingRetries(false);
+            }
+        }
+    };
+
+    const retriesKicker =
+        retriesNeedsRepair && retries === savedRetries
+            ? 'error'
+            : retriesDirty
+                ? 'edited'
+                : retriesLoadState === 'error'
+                    ? 'error'
+                    : retriesLoadState === 'loading' || retriesLoadState === 'idle'
+                        ? 'loading'
+                        : 'saved';
+    const retriesControlsDisabled = readOnly || !hasLoadedRetries;
+
    const handleAgentChange = (type: string, field: keyof Agent, value: Agent[keyof Agent]) => {
        setAgents(prev => ({
            ...prev,
@@ -283,6 +457,57 @@ export function NotificationsSection() {

    return (
        <div className="flex flex-col gap-6">
+            <fieldset disabled={readOnly} className="min-w-0 border-0 p-0 m-0">
+                <SettingsSection title="Delivery retries" kicker={retriesKicker}>
+                    <SettingsField
+                        label="Extra attempts"
+                        helper={
+                            retriesNeedsRepair
+                                ? 'Stored delivery retries value is invalid for this node. Runtime delivery uses 0 until you save a value from 0 to 3.'
+                                : retriesLoadState === 'error'
+                                    ? 'Could not load delivery retries for this node. Retry after the load succeeds; default 0 is not treated as saved until then.'
+                                    : 'Extra in-process attempts after a transient delivery failure (0 keeps single-shot). Fixed 1 second between attempts. Ambiguous timeouts can produce duplicate notifications.'
+                        }
+                    >
+                        <NumberChip
+                            value={retries}
+                            onChange={(v) => {
+                                retriesMutationGenRef.current += 1;
+                                setRetries(clampRetryExtras(v));
+                            }}
+                            suffix="extra"
+                            min={0}
+                            max={3}
+                            step={1}
+                            disabled={retriesControlsDisabled}
+                        />
+                    </SettingsField>
+                    <SettingsActions>
+                        {(hasLoadedRetries || retriesLoadState === 'error') && (
+                            <Button
+                                variant="outline"
+                                onClick={() => void fetchRetries()}
+                                disabled={readOnly || isSavingRetries || retriesLoadState === 'loading'}
+                            >
+                                {retriesLoadState === 'error' ? 'Retry load' : 'Reload'}
+                            </Button>
+                        )}
+                        <SettingsPrimaryButton
+                            onClick={() => void saveRetries()}
+                            disabled={retriesControlsDisabled || !retriesDirty || isSavingRetries}
+                        >
+                            {isSavingRetries ? (
+                                <>
+                                    <RefreshCw className="w-4 h-4 animate-spin" />
+                                    Saving
+                                </>
+                            ) : (
+                                'Save retries'
+                            )}
+                        </SettingsPrimaryButton>
+                    </SettingsActions>
+                </SettingsSection>
+            </fieldset>
            <Tabs value={notifTab} onValueChange={(v) => setNotifTab(v as ChannelType)} className="w-full">
                <TabsList className="w-full mb-4 grid grid-cols-4">
                    <TabsHighlight className="rounded-md bg-brand/20" transition={springs.snappy}>
@@ -96,7 +96,7 @@ function renderSection(
        case 'docker-storage': return <DockerStorageSection onDirtyChange={(d) => onDirtyChange('docker-storage', d)} />;
        case 'image-updates': return <UpdatesSection />;
        case 'fleet-mesh': return <FleetMeshSection onDirtyChange={(d) => onDirtyChange('fleet-mesh', d)} />;
-        case 'notifications': return <NotificationsSection />;
+        case 'notifications': return <NotificationsSection onDirtyChange={(d) => onDirtyChange('notifications', d)} />;
        case 'notification-routing': return <NotificationRoutingSection />;
        case 'notification-suppression': return (
            <NotificationSuppressionSection
@@ -25,6 +25,10 @@ const { masthead, nodeState } = vi.hoisted(() => ({
 vi.mock('@/context/NodeContext', () => ({
    useNodes: () => ({ activeNode: nodeState.activeNode }),
 }));
+const authState = { isAdmin: true };
+vi.mock('@/context/AuthContext', () => ({
+    useAuth: () => authState,
+}));
 vi.mock('../MastheadStatsContext', () => ({
    useMastheadStats: (stats: MastheadMetadataItem[] | null) => {
        masthead.last = stats;
@@ -64,11 +68,18 @@ describe('NotificationsSection', () => {
        mockedFetch.mockReset();
        masthead.last = null;
        nodeState.activeNode = { id: 1 };
-        mockedFetch.mockImplementation(async (url: string, opts?: { method?: string }) => {
+        authState.isAdmin = true;
+        mockedFetch.mockImplementation(async (url: string, opts?: { method?: string; nodeId?: number }) => {
            if (url === '/agents' && !opts?.method) return agentsResponse();
            if (url === '/agents' && opts?.method === 'POST') {
                return { ok: true, json: async () => ({}) };
            }
+            if (url === '/settings' && !opts?.method) {
+                return { ok: true, json: async () => ({ notification_dispatch_retries: '0' }) };
+            }
+            if (url === '/settings' && opts?.method === 'PATCH') {
+                return { ok: true, json: async () => ({ success: true }) };
+            }
            return { ok: true, json: async () => ([]) };
        });
    });
@@ -321,6 +332,9 @@ describe('NotificationsSection', () => {
                if (nodeState.activeNode.id === 1) return agentsResponse([REDACTED_APPRISE]);
                return agentsResponse([]);
            }
+            if (url === '/settings' && !opts?.method) {
+                return { ok: true, json: async () => ({ notification_dispatch_retries: '0' }) };
+            }
            return { ok: true, json: async () => ({}) };
        });

@@ -355,6 +369,9 @@ describe('NotificationsSection', () => {
                }
                return agentsResponse([]);
            }
+            if (url === '/settings' && !opts?.method) {
+                return { ok: true, json: async () => ({ notification_dispatch_retries: '0' }) };
+            }
            return { ok: true, json: async () => ({}) };
        });

@@ -376,4 +393,310 @@ describe('NotificationsSection', () => {
        expect(screen.getByLabelText(/Apprise endpoint/i)).toHaveValue('');
    });

+    it('preserves CHANNELS masthead and loads retries with explicit nodeId', async () => {
+        render(<NotificationsSection />);
+        await waitFor(() => expect(masthead.last?.[0]).toMatchObject({ label: 'CHANNELS', value: '1/4' }));
+        await waitFor(() =>
+            expect(mockedFetch.mock.calls.some(
+                ([url, opts]) => url === '/settings' && (opts as { nodeId?: number })?.nodeId === 1,
+            )).toBe(true),
+        );
+        expect(screen.getByText('Delivery retries')).toBeInTheDocument();
+        expect(screen.getByText('0')).toBeInTheDocument();
+    });
+
+    it('PATCHes only notification_dispatch_retries when saving retries', async () => {
+        render(<NotificationsSection />);
+        await waitFor(() => expect(screen.getByText('saved')).toBeInTheDocument());
+        const chipButton = screen.getByRole('button', { name: /0\s*extra/i });
+        await userEvent.click(chipButton);
+        const input = screen.getByRole('spinbutton');
+        await userEvent.clear(input);
+        await userEvent.type(input, '2');
+        await userEvent.keyboard('{Enter}');
+        await userEvent.click(screen.getByRole('button', { name: 'Save retries' }));
+        await waitFor(() => {
+            const patch = mockedFetch.mock.calls.find(
+                ([url, opts]) => url === '/settings' && (opts as { method?: string })?.method === 'PATCH',
+            );
+            expect(patch).toBeTruthy();
+            expect(JSON.parse((patch![1] as { body: string }).body)).toEqual({ notification_dispatch_retries: '2' });
+            expect((patch![1] as { nodeId?: number }).nodeId).toBe(1);
+        });
+        await waitFor(() => expect(screen.getByRole('button', { name: 'Save retries' })).toBeInTheDocument());
+        expect(screen.queryByRole('button', { name: /Saving/i })).toBeNull();
+        expect(findAgentsPost()).toBeUndefined();
+    });
+
+    it('disables delivery retries controls for non-admins', async () => {
+        authState.isAdmin = false;
+        render(<NotificationsSection />);
+        await waitFor(() => expect(screen.getByText('Delivery retries')).toBeInTheDocument());
+        expect(screen.getByRole('button', { name: 'Save retries' })).toBeDisabled();
+    });
+
+    it('ignores a stale settings body after a node switch', async () => {
+        let releaseNode1Settings: (() => void) | undefined;
+        const gate = new Promise<void>((resolve) => { releaseNode1Settings = resolve; });
+
+        mockedFetch.mockImplementation(async (url: string, opts?: { method?: string; nodeId?: number | null }) => {
+            if (url === '/agents' && !opts?.method) return agentsResponse([]);
+            if (url === '/settings' && !opts?.method) {
+                const targetId = opts?.nodeId ?? nodeState.activeNode.id;
+                if (targetId === 1) {
+                    return {
+                        ok: true,
+                        json: async () => {
+                            await gate;
+                            return { notification_dispatch_retries: '3' };
+                        },
+                    };
+                }
+                return { ok: true, json: async () => ({ notification_dispatch_retries: '0' }) };
+            }
+            return { ok: true, json: async () => ({}) };
+        });
+
+        const { rerender } = render(<NotificationsSection />);
+        nodeState.activeNode = { id: 2 };
+        rerender(<NotificationsSection />);
+        await waitFor(() =>
+            expect(mockedFetch.mock.calls.some(
+                ([url, opts]) => url === '/settings' && (opts as { nodeId?: number })?.nodeId === 2,
+            )).toBe(true),
+        );
+        releaseNode1Settings?.();
+        await new Promise((r) => setTimeout(r, 40));
+        expect(screen.getByRole('button', { name: /0\s*extra/i })).toBeInTheDocument();
+        expect(screen.queryByRole('button', { name: /3\s*extra/i })).toBeNull();
+    });
+
+    it('does not present default 0 as saved when settings GET fails', async () => {
+        mockedFetch.mockImplementation(async (url: string, opts?: { method?: string }) => {
+            if (url === '/agents' && !opts?.method) return agentsResponse([]);
+            if (url === '/settings' && !opts?.method) {
+                return { ok: false, status: 500, json: async () => ({ error: 'boom' }) };
+            }
+            return { ok: true, json: async () => ({}) };
+        });
+
+        render(<NotificationsSection />);
+        await waitFor(() => expect(screen.getByText('error')).toBeInTheDocument());
+        expect(screen.queryByText('saved')).toBeNull();
+        expect(screen.getByRole('button', { name: 'Save retries' })).toBeDisabled();
+        expect(screen.getByRole('button', { name: /0\s*extra/i })).toBeDisabled();
+        expect(screen.getByRole('button', { name: 'Retry load' })).toBeInTheDocument();
+    });
+
+    it('disables retry controls until the settings GET succeeds', async () => {
+        let releaseGet: (() => void) | undefined;
+        const gate = new Promise<void>((resolve) => { releaseGet = resolve; });
+
+        mockedFetch.mockImplementation(async (url: string, opts?: { method?: string }) => {
+            if (url === '/agents' && !opts?.method) return agentsResponse([]);
+            if (url === '/settings' && !opts?.method) {
+                return {
+                    ok: true,
+                    json: async () => {
+                        await gate;
+                        return { notification_dispatch_retries: '3' };
+                    },
+                };
+            }
+            return { ok: true, json: async () => ({}) };
+        });
+
+        render(<NotificationsSection />);
+        await waitFor(() => expect(screen.getByText('loading')).toBeInTheDocument());
+        expect(screen.queryByText('saved')).toBeNull();
+        expect(screen.getByRole('button', { name: /0\s*extra/i })).toBeDisabled();
+        expect(screen.getByRole('button', { name: 'Save retries' })).toBeDisabled();
+
+        releaseGet?.();
+        await waitFor(() => expect(screen.getByText('saved')).toBeInTheDocument());
+        expect(screen.getByRole('button', { name: /3\s*extra/i })).not.toBeDisabled();
+    });
+
+    it('keeps PATCH result when a deferred Reload GET returns stale data', async () => {
+        let releaseStale: (() => void) | undefined;
+        const staleGate = new Promise<void>((resolve) => { releaseStale = resolve; });
+        let settingsGetCount = 0;
+
+        mockedFetch.mockImplementation(async (url: string, opts?: { method?: string }) => {
+            if (url === '/agents' && !opts?.method) return agentsResponse([]);
+            if (url === '/settings' && !opts?.method) {
+                settingsGetCount += 1;
+                if (settingsGetCount === 1) {
+                    return { ok: true, json: async () => ({ notification_dispatch_retries: '0' }) };
+                }
+                return {
+                    ok: true,
+                    json: async () => {
+                        await staleGate;
+                        return { notification_dispatch_retries: '0' };
+                    },
+                };
+            }
+            if (url === '/settings' && opts?.method === 'PATCH') {
+                return { ok: true, json: async () => ({ success: true }) };
+            }
+            return { ok: true, json: async () => ({}) };
+        });
+
+        render(<NotificationsSection />);
+        await waitFor(() => expect(screen.getByText('saved')).toBeInTheDocument());
+
+        // Start a soft reload, then edit+save while that GET is still in flight.
+        await userEvent.click(screen.getByRole('button', { name: 'Reload' }));
+        await waitFor(() => expect(mockedFetch.mock.calls.filter(
+            ([url, opts]) => url === '/settings' && !(opts as { method?: string })?.method,
+        ).length).toBeGreaterThan(1));
+
+        await userEvent.click(screen.getByRole('button', { name: /0\s*extra/i }));
+        const input = screen.getByRole('spinbutton');
+        await userEvent.clear(input);
+        await userEvent.type(input, '2');
+        await userEvent.keyboard('{Enter}');
+        await userEvent.click(screen.getByRole('button', { name: 'Save retries' }));
+        await waitFor(() => expect(screen.getByRole('button', { name: /2\s*extra/i })).toBeInTheDocument());
+        await waitFor(() => expect(screen.getByText('saved')).toBeInTheDocument());
+
+        releaseStale?.();
+        await new Promise((r) => setTimeout(r, 40));
+        expect(screen.getByRole('button', { name: /2\s*extra/i })).toBeInTheDocument();
+        expect(screen.getByText('saved')).toBeInTheDocument();
+    });
+
+
+    it('clears Saving after edit-during-save supersedes the PATCH apply', async () => {
+        let releasePatch: (() => void) | undefined;
+        const patchGate = new Promise<void>((resolve) => { releasePatch = resolve; });
+
+        mockedFetch.mockImplementation(async (url: string, opts?: { method?: string }) => {
+            if (url === '/agents' && !opts?.method) return agentsResponse([]);
+            if (url === '/settings' && !opts?.method) {
+                return { ok: true, json: async () => ({ notification_dispatch_retries: '0' }) };
+            }
+            if (url === '/settings' && opts?.method === 'PATCH') {
+                await patchGate;
+                return { ok: true, json: async () => ({ success: true }) };
+            }
+            return { ok: true, json: async () => ({}) };
+        });
+
+        render(<NotificationsSection />);
+        await waitFor(() => expect(screen.getByText('saved')).toBeInTheDocument());
+
+        await userEvent.click(screen.getByRole('button', { name: /0\s*extra/i }));
+        let input = screen.getByRole('spinbutton');
+        await userEvent.clear(input);
+        await userEvent.type(input, '2');
+        await userEvent.keyboard('{Enter}');
+        await userEvent.click(screen.getByRole('button', { name: 'Save retries' }));
+        await waitFor(() => expect(screen.getByRole('button', { name: /Saving/i })).toBeInTheDocument());
+
+        // Edit while the PATCH is in flight (bumps mutation gen).
+        await userEvent.click(screen.getByRole('button', { name: /2\s*extra/i }));
+        input = screen.getByRole('spinbutton');
+        await userEvent.clear(input);
+        await userEvent.type(input, '3');
+        await userEvent.keyboard('{Enter}');
+
+        releasePatch?.();
+        await waitFor(() => expect(screen.getByRole('button', { name: 'Save retries' })).toBeInTheDocument());
+        expect(screen.queryByRole('button', { name: /Saving/i })).toBeNull();
+        expect(screen.getByRole('button', { name: /3\s*extra/i })).toBeInTheDocument();
+        expect(screen.getByText('edited')).toBeInTheDocument();
+    });
+
+    it('clears Saving when the active node changes during an in-flight PATCH', async () => {
+        let releasePatch: (() => void) | undefined;
+        const patchGate = new Promise<void>((resolve) => { releasePatch = resolve; });
+
+        mockedFetch.mockImplementation(async (url: string, opts?: { method?: string; nodeId?: number }) => {
+            if (url === '/agents' && !opts?.method) return agentsResponse([]);
+            if (url === '/settings' && !opts?.method) {
+                const id = opts?.nodeId ?? nodeState.activeNode.id;
+                return { ok: true, json: async () => ({ notification_dispatch_retries: id === 1 ? '0' : '1' }) };
+            }
+            if (url === '/settings' && opts?.method === 'PATCH') {
+                await patchGate;
+                return { ok: true, json: async () => ({ success: true }) };
+            }
+            return { ok: true, json: async () => ({}) };
+        });
+
+        const { rerender } = render(<NotificationsSection />);
+        await waitFor(() => expect(screen.getByText('saved')).toBeInTheDocument());
+
+        await userEvent.click(screen.getByRole('button', { name: /0\s*extra/i }));
+        const input = screen.getByRole('spinbutton');
+        await userEvent.clear(input);
+        await userEvent.type(input, '2');
+        await userEvent.keyboard('{Enter}');
+        await userEvent.click(screen.getByRole('button', { name: 'Save retries' }));
+        await waitFor(() => expect(screen.getByRole('button', { name: /Saving/i })).toBeInTheDocument());
+
+        nodeState.activeNode = { id: 2 };
+        rerender(<NotificationsSection />);
+        await waitFor(() => expect(screen.queryByRole('button', { name: /Saving/i })).toBeNull());
+        await waitFor(() => expect(screen.getByText('saved')).toBeInTheDocument());
+        expect(screen.getByRole('button', { name: 'Save retries' })).toBeInTheDocument();
+
+        releasePatch?.();
+        await new Promise((r) => setTimeout(r, 40));
+        expect(screen.queryByRole('button', { name: /Saving/i })).toBeNull();
+        expect(screen.getByRole('button', { name: /1\s*extra/i })).toBeInTheDocument();
+    });
+
+
+
+    it('treats invalid stored notification_dispatch_retries as error, not clamped saved', async () => {
+        mockedFetch.mockImplementation(async (url: string, opts?: { method?: string }) => {
+            if (url === '/agents' && !opts?.method) return agentsResponse([]);
+            if (url === '/settings' && !opts?.method) {
+                return { ok: true, json: async () => ({ notification_dispatch_retries: '9' }) };
+            }
+            if (url === '/settings' && opts?.method === 'PATCH') {
+                return { ok: true, json: async () => ({ success: true }) };
+            }
+            return { ok: true, json: async () => ({}) };
+        });
+
+        render(<NotificationsSection />);
+        await waitFor(() => expect(screen.getByText('error')).toBeInTheDocument());
+        expect(screen.queryByText('saved')).toBeNull();
+        expect(screen.getByText(/Stored delivery retries value is invalid/i)).toBeInTheDocument();
+        // Chip may show 0 as a draft, but Save must be enabled so the operator can repair.
+        expect(screen.getByRole('button', { name: 'Save retries' })).not.toBeDisabled();
+
+        await userEvent.click(screen.getByRole('button', { name: 'Save retries' }));
+        await waitFor(() => {
+            const patch = mockedFetch.mock.calls.find(
+                ([url, opts]) => url === '/settings' && (opts as { method?: string })?.method === 'PATCH',
+            );
+            expect(patch).toBeTruthy();
+            expect(JSON.parse((patch![1] as { body: string }).body)).toEqual({ notification_dispatch_retries: '0' });
+        });
+        await waitFor(() => expect(screen.getByText('saved')).toBeInTheDocument());
+        expect(screen.getByRole('button', { name: /0\s*extra/i })).toBeInTheDocument();
+    });
+
+    it('treats decimal stored notification_dispatch_retries as invalid, not truncated saved', async () => {
+        mockedFetch.mockImplementation(async (url: string, opts?: { method?: string }) => {
+            if (url === '/agents' && !opts?.method) return agentsResponse([]);
+            if (url === '/settings' && !opts?.method) {
+                return { ok: true, json: async () => ({ notification_dispatch_retries: '1.5' }) };
+            }
+            return { ok: true, json: async () => ({}) };
+        });
+
+        render(<NotificationsSection />);
+        await waitFor(() => expect(screen.getByText('error')).toBeInTheDocument());
+        expect(screen.queryByText('saved')).toBeNull();
+        expect(screen.queryByRole('button', { name: /1\s*extra/i })).toBeNull();
+        expect(screen.getByText(/Stored delivery retries value is invalid/i)).toBeInTheDocument();
+    });
+
+
 });
@@ -205,7 +205,7 @@ export const SETTINGS_ITEMS: readonly SettingsItemMeta[] = [
        group: 'notifications',
        label: 'Channels',
        description: 'Discord, Slack, Apprise, and custom webhook destinations for Sencho alerts.',
-        keywords: ['discord', 'slack', 'apprise', 'webhook', 'channels', 'destinations', 'alerts'],
+        keywords: ['discord', 'slack', 'apprise', 'webhook', 'channels', 'destinations', 'alerts', 'retry', 'retries'],
        tier: null,
        scope: 'node',
    },
@@ -22,6 +22,7 @@ export interface PatchableSettings {
    env_block_deploy_on_missing_required?: '0' | '1';
    auto_create_missing_external_networks?: '0' | '1';
    image_update_sidebar_indicators?: '0' | '1';
+    notification_dispatch_retries?: string;
 }

 export const DEFAULT_SETTINGS: PatchableSettings = {
@@ -48,6 +49,7 @@ export const DEFAULT_SETTINGS: PatchableSettings = {
    env_block_deploy_on_missing_required: '0',
    auto_create_missing_external_networks: '0',
    image_update_sidebar_indicators: '1',
+    notification_dispatch_retries: '0',
 };

 export type SectionId =