feat(blueprints): require confirmed rollout preview before reconcile (#1649)

* feat(blueprints): require confirmed rollout preview before reconcile Persist place/remove approval with an intent fingerprint and transition matrix so Apply, Retry, ticks, and pin cannot mutate the fleet until the operator confirms the current blast radius. Preview surfaces requirements, health, and informational in-flight rows without executing them. * fix(blueprints): silence unused retry nodeId lint error * test(blueprints): harden approval gate coverage and preview clarity Add real reconcileOne place/remove fan-out and STALE_GUARD regressions, surface reachability in the rollout dialog, align warning totals, and document the fail-closed upgrade pause. * test(blueprints): cover legacy approval schema migration Seed a pre-approval database with an enabled Blueprint and live deployment, run production DatabaseService startup, and assert pending null auth columns plus a fail-closed reconcile gate. * test(blueprints): clarify legacy approval migration fixture Extract seed/boot helpers so the migration regression reads as a linear upgrade path without changing assertions. * fix(blueprints): report apply outcomes and gate manual withdraw Return per-node reconcile outcomes from Confirm Apply, block create preview on unmanaged same-name stacks, and require an approved remove outcome for every manual withdraw or evict. * fix(blueprints): scope withdraw approval to destructive eviction Require remove approval only for snapshot/evict confirms and evict_blocked rows. Keep plain stateless standard withdraw as an immediate stop, and update withdraw-route tests to seed remove approval when needed.
2026-08-19 14:56:27 +00:00 · 2026-07-19 15:30:26 -04:00
parent 972f2b9483
commit d94e586af3
20 changed files with 3129 additions and 152 deletions
@@ -154,6 +154,14 @@ function BlueprintTile({ blueprint, onClick }: { blueprint: BlueprintListItem; o
                        <span className="text-warning">disabled</span>
                    </>
                )}
+                {blueprint.effectiveApproval && blueprint.effectiveApproval !== 'approved' && (
+                    <>
+                        <span>·</span>
+                        <span className={blueprint.effectiveApproval === 'reapproval_required' ? 'text-warning' : 'text-stat-subtitle'}>
+                            {blueprint.effectiveApproval === 'reapproval_required' ? 'reapproval required' : 'pending'}
+                        </span>
+                    </>
+                )}
            </div>
        </button>
    );
@@ -7,12 +7,12 @@
 * of them, so the sheet can never issue a request the API answers with 403.
 */
 import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { render, screen, fireEvent, waitFor } from '@testing-library/react';
+import { render, screen, fireEvent } from '@testing-library/react';
 import type { BlueprintSummary } from '@/lib/blueprintsApi';

 vi.mock('@/lib/blueprintsApi', async (importOriginal) => {
    const actual = await importOriginal<typeof import('@/lib/blueprintsApi')>();
-    return { ...actual, getBlueprint: vi.fn(), applyBlueprint: vi.fn() };
+    return { ...actual, getBlueprint: vi.fn(), applyBlueprint: vi.fn(), previewBlueprint: vi.fn() };
 });

 vi.mock('@/context/NodeContext', () => ({ useNodes: () => ({ nodes: [] }) }));
@@ -25,6 +25,12 @@ vi.mock('./BlueprintDeploymentTable', () => ({
    BlueprintDeploymentTable: () => <div data-testid="deployment-table" />,
 }));

+vi.mock('./RolloutPreviewDialog', () => ({
+    RolloutPreviewDialog: ({ open }: { open: boolean }) => (
+        open ? <div data-testid="rollout-preview-dialog">preview</div> : null
+    ),
+}));
+
 import { getBlueprint } from '@/lib/blueprintsApi';
 import { BlueprintDetail } from './BlueprintDetail';

@@ -48,6 +54,7 @@ function summary(): BlueprintSummary {
        },
        deployments: [],
        statusCounts: {},
+        effectiveApproval: 'pending',
    };
 }

@@ -101,32 +108,20 @@ describe('BlueprintDetail data fetching', () => {
        expect(vi.mocked(getBlueprint)).toHaveBeenLastCalledWith(2);
    });

-    it('keeps the loaded body on screen while a refresh is in flight', async () => {
-        let settleRefresh = () => {};
-        vi.mocked(getBlueprint)
-            .mockResolvedValueOnce(summary())
-            .mockImplementationOnce(
-                () => new Promise<BlueprintSummary>((resolve) => { settleRefresh = () => resolve(summary()); }),
-            );
-
+    it('opens the rollout preview dialog when Apply now is clicked', async () => {
        render(
            <BlueprintDetail blueprintId={1} open onOpenChange={noop} onChanged={noop} canEdit distinctLabels={[]} />,
        );
        expect(await screen.findByText('Show compose source')).toBeInTheDocument();
        const callsAfterLoad = vi.mocked(getBlueprint).mock.calls.length;

-        // Applying reloads the blueprint. The populated body must stay mounted instead
-        // of collapsing to the loading skeleton while that reload is in flight.
        fireEvent.click(screen.getByRole('button', { name: /apply now/i }));
-        await waitFor(() => expect(vi.mocked(getBlueprint).mock.calls.length).toBe(callsAfterLoad + 1));

-        // The deployment table only renders in the loaded body branch, never in the
-        // skeleton, so its presence proves the skeleton did not take over the refresh.
+        expect(screen.getByTestId('rollout-preview-dialog')).toBeInTheDocument();
+        // Opening the dialog must not refetch the detail sheet body.
+        expect(vi.mocked(getBlueprint)).toHaveBeenCalledTimes(callsAfterLoad);
        expect(screen.getByText('Show compose source')).toBeInTheDocument();
        expect(screen.getByTestId('deployment-table')).toBeInTheDocument();
-
-        settleRefresh();
-        await screen.findByText('Show compose source');
    });
 });

@@ -13,7 +13,6 @@ import {
    type WithdrawConfirm,
    type AcceptMode,
    getBlueprint,
-    applyBlueprint,
    updateBlueprint,
    deleteBlueprint,
    withdrawDeployment,
@@ -24,6 +23,7 @@ import { BlueprintEditor } from './BlueprintEditor';
 import { BlueprintDeploymentTable } from './BlueprintDeploymentTable';
 import { EvictionDialog } from './EvictionDialog';
 import { StateReviewDialog } from './StateReviewDialog';
+import { RolloutPreviewDialog } from './RolloutPreviewDialog';
 import { useNodes } from '@/context/NodeContext';
 import { formatTimeAgo } from '@/lib/relativeTime';

@@ -46,6 +46,7 @@ export function BlueprintDetail({ blueprintId, open, onOpenChange, onChanged, ca
    const [stateReviewTarget, setStateReviewTarget] = useState<{ nodeId: number; nodeName: string } | null>(null);
    const [deleteOpen, setDeleteOpen] = useState(false);
    const [deleteConfirmText, setDeleteConfirmText] = useState('');
+    const [previewOpen, setPreviewOpen] = useState(false);
    const { nodes } = useNodes();

    // Hold the latest onOpenChange without making it a refresh dependency. Parents
@@ -78,19 +79,9 @@ export function BlueprintDetail({ blueprintId, open, onOpenChange, onChanged, ca

    const blueprint = summary?.blueprint;

-    async function handleApply() {
-        if (!blueprint) return;
-        setSubmitting(true);
-        try {
-            await applyBlueprint(blueprint.id);
-            toast.success('Reconciliation triggered');
-            await refresh();
-            onChanged();
-        } catch (err) {
-            toast.error(err instanceof Error ? err.message : 'Failed to apply blueprint');
-        } finally {
-            setSubmitting(false);
-        }
+    async function handleRolloutApplied() {
+        await refresh();
+        onChanged();
    }

    async function handleSaveEdit(input: CreateBlueprintInput | UpdateBlueprintInput) {
@@ -156,7 +147,13 @@ export function BlueprintDetail({ blueprintId, open, onOpenChange, onChanged, ca
            await refresh();
            onChanged();
        } catch (err) {
-            toast.error(err instanceof Error ? err.message : 'Failed to withdraw');
+            const status = (err as Error & { status?: number }).status;
+            const message = err instanceof Error ? err.message : 'Failed to withdraw';
+            if (status === 409 && /approval|removal|stale/i.test(message)) {
+                toast.error('Confirm a remove rollout for this node before destructive eviction.');
+            } else {
+                toast.error(message);
+            }
        } finally {
            setBusyNodeId(null);
            setEvictTarget(null);
@@ -182,12 +179,12 @@ export function BlueprintDetail({ blueprintId, open, onOpenChange, onChanged, ca
    /**
     * Row-level retry isn't a backend primitive: the reconciler operates on the whole
     * blueprint, not a single deployment. We surface the row's "Retry" button and
-     * trigger a full reconciliation tick, which retries failed deployments naturally.
+     * open the rollout preview so the operator confirms the full plan.
     * The nodeId argument satisfies BlueprintDeploymentTable.onRetry's signature.
     */
    async function handleRetryRow(nodeId: number): Promise<void> {
        void nodeId;
-        await handleApply();
+        setPreviewOpen(true);
    }

    function openWithdraw(nodeId: number) {
@@ -206,8 +203,12 @@ export function BlueprintDetail({ blueprintId, open, onOpenChange, onChanged, ca
        ? `${describeSelector(blueprint.selector)} · ${blueprint.drift_mode} · rev ${blueprint.revision}`
        : (loading ? 'Loading…' : '');

+    const approvalLabel = summary?.effectiveApproval === 'reapproval_required'
+        ? 'reapproval required'
+        : summary?.effectiveApproval ?? 'pending';
+
    const footerContext = blueprint
-        ? `Updated ${formatTimeAgo(blueprint.updated_at)}${blueprint.enabled ? '' : ' · reconciler disabled'}`
+        ? `Updated ${formatTimeAgo(blueprint.updated_at)}${blueprint.enabled ? '' : ' · reconciler disabled'} · ${approvalLabel}`
        : undefined;

    const secondaryActions = blueprint && canEdit
@@ -238,7 +239,7 @@ export function BlueprintDetail({ blueprintId, open, onOpenChange, onChanged, ca
                primaryAction={blueprint && canEdit ? {
                    label: 'Apply now',
                    icon: Play,
-                    onClick: handleApply,
+                    onClick: () => setPreviewOpen(true),
                    disabled: submitting || !blueprint.enabled || editMode,
                } : undefined}
                secondaryActions={secondaryActions}
@@ -337,6 +338,15 @@ export function BlueprintDetail({ blueprintId, open, onOpenChange, onChanged, ca
                        onAccept={(mode) => performAccept(stateReviewTarget.nodeId, mode)}
                    />
                )}
+                {blueprint && (
+                    <RolloutPreviewDialog
+                        blueprintId={blueprint.id}
+                        blueprintName={blueprint.name}
+                        open={previewOpen}
+                        onOpenChange={setPreviewOpen}
+                        onApplied={handleRolloutApplied}
+                    />
+                )}
                {blueprint && (
                    <Modal open={deleteOpen} onOpenChange={(o) => { if (!o) { setDeleteOpen(false); setDeleteConfirmText(''); } }} size="md">
                        <ModalDestructiveHeader
@@ -0,0 +1,93 @@
+/**
+ * RolloutPreviewDialog rendering: reachability notes and full warning lists.
+ */
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { render, screen, waitFor } from '@testing-library/react';
+import type { BlueprintPreview } from '@/lib/blueprintsApi';
+
+vi.mock('@/lib/blueprintsApi', async (importOriginal) => {
+    const actual = await importOriginal<typeof import('@/lib/blueprintsApi')>();
+    return { ...actual, previewBlueprint: vi.fn(), applyBlueprint: vi.fn() };
+});
+
+vi.mock('@/components/ui/toast-store', () => ({
+    toast: { error: vi.fn(), success: vi.fn(), warning: vi.fn(), info: vi.fn(), loading: vi.fn(), dismiss: vi.fn() },
+}));
+
+import { previewBlueprint } from '@/lib/blueprintsApi';
+import { RolloutPreviewDialog } from './RolloutPreviewDialog';
+
+function previewFixture(overrides: Partial<BlueprintPreview> = {}): BlueprintPreview {
+    return {
+        blueprintId: 1,
+        classification: 'stateless',
+        matchedNodes: [{ id: 2, name: 'edge', type: 'remote' }],
+        plannedDeployments: [],
+        plannedDriftChecks: [],
+        plannedEvictions: [],
+        name: 'web',
+        revision: 1,
+        updatedAt: 0,
+        driftMode: 'observe',
+        stackName: 'web',
+        approvalStatus: 'pending',
+        effectiveApproval: 'pending',
+        planFingerprint: 'abc',
+        generatedAt: Date.now(),
+        summary: { safe: 0, warning: 2, blocker: 1, total: 1 },
+        changes: [{
+            nodeId: 2,
+            nodeName: 'edge',
+            nodeType: 'remote',
+            status: 'offline',
+            action: 'create',
+            severity: 'blocker',
+            kind: 'executor',
+            detail: 'New placement',
+            reachabilityNote: 'Remote node cached as offline or unknown',
+        }],
+        confirmableActions: [{ nodeId: 2, action: 'create' }],
+        executorActions: [{ nodeId: 2, action: 'create' }],
+        unauthorizedActions: [],
+        requirements: { variables: [], envFiles: [], composeSecrets: [] },
+        compatibilityWarnings: ['uses named volumes'],
+        healthNote: 'Reachability is from cached node status',
+        blockers: [{
+            id: 'change:2:create',
+            message: 'edge: New placement [remote/offline: Remote node cached as offline or unknown]',
+        }],
+        warnings: [
+            { id: 'compat:1', message: 'uses named volumes' },
+            { id: 'req:1', message: 'Required variable DB_PASSWORD' },
+        ],
+        ...overrides,
+    };
+}
+
+beforeEach(() => {
+    vi.mocked(previewBlueprint).mockResolvedValue(previewFixture());
+});
+
+describe('RolloutPreviewDialog', () => {
+    it('shows reachability note and does not truncate compatibility warnings', async () => {
+        render(
+            <RolloutPreviewDialog
+                blueprintId={1}
+                blueprintName="web"
+                open
+                onOpenChange={() => {}}
+                onApplied={() => {}}
+            />,
+        );
+
+        await waitFor(() => {
+            expect(screen.getAllByText(/Remote node cached as offline/i).length).toBeGreaterThanOrEqual(1);
+        });
+        expect(screen.getByText(/Warnings \(2\)/i)).toBeInTheDocument();
+        expect(screen.getByText('uses named volumes')).toBeInTheDocument();
+        expect(screen.getByText('Required variable DB_PASSWORD')).toBeInTheDocument();
+        expect(screen.getByText(/Blockers \(1\)/i)).toBeInTheDocument();
+        expect(screen.getByText(/\(remote\/offline\)/i)).toBeInTheDocument();
+        expect(screen.getByRole('button', { name: /confirm apply/i })).toBeDisabled();
+    });
+});
@@ -0,0 +1,229 @@
+import { useEffect, useRef, useState } from 'react';
+import { Modal, ModalHeader, ModalBody, ModalFooter } from '@/components/ui/modal';
+import { Button } from '@/components/ui/button';
+import { toast } from '@/components/ui/toast-store';
+import {
+    type BlueprintPreview,
+    previewBlueprint,
+    applyBlueprint,
+} from '@/lib/blueprintsApi';
+
+interface RolloutPreviewDialogProps {
+    blueprintId: number;
+    blueprintName: string;
+    open: boolean;
+    onOpenChange: (open: boolean) => void;
+    onApplied: () => void;
+}
+
+function approvalLabel(value: BlueprintPreview['effectiveApproval']): string {
+    if (value === 'reapproval_required') return 'reapproval required';
+    return value;
+}
+
+function sectionBorderClass(tone: 'destructive' | 'warning' | 'neutral'): string {
+    if (tone === 'destructive') return 'border-destructive/30 bg-destructive/5';
+    if (tone === 'warning') return 'border-warning/30 bg-warning/5';
+    return 'border-card-border bg-glass-highlight';
+}
+
+export function RolloutPreviewDialog({
+    blueprintId,
+    blueprintName,
+    open,
+    onOpenChange,
+    onApplied,
+}: RolloutPreviewDialogProps) {
+    const [preview, setPreview] = useState<BlueprintPreview | null>(null);
+    const [loading, setLoading] = useState(false);
+    const [submitting, setSubmitting] = useState(false);
+    const onOpenChangeRef = useRef(onOpenChange);
+    useEffect(() => { onOpenChangeRef.current = onOpenChange; }, [onOpenChange]);
+
+    useEffect(() => {
+        if (!open) {
+            setPreview(null);
+            setSubmitting(false);
+            return;
+        }
+        let cancelled = false;
+        setLoading(true);
+        setPreview(null);
+        previewBlueprint(blueprintId)
+            .then((result) => {
+                if (!cancelled) setPreview(result);
+            })
+            .catch((err) => {
+                if (cancelled) return;
+                toast.error(err instanceof Error ? err.message : 'Failed to preview rollout');
+                onOpenChangeRef.current(false);
+            })
+            .finally(() => {
+                if (!cancelled) setLoading(false);
+            });
+        return () => { cancelled = true; };
+    }, [open, blueprintId]);
+
+    const blocked = (preview?.summary.blocker ?? 0) > 0;
+    const canConfirm = !!preview && !loading && !submitting && !blocked;
+
+    async function handleConfirm() {
+        if (!preview) return;
+        setSubmitting(true);
+        try {
+            const result = await applyBlueprint(blueprintId, {
+                planFingerprint: preview.planFingerprint,
+                actions: preview.confirmableActions,
+            });
+            const { failed = 0, pending = 0 } = result.outcomeSummary ?? {};
+            if (failed > 0) {
+                toast.warning(result.message || 'Rollout confirmed with node failures');
+            } else if (pending > 0) {
+                toast.info(result.message || 'Rollout confirmed; some actions are still in progress');
+            } else {
+                toast.success(result.message || 'Rollout confirmed');
+            }
+            onApplied();
+            onOpenChange(false);
+        } catch (err) {
+            const message = err instanceof Error ? err.message : 'Failed to apply blueprint';
+            const status = (err as Error & { status?: number }).status;
+            if (status === 409) {
+                try {
+                    setPreview(await previewBlueprint(blueprintId));
+                } catch (refreshErr) {
+                    toast.error(refreshErr instanceof Error ? refreshErr.message : 'Failed to refresh preview');
+                }
+            }
+            toast.error(message);
+        } finally {
+            setSubmitting(false);
+        }
+    }
+
+    const hasRequirements = !!preview && (
+        preview.requirements.variables.length > 0
+        || preview.requirements.envFiles.length > 0
+        || preview.requirements.composeSecrets.length > 0
+    );
+
+    return (
+        <Modal open={open} onOpenChange={onOpenChange} size="lg">
+            <ModalHeader
+                kicker="BLUEPRINT · ROLLOUT PREVIEW"
+                title={`Confirm rollout: ${blueprintName}`}
+                description="Review the blast radius before authorizing place or remove outcomes."
+            />
+            <ModalBody>
+                {loading || !preview ? (
+                    <p className="text-sm text-muted-foreground">Computing preview…</p>
+                ) : (
+                    <div className="space-y-4 max-md:max-h-[60vh] max-md:overflow-y-auto">
+                        <p className="text-xs text-stat-subtitle">
+                            Enabled blueprints still need this confirmation before the reconciler mutates the fleet.
+                        </p>
+                        <div className="flex flex-wrap gap-3 text-xs font-mono uppercase tracking-[0.15em]">
+                            <span className="text-stat-value">Safe {preview.summary.safe}</span>
+                            <span className="text-warning">Warnings {preview.summary.warning}</span>
+                            <span className="text-destructive">Blockers {preview.summary.blocker}</span>
+                            <span className="text-stat-subtitle">{approvalLabel(preview.effectiveApproval)}</span>
+                        </div>
+                        <p className="text-xs text-stat-subtitle">{preview.healthNote}</p>
+                        {preview.blockers.length > 0 && (
+                            <Section title={`Blockers (${preview.blockers.length})`} tone="destructive">
+                                {preview.blockers.map(b => (
+                                    <li key={b.id} className="text-xs text-stat-value">{b.message}</li>
+                                ))}
+                            </Section>
+                        )}
+                        {preview.warnings.length > 0 && (
+                            <Section title={`Warnings (${preview.warnings.length})`} tone="warning">
+                                {preview.warnings.map(w => (
+                                    <li key={w.id} className="text-xs text-stat-value">{w.message}</li>
+                                ))}
+                            </Section>
+                        )}
+                        <Section title="Changes" tone="neutral">
+                            {preview.changes.map(c => {
+                                const nodeMeta = [c.nodeType, c.status].filter(Boolean).join('/');
+                                const reachNote = c.reachabilityNote && c.reachabilityNote !== 'Local node'
+                                    ? c.reachabilityNote
+                                    : null;
+                                return (
+                                    <li key={`${c.nodeId}:${c.action}`} className="text-xs text-stat-value">
+                                        <span className="font-mono">{c.nodeName}</span>
+                                        {nodeMeta ? (
+                                            <span className="text-stat-subtitle"> ({nodeMeta})</span>
+                                        ) : null}
+                                        {' · '}
+                                        {c.action}
+                                        {' · '}
+                                        {c.severity}
+                                        {': '}
+                                        {c.detail}
+                                        {reachNote ? (
+                                            <span className="text-stat-subtitle"> · {reachNote}</span>
+                                        ) : null}
+                                    </li>
+                                );
+                            })}
+                            {preview.changes.length === 0 && (
+                                <li className="text-xs text-stat-subtitle">No node actions in this plan.</li>
+                            )}
+                        </Section>
+                        {hasRequirements && (
+                            <Section title="Requirements" tone="neutral">
+                                {preview.requirements.variables.map(v => (
+                                    <li key={v.name} className="text-xs font-mono text-stat-value">
+                                        {`\${${v.name}}`}
+                                        {v.required ? ' required' : ''}
+                                        {v.likelySecret ? ' (likely secret)' : ''}
+                                    </li>
+                                ))}
+                                {preview.requirements.envFiles.map(f => (
+                                    <li key={f.path} className="text-xs font-mono text-stat-value">
+                                        env_file {f.path}{f.required ? ' required' : ''}
+                                    </li>
+                                ))}
+                                {preview.requirements.composeSecrets.map(s => (
+                                    <li key={s.name} className="text-xs font-mono text-stat-value">
+                                        secret {s.name}
+                                    </li>
+                                ))}
+                            </Section>
+                        )}
+                    </div>
+                )}
+            </ModalBody>
+            <ModalFooter
+                secondary={
+                    <Button variant="outline" size="sm" onClick={() => onOpenChange(false)} disabled={submitting}>
+                        Cancel
+                    </Button>
+                }
+                primary={
+                    <Button size="sm" onClick={() => void handleConfirm()} disabled={!canConfirm}>
+                        {submitting ? 'Applying…' : 'Confirm Apply'}
+                    </Button>
+                }
+            />
+        </Modal>
+    );
+}
+
+function Section({
+    title,
+    tone,
+    children,
+}: {
+    title: string;
+    tone: 'destructive' | 'warning' | 'neutral';
+    children: React.ReactNode;
+}) {
+    return (
+        <div className={`rounded-md border ${sectionBorderClass(tone)} px-3 py-2`}>
+            <div className="font-mono text-[10px] uppercase tracking-[0.2em] text-stat-subtitle mb-1.5">{title}</div>
+            <ul className="space-y-1 list-disc pl-4">{children}</ul>
+        </div>
+    );
+}