fix: bind fleet stop-by-label to the exact confirmed nodes and stacks (#1506)

A fleet stop re-matched stacks by label name at execution, so a stack that
gained the label between the operator's preview and confirmation could be
stopped even though it never appeared in the confirmation. A confirmed node
that was deleted after the preview also vanished from the results, letting the
remaining successes read as a clean stop.

The confirm flow now sends the exact node and stack list resolved in the
preview. Each node's stop is bound to that set: only stacks that are still
label-matched and confirmed are stopped, and a confirmed node missing from the
registry is reported as an explicit failure rather than dropped.
This commit is contained in:
Anso
2026-06-28 16:33:01 -04:00
committed by GitHub
parent 78a742fb44
commit 1dc12f7da8
7 changed files with 333 additions and 59 deletions
@@ -230,9 +230,10 @@ it('gates Stop fleet on a resolved preview, carries the node/stack list into the
await waitFor(() => {
const stopCall = mockedFetch.mock.calls.find(c => c[0] === '/fleet/labels/fleet-stop');
// The real stop binds execution to the nodes resolved in the preview: it
// carries their ids so a node that was unreachable then cannot be added later.
expect(JSON.parse(stopCall![1].body)).toEqual({ labelName: 'prod', dryRun: false, nodeIds: [1] });
// The real stop binds execution to the exact node + stack list resolved in
// the preview, so neither a reconnecting node nor a newly labelled stack can
// be added later.
expect(JSON.parse(stopCall![1].body)).toEqual({ labelName: 'prod', dryRun: false, targets: [{ nodeId: 1, stackNames: ['web'] }] });
});
// Per-node results render in the card (the "· 1 stack" label is unique to the
// results list, distinguishing it from the preview well).
@@ -527,3 +528,38 @@ it('renders an unreachable fleet-stop node as unreachable, not "no matching labe
expect(await screen.findByText(/edge-1 \(unreachable\)/)).toBeInTheDocument();
await waitFor(() => expect(toastWarning).toHaveBeenCalled());
});
it('reports a confirmed node that vanished as a failure, not a clean success', async () => {
const user = userEvent.setup();
mockedFetch.mockImplementation((url: string) => {
if (url === '/fleet/labels/match-preview') {
return Promise.resolve(jsonResponse(200, {
matchedNodes: 1, matchedStacks: 1, unreachableNodes: 0,
perNode: [{ nodeId: 1, nodeName: 'central', reachable: true, labelExists: true, stackCount: 1, stackNames: ['web'] }],
}));
}
if (url === '/fleet/labels/fleet-stop') {
// One stopped node plus a confirmed node that disappeared from the
// registry: the backend returns its confirmed stacks as failures.
return Promise.resolve(jsonResponse(200, {
results: [
{ nodeId: 1, nodeName: 'central', reachable: true, matched: true, stackResults: [{ stackName: 'web', success: true }] },
{ nodeId: 2, nodeName: 'Node 2', reachable: false, matched: false, error: 'Node no longer exists', stackResults: [{ stackName: 'api', success: false, error: 'Node no longer exists' }] },
],
}));
}
return Promise.resolve(jsonResponse(404, {}));
});
render(<LabelFleetStopCard />);
await user.type(screen.getByPlaceholderText('e.g. production'), 'prod');
const stopBtn = screen.getByRole('button', { name: 'Stop fleet' });
await waitFor(() => expect(stopBtn).toBeEnabled(), { timeout: 2000 });
await user.click(stopBtn);
const dialog = await screen.findByRole('alertdialog');
await user.click(within(dialog).getByRole('button', { name: 'Stop fleet' }));
// The vanished confirmed node renders as an unreachable failure and flips the
// outcome to a warning, so a partial stop never reads as a clean success.
expect(await screen.findByText(/Node 2 \(unreachable\)/)).toBeInTheDocument();
await waitFor(() => expect(toastWarning).toHaveBeenCalled());
expect(toastSuccess).not.toHaveBeenCalled();
});
@@ -190,7 +190,7 @@ export function LabelFleetStopCard() {
const blastTone = preview.kind === 'loading' || preview.kind === 'unavailable' ? 'muted' as const : undefined;
async function run(opts: { dryRun: boolean; nodeIds?: number[] }) {
async function run(opts: { dryRun: boolean; targets?: { nodeId: number; stackNames: string[] }[] }) {
const trimmed = labelName.trim();
if (!trimmed) return;
const verb = opts.dryRun ? 'Dry-running' : 'Stopping';
@@ -198,12 +198,13 @@ export function LabelFleetStopCard() {
setRunning(true);
setResults([]);
try {
// The real stop carries the node ids the operator confirmed in the preview
// so execution cannot expand to a node that was unreachable then and has
// since reconnected. The dry run sends no allowlist and scans the fleet.
// The real stop carries the exact node + stack list the operator confirmed
// in the preview, so execution cannot expand to a node that has since
// reconnected nor to a stack that gained the label after the preview. The
// dry run sends no allowlist and scans the fleet.
const res = await apiFetch('/fleet/labels/fleet-stop', {
method: 'POST',
body: JSON.stringify({ labelName: trimmed, dryRun: opts.dryRun, ...(opts.nodeIds ? { nodeIds: opts.nodeIds } : {}) }),
body: JSON.stringify({ labelName: trimmed, dryRun: opts.dryRun, ...(opts.targets ? { targets: opts.targets } : {}) }),
});
const body = await res.json().catch(() => ({}));
toast.dismiss(toastId);
@@ -373,7 +374,7 @@ export function LabelFleetStopCard() {
description="Sencho will stop the stacks listed below. Node labels are not used by this action. Services will be unavailable until restarted."
confirmLabel="Stop fleet"
confirming={running}
onConfirm={() => run({ dryRun: false, nodeIds: (resolvedTargets ?? []).map(t => t.nodeId) })}
onConfirm={() => run({ dryRun: false, targets: (resolvedTargets ?? []).map(t => ({ nodeId: t.nodeId, stackNames: t.stackNames })) })}
>
{resolvedTargets && resolvedTargets.length > 0 && <ResolvedTargetsList targets={resolvedTargets} />}
</ConfirmModal>
@@ -431,15 +432,16 @@ function renderPreviewSection(preview: PreviewState, trimmed: string) {
// The resolved node/stack list carried into the destructive confirm modal, so
// the operator confirms against the concrete blast radius rather than a label
// name. The node set is bound to this list (the stop sends these node ids and
// the backend acts on no others); only the per-node stacks are re-matched by
// label at execution, where state can still drift between preview and confirm.
// name. Both axes are bound to this list: the stop sends these node ids and the
// exact stacks per node, and the backend stops only stacks that are still
// label-matched AND in this set, so drift between preview and confirm (a node
// reconnecting, a stack newly labelled) cannot widen the blast radius.
function ResolvedTargetsList({ targets }: { targets: ResolvedTarget[] }) {
const stackCount = targets.reduce((n, t) => n + t.stackNames.length, 0);
return (
<div>
<div className={cn(KICKER, 'text-stat-icon mb-2 normal-case tracking-normal text-[11px]')}>
Will stop {stackCount} stack{stackCount === 1 ? '' : 's'} across {targets.length} node{targets.length === 1 ? '' : 's'}. The stacks on each node are re-matched by label at execution; the node set is fixed to those listed.
Will stop {stackCount} stack{stackCount === 1 ? '' : 's'} across {targets.length} node{targets.length === 1 ? '' : 's'}. Only these stacks are stopped, and only while they still carry the label.
</div>
<div className="max-h-[180px] overflow-y-auto rounded border border-card-border/60 bg-card/40 p-2">
<ul className="space-y-1.5">