fix: bind deploy progress, request, and health gate to the captured node (#1357)

* fix: bind deploy progress, request, and health gate to the captured node

A deploy/update/install/git-apply re-read the active node from localStorage
independently at three points: the progress WebSocket at mount, the POST at call
time, and the health-gate poll. If the active node changed between the click and
any of those, the operation, its live output, and its health verdict could
target different nodes, and the socket and POST splitting across nodes broke
output streaming.

Capture the operation's node once when it starts and thread it through every
leg. A new nodeId option on apiFetch overrides the active-node read, the
progress terminal takes a nodeId prop for its socket URL, the health gate polls
on the captured node, and a failed gate records its recovery entry only on the
node it ran on. The surface, the request, and the gate now always agree.

* fix: scope failed-gate recovery to the file list's node and harden node targeting

Addresses review findings on the captured-node binding:

- Track the node the stack file list was fetched for (filesNodeId) and record a
  failed gate's recovery entry only when it matches the gate's node. This closes
  a race where switching back to the gate's node could match a same-named stack
  from the previous node's still-loaded list before the new list lands, keying
  the record to the wrong file and blocking the correct one. refreshStacks now
  carries a sequence token so an out-of-order resolution cannot leave files and
  filesNodeId inconsistent.
- Make an explicit apiFetch nodeId authoritative over a caller-supplied
  x-node-id header.
- Add the missing stack-logs nodeId cases (null, and active-node fallback) to the
  terminal tests.
This commit is contained in:
Anso
2026-06-11 14:36:20 -04:00
committed by GitHub
parent a2fe58f62f
commit 48cebf9501
21 changed files with 617 additions and 51 deletions
@@ -71,7 +71,7 @@ beforeEach(() => {
});
const passedGate = (): HealthGateUiState => ({
stackName: 'web', gateId: 'g', trigger: 'update', status: 'passed', reason: null, windowSeconds: 90, startedAt: Date.now() - 90_000,
stackName: 'web', nodeId: null, gateId: 'g', trigger: 'update', status: 'passed', reason: null, windowSeconds: 90, startedAt: Date.now() - 90_000,
});
function renderBanner(activeNode: Node | null = null, panelStartedAt: number | null = Date.now() - 12_000) {
@@ -117,7 +117,7 @@ describe('StackOperationBanner', () => {
unmount();
mockPanelState = panel({ status: 'succeeded' });
mockHealthGate = { stackName: 'web', gateId: 'g', trigger: 'update', status: 'failed', reason: 'exited', windowSeconds: 90, startedAt: Date.now() };
mockHealthGate = { stackName: 'web', nodeId: null, gateId: 'g', trigger: 'update', status: 'failed', reason: 'exited', windowSeconds: 90, startedAt: Date.now() };
renderBanner();
expect(screen.queryByTestId('stack-operation-banner')).toBeNull();
});
@@ -137,7 +137,7 @@ describe('StackOperationBanner', () => {
it('shows the observing health gate and keeps present tense', () => {
mockPanelState = panel({ status: 'succeeded' });
mockHealthGate = { stackName: 'web', gateId: 'g', trigger: 'update', status: 'observing', reason: null, windowSeconds: 90, startedAt: Date.now() - 12_000 };
mockHealthGate = { stackName: 'web', nodeId: null, gateId: 'g', trigger: 'update', status: 'observing', reason: null, windowSeconds: 90, startedAt: Date.now() - 12_000 };
renderBanner();
expect(screen.getByText('Updating')).toBeInTheDocument();
expect(screen.getByText('Verifying health')).toBeInTheDocument();
@@ -146,7 +146,7 @@ describe('StackOperationBanner', () => {
it('shows the passed health gate', () => {
mockPanelState = panel({ status: 'succeeded' });
mockHealthGate = { stackName: 'web', gateId: 'g', trigger: 'update', status: 'passed', reason: null, windowSeconds: 90, startedAt: Date.now() - 90_000 };
mockHealthGate = { stackName: 'web', nodeId: null, gateId: 'g', trigger: 'update', status: 'passed', reason: null, windowSeconds: 90, startedAt: Date.now() - 90_000 };
renderBanner();
expect(screen.getByText('Health gate passed')).toBeInTheDocument();
expect(screen.getByText('Updated')).toBeInTheDocument();
@@ -154,7 +154,7 @@ describe('StackOperationBanner', () => {
it('shows the unknown health gate state with its reason', () => {
mockPanelState = panel({ status: 'succeeded' });
mockHealthGate = { stackName: 'web', gateId: 'g', trigger: 'update', status: 'unknown', reason: 'no healthcheck defined', windowSeconds: 90, startedAt: Date.now() };
mockHealthGate = { stackName: 'web', nodeId: null, gateId: 'g', trigger: 'update', status: 'unknown', reason: 'no healthcheck defined', windowSeconds: 90, startedAt: Date.now() };
renderBanner();
expect(screen.getByText('Health check unknown')).toBeInTheDocument();
expect(screen.getByText('no healthcheck defined')).toBeInTheDocument();
@@ -253,7 +253,7 @@ describe('StackOperationBanner', () => {
vi.useFakeTimers();
try {
mockPanelState = panel({ status: 'succeeded' });
mockHealthGate = { stackName: 'web', gateId: 'g', trigger: 'update', status: 'observing', reason: null, windowSeconds: 90, startedAt: Date.now() };
mockHealthGate = { stackName: 'web', nodeId: null, gateId: 'g', trigger: 'update', status: 'observing', reason: null, windowSeconds: 90, startedAt: Date.now() };
renderBanner();
act(() => { vi.advanceTimersByTime(8000); });
expect(onPanelClose).not.toHaveBeenCalled();
@@ -0,0 +1,46 @@
import { describe, it, expect } from 'vitest';
import { classifyFailedGate } from './failed-gate-recovery';
import type { HealthGateUiState } from '@/context/DeployFeedbackContext';
type Gate = Pick<HealthGateUiState, 'status' | 'nodeId' | 'stackName'>;
const gate = (over: Partial<Gate> = {}): Gate => ({ status: 'failed', nodeId: null, stackName: 'web', ...over });
describe('classifyFailedGate', () => {
it('skips when there is no gate', () => {
expect(classifyFailedGate(null, null, null, ['web.yml'])).toEqual({ kind: 'skip' });
});
it('skips a gate that has not failed', () => {
expect(classifyFailedGate(gate({ status: 'observing' }), null, null, ['web.yml'])).toEqual({ kind: 'skip' });
expect(classifyFailedGate(gate({ status: 'passed' }), null, null, ['web.yml'])).toEqual({ kind: 'skip' });
});
it('records on the local node when the gate ran locally and a file matches', () => {
expect(classifyFailedGate(gate({ nodeId: null }), null, null, ['web.yml'])).toEqual({ kind: 'record', stackFile: 'web.yml' });
});
it('records on a remote node when the gate, the active node, and the file list all match', () => {
expect(classifyFailedGate(gate({ nodeId: 3 }), 3, 3, ['web.yaml'])).toEqual({ kind: 'record', stackFile: 'web.yaml' });
});
it('skips when the gate ran on a different node than the active one', () => {
// Remote gate, active node is local: must not attach to a same-named local stack.
expect(classifyFailedGate(gate({ nodeId: 3 }), null, null, ['web.yml'])).toEqual({ kind: 'skip' });
// Local gate, active node is remote.
expect(classifyFailedGate(gate({ nodeId: null }), 3, 3, ['web.yml'])).toEqual({ kind: 'skip' });
// Two different remote nodes.
expect(classifyFailedGate(gate({ nodeId: 2 }), 5, 5, ['web.yml'])).toEqual({ kind: 'skip' });
});
it('skips when the active node matches but the loaded file list is still another node\'s', () => {
// Switch-back gap: active node is the gate's node again, but files (and its
// filesNodeId) have not refreshed yet, so the list belongs to the node we
// just left. A same-named stack there must not capture the recovery entry.
expect(classifyFailedGate(gate({ nodeId: 3 }), 3, 2, ['web.yml'])).toEqual({ kind: 'skip' });
expect(classifyFailedGate(gate({ nodeId: null }), null, 2, ['web.yml'])).toEqual({ kind: 'skip' });
});
it('reports no-file when the node and file list match but no stack file matches the name yet', () => {
expect(classifyFailedGate(gate({ nodeId: 3, stackName: 'web' }), 3, 3, ['other.yml'])).toEqual({ kind: 'no-file' });
});
});
@@ -0,0 +1,37 @@
import type { HealthGateUiState } from '@/context/DeployFeedbackContext';
/**
* Decide whether a failed health gate should record a recovery entry, and for
* which stack file. Extracted as a pure function so the cross-node guard (the
* load-bearing rule that a gate's failure records only on the node it ran on)
* is unit-testable without standing up the editor.
*
* - `skip`: not a failed gate, or it ran on a different node than the active one,
* or the loaded file list does not yet belong to the gate's node. Stack
* filenames repeat across nodes and recovery records are cleared on node
* switch, so recording against another node's list would attach the failure to
* the wrong stack. `filesNodeId` (the node `files` was fetched for) must match
* too: right after a switch the active node updates before the new list lands,
* and a name lookup against the stale list could resolve to a wrong filename.
* - `no-file`: the gate's node matches and its file list is loaded, but no stack
* file matches its name yet (the list may be mid-refresh). The caller leaves it
* unhandled so the effect retries once the files land.
* - `record`: record a recovery entry against `stackFile`.
*/
export type FailedGateOutcome =
| { kind: 'skip' }
| { kind: 'no-file' }
| { kind: 'record'; stackFile: string };
export function classifyFailedGate(
healthGate: Pick<HealthGateUiState, 'status' | 'nodeId' | 'stackName'> | null,
activeNodeId: number | null,
filesNodeId: number | null,
files: string[],
): FailedGateOutcome {
if (!healthGate || healthGate.status !== 'failed') return { kind: 'skip' };
// Record only while on the gate's node AND with that node's file list loaded.
if (healthGate.nodeId !== activeNodeId || healthGate.nodeId !== filesNodeId) return { kind: 'skip' };
const stackFile = files.find(f => f.replace(/\.(yml|yaml)$/, '') === healthGate.stackName);
return stackFile ? { kind: 'record', stackFile } : { kind: 'no-file' };
}
@@ -90,8 +90,11 @@ function makeOverlay(over: Partial<OverlayState> = {}): OverlayState {
} as unknown as OverlayState;
}
const runWithLog: Parameters<typeof useStackActions>[0]['runWithLog'] = async (_p, run) =>
run(Promise.resolve(), 'test-session');
let lastRunWithLogParams: Parameters<Parameters<typeof useStackActions>[0]['runWithLog']>[0] | null = null;
const runWithLog: Parameters<typeof useStackActions>[0]['runWithLog'] = async (params, run) => {
lastRunWithLogParams = params;
return run(Promise.resolve(), 'test-session');
};
function setup(over: {
editorState?: Partial<EditorState>;
@@ -197,6 +200,44 @@ describe('useStackActions.handleSaveAndDeploy', () => {
});
});
describe('useStackActions node binding', () => {
const mouseEvent = { preventDefault: vi.fn(), stopPropagation: vi.fn() } as unknown as React.MouseEvent;
beforeEach(() => {
vi.mocked(apiFetch).mockReset();
vi.mocked(apiFetch).mockResolvedValue(new Response(null, { status: 200 }));
lastRunWithLogParams = null;
});
function postCallFor(fragment: string): RequestInit | undefined {
const call = vi.mocked(apiFetch).mock.calls.find(
c => String(c[0]).includes(fragment) && (c[1] as RequestInit | undefined)?.method === 'POST',
);
return call?.[1] as RequestInit | undefined;
}
it('binds the deploy POST and the runWithLog session to the captured node', async () => {
const { result } = setup(); // activeNode.id = 1
await result.current.deployStack(mouseEvent);
expect(postCallFor('/deploy')).toEqual(expect.objectContaining({ nodeId: 1 }));
expect(lastRunWithLogParams).toEqual(expect.objectContaining({ nodeId: 1 }));
});
it('binds the update POST and the runWithLog session to the captured node', async () => {
const { result } = setup();
await result.current.updateStack(mouseEvent);
expect(postCallFor('/update')).toEqual(expect.objectContaining({ nodeId: 1 }));
expect(lastRunWithLogParams).toEqual(expect.objectContaining({ nodeId: 1 }));
});
it('binds a non-update action (restart) POST and session to the captured node', async () => {
const { result } = setup();
await result.current.restartStack(mouseEvent);
expect(postCallFor('/restart')).toEqual(expect.objectContaining({ nodeId: 1 }));
expect(lastRunWithLogParams).toEqual(expect.objectContaining({ nodeId: 1 }));
});
});
describe('useStackActions policy-block dialog wiring', () => {
const policyPayload = {
error: 'Policy "block-high" blocked deploy: 1 image(s) exceed HIGH',
@@ -654,6 +654,7 @@ export function useStackActions(options: UseStackActionsOptions) {
ignorePolicy: boolean,
started?: Promise<void>,
deploySessionId?: string,
opNodeId?: number | null,
): Promise<RunResult> => {
const previousStatus = stackListState.stackStatuses[stackFile];
const startedAt = Date.now();
@@ -663,7 +664,7 @@ export function useStackActions(options: UseStackActionsOptions) {
? `/stacks/${stackName}/deploy?ignorePolicy=true`
: `/stacks/${stackName}/deploy`;
if (started) await started;
const response = await apiFetch(path, withDeploySession(deploySessionId ?? '', { method: 'POST' }));
const response = await apiFetch(path, withDeploySession(deploySessionId ?? '', { method: 'POST', nodeId: opNodeId }));
if (!response.ok) {
const rawBody = await response.text();
if (response.status === 409) {
@@ -729,9 +730,12 @@ export function useStackActions(options: UseStackActionsOptions) {
const stackFile = stackListState.selectedFile;
const stackName = stackFile.replace(/\.(yml|yaml)$/, '');
stackListState.setStackAction(stackFile, 'deploy');
// Snapshot the node once so the request stays bound to it even if the active
// node changes while the operation is in flight.
const opNodeId = activeNode?.id ?? null;
try {
await runWithLog({ stackName, action: 'deploy', nodeId: activeNode?.id ?? null }, (started, ds) =>
runDeploy(stackName, stackFile, false, started, ds),
await runWithLog({ stackName, action: 'deploy', nodeId: opNodeId }, (started, ds) =>
runDeploy(stackName, stackFile, false, started, ds, opNodeId),
);
} finally {
stackListState.clearStackAction(stackFile);
@@ -764,9 +768,10 @@ export function useStackActions(options: UseStackActionsOptions) {
await rollbackStack(true);
} else {
stackListState.setStackAction(existingFile, 'deploy');
const opNodeId = activeNode?.id ?? null;
try {
await runWithLog({ stackName, action: 'deploy', nodeId: activeNode?.id ?? null }, (started, ds) =>
runDeploy(stackName, existingFile, true, started, ds),
await runWithLog({ stackName, action: 'deploy', nodeId: opNodeId }, (started, ds) =>
runDeploy(stackName, existingFile, true, started, ds, opNodeId),
);
} finally {
stackListState.clearStackAction(existingFile);
@@ -895,14 +900,17 @@ export function useStackActions(options: UseStackActionsOptions) {
const startedAt = Date.now();
stackListState.setStackAction(stackFile, action);
stackListState.setOptimisticStatus(stackFile, optimisticStatus);
// Snapshot the node once so stop/restart/update stays bound to it even if
// the active node changes while the operation is in flight.
const opNodeId = activeNode?.id ?? null;
try {
await runWithLog({ stackName, action, nodeId: activeNode?.id ?? null }, async (started, ds) => {
await runWithLog({ stackName, action, nodeId: opNodeId }, async (started, ds) => {
await started;
try {
const url = ignorePolicy
? `/stacks/${stackName}/${endpoint}?ignorePolicy=true`
: `/stacks/${stackName}/${endpoint}`;
const response = await apiFetch(url, withDeploySession(ds, { method: 'POST' }));
const response = await apiFetch(url, withDeploySession(ds, { method: 'POST', nodeId: opNodeId }));
if (!response.ok) {
const errText = await response.text();
if (response.status === 409) {
@@ -33,10 +33,20 @@ export function useStackListState() {
const { nodes, activeNode } = useNodes();
const [files, setFiles] = useState<string[]>([]);
// Node the current `files` list belongs to (null = local). Stamped together
// with `files` from the node active when the fetch started, so a consumer can
// tell whether the list it is reading is the one it expects, even during the
// async gap right after a node switch when `files` still holds the old node's
// entries. Filenames repeat across nodes, so a name lookup against the wrong
// list would resolve to the wrong file.
const [filesNodeId, setFilesNodeId] = useState<number | null>(null);
const [selectedFile, setSelectedFile] = useState<string | null>(null);
const [isLoading, setIsLoading] = useState(false);
const [stackActions, setStackActions] = useState<Record<string, StackAction>>({});
const stackActionsRef = useRef<Record<string, StackAction>>({});
// Monotonic token per refreshStacks call; lets a superseded fetch skip its
// state writes so a rapid node switch cannot leave a stale files/filesNodeId.
const fetchSeqRef = useRef(0);
// Per-stack terminal failure records driving the in-detail recovery panel.
// In-memory only. Node scoping is enforced by the caller, which clears these
@@ -133,18 +143,28 @@ export function useStackListState() {
const refreshStacks = async (background = false): Promise<string[]> => {
if (!background) setIsLoading(true);
// Snapshot the node this fetch targets and a sequence token so a superseded
// or out-of-order resolution (from a rapid node switch) cannot overwrite a
// newer node's list, keeping `files` and `filesNodeId` consistent.
const fetchNodeId = activeNode?.id ?? null;
const mySeq = ++fetchSeqRef.current;
const stale = () => fetchSeqRef.current !== mySeq;
try {
const res = await apiFetch('/stacks');
if (stale()) return [];
if (!res.ok) {
setFiles([]);
setFilesNodeId(fetchNodeId);
return [];
}
const data = await res.json();
const fileList: string[] = Array.isArray(data) ? data : [];
setFiles(fileList);
setFilesNodeId(fetchNodeId);
// Fetch all stack statuses in a single bulk call (falls back to per-stack queries for older remote nodes)
const statusRes = await apiFetch('/stacks/statuses');
if (stale()) return fileList;
let bulkStatuses: Record<string, 'running' | 'exited' | 'unknown'> | null = null;
const bulkPorts: Record<string, number | undefined> = {};
if (statusRes.ok) {
@@ -194,8 +214,10 @@ export function useStackListState() {
refreshLabels();
return fileList;
} catch (error) {
if (stale()) return [];
console.error('Failed to refresh stacks:', error);
setFiles([]);
setFilesNodeId(fetchNodeId);
return [];
} finally {
setIsLoading(false);
@@ -341,7 +363,7 @@ export function useStackListState() {
}, [remoteStackResults, nodes]);
return {
files, setFiles,
files, setFiles, filesNodeId,
selectedFile, setSelectedFile,
isLoading, setIsLoading,
stackActions, stackActionsRef,