fix(stacks): serialize concurrent lifecycle operations per stack (#1182)

* fix(stacks): serialize concurrent lifecycle operations per stack

Two simultaneous POSTs to /api/stacks/:name/{deploy,down,restart,stop,
start,update} could race against the same compose project, doubling
notifications, doubling post-deploy scans, and corrupting the
atomic-deploy backup snapshot. Each lifecycle route now acquires a
per-(nodeId, stackName) in-process lock; the second caller gets 409
with {code: 'stack_op_in_progress', inProgress: {action, startedAt,
user}} and the frontend surfaces a "X is already deploying" toast.

The lock is process-local on purpose: it shares a lifetime with the
docker compose child process. A Sencho restart clears all locks, which
matches the truth that an in-flight compose op is gone too.

The existing policy-block 409 is shape-distinguishable (has policy /
violations) and continues to work; the frontend checks the new code
discriminator first before falling through to policy handling.

* chore(stacks): validate action enum in 409 parser; cover start collision

The frontend parseStackOpInProgress used to cast the parsed action
directly to StackOpAction. A backend bug or spoofed payload returning
action='wibble' would slip through. Validate against the known enum
set before returning the parsed info.

Adds an integration test for the deploy-blocks-while-start-in-flight
case so all six lifecycle verbs have collision coverage (the existing
suite covered deploy/down/restart/stop/update; start was indirect).
This commit is contained in:
Anso
2026-05-24 01:12:07 -04:00
committed by GitHub
parent 8ba88755b1
commit 60ecd574b3
5 changed files with 621 additions and 20 deletions
@@ -19,6 +19,27 @@ interface RunResult {
type StackActionError = Error & { rolledBack?: boolean };
type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update';
interface StackOpInProgressInfo {
action: StackOpAction;
startedAt: number;
user: string;
}
const STACK_OP_PRESENT_PARTICIPLE: Record<StackOpAction, string> = {
deploy: 'deploying',
down: 'stopping',
restart: 'restarting',
stop: 'stopping',
start: 'starting',
update: 'updating',
};
const VALID_STACK_OP_ACTIONS: ReadonlySet<string> = new Set(
Object.keys(STACK_OP_PRESENT_PARTICIPLE),
);
type EditorState = ReturnType<typeof useEditorViewState>;
type StackListState = ReturnType<typeof useStackListState>;
type NavState = ReturnType<typeof useViewNavigationState>;
@@ -42,6 +63,35 @@ interface UseStackActionsOptions {
const isRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === 'object' && value !== null;
const parseStackOpInProgress = (rawBody: string): StackOpInProgressInfo | null => {
try {
const parsed: unknown = JSON.parse(rawBody);
if (!isRecord(parsed) || parsed.code !== 'stack_op_in_progress') return null;
const inProgress = parsed.inProgress;
if (
!isRecord(inProgress) ||
typeof inProgress.action !== 'string' ||
typeof inProgress.startedAt !== 'number' ||
!VALID_STACK_OP_ACTIONS.has(inProgress.action)
) {
return null;
}
return {
action: inProgress.action as StackOpAction,
startedAt: inProgress.startedAt,
user: typeof inProgress.user === 'string' ? inProgress.user : '',
};
} catch {
return null;
}
};
const stackOpInProgressMessage = (stackName: string, info: StackOpInProgressInfo): string => {
const verb = STACK_OP_PRESENT_PARTICIPLE[info.action] ?? 'busy';
const actor = info.user && info.user !== 'system' ? ` (started by ${info.user})` : '';
return `${stackName} is already ${verb}${actor}.`;
};
const parseStackActionError = (rawBody: string, fallback: string): StackActionError => {
let message = rawBody || fallback;
let rolledBack = false;
@@ -388,6 +438,17 @@ export function useStackActions(options: UseStackActionsOptions) {
if (!response.ok) {
const rawBody = await response.text();
if (response.status === 409) {
const inProgress = parseStackOpInProgress(rawBody);
if (inProgress) {
const message = stackOpInProgressMessage(stackName, inProgress);
if (previousStatus !== undefined)
stackListState.setOptimisticStatus(
stackFile,
previousStatus as 'running' | 'exited',
);
toast.error(message);
return { ok: false, errorMessage: message };
}
let parsed: PolicyBlockPayload | null = null;
try {
parsed = JSON.parse(rawBody) as PolicyBlockPayload;
@@ -578,6 +639,14 @@ export function useStackActions(options: UseStackActionsOptions) {
const response = await apiFetch(`/stacks/${stackName}/${endpoint}`, { method: 'POST' });
if (!response.ok) {
const errText = await response.text();
if (response.status === 409) {
const inProgress = parseStackOpInProgress(errText);
if (inProgress) {
const message = stackOpInProgressMessage(stackName, inProgress);
toast.error(message);
return { ok: false as const, errorMessage: message };
}
}
const actionError = parseStackActionError(errText, `${action} failed`);
return {
ok: false as const,
@@ -728,6 +797,13 @@ export function useStackActions(options: UseStackActionsOptions) {
const response = await apiFetch(`/stacks/${stackName}/${endpoint}`, { method: 'POST' });
if (!response.ok) {
const errText = await response.text();
if (response.status === 409) {
const inProgress = parseStackOpInProgress(errText);
if (inProgress) {
toast.error(stackOpInProgressMessage(stackName, inProgress));
return;
}
}
throw parseStackActionError(errText, `${action} failed`);
}
toast.success(`Stack ${action}ed successfully!`);