gsadmin/sencho
1
0
Fork 0
mirror of https://github.com/Studio-Saelix/sencho.git synced 2026-08-28 11:17:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(stacks): serialize concurrent lifecycle operations per stack (#1182)

Browse Source 
* fix(stacks): serialize concurrent lifecycle operations per stack

Two simultaneous POSTs to /api/stacks/:name/{deploy,down,restart,stop,
start,update} could race against the same compose project, doubling
notifications, doubling post-deploy scans, and corrupting the
atomic-deploy backup snapshot. Each lifecycle route now acquires a
per-(nodeId, stackName) in-process lock; the second caller gets 409
with {code: 'stack_op_in_progress', inProgress: {action, startedAt,
user}} and the frontend surfaces a "X is already deploying" toast.

The lock is process-local on purpose: it shares a lifetime with the
docker compose child process. A Sencho restart clears all locks, which
matches the truth that an in-flight compose op is gone too.

The existing policy-block 409 is shape-distinguishable (has policy /
violations) and continues to work; the frontend checks the new code
discriminator first before falling through to policy handling.

* chore(stacks): validate action enum in 409 parser; cover start collision

The frontend parseStackOpInProgress used to cast the parsed action
directly to StackOpAction. A backend bug or spoofed payload returning
action='wibble' would slip through. Validate against the known enum
set before returning the parsed info.

Adds an integration test for the deploy-blocks-while-start-in-flight
case so all six lifecycle verbs have collision coverage (the existing
suite covered deploy/down/restart/stop/update; start was indirect).
This commit is contained in:
Anso
2026-05-24 01:12:07 -04:00
committed by GitHub
parent 8ba88755b1
commit 60ecd574b3
5 changed files with 621 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/routes/stacks.ts
+82 -20
View File
@@ -14,6 +14,7 @@ import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement';
import { requirePermission } from '../middleware/permissions';
import { requirePaid, requireAdmin, effectiveTier } from '../middleware/tierGates';
import { NotificationService, type NotificationCategory } from '../services/NotificationService';
import { StackOpLockService, type StackOpAction } from '../services/StackOpLockService';
import { isValidGitSourcePath, isValidStackName, isValidServiceName, isPathWithinBase, isValidRelativeStackPath } from '../utils/validation';
import { getErrorMessage } from '../utils/errors';
import { isDebugEnabled } from '../utils/debug';
@@ -37,6 +38,42 @@ function notifyActionSuccess(category: NotificationCategory, message: string, st
.catch(err => console.error('[Stacks] Failed to dispatch activity for %s:', sanitizeForLog(stackName), err));
}
const STACK_OP_PRESENT_PARTICIPLE: Record<StackOpAction, string> = {
deploy: 'deploying',
down: 'stopping',
restart: 'restarting',
stop: 'stopping',
start: 'starting',
update: 'updating',
};
function tryAcquireStackOpLock(
req: Request,
res: Response,
stackName: string,
action: StackOpAction,
): boolean {
const user = req.user?.username ?? 'system';
const result = StackOpLockService.getInstance().tryAcquire(req.nodeId, stackName, action, user);
if (!result.acquired) {
res.status(409).json({
error: `${stackName} is already ${STACK_OP_PRESENT_PARTICIPLE[result.existing.action]}`,
code: 'stack_op_in_progress',
inProgress: {
action: result.existing.action,
startedAt: result.existing.startedAt,
user: result.existing.user,
},
});
return false;
}
return true;
}
function releaseStackOpLock(req: Request, stackName: string): void {
StackOpLockService.getInstance().release(req.nodeId, stackName);
}
async function requireStackExists(nodeId: number, stackName: string, res: Response): Promise<boolean> {
if (!isValidStackName(stackName)) {
res.status(400).json({ error: 'Invalid stack name' });
@@ -640,6 +677,8 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => {
const stackName = req.params.stackName as string;
if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
// Lock held below. All early-returns must stay inside the try so finally fires.
if (!tryAcquireStackOpLock(req, res, stackName, 'deploy')) return;
try {
if (!(await runPolicyGate(req, res, stackName, req.nodeId))) return;
const skipScan = req.body?.skip_scan === true;
@@ -669,7 +708,9 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => {
}
const message = getErrorMessage(error, 'Failed to deploy stack');
notifyActionFailure('deploy', stackName, error);
res.status(500).json({ error: message, rolledBack });
if (!res.headersSent) res.status(500).json({ error: message, rolledBack });
} finally {
releaseStackOpLock(req, stackName);
}
});
@@ -677,6 +718,8 @@ stacksRouter.post('/:stackName/down', async (req: Request, res: Response) => {
const stackName = req.params.stackName as string;
if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
// Lock held below. All early-returns must stay inside the try so finally fires.
if (!tryAcquireStackOpLock(req, res, stackName, 'down')) return;
try {
await ComposeService.getInstance(req.nodeId).runCommand(stackName, 'down', getTerminalWs());
invalidateNodeCaches(req.nodeId);
@@ -685,7 +728,9 @@ stacksRouter.post('/:stackName/down', async (req: Request, res: Response) => {
} catch (error: unknown) {
console.error('[Stacks] Down failed: %s', sanitizeForLog(stackName), error);
notifyActionFailure('down', stackName, error);
res.status(500).json({ error: 'Failed to start command' });
if (!res.headersSent) res.status(500).json({ error: 'Failed to start command' });
} finally {
releaseStackOpLock(req, stackName);
}
});
@@ -729,25 +774,36 @@ async function bulkContainerOp(
): Promise<void> {
const stackName = req.params.stackName as string;
if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
const titleCase = action.charAt(0).toUpperCase() + action.slice(1);
const outcome = await containerActionForStack(req.nodeId, stackName, action);
// Lock held below. All early-returns must stay inside the try so finally fires.
if (!tryAcquireStackOpLock(req, res, stackName, action)) return;
try {
const titleCase = action.charAt(0).toUpperCase() + action.slice(1);
const outcome = await containerActionForStack(req.nodeId, stackName, action);
if (outcome.kind === 'no-containers') {
res.status(404).json({ error: 'No containers found for this stack.' });
return;
}
if (outcome.kind === 'error') {
console.error('[Stacks] %s failed: %s %s', sanitizeForLog(titleCase), sanitizeForLog(stackName), sanitizeForLog(outcome.message));
if (action !== 'start') notifyActionFailure(action, stackName, new Error(outcome.message));
res.status(500).json({ error: outcome.message });
return;
}
if (outcome.kind === 'no-containers') {
res.status(404).json({ error: 'No containers found for this stack.' });
return;
}
if (outcome.kind === 'error') {
console.error('[Stacks] %s failed: %s %s', sanitizeForLog(titleCase), sanitizeForLog(stackName), sanitizeForLog(outcome.message));
if (action !== 'start') notifyActionFailure(action, stackName, new Error(outcome.message));
res.status(500).json({ error: outcome.message });
return;
}
invalidateNodeCaches(req.nodeId);
console.log(`[Stacks] ${titleCase} completed: ${sanitizeForLog(stackName)} (${outcome.count} containers)`);
res.json({ success: true, message: `${titleCase} completed via Engine API.` });
const { category, pastTense } = CONTAINER_ACTION_META[action];
notifyActionSuccess(category, `${stackName} ${pastTense}`, stackName, req.user?.username ?? 'system');
invalidateNodeCaches(req.nodeId);
console.log(`[Stacks] ${titleCase} completed: ${sanitizeForLog(stackName)} (${outcome.count} containers)`);
res.json({ success: true, message: `${titleCase} completed via Engine API.` });
const { category, pastTense } = CONTAINER_ACTION_META[action];
notifyActionSuccess(category, `${stackName} ${pastTense}`, stackName, req.user?.username ?? 'system');
} catch (error: unknown) {
console.error('[Stacks] %s threw unexpectedly: %s', sanitizeForLog(action), sanitizeForLog(stackName), error);
if (!res.headersSent) {
res.status(500).json({ error: getErrorMessage(error, `Failed to ${action} stack`) });
}
} finally {
releaseStackOpLock(req, stackName);
}
}
stacksRouter.post('/:stackName/restart', (req, res) => bulkContainerOp(req, res, 'restart'));
@@ -824,6 +880,8 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
const stackName = req.params.stackName as string;
if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
// Lock held below. All early-returns must stay inside the try so finally fires.
if (!tryAcquireStackOpLock(req, res, stackName, 'update')) return;
try {
if (!(await runPolicyGate(req, res, stackName, req.nodeId))) return;
const skipScan = req.body?.skip_scan === true;
@@ -861,7 +919,11 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
console.warn(`[Stacks] Update failed, rollback did not complete: ${sanitizeForLog(stackName)}`);
}
notifyActionFailure('update', stackName, error);
res.status(500).json({ error: getErrorMessage(error, 'Failed to update'), rolledBack });
if (!res.headersSent) {
res.status(500).json({ error: getErrorMessage(error, 'Failed to update'), rolledBack });
}
} finally {
releaseStackOpLock(req, stackName);
}
});