From 60ecd574b3488aff53c0c417d1cd8087251b763a Mon Sep 17 00:00:00 2001 From: Anso Date: Sun, 24 May 2026 01:12:07 -0400 Subject: [PATCH] fix(stacks): serialize concurrent lifecycle operations per stack (#1182) * fix(stacks): serialize concurrent lifecycle operations per stack Two simultaneous POSTs to /api/stacks/:name/{deploy,down,restart,stop, start,update} could race against the same compose project, doubling notifications, doubling post-deploy scans, and corrupting the atomic-deploy backup snapshot. Each lifecycle route now acquires a per-(nodeId, stackName) in-process lock; the second caller gets 409 with {code: 'stack_op_in_progress', inProgress: {action, startedAt, user}} and the frontend surfaces a "X is already deploying" toast. The lock is process-local on purpose: it shares a lifetime with the docker compose child process. A Sencho restart clears all locks, which matches the truth that an in-flight compose op is gone too. The existing policy-block 409 is shape-distinguishable (has policy / violations) and continues to work; the frontend checks the new code discriminator first before falling through to policy handling. * chore(stacks): validate action enum in 409 parser; cover start collision The frontend parseStackOpInProgress used to cast the parsed action directly to StackOpAction. A backend bug or spoofed payload returning action='wibble' would slip through. Validate against the known enum set before returning the parsed info. Adds an integration test for the deploy-blocks-while-start-in-flight case so all six lifecycle verbs have collision coverage (the existing suite covered deploy/down/restart/stop/update; start was indirect). --- .../__tests__/stack-op-lock-routes.test.ts | 297 ++++++++++++++++++ .../__tests__/stack-op-lock-service.test.ts | 96 ++++++ backend/src/routes/stacks.ts | 102 ++++-- backend/src/services/StackOpLockService.ts | 70 +++++ .../EditorLayout/hooks/useStackActions.ts | 76 +++++ 5 files changed, 621 insertions(+), 20 deletions(-) create mode 100644 backend/src/__tests__/stack-op-lock-routes.test.ts create mode 100644 backend/src/__tests__/stack-op-lock-service.test.ts create mode 100644 backend/src/services/StackOpLockService.ts diff --git a/backend/src/__tests__/stack-op-lock-routes.test.ts b/backend/src/__tests__/stack-op-lock-routes.test.ts new file mode 100644 index 00000000..d9089824 --- /dev/null +++ b/backend/src/__tests__/stack-op-lock-routes.test.ts @@ -0,0 +1,297 @@ +/** + * Integration tests for the per-(nodeId, stackName) lifecycle mutex. + * + * The mutex prevents two simultaneous compose actions from racing against the + * same stack. The second caller receives 409 with a structured envelope so the + * frontend can show "X is already deploying" instead of doubling up. + * + * Each lifecycle route (deploy, down, restart, stop, start, update) is + * exercised: the first request is held mid-call via a deferred promise so the + * second request lands while the lock is still held, asserting both 409 and + * the expected `{code, inProgress}` payload. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import request from 'supertest'; +import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; + +const { + mockDeployStack, + mockRunCommand, + mockUpdateStack, + mockGetContainersByStack, + mockRestartContainer, + mockStopContainer, + mockStartContainer, +} = vi.hoisted(() => ({ + mockDeployStack: vi.fn(), + mockRunCommand: vi.fn(), + mockUpdateStack: vi.fn(), + mockGetContainersByStack: vi.fn(), + mockRestartContainer: vi.fn(), + mockStopContainer: vi.fn(), + mockStartContainer: vi.fn(), +})); + +vi.mock('../services/ComposeService', async () => { + const actual = await vi.importActual( + '../services/ComposeService', + ); + return { + ...actual, + ComposeService: { + ...actual.ComposeService, + getInstance: () => ({ + deployStack: mockDeployStack, + runCommand: mockRunCommand, + updateStack: mockUpdateStack, + }), + }, + }; +}); + +vi.mock('../services/DockerController', async () => { + const actual = await vi.importActual( + '../services/DockerController', + ); + return { + ...actual, + default: { + ...actual.default, + getInstance: () => ({ + getContainersByStack: mockGetContainersByStack, + restartContainer: mockRestartContainer, + stopContainer: mockStopContainer, + startContainer: mockStartContainer, + }), + }, + }; +}); + +vi.mock('../services/FileSystemService', () => ({ + FileSystemService: { + getInstance: () => ({ + getBaseDir: () => '/tmp/compose', + hasComposeFile: vi.fn().mockResolvedValue(true), + }), + }, +})); + +let tmpDir: string; +let app: import('express').Express; +let authCookie: string; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + authCookie = await loginAsTestAdmin(app); + + const { NotificationService } = await import('../services/NotificationService'); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); +}); + +afterAll(() => { + vi.restoreAllMocks(); + cleanupTestDb(tmpDir); +}); + +beforeEach(async () => { + mockDeployStack.mockReset(); + mockRunCommand.mockReset(); + mockUpdateStack.mockReset(); + mockGetContainersByStack.mockReset(); + mockRestartContainer.mockReset(); + mockStopContainer.mockReset(); + mockStartContainer.mockReset(); + const { StackOpLockService } = await import('../services/StackOpLockService'); + StackOpLockService.resetForTests(); +}); + +interface Deferred { + promise: Promise; + resolve: (value: T) => void; + reject: (err: unknown) => void; +} + +function deferred(): Deferred { + let resolve!: (value: T) => void; + let reject!: (err: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +describe('Stack lifecycle mutex', () => { + it('returns 409 with stack_op_in_progress when a deploy is already running', async () => { + const gate = deferred(); + mockDeployStack.mockImplementationOnce(() => gate.promise); + + const first = request(app) + .post('/api/stacks/web/deploy') + .set('Cookie', authCookie) + .send({ skip_scan: true }) + .then(r => r); + + await vi.waitFor(() => expect(mockDeployStack).toHaveBeenCalled()); + + const second = await request(app) + .post('/api/stacks/web/deploy') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + + expect(second.status).toBe(409); + expect(second.body).toMatchObject({ + code: 'stack_op_in_progress', + inProgress: { action: 'deploy' }, + }); + expect(second.body.error).toMatch(/already deploying/i); + expect(typeof second.body.inProgress.startedAt).toBe('number'); + + gate.resolve(); + const firstRes = await first; + expect(firstRes.status).toBe(200); + }); + + it('releases the lock after a successful deploy so the next request acquires', async () => { + mockDeployStack.mockResolvedValueOnce(undefined); + const first = await request(app) + .post('/api/stacks/web/deploy') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + expect(first.status).toBe(200); + + mockDeployStack.mockResolvedValueOnce(undefined); + const second = await request(app) + .post('/api/stacks/web/deploy') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + expect(second.status).toBe(200); + }); + + it('releases the lock after a failed deploy', async () => { + mockDeployStack.mockRejectedValueOnce(new Error('image pull failed')); + const first = await request(app) + .post('/api/stacks/web/deploy') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + expect(first.status).toBe(500); + + mockDeployStack.mockResolvedValueOnce(undefined); + const second = await request(app) + .post('/api/stacks/web/deploy') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + expect(second.status).toBe(200); + }); + + it('blocks restart while a deploy is in flight on the same stack', async () => { + const gate = deferred(); + mockDeployStack.mockImplementationOnce(() => gate.promise); + + const deploy = request(app) + .post('/api/stacks/web/deploy') + .set('Cookie', authCookie) + .send({ skip_scan: true }) + .then(r => r); + await vi.waitFor(() => expect(mockDeployStack).toHaveBeenCalled()); + + const restart = await request(app) + .post('/api/stacks/web/restart') + .set('Cookie', authCookie); + expect(restart.status).toBe(409); + expect(restart.body.code).toBe('stack_op_in_progress'); + expect(restart.body.inProgress.action).toBe('deploy'); + + gate.resolve(); + await deploy; + }); + + it('allows concurrent ops on different stacks', async () => { + const gate = deferred(); + mockDeployStack.mockImplementation(() => gate.promise); + + const webDeploy = request(app) + .post('/api/stacks/web/deploy') + .set('Cookie', authCookie) + .send({ skip_scan: true }) + .then(r => r); + await vi.waitFor(() => expect(mockDeployStack).toHaveBeenCalledTimes(1)); + + const apiDeploy = request(app) + .post('/api/stacks/api/deploy') + .set('Cookie', authCookie) + .send({ skip_scan: true }) + .then(r => r); + await vi.waitFor(() => expect(mockDeployStack).toHaveBeenCalledTimes(2)); + + gate.resolve(); + const [webRes, apiRes] = await Promise.all([webDeploy, apiDeploy]); + expect(webRes.status).toBe(200); + expect(apiRes.status).toBe(200); + }); + + it('blocks update while down is in flight', async () => { + const gate = deferred(); + mockRunCommand.mockImplementationOnce(() => gate.promise); + + const down = request(app) + .post('/api/stacks/web/down') + .set('Cookie', authCookie) + .then(r => r); + await vi.waitFor(() => expect(mockRunCommand).toHaveBeenCalled()); + + const update = await request(app) + .post('/api/stacks/web/update') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + expect(update.status).toBe(409); + expect(update.body.inProgress.action).toBe('down'); + + gate.resolve(); + await down; + }); + + it('returns 409 for deploy while start is in flight', async () => { + mockGetContainersByStack.mockResolvedValue([{ Id: 'c1' }]); + const gate = deferred(); + mockStartContainer.mockImplementationOnce(() => gate.promise); + + const start = request(app) + .post('/api/stacks/web/start') + .set('Cookie', authCookie) + .then(r => r); + await vi.waitFor(() => expect(mockStartContainer).toHaveBeenCalled()); + + const deploy = await request(app) + .post('/api/stacks/web/deploy') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + expect(deploy.status).toBe(409); + expect(deploy.body.inProgress.action).toBe('start'); + + gate.resolve(); + await start; + }); + + it('returns 409 for stop while restart is in flight', async () => { + mockGetContainersByStack.mockResolvedValue([{ Id: 'c1' }]); + const gate = deferred(); + mockRestartContainer.mockImplementationOnce(() => gate.promise); + + const restart = request(app) + .post('/api/stacks/web/restart') + .set('Cookie', authCookie) + .then(r => r); + await vi.waitFor(() => expect(mockRestartContainer).toHaveBeenCalled()); + + const stop = await request(app) + .post('/api/stacks/web/stop') + .set('Cookie', authCookie); + expect(stop.status).toBe(409); + expect(stop.body.inProgress.action).toBe('restart'); + + gate.resolve(); + await restart; + }); +}); diff --git a/backend/src/__tests__/stack-op-lock-service.test.ts b/backend/src/__tests__/stack-op-lock-service.test.ts new file mode 100644 index 00000000..2e41d9ed --- /dev/null +++ b/backend/src/__tests__/stack-op-lock-service.test.ts @@ -0,0 +1,96 @@ +/** + * Unit tests for StackOpLockService. + * + * The service is the in-memory mutex behind the 409 fast-fail for concurrent + * stack lifecycle operations. Each test resets the singleton via + * `resetForTests()` so state doesn't leak between cases. + */ +import { describe, it, expect, beforeEach } from 'vitest'; +import { StackOpLockService } from '../services/StackOpLockService'; + +beforeEach(() => { + StackOpLockService.resetForTests(); +}); + +describe('StackOpLockService', () => { + it('returns a singleton instance', () => { + const a = StackOpLockService.getInstance(); + const b = StackOpLockService.getInstance(); + expect(a).toBe(b); + }); + + it('acquires a lock when the slot is empty', () => { + const svc = StackOpLockService.getInstance(); + const result = svc.tryAcquire(1, 'web', 'deploy', 'admin'); + expect(result.acquired).toBe(true); + expect(svc.size()).toBe(1); + }); + + it('returns acquired=false with the existing lock when already held', () => { + const svc = StackOpLockService.getInstance(); + svc.tryAcquire(1, 'web', 'deploy', 'admin'); + const result = svc.tryAcquire(1, 'web', 'restart', 'bob'); + expect(result.acquired).toBe(false); + if (!result.acquired) { + expect(result.existing.action).toBe('deploy'); + expect(result.existing.user).toBe('admin'); + expect(typeof result.existing.startedAt).toBe('number'); + } + }); + + it('different stacks on the same node lock independently', () => { + const svc = StackOpLockService.getInstance(); + expect(svc.tryAcquire(1, 'web', 'deploy', 'admin').acquired).toBe(true); + expect(svc.tryAcquire(1, 'api', 'deploy', 'admin').acquired).toBe(true); + expect(svc.size()).toBe(2); + }); + + it('same stack name on different nodes locks independently', () => { + const svc = StackOpLockService.getInstance(); + expect(svc.tryAcquire(1, 'web', 'deploy', 'admin').acquired).toBe(true); + expect(svc.tryAcquire(2, 'web', 'deploy', 'admin').acquired).toBe(true); + expect(svc.size()).toBe(2); + }); + + it('release frees the slot so the next caller acquires', () => { + const svc = StackOpLockService.getInstance(); + svc.tryAcquire(1, 'web', 'deploy', 'admin'); + svc.release(1, 'web'); + expect(svc.size()).toBe(0); + expect(svc.tryAcquire(1, 'web', 'restart', 'bob').acquired).toBe(true); + }); + + it('release on an unheld key is a no-op', () => { + const svc = StackOpLockService.getInstance(); + expect(() => svc.release(1, 'nope')).not.toThrow(); + expect(svc.size()).toBe(0); + }); + + it('get returns the lock contents or undefined', () => { + const svc = StackOpLockService.getInstance(); + expect(svc.get(1, 'web')).toBeUndefined(); + svc.tryAcquire(1, 'web', 'update', 'eve'); + const lock = svc.get(1, 'web'); + expect(lock?.action).toBe('update'); + expect(lock?.user).toBe('eve'); + }); + + it('resetForTests clears all state and returns a fresh instance', () => { + const before = StackOpLockService.getInstance(); + before.tryAcquire(1, 'web', 'deploy', 'admin'); + StackOpLockService.resetForTests(); + const after = StackOpLockService.getInstance(); + expect(after).not.toBe(before); + expect(after.size()).toBe(0); + }); + + it('records the lock startedAt as a recent timestamp', () => { + const svc = StackOpLockService.getInstance(); + const t0 = Date.now(); + svc.tryAcquire(1, 'web', 'deploy', 'admin'); + const lock = svc.get(1, 'web'); + expect(lock).toBeDefined(); + expect(lock!.startedAt).toBeGreaterThanOrEqual(t0); + expect(lock!.startedAt).toBeLessThanOrEqual(Date.now()); + }); +}); diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index e39f471e..0eb41b21 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -14,6 +14,7 @@ import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement'; import { requirePermission } from '../middleware/permissions'; import { requirePaid, requireAdmin, effectiveTier } from '../middleware/tierGates'; import { NotificationService, type NotificationCategory } from '../services/NotificationService'; +import { StackOpLockService, type StackOpAction } from '../services/StackOpLockService'; import { isValidGitSourcePath, isValidStackName, isValidServiceName, isPathWithinBase, isValidRelativeStackPath } from '../utils/validation'; import { getErrorMessage } from '../utils/errors'; import { isDebugEnabled } from '../utils/debug'; @@ -37,6 +38,42 @@ function notifyActionSuccess(category: NotificationCategory, message: string, st .catch(err => console.error('[Stacks] Failed to dispatch activity for %s:', sanitizeForLog(stackName), err)); } +const STACK_OP_PRESENT_PARTICIPLE: Record = { + deploy: 'deploying', + down: 'stopping', + restart: 'restarting', + stop: 'stopping', + start: 'starting', + update: 'updating', +}; + +function tryAcquireStackOpLock( + req: Request, + res: Response, + stackName: string, + action: StackOpAction, +): boolean { + const user = req.user?.username ?? 'system'; + const result = StackOpLockService.getInstance().tryAcquire(req.nodeId, stackName, action, user); + if (!result.acquired) { + res.status(409).json({ + error: `${stackName} is already ${STACK_OP_PRESENT_PARTICIPLE[result.existing.action]}`, + code: 'stack_op_in_progress', + inProgress: { + action: result.existing.action, + startedAt: result.existing.startedAt, + user: result.existing.user, + }, + }); + return false; + } + return true; +} + +function releaseStackOpLock(req: Request, stackName: string): void { + StackOpLockService.getInstance().release(req.nodeId, stackName); +} + async function requireStackExists(nodeId: number, stackName: string, res: Response): Promise { if (!isValidStackName(stackName)) { res.status(400).json({ error: 'Invalid stack name' }); @@ -640,6 +677,8 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => { const stackName = req.params.stackName as string; if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return; if (!(await requireStackExists(req.nodeId, stackName, res))) return; + // Lock held below. All early-returns must stay inside the try so finally fires. + if (!tryAcquireStackOpLock(req, res, stackName, 'deploy')) return; try { if (!(await runPolicyGate(req, res, stackName, req.nodeId))) return; const skipScan = req.body?.skip_scan === true; @@ -669,7 +708,9 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => { } const message = getErrorMessage(error, 'Failed to deploy stack'); notifyActionFailure('deploy', stackName, error); - res.status(500).json({ error: message, rolledBack }); + if (!res.headersSent) res.status(500).json({ error: message, rolledBack }); + } finally { + releaseStackOpLock(req, stackName); } }); @@ -677,6 +718,8 @@ stacksRouter.post('/:stackName/down', async (req: Request, res: Response) => { const stackName = req.params.stackName as string; if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return; if (!(await requireStackExists(req.nodeId, stackName, res))) return; + // Lock held below. All early-returns must stay inside the try so finally fires. + if (!tryAcquireStackOpLock(req, res, stackName, 'down')) return; try { await ComposeService.getInstance(req.nodeId).runCommand(stackName, 'down', getTerminalWs()); invalidateNodeCaches(req.nodeId); @@ -685,7 +728,9 @@ stacksRouter.post('/:stackName/down', async (req: Request, res: Response) => { } catch (error: unknown) { console.error('[Stacks] Down failed: %s', sanitizeForLog(stackName), error); notifyActionFailure('down', stackName, error); - res.status(500).json({ error: 'Failed to start command' }); + if (!res.headersSent) res.status(500).json({ error: 'Failed to start command' }); + } finally { + releaseStackOpLock(req, stackName); } }); @@ -729,25 +774,36 @@ async function bulkContainerOp( ): Promise { const stackName = req.params.stackName as string; if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return; - const titleCase = action.charAt(0).toUpperCase() + action.slice(1); - const outcome = await containerActionForStack(req.nodeId, stackName, action); + // Lock held below. All early-returns must stay inside the try so finally fires. + if (!tryAcquireStackOpLock(req, res, stackName, action)) return; + try { + const titleCase = action.charAt(0).toUpperCase() + action.slice(1); + const outcome = await containerActionForStack(req.nodeId, stackName, action); - if (outcome.kind === 'no-containers') { - res.status(404).json({ error: 'No containers found for this stack.' }); - return; - } - if (outcome.kind === 'error') { - console.error('[Stacks] %s failed: %s %s', sanitizeForLog(titleCase), sanitizeForLog(stackName), sanitizeForLog(outcome.message)); - if (action !== 'start') notifyActionFailure(action, stackName, new Error(outcome.message)); - res.status(500).json({ error: outcome.message }); - return; - } + if (outcome.kind === 'no-containers') { + res.status(404).json({ error: 'No containers found for this stack.' }); + return; + } + if (outcome.kind === 'error') { + console.error('[Stacks] %s failed: %s %s', sanitizeForLog(titleCase), sanitizeForLog(stackName), sanitizeForLog(outcome.message)); + if (action !== 'start') notifyActionFailure(action, stackName, new Error(outcome.message)); + res.status(500).json({ error: outcome.message }); + return; + } - invalidateNodeCaches(req.nodeId); - console.log(`[Stacks] ${titleCase} completed: ${sanitizeForLog(stackName)} (${outcome.count} containers)`); - res.json({ success: true, message: `${titleCase} completed via Engine API.` }); - const { category, pastTense } = CONTAINER_ACTION_META[action]; - notifyActionSuccess(category, `${stackName} ${pastTense}`, stackName, req.user?.username ?? 'system'); + invalidateNodeCaches(req.nodeId); + console.log(`[Stacks] ${titleCase} completed: ${sanitizeForLog(stackName)} (${outcome.count} containers)`); + res.json({ success: true, message: `${titleCase} completed via Engine API.` }); + const { category, pastTense } = CONTAINER_ACTION_META[action]; + notifyActionSuccess(category, `${stackName} ${pastTense}`, stackName, req.user?.username ?? 'system'); + } catch (error: unknown) { + console.error('[Stacks] %s threw unexpectedly: %s', sanitizeForLog(action), sanitizeForLog(stackName), error); + if (!res.headersSent) { + res.status(500).json({ error: getErrorMessage(error, `Failed to ${action} stack`) }); + } + } finally { + releaseStackOpLock(req, stackName); + } } stacksRouter.post('/:stackName/restart', (req, res) => bulkContainerOp(req, res, 'restart')); @@ -824,6 +880,8 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => { const stackName = req.params.stackName as string; if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return; if (!(await requireStackExists(req.nodeId, stackName, res))) return; + // Lock held below. All early-returns must stay inside the try so finally fires. + if (!tryAcquireStackOpLock(req, res, stackName, 'update')) return; try { if (!(await runPolicyGate(req, res, stackName, req.nodeId))) return; const skipScan = req.body?.skip_scan === true; @@ -861,7 +919,11 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => { console.warn(`[Stacks] Update failed, rollback did not complete: ${sanitizeForLog(stackName)}`); } notifyActionFailure('update', stackName, error); - res.status(500).json({ error: getErrorMessage(error, 'Failed to update'), rolledBack }); + if (!res.headersSent) { + res.status(500).json({ error: getErrorMessage(error, 'Failed to update'), rolledBack }); + } + } finally { + releaseStackOpLock(req, stackName); } }); diff --git a/backend/src/services/StackOpLockService.ts b/backend/src/services/StackOpLockService.ts new file mode 100644 index 00000000..224b8a21 --- /dev/null +++ b/backend/src/services/StackOpLockService.ts @@ -0,0 +1,70 @@ +/** + * Tracks in-flight stack lifecycle operations (deploy, down, restart, stop, + * start, update) per (nodeId, stackName). A second request to the same stack + * while the first is still running returns 409 instead of racing the first. + * + * State is intentionally process-local: a Sencho restart clears all locks, + * which matches the lifecycle of any in-flight `docker compose` child process. + */ + +export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update'; + +export interface StackOpLock { + action: StackOpAction; + startedAt: number; + user: string; +} + +interface AcquireSuccess { + acquired: true; +} + +interface AcquireConflict { + acquired: false; + existing: StackOpLock; +} + +export type AcquireResult = AcquireSuccess | AcquireConflict; + +export class StackOpLockService { + private static instance: StackOpLockService; + private readonly locks = new Map(); + + public static getInstance(): StackOpLockService { + if (!this.instance) this.instance = new StackOpLockService(); + return this.instance; + } + + public static resetForTests(): void { + this.instance = new StackOpLockService(); + } + + private key(nodeId: number, stackName: string): string { + return `${nodeId}:${stackName}`; + } + + public tryAcquire( + nodeId: number, + stackName: string, + action: StackOpAction, + user: string, + ): AcquireResult { + const k = this.key(nodeId, stackName); + const existing = this.locks.get(k); + if (existing) return { acquired: false, existing }; + this.locks.set(k, { action, startedAt: Date.now(), user }); + return { acquired: true }; + } + + public release(nodeId: number, stackName: string): void { + this.locks.delete(this.key(nodeId, stackName)); + } + + public get(nodeId: number, stackName: string): StackOpLock | undefined { + return this.locks.get(this.key(nodeId, stackName)); + } + + public size(): number { + return this.locks.size; + } +} diff --git a/frontend/src/components/EditorLayout/hooks/useStackActions.ts b/frontend/src/components/EditorLayout/hooks/useStackActions.ts index ff9d7b7a..433e8cc7 100644 --- a/frontend/src/components/EditorLayout/hooks/useStackActions.ts +++ b/frontend/src/components/EditorLayout/hooks/useStackActions.ts @@ -19,6 +19,27 @@ interface RunResult { type StackActionError = Error & { rolledBack?: boolean }; +type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update'; + +interface StackOpInProgressInfo { + action: StackOpAction; + startedAt: number; + user: string; +} + +const STACK_OP_PRESENT_PARTICIPLE: Record = { + deploy: 'deploying', + down: 'stopping', + restart: 'restarting', + stop: 'stopping', + start: 'starting', + update: 'updating', +}; + +const VALID_STACK_OP_ACTIONS: ReadonlySet = new Set( + Object.keys(STACK_OP_PRESENT_PARTICIPLE), +); + type EditorState = ReturnType; type StackListState = ReturnType; type NavState = ReturnType; @@ -42,6 +63,35 @@ interface UseStackActionsOptions { const isRecord = (value: unknown): value is Record => typeof value === 'object' && value !== null; +const parseStackOpInProgress = (rawBody: string): StackOpInProgressInfo | null => { + try { + const parsed: unknown = JSON.parse(rawBody); + if (!isRecord(parsed) || parsed.code !== 'stack_op_in_progress') return null; + const inProgress = parsed.inProgress; + if ( + !isRecord(inProgress) || + typeof inProgress.action !== 'string' || + typeof inProgress.startedAt !== 'number' || + !VALID_STACK_OP_ACTIONS.has(inProgress.action) + ) { + return null; + } + return { + action: inProgress.action as StackOpAction, + startedAt: inProgress.startedAt, + user: typeof inProgress.user === 'string' ? inProgress.user : '', + }; + } catch { + return null; + } +}; + +const stackOpInProgressMessage = (stackName: string, info: StackOpInProgressInfo): string => { + const verb = STACK_OP_PRESENT_PARTICIPLE[info.action] ?? 'busy'; + const actor = info.user && info.user !== 'system' ? ` (started by ${info.user})` : ''; + return `${stackName} is already ${verb}${actor}.`; +}; + const parseStackActionError = (rawBody: string, fallback: string): StackActionError => { let message = rawBody || fallback; let rolledBack = false; @@ -388,6 +438,17 @@ export function useStackActions(options: UseStackActionsOptions) { if (!response.ok) { const rawBody = await response.text(); if (response.status === 409) { + const inProgress = parseStackOpInProgress(rawBody); + if (inProgress) { + const message = stackOpInProgressMessage(stackName, inProgress); + if (previousStatus !== undefined) + stackListState.setOptimisticStatus( + stackFile, + previousStatus as 'running' | 'exited', + ); + toast.error(message); + return { ok: false, errorMessage: message }; + } let parsed: PolicyBlockPayload | null = null; try { parsed = JSON.parse(rawBody) as PolicyBlockPayload; @@ -578,6 +639,14 @@ export function useStackActions(options: UseStackActionsOptions) { const response = await apiFetch(`/stacks/${stackName}/${endpoint}`, { method: 'POST' }); if (!response.ok) { const errText = await response.text(); + if (response.status === 409) { + const inProgress = parseStackOpInProgress(errText); + if (inProgress) { + const message = stackOpInProgressMessage(stackName, inProgress); + toast.error(message); + return { ok: false as const, errorMessage: message }; + } + } const actionError = parseStackActionError(errText, `${action} failed`); return { ok: false as const, @@ -728,6 +797,13 @@ export function useStackActions(options: UseStackActionsOptions) { const response = await apiFetch(`/stacks/${stackName}/${endpoint}`, { method: 'POST' }); if (!response.ok) { const errText = await response.text(); + if (response.status === 409) { + const inProgress = parseStackOpInProgress(errText); + if (inProgress) { + toast.error(stackOpInProgressMessage(stackName, inProgress)); + return; + } + } throw parseStackActionError(errText, `${action} failed`); } toast.success(`Stack ${action}ed successfully!`);