From 009ec43638455f42428424629ffb1cfe074786f1 Mon Sep 17 00:00:00 2001 From: Anso Date: Sun, 24 May 2026 15:43:36 -0400 Subject: [PATCH] feat(stacks): structured 503 docker_unavailable envelope + disconnect tests (#1191) Stack lifecycle routes used to surface raw ECONNREFUSED text to the client whenever the Docker daemon was unreachable. The frontend had no way to distinguish "daemon down" from any other 500 and would render the raw error message. Detect daemon-reachability failures inside the route layer and surface a structured envelope so the UI can render a dedicated "Docker is down" state and operators can branch on a stable code: HTTP 503 { error: , code: 'docker_unavailable' } Detection lives in isDockerUnavailableError (exported from routes/stacks.ts). The match is intentionally permissive across error shapes Dockerode and the docker compose CLI produce: NodeJS ECONNREFUSED errors with .code, ENOENT on docker.sock, and the CLI's "Cannot connect to the Docker daemon" string. The helper is unit-tested in isolation as well as exercised end-to-end through the route. Applied to the five lifecycle routes that can hit the daemon-down path: POST /api/stacks/:name/restart (via bulkContainerOp) POST /api/stacks/:name/stop (via bulkContainerOp) POST /api/stacks/:name/start (via bulkContainerOp) POST /api/stacks/:name/deploy POST /api/stacks/:name/down POST /api/stacks/:name/update Adds ContainerActionOutcome variant 'docker-unavailable' so the route can branch on the typed outcome rather than string-matching error messages a second time. 11 integration tests in stack-docker-disconnect.test.ts cover: - isDockerUnavailableError matches ECONNREFUSED, CLI text, ENOENT on docker.sock; rejects unrelated errors and null/undefined. - restart/stop/start return 503 + code on Dockerode listContainers refusing. - deploy/down/update return 503 + code when ComposeService rejects with daemon-down error. - Unrelated deploy failures (YAML parse error) still return 500 without the code, confirming the discriminator is correctly scoped. Resolves L-3 from the stack-management audit. --- .../__tests__/stack-docker-disconnect.test.ts | 238 ++++++++++++++++++ backend/src/routes/stacks.ts | 54 +++- 2 files changed, 289 insertions(+), 3 deletions(-) create mode 100644 backend/src/__tests__/stack-docker-disconnect.test.ts diff --git a/backend/src/__tests__/stack-docker-disconnect.test.ts b/backend/src/__tests__/stack-docker-disconnect.test.ts new file mode 100644 index 00000000..4a33625d --- /dev/null +++ b/backend/src/__tests__/stack-docker-disconnect.test.ts @@ -0,0 +1,238 @@ +/** + * Integration tests for Docker-daemon disconnect handling on stack lifecycle + * routes. When the engine socket is unreachable, the lifecycle routes must + * return a structured 503 envelope with `code: docker_unavailable` so the UI + * can surface a "Docker is down" message instead of raw ECONNREFUSED text. + * + * Disconnect is simulated by injecting ECONNREFUSED at the Dockerode test + * seam (DockerController.getInstance(...).getContainersByStack throws) and at + * the ComposeService spawn path (deploy/down/update rejection with the same + * error shape). Both shapes flow through isDockerUnavailableError in the + * route layer. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import request from 'supertest'; +import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; +import { isDockerUnavailableError } from '../routes/stacks'; + +const { + mockDeployStack, + mockRunCommand, + mockUpdateStack, + mockGetContainersByStack, + mockRestartContainer, + mockStopContainer, + mockStartContainer, +} = vi.hoisted(() => ({ + mockDeployStack: vi.fn(), + mockRunCommand: vi.fn(), + mockUpdateStack: vi.fn(), + mockGetContainersByStack: vi.fn(), + mockRestartContainer: vi.fn(), + mockStopContainer: vi.fn(), + mockStartContainer: vi.fn(), +})); + +vi.mock('../services/ComposeService', async () => { + const actual = await vi.importActual( + '../services/ComposeService', + ); + return { + ...actual, + ComposeService: { + ...actual.ComposeService, + getInstance: () => ({ + deployStack: mockDeployStack, + runCommand: mockRunCommand, + updateStack: mockUpdateStack, + }), + }, + }; +}); + +vi.mock('../services/DockerController', async () => { + const actual = await vi.importActual( + '../services/DockerController', + ); + return { + ...actual, + default: { + ...actual.default, + getInstance: () => ({ + getContainersByStack: mockGetContainersByStack, + restartContainer: mockRestartContainer, + stopContainer: mockStopContainer, + startContainer: mockStartContainer, + }), + }, + }; +}); + +vi.mock('../services/FileSystemService', () => ({ + FileSystemService: { + getInstance: () => ({ + getBaseDir: () => '/tmp/compose', + hasComposeFile: vi.fn().mockResolvedValue(true), + }), + }, +})); + +let tmpDir: string; +let app: import('express').Express; +let authCookie: string; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + authCookie = await loginAsTestAdmin(app); + + const { NotificationService } = await import('../services/NotificationService'); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); +}); + +afterAll(() => { + vi.restoreAllMocks(); + cleanupTestDb(tmpDir); +}); + +beforeEach(() => { + mockDeployStack.mockReset(); + mockRunCommand.mockReset(); + mockUpdateStack.mockReset(); + mockGetContainersByStack.mockReset(); + mockRestartContainer.mockReset(); + mockStopContainer.mockReset(); + mockStartContainer.mockReset(); +}); + +/** Build an Error that mimics the shape Dockerode throws on socket failure. */ +function dockerSocketDownError(): NodeJS.ErrnoException { + const err = Object.assign(new Error('connect ECONNREFUSED /var/run/docker.sock'), { + code: 'ECONNREFUSED', + errno: -111, + syscall: 'connect', + address: '/var/run/docker.sock', + }) as NodeJS.ErrnoException; + return err; +} + +/** Build an Error that mimics docker compose CLI output when dockerd is down. */ +function composeDaemonDownError(): Error { + return new Error( + 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?', + ); +} + +describe('isDockerUnavailableError helper', () => { + it('matches plain ECONNREFUSED node errors', () => { + expect(isDockerUnavailableError(dockerSocketDownError())).toBe(true); + }); + + it('matches docker compose CLI "cannot connect to the Docker daemon" output', () => { + expect(isDockerUnavailableError(composeDaemonDownError())).toBe(true); + }); + + it('matches docker.sock ENOENT errors', () => { + const err = Object.assign(new Error('ENOENT: no such file or directory, connect /var/run/docker.sock'), { + code: 'ENOENT', + }); + expect(isDockerUnavailableError(err)).toBe(true); + }); + + it('does not match unrelated errors', () => { + expect(isDockerUnavailableError(new Error('image pull failed: manifest unknown'))).toBe(false); + expect(isDockerUnavailableError(new Error('compose YAML parse error'))).toBe(false); + expect(isDockerUnavailableError(null)).toBe(false); + expect(isDockerUnavailableError(undefined)).toBe(false); + }); +}); + +describe('POST /api/stacks/:name/restart with daemon down', () => { + it('returns 503 with code: docker_unavailable when Dockerode listContainers refuses', async () => { + mockGetContainersByStack.mockRejectedValue(dockerSocketDownError()); + + const res = await request(app) + .post('/api/stacks/web/restart') + .set('Cookie', authCookie); + + expect(res.status).toBe(503); + expect(res.body).toMatchObject({ code: 'docker_unavailable' }); + expect(res.body.error).toMatch(/ECONNREFUSED|docker daemon|unreachable/i); + }); + + it('still returns 503 on stop when the daemon is down', async () => { + mockGetContainersByStack.mockRejectedValue(dockerSocketDownError()); + + const res = await request(app) + .post('/api/stacks/web/stop') + .set('Cookie', authCookie); + + expect(res.status).toBe(503); + expect(res.body.code).toBe('docker_unavailable'); + }); + + it('still returns 503 on start when the daemon is down', async () => { + mockGetContainersByStack.mockRejectedValue(dockerSocketDownError()); + + const res = await request(app) + .post('/api/stacks/web/start') + .set('Cookie', authCookie); + + expect(res.status).toBe(503); + expect(res.body.code).toBe('docker_unavailable'); + }); +}); + +describe('POST /api/stacks/:name/deploy with daemon down', () => { + it('returns 503 with code: docker_unavailable when ComposeService surfaces the daemon error', async () => { + mockDeployStack.mockRejectedValue(composeDaemonDownError()); + + const res = await request(app) + .post('/api/stacks/web/deploy') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + + expect(res.status).toBe(503); + expect(res.body.code).toBe('docker_unavailable'); + expect(res.body.error).toMatch(/Cannot connect to the Docker daemon|unreachable/i); + }); + + it('still returns 500 (not 503) for unrelated deploy failures', async () => { + mockDeployStack.mockRejectedValue(new Error('compose YAML parse error')); + + const res = await request(app) + .post('/api/stacks/web/deploy') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + + expect(res.status).toBe(500); + expect(res.body.code).toBeUndefined(); + }); +}); + +describe('POST /api/stacks/:name/down with daemon down', () => { + it('returns 503 with code: docker_unavailable when runCommand surfaces the daemon error', async () => { + mockRunCommand.mockRejectedValue(composeDaemonDownError()); + + const res = await request(app) + .post('/api/stacks/web/down') + .set('Cookie', authCookie); + + expect(res.status).toBe(503); + expect(res.body.code).toBe('docker_unavailable'); + }); +}); + +describe('POST /api/stacks/:name/update with daemon down', () => { + it('returns 503 with code: docker_unavailable when updateStack surfaces the daemon error', async () => { + mockUpdateStack.mockRejectedValue(composeDaemonDownError()); + + const res = await request(app) + .post('/api/stacks/web/update') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + + expect(res.status).toBe(503); + expect(res.body.code).toBe('docker_unavailable'); + }); +}); diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index b01e7fe5..64e50398 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -754,7 +754,13 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => { } const message = getErrorMessage(error, 'Failed to deploy stack'); notifyActionFailure('deploy', stackName, error); - if (!res.headersSent) res.status(500).json({ error: message, rolledBack }); + if (!res.headersSent) { + if (isDockerUnavailableError(error)) { + res.status(503).json({ error: message, code: 'docker_unavailable', rolledBack }); + } else { + res.status(500).json({ error: message, rolledBack }); + } + } } finally { releaseStackOpLock(req, stackName); } @@ -775,7 +781,13 @@ stacksRouter.post('/:stackName/down', async (req: Request, res: Response) => { } catch (error: unknown) { console.error('[Stacks] Down failed: %s', sanitizeForLog(stackName), error); notifyActionFailure('down', stackName, error); - if (!res.headersSent) res.status(500).json({ error: 'Failed to start command' }); + if (!res.headersSent) { + if (isDockerUnavailableError(error)) { + res.status(503).json({ error: getErrorMessage(error, 'Docker daemon is unreachable'), code: 'docker_unavailable' }); + } else { + res.status(500).json({ error: 'Failed to start command' }); + } + } } finally { releaseStackOpLock(req, stackName); } @@ -792,8 +804,31 @@ const CONTAINER_ACTION_META: Record op(c.Id))); return { kind: 'ok', count: containers.length }; } catch (error: unknown) { + if (isDockerUnavailableError(error)) { + return { kind: 'docker-unavailable', message: getErrorMessage(error, 'Docker daemon is unreachable') }; + } return { kind: 'error', message: getErrorMessage(error, `Failed to ${action} containers`) }; } } @@ -832,6 +870,12 @@ async function bulkContainerOp( res.status(404).json({ error: 'No containers found for this stack.' }); return; } + if (outcome.kind === 'docker-unavailable') { + console.error('[Stacks] %s failed: docker unavailable for %s', sanitizeForLog(titleCase), sanitizeForLog(stackName)); + if (action !== 'start') notifyActionFailure(action, stackName, new Error(outcome.message)); + res.status(503).json({ error: outcome.message, code: 'docker_unavailable' }); + return; + } if (outcome.kind === 'error') { console.error('[Stacks] %s failed: %s %s', sanitizeForLog(titleCase), sanitizeForLog(stackName), sanitizeForLog(outcome.message)); if (action !== 'start') notifyActionFailure(action, stackName, new Error(outcome.message)); @@ -968,7 +1012,11 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => { } notifyActionFailure('update', stackName, error); if (!res.headersSent) { - res.status(500).json({ error: getErrorMessage(error, 'Failed to update'), rolledBack }); + if (isDockerUnavailableError(error)) { + res.status(503).json({ error: getErrorMessage(error, 'Docker daemon is unreachable'), code: 'docker_unavailable', rolledBack }); + } else { + res.status(500).json({ error: getErrorMessage(error, 'Failed to update'), rolledBack }); + } } } finally { releaseStackOpLock(req, stackName);