feat(stacks): structured 503 docker_unavailable envelope + disconnect tests (#1191)

Stack lifecycle routes used to surface raw ECONNREFUSED text to the
client whenever the Docker daemon was unreachable. The frontend had no
way to distinguish "daemon down" from any other 500 and would render
the raw error message.

Detect daemon-reachability failures inside the route layer and surface
a structured envelope so the UI can render a dedicated "Docker is down"
state and operators can branch on a stable code:

  HTTP 503 { error: <message>, code: 'docker_unavailable' }

Detection lives in isDockerUnavailableError (exported from
routes/stacks.ts). The match is intentionally permissive across error
shapes Dockerode and the docker compose CLI produce: NodeJS ECONNREFUSED
errors with .code, ENOENT on docker.sock, and the CLI's
"Cannot connect to the Docker daemon" string. The helper is unit-tested
in isolation as well as exercised end-to-end through the route.

Applied to the five lifecycle routes that can hit the daemon-down path:
  POST /api/stacks/:name/restart  (via bulkContainerOp)
  POST /api/stacks/:name/stop     (via bulkContainerOp)
  POST /api/stacks/:name/start    (via bulkContainerOp)
  POST /api/stacks/:name/deploy
  POST /api/stacks/:name/down
  POST /api/stacks/:name/update

Adds ContainerActionOutcome variant 'docker-unavailable' so the route
can branch on the typed outcome rather than string-matching error
messages a second time.

11 integration tests in stack-docker-disconnect.test.ts cover:
  - isDockerUnavailableError matches ECONNREFUSED, CLI text, ENOENT on
    docker.sock; rejects unrelated errors and null/undefined.
  - restart/stop/start return 503 + code on Dockerode listContainers
    refusing.
  - deploy/down/update return 503 + code when ComposeService rejects
    with daemon-down error.
  - Unrelated deploy failures (YAML parse error) still return 500
    without the code, confirming the discriminator is correctly scoped.

Resolves L-3 from the stack-management audit.

backend/src/__tests__/stack-docker-disconnect.test.ts

@@ -0,0 +1,238 @@
+/**
+ * Integration tests for Docker-daemon disconnect handling on stack lifecycle
+ * routes. When the engine socket is unreachable, the lifecycle routes must
+ * return a structured 503 envelope with `code: docker_unavailable` so the UI
+ * can surface a "Docker is down" message instead of raw ECONNREFUSED text.
+ *
+ * Disconnect is simulated by injecting ECONNREFUSED at the Dockerode test
+ * seam (DockerController.getInstance(...).getContainersByStack throws) and at
+ * the ComposeService spawn path (deploy/down/update rejection with the same
+ * error shape). Both shapes flow through isDockerUnavailableError in the
+ * route layer.
+ */
+import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
+import request from 'supertest';
+import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
+import { isDockerUnavailableError } from '../routes/stacks';
+
+const {
+  mockDeployStack,
+  mockRunCommand,
+  mockUpdateStack,
+  mockGetContainersByStack,
+  mockRestartContainer,
+  mockStopContainer,
+  mockStartContainer,
+} = vi.hoisted(() => ({
+  mockDeployStack: vi.fn(),
+  mockRunCommand: vi.fn(),
+  mockUpdateStack: vi.fn(),
+  mockGetContainersByStack: vi.fn(),
+  mockRestartContainer: vi.fn(),
+  mockStopContainer: vi.fn(),
+  mockStartContainer: vi.fn(),
+}));
+
+vi.mock('../services/ComposeService', async () => {
+  const actual = await vi.importActual<typeof import('../services/ComposeService')>(
+    '../services/ComposeService',
+  );
+  return {
+    ...actual,
+    ComposeService: {
+      ...actual.ComposeService,
+      getInstance: () => ({
+        deployStack: mockDeployStack,
+        runCommand: mockRunCommand,
+        updateStack: mockUpdateStack,
+      }),
+    },
+  };
+});
+
+vi.mock('../services/DockerController', async () => {
+  const actual = await vi.importActual<typeof import('../services/DockerController')>(
+    '../services/DockerController',
+  );
+  return {
+    ...actual,
+    default: {
+      ...actual.default,
+      getInstance: () => ({
+        getContainersByStack: mockGetContainersByStack,
+        restartContainer: mockRestartContainer,
+        stopContainer: mockStopContainer,
+        startContainer: mockStartContainer,
+      }),
+    },
+  };
+});
+
+vi.mock('../services/FileSystemService', () => ({
+  FileSystemService: {
+    getInstance: () => ({
+      getBaseDir: () => '/tmp/compose',
+      hasComposeFile: vi.fn().mockResolvedValue(true),
+    }),
+  },
+}));
+
+let tmpDir: string;
+let app: import('express').Express;
+let authCookie: string;
+
+beforeAll(async () => {
+  tmpDir = await setupTestDb();
+  ({ app } = await import('../index'));
+  authCookie = await loginAsTestAdmin(app);
+
+  const { NotificationService } = await import('../services/NotificationService');
+  vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
+});
+
+afterAll(() => {
+  vi.restoreAllMocks();
+  cleanupTestDb(tmpDir);
+});
+
+beforeEach(() => {
+  mockDeployStack.mockReset();
+  mockRunCommand.mockReset();
+  mockUpdateStack.mockReset();
+  mockGetContainersByStack.mockReset();
+  mockRestartContainer.mockReset();
+  mockStopContainer.mockReset();
+  mockStartContainer.mockReset();
+});
+
+/** Build an Error that mimics the shape Dockerode throws on socket failure. */
+function dockerSocketDownError(): NodeJS.ErrnoException {
+  const err = Object.assign(new Error('connect ECONNREFUSED /var/run/docker.sock'), {
+    code: 'ECONNREFUSED',
+    errno: -111,
+    syscall: 'connect',
+    address: '/var/run/docker.sock',
+  }) as NodeJS.ErrnoException;
+  return err;
+}
+
+/** Build an Error that mimics docker compose CLI output when dockerd is down. */
+function composeDaemonDownError(): Error {
+  return new Error(
+    'Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?',
+  );
+}
+
+describe('isDockerUnavailableError helper', () => {
+  it('matches plain ECONNREFUSED node errors', () => {
+    expect(isDockerUnavailableError(dockerSocketDownError())).toBe(true);
+  });
+
+  it('matches docker compose CLI "cannot connect to the Docker daemon" output', () => {
+    expect(isDockerUnavailableError(composeDaemonDownError())).toBe(true);
+  });
+
+  it('matches docker.sock ENOENT errors', () => {
+    const err = Object.assign(new Error('ENOENT: no such file or directory, connect /var/run/docker.sock'), {
+      code: 'ENOENT',
+    });
+    expect(isDockerUnavailableError(err)).toBe(true);
+  });
+
+  it('does not match unrelated errors', () => {
+    expect(isDockerUnavailableError(new Error('image pull failed: manifest unknown'))).toBe(false);
+    expect(isDockerUnavailableError(new Error('compose YAML parse error'))).toBe(false);
+    expect(isDockerUnavailableError(null)).toBe(false);
+    expect(isDockerUnavailableError(undefined)).toBe(false);
+  });
+});
+
+describe('POST /api/stacks/:name/restart with daemon down', () => {
+  it('returns 503 with code: docker_unavailable when Dockerode listContainers refuses', async () => {
+    mockGetContainersByStack.mockRejectedValue(dockerSocketDownError());
+
+    const res = await request(app)
+      .post('/api/stacks/web/restart')
+      .set('Cookie', authCookie);
+
+    expect(res.status).toBe(503);
+    expect(res.body).toMatchObject({ code: 'docker_unavailable' });
+    expect(res.body.error).toMatch(/ECONNREFUSED|docker daemon|unreachable/i);
+  });
+
+  it('still returns 503 on stop when the daemon is down', async () => {
+    mockGetContainersByStack.mockRejectedValue(dockerSocketDownError());
+
+    const res = await request(app)
+      .post('/api/stacks/web/stop')
+      .set('Cookie', authCookie);
+
+    expect(res.status).toBe(503);
+    expect(res.body.code).toBe('docker_unavailable');
+  });
+
+  it('still returns 503 on start when the daemon is down', async () => {
+    mockGetContainersByStack.mockRejectedValue(dockerSocketDownError());
+
+    const res = await request(app)
+      .post('/api/stacks/web/start')
+      .set('Cookie', authCookie);
+
+    expect(res.status).toBe(503);
+    expect(res.body.code).toBe('docker_unavailable');
+  });
+});
+
+describe('POST /api/stacks/:name/deploy with daemon down', () => {
+  it('returns 503 with code: docker_unavailable when ComposeService surfaces the daemon error', async () => {
+    mockDeployStack.mockRejectedValue(composeDaemonDownError());
+
+    const res = await request(app)
+      .post('/api/stacks/web/deploy')
+      .set('Cookie', authCookie)
+      .send({ skip_scan: true });
+
+    expect(res.status).toBe(503);
+    expect(res.body.code).toBe('docker_unavailable');
+    expect(res.body.error).toMatch(/Cannot connect to the Docker daemon|unreachable/i);
+  });
+
+  it('still returns 500 (not 503) for unrelated deploy failures', async () => {
+    mockDeployStack.mockRejectedValue(new Error('compose YAML parse error'));
+
+    const res = await request(app)
+      .post('/api/stacks/web/deploy')
+      .set('Cookie', authCookie)
+      .send({ skip_scan: true });
+
+    expect(res.status).toBe(500);
+    expect(res.body.code).toBeUndefined();
+  });
+});
+
+describe('POST /api/stacks/:name/down with daemon down', () => {
+  it('returns 503 with code: docker_unavailable when runCommand surfaces the daemon error', async () => {
+    mockRunCommand.mockRejectedValue(composeDaemonDownError());
+
+    const res = await request(app)
+      .post('/api/stacks/web/down')
+      .set('Cookie', authCookie);
+
+    expect(res.status).toBe(503);
+    expect(res.body.code).toBe('docker_unavailable');
+  });
+});
+
+describe('POST /api/stacks/:name/update with daemon down', () => {
+  it('returns 503 with code: docker_unavailable when updateStack surfaces the daemon error', async () => {
+    mockUpdateStack.mockRejectedValue(composeDaemonDownError());
+
+    const res = await request(app)
+      .post('/api/stacks/web/update')
+      .set('Cookie', authCookie)
+      .send({ skip_scan: true });
+
+    expect(res.status).toBe(503);
+    expect(res.body.code).toBe('docker_unavailable');
+  });
+});

backend/src/routes/stacks.ts

@@ -754,7 +754,13 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => {
     }
     const message = getErrorMessage(error, 'Failed to deploy stack');
     notifyActionFailure('deploy', stackName, error);
-    if (!res.headersSent) res.status(500).json({ error: message, rolledBack });
+    if (!res.headersSent) {
+      if (isDockerUnavailableError(error)) {
+        res.status(503).json({ error: message, code: 'docker_unavailable', rolledBack });
+      } else {
+        res.status(500).json({ error: message, rolledBack });
+      }
+    }
   } finally {
     releaseStackOpLock(req, stackName);
   }
@@ -775,7 +781,13 @@ stacksRouter.post('/:stackName/down', async (req: Request, res: Response) => {
   } catch (error: unknown) {
     console.error('[Stacks] Down failed: %s', sanitizeForLog(stackName), error);
     notifyActionFailure('down', stackName, error);
-    if (!res.headersSent) res.status(500).json({ error: 'Failed to start command' });
+    if (!res.headersSent) {
+      if (isDockerUnavailableError(error)) {
+        res.status(503).json({ error: getErrorMessage(error, 'Docker daemon is unreachable'), code: 'docker_unavailable' });
+      } else {
+        res.status(500).json({ error: 'Failed to start command' });
+      }
+    }
   } finally {
     releaseStackOpLock(req, stackName);
   }
@@ -792,8 +804,31 @@ const CONTAINER_ACTION_META: Record<StackContainerAction, { category: Notificati
 export type ContainerActionOutcome =
   | { kind: 'ok'; count: number }
   | { kind: 'no-containers' }
+  | { kind: 'docker-unavailable'; message: string }
   | { kind: 'error'; message: string };
 
+/**
+ * Returns true when the error looks like a Docker-engine reachability
+ * failure (socket gone, daemon down, connection refused). The string match
+ * is intentionally permissive: Dockerode wraps the underlying Node error
+ * differently depending on the transport (unix socket vs tcp vs ssh) and
+ * the OS, so a code-only check (`err.code === 'ECONNREFUSED'`) would miss
+ * some shapes.
+ */
+export function isDockerUnavailableError(error: unknown): boolean {
+  if (!error) return false;
+  const err = error as NodeJS.ErrnoException & { errno?: number };
+  if (err.code === 'ECONNREFUSED' || err.code === 'ENOENT' && /docker\.sock/i.test(err.message ?? '')) {
+    return true;
+  }
+  const message = String(err.message ?? '').toLowerCase();
+  return (
+    message.includes('econnrefused') ||
+    message.includes('cannot connect to the docker daemon') ||
+    message.includes('docker.sock') && (message.includes('connect') || message.includes('no such'))
+  );
+}
+
 export async function containerActionForStack(
   nodeId: number,
   stackName: string,
@@ -810,6 +845,9 @@ export async function containerActionForStack(
     await Promise.all(containers.map(c => op(c.Id)));
     return { kind: 'ok', count: containers.length };
   } catch (error: unknown) {
+    if (isDockerUnavailableError(error)) {
+      return { kind: 'docker-unavailable', message: getErrorMessage(error, 'Docker daemon is unreachable') };
+    }
     return { kind: 'error', message: getErrorMessage(error, `Failed to ${action} containers`) };
   }
 }
@@ -832,6 +870,12 @@ async function bulkContainerOp(
       res.status(404).json({ error: 'No containers found for this stack.' });
       return;
     }
+    if (outcome.kind === 'docker-unavailable') {
+      console.error('[Stacks] %s failed: docker unavailable for %s', sanitizeForLog(titleCase), sanitizeForLog(stackName));
+      if (action !== 'start') notifyActionFailure(action, stackName, new Error(outcome.message));
+      res.status(503).json({ error: outcome.message, code: 'docker_unavailable' });
+      return;
+    }
     if (outcome.kind === 'error') {
       console.error('[Stacks] %s failed: %s %s', sanitizeForLog(titleCase), sanitizeForLog(stackName), sanitizeForLog(outcome.message));
       if (action !== 'start') notifyActionFailure(action, stackName, new Error(outcome.message));
@@ -968,7 +1012,11 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
     }
     notifyActionFailure('update', stackName, error);
     if (!res.headersSent) {
-      res.status(500).json({ error: getErrorMessage(error, 'Failed to update'), rolledBack });
+      if (isDockerUnavailableError(error)) {
+        res.status(503).json({ error: getErrorMessage(error, 'Docker daemon is unreachable'), code: 'docker_unavailable', rolledBack });
+      } else {
+        res.status(500).json({ error: getErrorMessage(error, 'Failed to update'), rolledBack });
+      }
     }
   } finally {
     releaseStackOpLock(req, stackName);