From 4c93947004ff0e141c4a3aa96356106430e97888 Mon Sep 17 00:00:00 2001 From: Anso Date: Fri, 14 Aug 2026 09:24:48 -0400 Subject: [PATCH] fix: stop compose-ps spam after deleting a stack (#1831) * fix: stop compose-ps spam after deleting a stack A live logs WebSocket kept calling docker compose ps against the removed stack directory every 2s. Missing cwd surfaces as spawn ENOENT, which was logged as Docker CLI unavailable. Skip compose when the dir is gone and idle the log stream instead of retrying. * fix: contain stack directory stats inside the compose root Resolve the stack path against the node's compose directory and refuse paths that escape it before calling fs.stat, matching the existing filesystem containment pattern at other sinks. --- backend/src/__tests__/compose-service.test.ts | 83 + .../src/__tests__/docker-controller.test.ts | 71 + backend/src/services/ComposeService.ts | 3284 +++++++++-------- backend/src/services/DockerController.ts | 26 + 4 files changed, 1833 insertions(+), 1631 deletions(-) diff --git a/backend/src/__tests__/compose-service.test.ts b/backend/src/__tests__/compose-service.test.ts index fd012439..ea2bba1e 100644 --- a/backend/src/__tests__/compose-service.test.ts +++ b/backend/src/__tests__/compose-service.test.ts @@ -35,6 +35,7 @@ const { mockCompensateWithCandidate, mockBuildUnifiedHeldImagePredicate, mockGetRecovery, + mockFsStat, } = vi.hoisted(() => ({ mockSpawn: vi.fn(), mockGetContainersByStack: vi.fn().mockResolvedValue([]), @@ -87,6 +88,7 @@ const { mockCompensateWithCandidate: vi.fn().mockResolvedValue(true), mockBuildUnifiedHeldImagePredicate: vi.fn().mockReturnValue(() => false), mockGetRecovery: vi.fn().mockReturnValue(undefined), + mockFsStat: vi.fn().mockResolvedValue({ isDirectory: () => true }), })); vi.mock('child_process', () => ({ spawn: mockSpawn, execFile: vi.fn() })); @@ -97,11 +99,13 @@ vi.mock('fs', () => ({ writeFileSync: (...args: unknown[]) => mockWriteFileSync(...args), unlinkSync: (...args: unknown[]) => mockUnlinkSync(...args), rmdirSync: (...args: unknown[]) => mockRmdirSync(...args), + promises: { stat: (...args: unknown[]) => mockFsStat(...args) }, }, mkdtempSync: (...args: unknown[]) => mockMkdtempSync(...args), writeFileSync: (...args: unknown[]) => mockWriteFileSync(...args), unlinkSync: (...args: unknown[]) => mockUnlinkSync(...args), rmdirSync: (...args: unknown[]) => mockRmdirSync(...args), + promises: { stat: (...args: unknown[]) => mockFsStat(...args) }, })); vi.mock('../services/NodeRegistry', () => ({ @@ -316,6 +320,7 @@ beforeEach(() => { mockIsMeshStackEnabled.mockReturnValue(false); mockCompensateWithCandidate.mockResolvedValue(true); mockBuildUnifiedHeldImagePredicate.mockReturnValue(() => false); + mockFsStat.mockResolvedValue({ isDirectory: () => true }); delete process.env.SENCHO_MODE; vi.useFakeTimers({ shouldAdvanceTime: true }); }); @@ -1709,6 +1714,84 @@ describe('ComposeService - streamLogs', () => { expect.anything(), ); }); + + it('does not reschedule when the stack directory is gone', async () => { + mockFsStat.mockRejectedValue(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })); + const ws = createMockWs(); + const svc = ComposeService.getInstance(1); + + svc.streamLogs('mystack', ws); + await vi.waitFor(() => expect(ws.send).toHaveBeenCalled()); + + const sent = (ws.send as ReturnType).mock.calls + .flatMap((c) => String(c[0]).split(/\r?\n/)) + .filter(Boolean); + expect(sent.some((line) => line.includes('Stack directory is gone'))).toBe(true); + expect(mockGetContainersByStack).not.toHaveBeenCalled(); + expect(ws.send).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(4000); + expect(mockGetContainersByStack).not.toHaveBeenCalled(); + expect(ws.send).toHaveBeenCalledTimes(1); + expect(ws.close).not.toHaveBeenCalled(); + }); + + it('retries every 2s when the stack dir exists and no containers are found', async () => { + mockGetContainersByStack.mockResolvedValue([]); + const ws = createMockWs(); + const svc = ComposeService.getInstance(1); + + svc.streamLogs('mystack', ws); + await vi.waitFor(() => expect(mockGetContainersByStack).toHaveBeenCalledTimes(1)); + + await vi.advanceTimersByTimeAsync(2000); + expect(mockGetContainersByStack).toHaveBeenCalledTimes(2); + + const sent = (ws.send as ReturnType).mock.calls + .flatMap((c) => String(c[0])) + .join(''); + expect(sent).toContain('No containers found. Waiting for activity'); + }); + + it('stops retrying after the websocket closes', async () => { + mockGetContainersByStack.mockResolvedValue([]); + const ws = createMockWs(); + const svc = ComposeService.getInstance(1); + + svc.streamLogs('mystack', ws); + await vi.waitFor(() => expect(mockGetContainersByStack).toHaveBeenCalledTimes(1)); + + ws.readyState = 3; + ws.emit('close'); + await vi.advanceTimersByTimeAsync(4000); + expect(mockGetContainersByStack).toHaveBeenCalledTimes(1); + }); + + it('stops the empty-container retry after the stack directory disappears', async () => { + mockGetContainersByStack.mockResolvedValue([]); + const ws = createMockWs(); + const svc = ComposeService.getInstance(1); + + svc.streamLogs('mystack', ws); + await vi.waitFor(() => expect(mockGetContainersByStack).toHaveBeenCalledTimes(1)); + + mockFsStat.mockRejectedValue(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })); + await vi.advanceTimersByTimeAsync(2000); + + expect(mockGetContainersByStack).toHaveBeenCalledTimes(1); + const sent = (ws.send as ReturnType).mock.calls + .flatMap((c) => String(c[0])) + .join(''); + expect(sent).toContain('No containers found. Waiting for activity'); + expect(sent).toContain('Stack directory is gone'); + expect(ws.close).not.toHaveBeenCalled(); + + const sendsAfterGone = (ws.send as ReturnType).mock.calls.length; + await vi.advanceTimersByTimeAsync(4000); + expect(mockGetContainersByStack).toHaveBeenCalledTimes(1); + expect(ws.send).toHaveBeenCalledTimes(sendsAfterGone); + expect(ws.close).not.toHaveBeenCalled(); + }); }); diff --git a/backend/src/__tests__/docker-controller.test.ts b/backend/src/__tests__/docker-controller.test.ts index 48c648c7..d5eee6b6 100644 --- a/backend/src/__tests__/docker-controller.test.ts +++ b/backend/src/__tests__/docker-controller.test.ts @@ -78,6 +78,7 @@ beforeEach(() => { // holds re-spy after this beforeEach runs. vi.spyOn(StackUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set()); vi.spyOn(ServiceUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set()); + vi.spyOn(fs, 'stat').mockResolvedValue({ isDirectory: () => true } as Awaited>); }); /** Hold specific images via the stack-side recovery service for one test. */ @@ -2478,3 +2479,73 @@ describe('DockerController - smartFallback stack-dir evidence', () => { }); }); +describe('DockerController - missing stack directory', () => { + const enoent = () => Object.assign(new Error('ENOENT: no such file or directory'), { code: 'ENOENT' }); + + beforeEach(() => { + vi.spyOn(fs, 'stat').mockRejectedValue(enoent()); + }); + + it('getContainersByStack returns [] without spawning compose', async () => { + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const dc = DockerController.getInstance(1); + const fetchSpy = spyOrphanDc(dc, 'fetchComposePsContainers'); + + await expect(dc.getContainersByStack('gone-stack')).resolves.toEqual([]); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(mockExecFileAsync).not.toHaveBeenCalled(); + expect(errSpy).not.toHaveBeenCalled(); + fetchSpy.mockRestore(); + errSpy.mockRestore(); + }); + + it('getLegacyOrphanContainersByStack returns [] without spawning compose', async () => { + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const dc = DockerController.getInstance(1); + const fetchSpy = spyOrphanDc(dc, 'fetchComposePsContainers'); + const fallbackSpy = spyOrphanDc(dc, 'smartFallback'); + + await expect(dc.getLegacyOrphanContainersByStack('gone-stack')).resolves.toEqual([]); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(fallbackSpy).not.toHaveBeenCalled(); + expect(mockExecFileAsync).not.toHaveBeenCalled(); + expect(errSpy).not.toHaveBeenCalled(); + fetchSpy.mockRestore(); + fallbackSpy.mockRestore(); + errSpy.mockRestore(); + }); + + it('classifyLegacyOrphansForUpdate returns classification_failed without spawning compose', async () => { + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const dc = DockerController.getInstance(1); + const fetchSpy = spyOrphanDc(dc, 'fetchComposePsContainers'); + + await expect(dc.classifyLegacyOrphansForUpdate('gone-stack')).resolves.toEqual({ + status: 'classification_failed', + error: 'Stack directory is gone', + }); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(mockExecFileAsync).not.toHaveBeenCalled(); + expect(errSpy).not.toHaveBeenCalled(); + fetchSpy.mockRestore(); + errSpy.mockRestore(); + }); + + it('still logs Docker CLI unavailable when the stack dir exists and compose ENOENT fires', async () => { + vi.spyOn(fs, 'stat').mockResolvedValue({ isDirectory: () => true } as Awaited>); + vi.spyOn(os, 'freemem').mockReturnValue(2 * 1024 * 1024 * 1024); + vi.spyOn(os, 'totalmem').mockReturnValue(8 * 1024 * 1024 * 1024); + mockExecFileAsync.mockRejectedValue(Object.assign(new Error('spawn docker ENOENT'), { code: 'ENOENT' })); + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + const dc = DockerController.getInstance(1); + await dc.getContainersByStack('my-stack'); + + expect(mockExecFileAsync).toHaveBeenCalled(); + const logged = errSpy.mock.calls.map((c) => c.map(String).join(' ')).join('\n'); + expect(logged).toMatch(/Docker Compose Error for/); + expect(logged).toContain('Docker CLI unavailable on this node'); + errSpy.mockRestore(); + }); +}); + diff --git a/backend/src/services/ComposeService.ts b/backend/src/services/ComposeService.ts index 30853e37..b425e94f 100644 --- a/backend/src/services/ComposeService.ts +++ b/backend/src/services/ComposeService.ts @@ -1,1631 +1,1653 @@ -import { spawn } from 'child_process'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; -import WebSocket from 'ws'; -import DockerController from './DockerController'; -import { DatabaseService } from './DatabaseService'; -import { FileSystemService } from './FileSystemService'; -import { MeshService } from './MeshService'; -import { LogFormatter } from './LogFormatter'; -import { NodeRegistry } from './NodeRegistry'; -import { RegistryService } from './RegistryService'; -import { DriftLedgerService } from './DriftLedgerService'; -import SelfIdentityService from './SelfIdentityService'; -import { parseEffectiveModel } from './preflight/effectiveModel'; -import { deriveStackExposure } from './preflight/exposure'; - -import { isDebugEnabled } from '../utils/debug'; -import { getErrorMessage } from '../utils/errors'; -import { normalizeContainerName } from '../utils/log-parsing'; -import { describeSpawnError } from '../utils/spawnErrors'; -import { isPathWithinBase, isValidStackName, isValidRelativeStackPath } from '../utils/validation'; -import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs'; -import type { RollbackInvocationRecord } from '../types/rollbackGeneration'; -import { parseMissingRequiredVars } from '../helpers/envVarParse'; -import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog'; -import { pathsMatch, resolveHostBindPath } from '../utils/composePathMapping'; -import { loadStackBuildServices } from './ImageUpdateService'; -import { resolveMissingExternalNetworks } from './network/resolveMissingExternalNetworks'; -import { - MissingExternalNetworksError, - type DeployInvocationContext, -} from './network/missingExternalNetworksError'; -import { buildUnifiedHeldImagePredicate } from './recoveryHeldImages'; -import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; -import type { NotificationCategory } from './NotificationService'; - -/** True when a generation capture recorded an invocation object (prefix may be empty). */ -function hasUsableCapturedInvocation( - invocation: RollbackInvocationRecord | null | undefined, -): invocation is RollbackInvocationRecord { - return invocation != null; -} - -function recordNetworkAutoCreatedActivity( - nodeId: number, - stackName: string, - createdNames: string[], - level: 'info' | 'warning', - ctx?: DeployInvocationContext, -): void { - if (createdNames.length === 0) return; - const names = [...createdNames].sort((a, b) => a.localeCompare(b)).join(', '); - const source = ctx?.source ?? 'manual'; - try { - DatabaseService.getInstance().addNotificationHistory(nodeId, { - level, - category: 'network_auto_created' as NotificationCategory, - message: `Auto-created external network(s) for ${stackName}: ${names} (source: ${source})`, - timestamp: Date.now(), - stack_name: stackName, - actor_username: ctx?.actor ?? null, - }); - } catch (error) { - console.error( - '[ComposeService] Failed to record network_auto_created activity for %s:', - sanitizeForLog(stackName), - sanitizeForLog(getErrorMessage(error, 'unknown')), - ); - } -} - -export class ComposeRollbackError extends Error { - public readonly rollbackAttempted: boolean; - public readonly rolledBack: boolean; - public readonly originalError: unknown; - - constructor(originalError: unknown, rollbackAttempted: boolean, rolledBack: boolean) { - super(getErrorMessage(originalError, 'Compose operation failed')); - this.name = 'ComposeRollbackError'; - this.rollbackAttempted = rollbackAttempted; - this.rolledBack = rolledBack; - this.originalError = originalError; - Object.setPrototypeOf(this, ComposeRollbackError.prototype); - } -} - -export function getComposeRollbackInfo(error: unknown): { attempted: boolean; rolledBack: boolean } | null { - if (!(error instanceof ComposeRollbackError)) { - return null; - } - return { attempted: error.rollbackAttempted, rolledBack: error.rolledBack }; -} - -function isNonFatalCompensationError(error: unknown): boolean { - const code = (error as { code?: string }).code; - return code === 'HELD_IMAGE_MISSING' || code === 'RECOVERY_PROBE_FAILED'; -} - -async function compensateOrSwallow(compensate: () => Promise): Promise { - try { - return await compensate(); - } catch (compError) { - if (!isNonFatalCompensationError(compError)) throw compError; - return false; - } -} - -const DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS = 30 * 60 * 1000; - -/** Public so other services (e.g. recovery claim leases) can size their own timers off the same ceiling without depending on a private module-local. */ -export function getComposeCommandTimeoutMs(): number { - const configured = Number(process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS); - if (Number.isFinite(configured) && configured > 0) { - return configured; - } - return DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS; -} - -// Idle backstop for long-running pull/recreate steps: if the child emits no -// output for this window while still running, the step is treated as stalled -// and terminated, so a hung `docker compose pull` surfaces a fast failure -// instead of spinning until the much longer command timeout above. Conservative -// by default because a working pull can be briefly silent while a large layer -// extracts; operators on slow links or heavy local builds can raise it. -const DEFAULT_COMPOSE_STALL_TIMEOUT_MS = 10 * 60 * 1000; - -function getComposeStallTimeoutMs(): number { - const configured = Number(process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS); - if (Number.isFinite(configured) && configured > 0) { - return configured; - } - return DEFAULT_COMPOSE_STALL_TIMEOUT_MS; -} - -/** - * ComposeService - local docker compose CLI execution. - * - * In the Distributed API model, remote node compose operations are handled - * by the remote Sencho instance. This service only executes commands locally. - */ -export class ComposeService { - private baseDir: string; - private nodeId: number; - - constructor(nodeId?: number) { - this.nodeId = nodeId ?? NodeRegistry.getInstance().getDefaultNodeId(); - this.baseDir = NodeRegistry.getInstance().getComposeDir(this.nodeId); - } - - public static getInstance(nodeId?: number): ComposeService { - return new ComposeService(nodeId); - } - - /** - * Build the authored `docker compose` argument list for a stack: the validated - * multi-file deploy prefix (ordered `-f` files + `-p ` + - * `--project-directory`) for a Git source with an applied multi-file spec, then - * the Sencho Mesh override file last (highest `-f` precedence) when the stack is - * opted into the mesh, then the action. Single-file / non-git stacks get no file - * prefix, so docker compose's built-in discovery resolves the root compose.yaml, - * byte-identical to the pre-multi-file behavior. The user's source files are - * never mutated. Lifecycle commands (deploy, update, stop/start/restart/down) - * route through this method, so they share one file prefix plus the mesh override. - * Image scans (listStackImages) and the Compose Doctor (renderConfig) reuse the - * same `authoredComposeFileArgs` prefix directly but intentionally omit the mesh - * override, rendering the user's authored model without mesh injection. - */ - /** Public wrapper for dual-arg assembly and recovery Compose invocations. */ - public async buildAuthoredComposeArgs(stackName: string, action: string[]): Promise { - return this.authoredComposeArgs(stackName, action); - } - - public async validateStackForMutation(stackName: string): Promise { - await this.assertRequiredEnvPresent(stackName); - await this.assertSafePilotBindMapping(stackName); - } - - /** - * Render/validate the exact Compose invocation used by mutating operations - * (authored files, env pins, and generated Mesh override) before capture. - */ - public async validateExactComposeInvocation(stackName: string): Promise { - if (!isValidStackName(stackName)) { - throw new Error('Invalid stack path'); - } - const baseResolved = path.resolve(this.baseDir); - const stackDir = path.resolve(baseResolved, stackName); - if (!stackDir.startsWith(baseResolved + path.sep)) { - throw new Error('Invalid stack path'); - } - const args = await this.authoredComposeArgs(stackName, ['config', '--quiet']); - await this.execute('docker', args, stackDir, undefined, true); - } - - private async authoredComposeArgs(stackName: string, action: string[]): Promise { - const args: string[] = ['compose']; - const filePrefix = authoredComposeFileArgs(stackName, this.nodeId); - args.push(...filePrefix); - // Pin env resolution to the root .env when a context dir shifts the project - // directory, so deploy/update resolve the same effective config the validator did. - args.push(...await authoredComposeEnvFileArgs(stackName, this.nodeId)); - - const meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(this.nodeId, stackName); - let overridePath: string | null = null; - try { - overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName); - } catch (err) { - if (meshEnabled) { - throw err instanceof Error - ? err - : new Error(`Mesh override generation failed: ${String(err)}`); - } - console.warn('[ComposeService] mesh override skipped:', sanitizeForLog((err as Error).message)); - } - if (meshEnabled && !overridePath) { - throw new Error( - `Mesh override is required for stack "${stackName}" but could not be generated`, - ); - } - if (overridePath) { - if (filePrefix.length === 0) { - // Single-file stack: passing any -f disables compose's auto-discovery, so name - // the base file explicitly, then re-add the user's implicit override (if any) so - // it is not silently dropped, before layering the mesh override on top. - const fsSvc = FileSystemService.getInstance(this.nodeId); - const baseFilename = await fsSvc.getComposeFilename(stackName); - args.push('-f', baseFilename); - let userOverride: string | null = null; - try { - userOverride = await fsSvc.getOverrideFilename(stackName); - } catch (err) { - // Containment-guard rejections (bad stack name / symlink escape) are hard errors: - // abort the deploy rather than degrade. The "no override" case returns null rather - // than throwing, so any other throw is transient I/O: drop the override and proceed - // (logging the consequence) instead of failing the deploy. - const code = (err as { code?: string }).code; - if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') { - throw err; - } - console.warn('[ComposeService] could not resolve user compose override; deploying without it:', sanitizeForLog((err as Error).message)); - } - if (userOverride) { - args.push('-f', userOverride); - } - } - args.push('-f', overridePath); - } - args.push(...action); - return args; - } - - private execute( - command: string, - args: string[], - cwd: string, - ws?: WebSocket, - throwOnError = true, - env?: Record, - // When set, terminate the child if it emits no output for this long while - // still running (idle stall backstop). Appended last so the existing - // registry-auth call sites that pass `env` are unaffected. - idleTimeoutMs?: number - ): Promise { - return new Promise((resolve, reject) => { - const child = spawn(command, args, { - cwd, - env: env ?? { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' - } - }); - - let errorLog = ''; - let settled = false; - let exited = false; - let pendingTerminationError: Error | null = null; - const timeoutMs = getComposeCommandTimeoutMs(); - let timeout: ReturnType | null = null; - let forceKillTimeout: ReturnType | null = null; - let idleTimeout: ReturnType | null = null; - - const sendOutput = (text: string) => { - if (ws && ws.readyState === WebSocket.OPEN) { - ws.send(text); - } - }; - - const cleanup = () => { - if (timeout) { - clearTimeout(timeout); - timeout = null; - } - if (forceKillTimeout) { - clearTimeout(forceKillTimeout); - forceKillTimeout = null; - } - if (idleTimeout) { - clearTimeout(idleTimeout); - idleTimeout = null; - } - }; - - const finish = (complete: () => void) => { - if (settled) return; - settled = true; - cleanup(); - complete(); - }; - - const terminateChild = (error: Error) => { - pendingTerminationError = pendingTerminationError ?? error; - if (exited) return; - try { - child.kill('SIGTERM'); - } catch (error) { - console.warn('[ComposeService] Failed to terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); - } - forceKillTimeout = setTimeout(() => { - if (exited) return; - try { - child.kill('SIGKILL'); - } catch (error) { - console.warn('[ComposeService] Failed to force terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); - } - }, 5000); - }; - - // Idle stall backstop. Armed once below and reset on every output chunk; - // if it ever fires, the step has been silent for idleTimeoutMs while still - // running, so terminate it. Never rearmed after a termination is pending or - // the child has exited, so it cannot re-fire during the SIGTERM grace. - const armIdleTimeout = () => { - if (idleTimeoutMs === undefined) return; - if (exited || settled || pendingTerminationError) return; - if (idleTimeout) clearTimeout(idleTimeout); - idleTimeout = setTimeout(() => { - const seconds = Math.round(idleTimeoutMs / 1000); - sendOutput(`=== No output for ${seconds}s; the operation appears stalled and was stopped ===\n`); - terminateChild(new Error(`STACK_STALLED_OUTPUT: no output for ${seconds}s`)); - }, idleTimeoutMs); - }; - - // The progress socket is output-only: a deploy/update/down is owned by the - // HTTP request that started it, so closing or losing the socket (the user - // minimizes the panel, navigates away, or the connection blips) must not - // terminate the compose process. Termination is driven solely by the - // command timeout here and the optional idle stall backstop above. - timeout = setTimeout(() => { - const message = `Command timed out after ${Math.round(timeoutMs / 1000)}s`; - sendOutput(`${message}\n`); - terminateChild(new Error(message)); - }, timeoutMs); - - armIdleTimeout(); - - const onData = (data: Buffer) => { - const text = data.toString(); - errorLog += text; - sendOutput(text); - armIdleTimeout(); - }; - - child.stdout.on('data', onData); - child.stderr.on('data', onData); - - child.on('close', (code: number | null) => { - exited = true; - finish(() => { - sendOutput(`Command exited with code ${code}\n`); - if (pendingTerminationError) { - if (throwOnError) reject(pendingTerminationError); - else resolve(); - return; - } - if (code === 0) resolve(); - else if (throwOnError) reject(new Error(redactSensitiveText(errorLog.trim()) || `Command failed with code ${code}`)); - else resolve(); - }); - }); - - child.on('error', (error: Error & { code?: string }) => { - exited = true; - finish(() => { - const mapped = describeSpawnError(error as NodeJS.ErrnoException, { command }); - const message = redactSensitiveText(mapped.message); - sendOutput(`Error: ${message}\n`); - if (mapped.isLowMemory) { - console.warn('[ComposeService] spawn failed under memory pressure:', message); - } - if (pendingTerminationError) { - if (throwOnError) reject(pendingTerminationError); - else resolve(); - return; - } - if (throwOnError) reject(new Error(message)); - else resolve(); - }); - }); - }); - } - - private async withRegistryAuth( - fn: (env: Record) => Promise, - sendOutput?: (data: string) => void, - ): Promise { - const registries = DatabaseService.getInstance().getRegistries(); - if (registries.length === 0) { - return fn({ - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }); - } - - const { config, warnings } = await RegistryService.getInstance().resolveDockerConfig(); - if (warnings.length > 0 && sendOutput) { - for (const warning of warnings) { - sendOutput(`[Sencho] Warning: ${warning}\n`); - } - } - - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-docker-')); - const configPath = path.join(tmpDir, 'config.json'); - - try { - fs.writeFileSync(configPath, JSON.stringify(config), { mode: 0o600 }); - return await fn({ - ...process.env, - DOCKER_CONFIG: tmpDir, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }); - } finally { - // Best-effort cleanup; each step runs independently so a file that was never - // written (e.g., writeFileSync threw) does not prevent the directory removal. - try { fs.unlinkSync(configPath); } catch { /* file may not exist */ } - try { fs.rmdirSync(tmpDir); } catch (e) { - console.warn('[ComposeService] Could not remove temp Docker config dir:', (e as Error).message); - } - } - } - - private async createAtomicBackup( - stackName: string, - operation: 'deployment' | 'update', - sendOutput: (data: string) => void, - ): Promise { - try { - const fsSvc = FileSystemService.getInstance(this.nodeId); - await fsSvc.backupStackFiles(stackName); - sendOutput(`=== Backup created for atomic ${operation} ===\n`); - } catch (error) { - console.error('Atomic backup failed for %s:', sanitizeForLog(stackName), getErrorMessage(error, 'unknown error')); - sendOutput(`=== Atomic ${operation} backup failed. Operation aborted ===\n`); - throw new Error(`Atomic ${operation} backup failed: ${getErrorMessage(error, 'unknown error')}`); - } - } - - private async restoreAtomicBackup( - stackName: string, - stackDir: string, - ws: WebSocket | undefined, - sendOutput: (data: string) => void, - ): Promise { - try { - const fsSvc = FileSystemService.getInstance(this.nodeId); - await fsSvc.restoreStackFiles(stackName); - await this.withRegistryAuth(async (env) => { - await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env); - }, sendOutput); - sendOutput('=== Restored previous compose and env files ===\n'); - return true; - } catch (rollbackError) { - console.error('Rollback failed for %s:', sanitizeForLog(stackName), getErrorMessage(rollbackError, 'unknown error')); - sendOutput('=== Rollback failed. Manual intervention may be required ===\n'); - return false; - } - } - - private createContainerCrashError(exitCode: number): Error { - return new Error( - `CONTAINER_CRASHED\nExit Code: ${exitCode}\nContainer exited after deployment. Check container logs for details.` - ); - } - - async runCommand(stackName: string, action: 'down' | 'start' | 'stop' | 'restart', ws?: WebSocket): Promise { - const stackDir = path.join(this.baseDir, stackName); - await this.execute('docker', await this.authoredComposeArgs(stackName, [action]), stackDir, ws); - } - - /** Interactive compose down (Take down UI / POST /down). Plain `down` by default. */ - async runDown(stackName: string, options?: { removeVolumes?: boolean }, ws?: WebSocket): Promise { - const stackDir = path.join(this.baseDir, stackName); - const args = options?.removeVolumes ? ['down', '--volumes'] : ['down']; - await this.execute('docker', await this.authoredComposeArgs(stackName, args), stackDir, ws); - } - - /** - * Opt-in guard: when `env_block_deploy_on_missing_required` is enabled, refuse a - * deploy whose required `${VAR:?err}` variables are unset OR empty, before any - * backup, cleanup, pull, or `up` runs. Compose's own resolution is authoritative - * (it passes process.env), and on the failing path it emits no rendered model, so - * no env value is materialized. Default off and any settings-read failure both - * fall through without blocking. - */ - private async assertRequiredEnvPresent(stackName: string): Promise { - let enabled = false; - try { - enabled = DatabaseService.getInstance().getGlobalSettings()['env_block_deploy_on_missing_required'] === '1'; - } catch { - return; // safe default: a settings-read failure never blocks a deploy - } - if (!enabled) return; - const result = await this.renderConfig(stackName); - const missing = parseMissingRequiredVars(result.stderr); - if (missing.length === 0) return; - const plural = missing.length > 1; - throw new Error( - `Deploy blocked: required environment variable${plural ? 's' : ''} ${missing.join(', ')} ` + - `${plural ? 'are' : 'is'} missing. Define ${plural ? 'them' : 'it'} in a .env or env_file, then deploy again.`, - ); - } - - private async assertSafePilotBindMapping(stackName: string): Promise { - if (process.env.SENCHO_MODE !== 'pilot') return; - - let mounts: Array<{ source: string; destination: string }> | null; - try { - mounts = await SelfIdentityService.getInstance().getBindMounts(); - } catch (error) { - console.warn('[ComposeService] Could not verify pilot compose path mapping:', sanitizeForLog(getErrorMessage(error, 'unknown'))); - return; - } - if (mounts === null) return; - - const composeDir = path.resolve(this.baseDir); - const hostComposeDir = resolveHostBindPath(composeDir, mounts); - if (!hostComposeDir || pathsMatch(hostComposeDir, composeDir)) return; - - const rendered = await this.renderConfig(stackName); - if (rendered.rendered === null) return; - - let parsed: unknown; - try { - parsed = JSON.parse(rendered.rendered); - } catch (error) { - console.warn('[ComposeService] Could not inspect rendered binds for pilot path safety:', sanitizeForLog(getErrorMessage(error, 'unknown'))); - return; - } - const model = parseEffectiveModel(parsed, stackName); - const unsafeBind = model.services - .flatMap((service) => service.binds) - .find((bind) => isPathWithinBase(path.resolve(bind.source), composeDir)); - if (!unsafeBind) return; - - throw new Error( - `Deploy blocked: relative bind mounts resolve under ${composeDir}, but the host path is ${hostComposeDir}. ` + - `Use a 1:1 mount with the same absolute path on the host and in the Pilot Agent, then retry.`, - ); - } - - /** - * Missing-external gate: after env/Pilot asserts, before atomic backup. - * Creates safe bridge networks only when the opt-in setting is on. - */ - private async ensureExternalNetworksForDeploy( - stackName: string, - ctx?: DeployInvocationContext, - ): Promise { - const resolved = await resolveMissingExternalNetworks(this.nodeId, stackName); - if (resolved.status === 'render_unavailable') { - throw new MissingExternalNetworksError({ - kind: 'unavailable', - message: 'Sencho could not render this stack\'s Compose model to check external networks.', - }); - } - if (resolved.status === 'runtime_unavailable') { - // No declared externals: nothing to verify; proceed. - if (resolved.declaredExternalCount === 0) return; - throw new MissingExternalNetworksError({ - kind: 'unavailable', - message: 'Sencho could not read Docker networking state to check external networks.', - }); - } - - if (resolved.networks.length === 0) return; - - const unsafe = resolved.networks.filter((n) => !n.safe); - if (unsafe.length > 0) { - throw new MissingExternalNetworksError({ - kind: 'unsupported', - message: 'One or more missing external networks cannot be created safely by Sencho.', - networks: resolved.networks, - }); - } - - if (!resolved.autoCreateEnabled) { - throw new MissingExternalNetworksError({ - kind: 'prompt', - message: 'One or more external networks required by this stack are missing on this node.', - networks: resolved.networks, - }); - } - - const docker = DockerController.getInstance(this.nodeId); - const createdNames: string[] = []; - const recordCreatedNetworks = (level: 'info' | 'warning') => { - if (createdNames.length === 0) return; - invalidateNodeCaches(this.nodeId); - recordNetworkAutoCreatedActivity(this.nodeId, stackName, createdNames, level, ctx); - }; - - for (const network of resolved.networks) { - try { - await docker.createNetwork({ Name: network.name, Driver: 'bridge' }); - createdNames.push(network.name); - } catch (createErr) { - // Authoritative re-check: continue only if the network now exists. - let exists = false; - try { - const knownStacks = await FileSystemService.getInstance(this.nodeId).getStacks(); - const snapshot = await docker.getDependencySnapshot(knownStacks); - exists = snapshot.networks.some((n) => n.name === network.name); - } catch (snapErr) { - console.warn( - '[ComposeService] Post-create snapshot failed for %s:', - sanitizeForLog(network.name), - sanitizeForLog(getErrorMessage(snapErr, 'unknown')), - ); - } - if (!exists) { - recordCreatedNetworks('warning'); - throw new MissingExternalNetworksError({ - kind: 'create_failed', - message: `Failed to create external network "${network.name}".`, - networks: resolved.networks, - createdNames, - remainingNames: resolved.networks - .map((missingNetwork) => missingNetwork.name) - .filter((name) => !createdNames.includes(name)), - }); - } - // Race-existing: do not record in createdNames. - } - } - - // Re-resolve before Compose. - const recheck = await resolveMissingExternalNetworks(this.nodeId, stackName); - if (recheck.status !== 'ok' || recheck.networks.length > 0) { - recordCreatedNetworks('warning'); - throw new MissingExternalNetworksError({ - kind: recheck.status === 'ok' ? 'create_failed' : 'unavailable', - message: 'External networks were still missing after automatic creation.', - networks: recheck.networks, - createdNames, - remainingNames: recheck.networks.map((n) => n.name), - }); - } - - recordCreatedNetworks('info'); - } - - async deployStack( - stackName: string, - ws?: WebSocket, - atomic?: boolean, - ctx?: DeployInvocationContext, - ): Promise<{ recoveryId: string | null }> { - await this.assertRequiredEnvPresent(stackName); - await this.assertSafePilotBindMapping(stackName); - await this.ensureExternalNetworksForDeploy(stackName, ctx); - - const stackDir = path.join(this.baseDir, stackName); - const debug = isDebugEnabled(); - const t0 = Date.now(); - if (debug) console.debug('[ComposeService:debug] deployStack', { stackName, stackDir, atomic }); - const sendOutput = (data: string) => { - if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); - }; - - const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); - const recoverySvc = atomic ? StackUpdateRecoveryService.getInstance() : null; - let recoveryId: string | null = null; - let handedOff = false; - - if (atomic && recoverySvc) { - sendOutput('=== Capturing rollback generation for atomic deploy ===\n'); - const candidate = await recoverySvc.captureCandidate({ - nodeId: this.nodeId, - stackName, - createdBy: 'atomic-deploy', - operationKind: 'deployment', - }); - recoveryId = candidate.id; - if (!recoverySvc.markAcquired(candidate.id)) { - await recoverySvc.abandon(candidate.id); - throw new Error('Failed to mark recovery generation as acquired'); - } - if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) { - await recoverySvc.abandon(candidate.id); - throw new Error('Failed to hand off recovery generation'); - } - handedOff = true; - if (!recoverySvc.markReconciling(candidate.id)) { - throw new Error('Failed to mark recovery generation as reconciling after handoff'); - } - } - - try { - try { - const dockerController = DockerController.getInstance(this.nodeId); - const legacyOrphans = await dockerController.getLegacyOrphanContainersByStack(stackName); - if (legacyOrphans.length > 0) { - sendOutput(`=== Cleaning up legacy orphan containers before deployment ===\n`); - await dockerController.removeContainers(legacyOrphans.map((c) => c.Id)); - } - } catch (e) { - console.warn('Failed to clean up legacy containers for %s:', sanitizeForLog(stackName), e); - } - - await this.withRegistryAuth(async (env) => { - await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs()); - }, sendOutput); - - // Post-Deploy Health Probe - await new Promise(resolve => setTimeout(resolve, 3000)); - - const dockerController = DockerController.getInstance(this.nodeId); - const containers = await dockerController.getDocker().listContainers({ - all: true, - filters: { label: [`com.docker.compose.project=${stackName}`] } - }); - - for (const containerInfo of containers) { - if (containerInfo.State === 'exited') { - const container = dockerController.getDocker().getContainer(containerInfo.Id); - const inspectData = await container.inspect(); - const exitCode = inspectData.State.ExitCode; - - if (exitCode !== 0) { - throw this.createContainerCrashError(exitCode); - } - } - } - - if (atomic && recoverySvc && recoveryId) { - if (!recoverySvc.markImmediateVerified(recoveryId)) { - console.warn( - '[ComposeService] Could not CAS immediate_verified for recovery %s', - sanitizeForLog(recoveryId), - ); - } - } - if (debug) console.debug(`[ComposeService:debug] deployStack completed in ${Date.now() - t0}ms`, { stackName }); - } catch (deployError) { - if (atomic && recoverySvc && handedOff && recoveryId) { - sendOutput('\n=== Deployment failed - restoring previous runtime from recovery generation ===\n'); - const generationId = recoveryId; - const rolledBack = await compensateOrSwallow(() => - recoverySvc.compensateWithCandidate( - generationId, - (overridePath, invocation) => this.composeUpWithRecoveryOverride( - stackName, - overridePath, - ws, - invocation, - ), - ), - ); - throw new ComposeRollbackError(deployError, true, rolledBack); - } - if (atomic && recoverySvc && recoveryId && !handedOff) { - await recoverySvc.abandon(recoveryId); - } - throw deployError; - } - // Reached only on a successful deploy (the catch above always rethrows). Record - // the drift baseline here so every deploy path gets one, not just the manual - // route: bulk, Git-source, App Store, scheduler, and webhook deploys all funnel - // through this method. Internally guarded; awaited so it cannot race later work. - await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName); - // Reconcile the ledger against the just-deployed runtime: findings this deploy - // fixed are resolved and any it left are recorded (and surfaced in the activity - // feed) now, instead of waiting for someone to open the Drift tab. The rollback - // route re-deploys through this method, so it is covered; a failed atomic deploy - // instead restores the previous files and throws above, so that recovery path - // reconciles on its next deploy or scan, not here. Best-effort internally. - await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName); - // Refresh the exposure cache so posture reflects the just-deployed model. - // Best-effort: a refresh failure logs a warning but never fails the deploy. - try { - await this.refreshExposureCache(stackName); - } catch (err) { - console.warn('[ComposeService] Exposure refresh failed after deploy for %s:', - sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown'))); - } - return { recoveryId }; - } - - streamLogs(stackName: string, ws: WebSocket) { - let isClosed = false; - let isFirstRun = true; - let isWaitingForActivity = false; - - ws.on('close', () => { isClosed = true; }); - - const startStream = async () => { - if (isClosed || ws.readyState !== WebSocket.OPEN) return; - - try { - const dockerController = DockerController.getInstance(this.nodeId); - const containers = await dockerController.getContainersByStack(stackName); - - if (!containers || containers.length === 0) { - if (!isWaitingForActivity) { - ws.send(`\r\n\x1b[33m[Sencho] No containers found. Waiting for activity...\x1b[0m\r\n`); - isWaitingForActivity = true; - } - setTimeout(startStream, 2000); - return; - } - - const runningContainers = containers.filter((c: any) => c.State === 'running'); - - if (!isFirstRun && runningContainers.length === 0) { - if (!isWaitingForActivity) { - ws.send(`\r\n\x1b[33m[Sencho] Log stream ended. Waiting for container activity...\x1b[0m\r\n`); - isWaitingForActivity = true; - } - setTimeout(startStream, 2000); - return; - } - - const containersToLog = isFirstRun ? containers : runningContainers; - isFirstRun = false; - isWaitingForActivity = false; - - let activeProcesses = 0; - let streamEndedHandled = false; - const localProcesses: ReturnType[] = []; - - const onWsClose = () => { - localProcesses.forEach(cp => { try { cp.kill(); } catch { } }); - }; - - ws.on('close', onWsClose); - - const handleProcessEnd = () => { - activeProcesses--; - if (activeProcesses <= 0 && !streamEndedHandled) { - streamEndedHandled = true; - ws.removeListener('close', onWsClose); - if (!isClosed && ws.readyState === WebSocket.OPEN) { - setTimeout(startStream, 1000); - } - } - }; - - for (const container of containersToLog) { - const rawName = container.Names?.[0]?.replace(/^\//, '') || container.Id; - const displayName = normalizeContainerName(rawName, stackName); - activeProcesses++; - let lineBuffer = ''; - - const sendOutput = (data: Buffer) => { - if (ws.readyState === WebSocket.OPEN) { - lineBuffer += data.toString(); - const lines = lineBuffer.split(/\r?\n/); - lineBuffer = lines.pop() || ''; - for (const line of lines) { - ws.send(LogFormatter.process(`${displayName} | ${line}`) + '\r\n'); - } - } - }; - - const flushBuffer = () => { - if (lineBuffer && ws.readyState === WebSocket.OPEN) { - ws.send(LogFormatter.process(`${displayName} | ${lineBuffer}`) + '\r\n'); - lineBuffer = ''; - } - }; - - const child = spawn('docker', ['logs', '-f', '-t', '--tail', '100', rawName], { - env: { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' - } - }); - localProcesses.push(child); - child.stdout.on('data', sendOutput); - child.stderr.on('data', sendOutput); - child.on('error', handleProcessEnd); - child.on('close', () => { - flushBuffer(); - handleProcessEnd(); - }); - } - } catch (err) { - if (!isClosed && ws.readyState === WebSocket.OPEN) { - if (!isWaitingForActivity) { - ws.send(`\r\n\x1b[31m[Sencho] Error tracking containers. Retrying...\x1b[0m\r\n`); - isWaitingForActivity = true; - } - setTimeout(startStream, 2000); - } - } - }; - - startStream(); - } - - - /** - * Pinned compose-up with a recovery override layered last - * (`--pull never --no-build`). Used by manual rollback and deploy/update - * compensation. When `invocation` is set, Compose args come from the - * generation capture instead of the live database-derived invocation. - */ - async composeUpWithRecoveryOverride( - stackName: string, - overridePath: string, - ws?: WebSocket, - invocation?: RollbackInvocationRecord | null, - ): Promise { - const stackDir = path.join(this.baseDir, stackName); - const sendOutput = (data: string) => { - if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); - }; - await this.withRegistryAuth(async (env) => { - await this.execute( - 'docker', - await this.buildComposeArgsWithRecoveryOverride( - stackName, - ['up', '-d', '--remove-orphans', '--pull', 'never', '--no-build'], - overridePath, - invocation ?? null, - ), - stackDir, - ws, - true, - env, - getComposeStallTimeoutMs(), - ); - }, sendOutput); - } - - /** - * Authored (+ mesh) compose args with an optional recovery override layered LAST. - * When `invocation` is provided, use its validated captured prefix so recovery - * does not mix restored files with the live deploy-spec / project-env selection. - */ - public async buildComposeArgsWithRecoveryOverride( - stackName: string, - action: string[], - recoveryOverridePath: string | null, - invocation?: RollbackInvocationRecord | null, - ): Promise { - const useCaptured = hasUsableCapturedInvocation(invocation); - const out = useCaptured - ? ['compose', ...this.composePrefixFromCapturedInvocation(stackName, invocation)] - : await this.authoredComposeArgsPrefix(stackName); - - if (useCaptured && invocation.meshEnabled) { - await this.appendCapturedMeshLayer(stackName, out); - } - - if (recoveryOverridePath) { - // Captured invocations already encode their file set; live args may need - // an explicit base (+ user override) so docker compose accepts a trailing -f. - await this.ensureExplicitComposeFiles(stackName, out, !useCaptured); - out.push('-f', recoveryOverridePath); - } - out.push(...action); - return out; - } - - /** - * Re-apply the Mesh override layer when the generation captured Mesh as enabled. - * Uses live ensureStackOverride (same absolute override path semantics as deploy). - */ - private async appendCapturedMeshLayer(stackName: string, args: string[]): Promise { - const overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName); - if (!overridePath) { - throw new Error( - `Captured Mesh-enabled invocation cannot regenerate mesh override for stack "${stackName}"`, - ); - } - await this.ensureExplicitComposeFiles(stackName, args, true); - args.push('-f', overridePath); - } - - /** Slice the global-flag prefix from authoredComposeArgs (no action tokens). */ - private async authoredComposeArgsPrefix(stackName: string): Promise { - const withSentinel = await this.authoredComposeArgs(stackName, ['__SENCHO_ACTION_SENTINEL__']); - const idx = withSentinel.indexOf('__SENCHO_ACTION_SENTINEL__'); - const prefix = idx >= 0 ? withSentinel.slice(0, idx) : withSentinel; - return [...prefix]; - } - - /** - * When args lack `-f`, pin the stack compose file (and optionally the user - * override) so a trailing override layer is accepted by docker compose. - */ - private async ensureExplicitComposeFiles( - stackName: string, - args: string[], - includeUserOverride: boolean, - ): Promise { - if (args.includes('-f')) return; - const fsSvc = FileSystemService.getInstance(this.nodeId); - args.push('-f', await fsSvc.getComposeFilename(stackName)); - if (!includeUserOverride) return; - try { - const userOverride = await fsSvc.getOverrideFilename(stackName); - if (userOverride) args.push('-f', userOverride); - } catch (err) { - const code = (err as { code?: string }).code; - if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') { - throw err; - } - console.warn( - '[ComposeService] could not resolve user compose override while pinning compose files:', - sanitizeForLog((err as Error).message), - ); - } - } - - private resolveValidatedStackDir(stackName: string): string { - const stackDir = path.resolve(this.baseDir, stackName); - if (!isPathWithinBase(stackDir, this.baseDir) || path.resolve(this.baseDir) === stackDir) { - throw new Error('Invalid stack path'); - } - return stackDir; - } - - /** - * Rebuild a spawn-safe compose global-flag prefix from a generation's - * captured invocation. Relative -f / --project-directory paths and absolute - * --env-file paths must stay inside the stack directory. - */ - private composePrefixFromCapturedInvocation( - stackName: string, - invocation: RollbackInvocationRecord, - ): string[] { - const stackDir = this.resolveValidatedStackDir(stackName); - // Empty prefix is valid (single-file auto-discovery at capture time). - const raw = [...invocation.composeArgsPrefix]; - const out: string[] = []; - for (let i = 0; i < raw.length; i++) { - const token = raw[i]; - if (token === '-f' || token === '--file') { - const file = raw[++i]; - if (!file || !isValidRelativeStackPath(file)) { - throw new Error(`Invalid captured compose file path for stack "${stackName}"`); - } - if (!isPathWithinBase(path.resolve(stackDir, file), stackDir)) { - throw new Error(`Captured compose file path escapes stack directory for "${stackName}"`); - } - out.push('-f', file); - continue; - } - if (token === '--env-file') { - const envPath = raw[++i]; - if (!envPath) { - throw new Error(`Invalid captured --env-file for stack "${stackName}"`); - } - const abs = path.resolve(envPath); - if (!isPathWithinBase(abs, stackDir)) { - throw new Error(`Captured env-file path escapes stack directory for "${stackName}"`); - } - out.push('--env-file', abs); - continue; - } - if (token === '--project-directory') { - const dir = raw[++i]; - if (!dir) { - throw new Error(`Invalid captured --project-directory for stack "${stackName}"`); - } - const abs = path.isAbsolute(dir) ? path.resolve(dir) : path.resolve(stackDir, dir); - if (!isPathWithinBase(abs, stackDir)) { - throw new Error(`Captured project-directory escapes stack directory for "${stackName}"`); - } - out.push('--project-directory', abs); - continue; - } - if (token === '-p' || token === '--project-name') { - const name = raw[++i]; - if (!name || name !== (invocation.projectName || stackName)) { - throw new Error(`Captured project name mismatch for stack "${stackName}"`); - } - out.push('-p', stackName); - continue; - } - throw new Error(`Unsupported captured compose flag "${token}" for stack "${stackName}"`); - } - return out; - } - - async updateStack( - stackName: string, - ws?: WebSocket, - atomic?: boolean, - ): Promise<{ recoveryId: string | null }> { - await this.assertRequiredEnvPresent(stackName); - await this.assertSafePilotBindMapping(stackName); - const stackDir = path.join(this.baseDir, stackName); - const debug = isDebugEnabled(); - const t0 = Date.now(); - if (debug) console.debug('[ComposeService:debug] updateStack', { stackName, stackDir, atomic }); - const sendOutput = (data: string) => { - if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); - }; - - // Dynamic import avoids a static cycle (recovery imports getComposeCommandTimeoutMs). - const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); - const recoverySvc = StackUpdateRecoveryService.getInstance(); - let recoveryId: string | null = null; - let handedOff = false; - - try { - sendOutput('=== Validating stack for update ===\n'); - sendOutput('=== Capturing rollback generation ===\n'); - const candidate = await recoverySvc.captureCandidate({ - nodeId: this.nodeId, - stackName, - createdBy: null, - operationKind: 'update', - }); - recoveryId = candidate.id; - - const buildServices = await loadStackBuildServices(this.nodeId, stackName); - const buildAware = buildServices.length > 0; - - try { - await this.withRegistryAuth(async (env) => { - if (buildAware) { - sendOutput('=== Building images ===\n'); - await this.execute( - 'docker', - await this.authoredComposeArgs(stackName, ['build', '--pull']), - stackDir, ws, true, env, getComposeStallTimeoutMs(), - ); - sendOutput('=== Pulling registry images ===\n'); - await this.execute( - 'docker', - await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']), - stackDir, ws, true, env, getComposeStallTimeoutMs(), - ); - } else { - sendOutput('=== Pulling latest images ===\n'); - await this.execute( - 'docker', - await this.authoredComposeArgs(stackName, ['pull']), - stackDir, ws, true, env, getComposeStallTimeoutMs(), - ); - } - }, sendOutput); - } catch (acquireError) { - // Acquisition failure: abandon candidate; leave runtime untouched. - await recoverySvc.abandon(candidate.id); - recoveryId = null; - throw acquireError; - } - - if (!recoverySvc.markAcquired(candidate.id)) { - await recoverySvc.abandon(candidate.id); - throw new Error('Failed to mark recovery generation as acquired'); - } - - const dockerController = DockerController.getInstance(this.nodeId); - sendOutput('=== Classifying legacy orphans ===\n'); - const classified = await dockerController.classifyLegacyOrphansForUpdate(stackName); - if (classified.status === 'classification_failed') { - await recoverySvc.abandon(candidate.id); - recoveryId = null; - throw new Error(`Legacy orphan classification failed: ${classified.error}`); - } - - if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) { - await recoverySvc.abandon(candidate.id); - recoveryId = null; - throw new Error('Failed to hand off recovery generation'); - } - handedOff = true; - if (!recoverySvc.markReconciling(candidate.id)) { - throw new Error('Failed to mark recovery generation as reconciling after handoff'); - } - - if (classified.status === 'orphans') { - sendOutput(`=== Removing ${classified.ids.length} legacy orphan container(s) ===\n`); - const results = await dockerController.removeContainers(classified.ids); - const failed = results.filter((r) => !r.success); - if (failed.length > 0) { - throw new Error( - `Failed to remove ${failed.length} legacy orphan container(s) after handoff`, - ); - } - } - - await this.withRegistryAuth(async (env) => { - sendOutput('=== Recreating containers ===\n'); - await this.execute( - 'docker', - await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), - stackDir, ws, true, env, getComposeStallTimeoutMs(), - ); - }, sendOutput); - - // Immediate verification probe - await new Promise((resolve) => setTimeout(resolve, 3000)); - - const containers = await dockerController.getDocker().listContainers({ - all: true, - filters: { label: [`com.docker.compose.project=${stackName}`] }, - }); - - for (const containerInfo of containers) { - if (containerInfo.State === 'exited') { - const container = dockerController.getDocker().getContainer(containerInfo.Id); - const inspectData = await container.inspect(); - const exitCode = inspectData.State.ExitCode; - if (exitCode !== 0) { - throw this.createContainerCrashError(exitCode); - } - } - } - - if (!recoverySvc.markImmediateVerified(candidate.id)) { - console.warn( - '[ComposeService] Could not CAS immediate_verified for recovery %s', - sanitizeForLog(candidate.id), - ); - } - - sendOutput('=== Stack updated successfully ===\n'); - - // Defer prune until gate retention / gate link: only prune when no active holds - // would be violated. Still honor prune_on_update, but use unified holds so - // candidate/current rollback images are retained. - try { - const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1'; - if (pruneOnUpdate) { - const isImageHeld = buildUnifiedHeldImagePredicate(this.nodeId); - const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld); - const reclaimed = result.reclaimedBytes > 0 - ? ` · reclaimed ${(result.reclaimedBytes / (1024 * 1024)).toFixed(1)} MB` - : ''; - sendOutput(`=== Pruned dangling images${reclaimed} ===\n`); - } - } catch (pruneError) { - console.warn( - 'Failed to prune dangling images after update for %s:', - sanitizeForLog(stackName), - pruneError, - ); - } - - if (debug) { - console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName }); - } - } catch (updateError) { - if (!handedOff && recoveryId) { - await recoverySvc.abandon(recoveryId); - recoveryId = null; - } - if (handedOff && recoveryId) { - sendOutput('\n=== Update failed - restoring previous runtime from recovery generation ===\n'); - const generationId = recoveryId; - const rolledBack = await compensateOrSwallow(() => - recoverySvc.compensateWithCandidate( - generationId, - (overridePath, invocation) => this.composeUpWithRecoveryOverride( - stackName, - overridePath, - ws, - invocation, - ), - ), - ); - throw new ComposeRollbackError(updateError, true, rolledBack); - } - // Pre-handoff failure: abandon already handled on acquire/classify; runtime untouched. - throw updateError; - } - - await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName); - await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName); - try { - await this.refreshExposureCache(stackName); - } catch (err) { - console.warn( - '[ComposeService] Exposure refresh failed after update for %s:', - sanitizeForLog(stackName), - sanitizeForLog(getErrorMessage(err, 'unknown')), - ); - } - return { recoveryId }; - } - - /** - * Service-scoped update: pull (or `build --pull` for a build-backed service) - * and recreate a single service's replicas in place. Always - * `--no-deps --force-recreate`, never `--remove-orphans`, so sibling services - * keep their container ids and StartedAt. No drift re-baseline and no dangling - * prune here; the orchestrator owns per-service post-update reconciliation. - */ - async updateService(stackName: string, serviceName: string, hasBuild: boolean, ws?: WebSocket): Promise { - await this.assertRequiredEnvPresent(stackName); - await this.assertSafePilotBindMapping(stackName); - const stackDir = path.join(this.baseDir, stackName); - const sendOutput = (data: string) => { - if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); - }; - await this.withRegistryAuth(async (env) => { - if (hasBuild) { - sendOutput(`=== Building ${serviceName} ===\n`); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['build', '--pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); - } else { - sendOutput(`=== Pulling ${serviceName} ===\n`); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); - } - sendOutput(`=== Recreating ${serviceName} ===\n`); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); - }, sendOutput); - } - - /** - * Recreate a single service from the image already present locally, without - * pulling or building. Used by service restore after the recovery image id has - * been retagged onto the declared ref (`--pull never --no-build` so Compose - * uses the just-retagged local image). Always `--no-deps --force-recreate`, - * never `--remove-orphans`. - */ - async recreateServiceFromLocal(stackName: string, serviceName: string, ws?: WebSocket): Promise { - await this.assertRequiredEnvPresent(stackName); - await this.assertSafePilotBindMapping(stackName); - const stackDir = path.join(this.baseDir, stackName); - const sendOutput = (data: string) => { - if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); - }; - await this.withRegistryAuth(async (env) => { - sendOutput(`=== Restoring ${serviceName} ===\n`); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', '--pull', 'never', '--no-build', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); - }, sendOutput); - } - - public async downStack(stackName: string, options?: { removeVolumes?: boolean }): Promise { - const stackPath = path.join(this.baseDir, stackName); - try { - const args = options?.removeVolumes - ? ['down', '--volumes', '--remove-orphans'] - : ['down', '--remove-orphans']; - await this.execute('docker', await this.authoredComposeArgs(stackName, args), stackPath, undefined, false); - } catch (error) { - console.warn(`[Teardown] Docker down failed or nothing to clean up for ${sanitizeForLog(stackName)}`); - } - } - - /** - * Enumerate image references declared in a stack's compose file. - * - * Used by the pre-deploy policy gate to decide which images to scan before - * `docker compose up` runs. Path traversal is guarded against the node's - * compose base directory; missing / unreadable compose files or `.env` - * interpolation failures surface as a rejected Promise so the gate can - * block the deploy rather than silently allow it. - */ - public async listStackImages( - stackName: string, - invocation?: RollbackInvocationRecord | null, - ): Promise { - if (!isValidStackName(stackName)) { - throw new Error('Invalid stack path'); - } - const stackDir = this.resolveValidatedStackDir(stackName); - // Prefer a generation-captured invocation so restored-target policy does not - // scan with the live deploy-spec / project-env selection. - const useCaptured = hasUsableCapturedInvocation(invocation); - const fileAndEnvPrefix = useCaptured - ? [...this.composePrefixFromCapturedInvocation(stackName, invocation)] - : [ - ...authoredComposeFileArgs(stackName, this.nodeId), - ...(await authoredComposeEnvFileArgs(stackName, this.nodeId)), - ]; - if (useCaptured && invocation.meshEnabled) { - await this.appendCapturedMeshLayer(stackName, fileAndEnvPrefix); - } - const stdout = await this.captureCompose([...fileAndEnvPrefix, 'config', '--images'], stackDir); - const seen = new Set(); - const images: string[] = []; - for (const raw of stdout.split(/\r?\n/)) { - const line = raw.trim(); - if (!line) continue; - if (line.startsWith('sha256:')) continue; - if (seen.has(line)) continue; - seen.add(line); - images.push(line); - } - return images; - } - - /** Render the effective Compose model and cache the per-stack exposure - * descriptor so the Security posture can join exposed images against - * vulnerability findings without re-rendering config on every poll. - * Best-effort: render or parse failure logs a warning and keeps the - * prior cached descriptor, never failing the deploy. */ - private async refreshExposureCache(stackName: string): Promise { - const result = await this.renderConfig(stackName); - if (result.rendered === null) { - console.warn('[ComposeService] Exposure cache skipped for %s: model not renderable', - sanitizeForLog(stackName)); - return; - } - let parsed: unknown; - try { - parsed = JSON.parse(result.rendered); - } catch { - console.warn('[ComposeService] Exposure cache skipped for %s: unparseable model JSON', - sanitizeForLog(stackName)); - return; - } - const model = parseEffectiveModel(parsed, stackName); - const descriptor = deriveStackExposure(model, stackName, Date.now()); - DatabaseService.getInstance().upsertStackExposure( - this.nodeId, - stackName, - JSON.stringify(descriptor), - descriptor.computedAt, - ); - } - - private captureCompose(args: string[], cwd: string): Promise { - return new Promise((resolve, reject) => { - const child = spawn('docker', ['compose', ...args], { - cwd, - env: { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }, - }); - let stdout = ''; - let stderr = ''; - child.stdout.on('data', (data: Buffer) => { stdout += data.toString(); }); - child.stderr.on('data', (data: Buffer) => { stderr += data.toString(); }); - child.on('error', (err: NodeJS.ErrnoException) => { - const mapped = describeSpawnError(err, { command: 'docker compose' }); - if (mapped.isLowMemory) { - console.warn('[ComposeService] captureCompose spawn failed under memory pressure:', mapped.message); - } - reject(new Error(mapped.message)); - }); - child.on('close', (code) => { - if (code === 0) resolve(stdout); - else reject(new Error(stderr.trim() || `docker compose ${args.join(' ')} failed with code ${code}`)); - }); - }); - } - - /** - * Render the effective compose model as YAML (the default `docker compose - * config` output) with the exact authored invocation and NO mesh override. - * Used by the Git source detach/export contract: the rendered model becomes - * the stack's single compose.yaml. Throws when the render fails or times - * out, so the detach transaction aborts before anything changes. - */ - public async renderComposeYaml(stackName: string): Promise { - if (!isValidStackName(stackName)) { - throw new Error('Invalid stack path'); - } - const baseResolved = path.resolve(this.baseDir); - const stackDir = path.resolve(baseResolved, stackName); - if (!stackDir.startsWith(baseResolved + path.sep)) { - throw new Error('Invalid stack path'); - } - let filePrefix: string[]; - try { - filePrefix = authoredComposeFileArgs(stackName, this.nodeId); - } catch (err) { - throw err instanceof Error ? err : new Error(String(err)); - } - const envFileArgs = await authoredComposeEnvFileArgs(stackName, this.nodeId); - const child = spawn('docker', ['compose', ...filePrefix, ...envFileArgs, 'config'], { - cwd: stackDir, - env: { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }, - }); - return new Promise((resolve, reject) => { - const MAX_OUTPUT = 5 * 1024 * 1024; // 5 MiB cap on each stream - const TIMEOUT_MS = 30_000; - // Accumulate Buffer chunks and decode ONCE at the end: chunk-wise - // toString() can split a multi-byte UTF-8 sequence across a chunk - // boundary and mangle non-ASCII values. - const outChunks: Buffer[] = []; - let outBytes = 0; - let stderr = ''; - let capped = false; - let settled = false; - const timer = setTimeout(() => { - settled = true; - clearTimeout(timer); - try { - child.kill('SIGKILL'); - } catch { - // best effort - } - reject(new Error(`docker compose config timed out after ${TIMEOUT_MS / 1000}s`)); - }, TIMEOUT_MS); - const finish = (error: Error | null) => { - if (settled) return; - settled = true; - clearTimeout(timer); - if (error) reject(error); - else resolve(Buffer.concat(outChunks).toString('utf8')); - }; - child.stdout.on('data', (data: Buffer) => { - if (capped) return; - outBytes += data.length; - if (outBytes > MAX_OUTPUT) { - // A truncated model frequently still parses as YAML; overwriting a - // working compose.yaml with it would be silent corruption. The cap is - // an error, not a truncation. - capped = true; - settled = true; - clearTimeout(timer); - try { - child.kill('SIGKILL'); - } catch { - // best effort - } - reject(new Error(`docker compose config output exceeded ${MAX_OUTPUT} bytes`)); - return; - } - outChunks.push(data); - }); - child.stderr.on('data', (data: Buffer) => { - if (stderr.length < MAX_OUTPUT) stderr += data.toString(); - }); - child.on('close', (code) => { - if (capped) return; - if (code === 0) finish(null); - else finish(new Error(stderr.trim() || `docker compose config exited with code ${code}`)); - }); - child.on('error', (err) => finish(err)); - }); - } - - /** - * Render the fully-resolved effective Compose model via `docker compose - * config --format json`. This is the AUTHORED model: it does NOT splice in - * the Sencho Mesh override, so it stays read-only (the override is - * write-generated) and reflects what the user actually edits. The override - * would also add the managed `sencho_mesh` external network and per-service - * mesh attachments, which would make preflight emit a false "external network - * not found" finding, so rendering the authored model is both safer and more - * accurate here. - * Captures stderr (where Compose reports unset variables) and never rejects - * on a non-zero exit, so the Compose Doctor can turn a failed render into a - * finding rather than an exception. Bounded by a timeout and an output cap. - * Rejects only when the docker binary cannot be spawned. - */ - public async renderConfig( - stackName: string, - ): Promise<{ rendered: string | null; stderr: string; code: number | null; timedOut: boolean }> { - if (!isValidStackName(stackName)) { - throw new Error('Invalid stack path'); - } - // Canonical inline js/path-injection barrier, kept in the same scope as the - // spawn cwd sink below. CodeQL credits neither the wrapped isPathWithinBase - // helper nor a barrier separated from the sink by the Promise-executor - // closure, so the spawn is hoisted out of the executor. startsWith already - // rejects the base dir itself, since base does not start with base + sep. - const baseResolved = path.resolve(this.baseDir); - const stackDir = path.resolve(baseResolved, stackName); - if (!stackDir.startsWith(baseResolved + path.sep)) { - throw new Error('Invalid stack path'); - } - // Render the authored multi-file model (no mesh override) so the Compose Doctor - // sees every override file; single-file stacks get an empty prefix. The env-file - // flag keeps render resolving the same root .env the validator and deploy use. - let filePrefix: string[]; - try { - filePrefix = authoredComposeFileArgs(stackName, this.nodeId); - } catch (err) { - throw err instanceof Error ? err : new Error(String(err)); - } - const envFileArgs = await authoredComposeEnvFileArgs(stackName, this.nodeId); - const child = spawn('docker', ['compose', ...filePrefix, ...envFileArgs, 'config', '--format', 'json'], { - cwd: stackDir, - env: { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }, - }); - return new Promise((resolve, reject) => { - const MAX_OUTPUT = 5 * 1024 * 1024; // 5 MiB cap on each stream - const TIMEOUT_MS = 20_000; - let stdout = ''; - let stderr = ''; - let timedOut = false; - let capped = false; - let settled = false; - const timer = setTimeout(() => { timedOut = true; child.kill('SIGKILL'); }, TIMEOUT_MS); - const finish = (result: { rendered: string | null; stderr: string; code: number | null; timedOut: boolean }) => { - if (settled) return; - settled = true; - clearTimeout(timer); - resolve(result); - }; - child.stdout.on('data', (data: Buffer) => { - stdout += data.toString(); - if (stdout.length > MAX_OUTPUT && !capped) { capped = true; child.kill('SIGKILL'); } - }); - child.stderr.on('data', (data: Buffer) => { - if (stderr.length < MAX_OUTPUT) stderr += data.toString(); - }); - child.on('error', (err: NodeJS.ErrnoException) => { - if (settled) return; - settled = true; - clearTimeout(timer); - reject(new Error(describeSpawnError(err, { command: 'docker compose' }).message)); - }); - child.on('close', (code) => { - if (timedOut) finish({ rendered: null, stderr: stderr.trim() || 'docker compose config timed out', code, timedOut: true }); - else if (capped) finish({ rendered: null, stderr: 'Rendered model exceeded the size limit', code, timedOut: false }); - else if (code === 0) finish({ rendered: stdout, stderr, code, timedOut: false }); - else finish({ rendered: null, stderr: stderr.trim() || `docker compose config failed with code ${code}`, code, timedOut: false }); - }); - }); - } -} +import { spawn } from 'child_process'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import WebSocket from 'ws'; +import DockerController from './DockerController'; +import { DatabaseService } from './DatabaseService'; +import { FileSystemService } from './FileSystemService'; +import { MeshService } from './MeshService'; +import { LogFormatter } from './LogFormatter'; +import { NodeRegistry } from './NodeRegistry'; +import { RegistryService } from './RegistryService'; +import { DriftLedgerService } from './DriftLedgerService'; +import SelfIdentityService from './SelfIdentityService'; +import { parseEffectiveModel } from './preflight/effectiveModel'; +import { deriveStackExposure } from './preflight/exposure'; + +import { isDebugEnabled } from '../utils/debug'; +import { getErrorMessage } from '../utils/errors'; +import { normalizeContainerName } from '../utils/log-parsing'; +import { describeSpawnError } from '../utils/spawnErrors'; +import { isPathWithinBase, isValidStackName, isValidRelativeStackPath } from '../utils/validation'; +import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs'; +import type { RollbackInvocationRecord } from '../types/rollbackGeneration'; +import { parseMissingRequiredVars } from '../helpers/envVarParse'; +import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog'; +import { pathsMatch, resolveHostBindPath } from '../utils/composePathMapping'; +import { loadStackBuildServices } from './ImageUpdateService'; +import { resolveMissingExternalNetworks } from './network/resolveMissingExternalNetworks'; +import { + MissingExternalNetworksError, + type DeployInvocationContext, +} from './network/missingExternalNetworksError'; +import { buildUnifiedHeldImagePredicate } from './recoveryHeldImages'; +import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; +import type { NotificationCategory } from './NotificationService'; + +/** True when a generation capture recorded an invocation object (prefix may be empty). */ +function hasUsableCapturedInvocation( + invocation: RollbackInvocationRecord | null | undefined, +): invocation is RollbackInvocationRecord { + return invocation != null; +} + +function recordNetworkAutoCreatedActivity( + nodeId: number, + stackName: string, + createdNames: string[], + level: 'info' | 'warning', + ctx?: DeployInvocationContext, +): void { + if (createdNames.length === 0) return; + const names = [...createdNames].sort((a, b) => a.localeCompare(b)).join(', '); + const source = ctx?.source ?? 'manual'; + try { + DatabaseService.getInstance().addNotificationHistory(nodeId, { + level, + category: 'network_auto_created' as NotificationCategory, + message: `Auto-created external network(s) for ${stackName}: ${names} (source: ${source})`, + timestamp: Date.now(), + stack_name: stackName, + actor_username: ctx?.actor ?? null, + }); + } catch (error) { + console.error( + '[ComposeService] Failed to record network_auto_created activity for %s:', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + } +} + +export class ComposeRollbackError extends Error { + public readonly rollbackAttempted: boolean; + public readonly rolledBack: boolean; + public readonly originalError: unknown; + + constructor(originalError: unknown, rollbackAttempted: boolean, rolledBack: boolean) { + super(getErrorMessage(originalError, 'Compose operation failed')); + this.name = 'ComposeRollbackError'; + this.rollbackAttempted = rollbackAttempted; + this.rolledBack = rolledBack; + this.originalError = originalError; + Object.setPrototypeOf(this, ComposeRollbackError.prototype); + } +} + +export function getComposeRollbackInfo(error: unknown): { attempted: boolean; rolledBack: boolean } | null { + if (!(error instanceof ComposeRollbackError)) { + return null; + } + return { attempted: error.rollbackAttempted, rolledBack: error.rolledBack }; +} + +function isNonFatalCompensationError(error: unknown): boolean { + const code = (error as { code?: string }).code; + return code === 'HELD_IMAGE_MISSING' || code === 'RECOVERY_PROBE_FAILED'; +} + +async function compensateOrSwallow(compensate: () => Promise): Promise { + try { + return await compensate(); + } catch (compError) { + if (!isNonFatalCompensationError(compError)) throw compError; + return false; + } +} + +const DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS = 30 * 60 * 1000; + +/** Public so other services (e.g. recovery claim leases) can size their own timers off the same ceiling without depending on a private module-local. */ +export function getComposeCommandTimeoutMs(): number { + const configured = Number(process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS); + if (Number.isFinite(configured) && configured > 0) { + return configured; + } + return DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS; +} + +// Idle backstop for long-running pull/recreate steps: if the child emits no +// output for this window while still running, the step is treated as stalled +// and terminated, so a hung `docker compose pull` surfaces a fast failure +// instead of spinning until the much longer command timeout above. Conservative +// by default because a working pull can be briefly silent while a large layer +// extracts; operators on slow links or heavy local builds can raise it. +const DEFAULT_COMPOSE_STALL_TIMEOUT_MS = 10 * 60 * 1000; + +function getComposeStallTimeoutMs(): number { + const configured = Number(process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS); + if (Number.isFinite(configured) && configured > 0) { + return configured; + } + return DEFAULT_COMPOSE_STALL_TIMEOUT_MS; +} + +/** + * ComposeService - local docker compose CLI execution. + * + * In the Distributed API model, remote node compose operations are handled + * by the remote Sencho instance. This service only executes commands locally. + */ +export class ComposeService { + private baseDir: string; + private nodeId: number; + + constructor(nodeId?: number) { + this.nodeId = nodeId ?? NodeRegistry.getInstance().getDefaultNodeId(); + this.baseDir = NodeRegistry.getInstance().getComposeDir(this.nodeId); + } + + public static getInstance(nodeId?: number): ComposeService { + return new ComposeService(nodeId); + } + + /** + * Build the authored `docker compose` argument list for a stack: the validated + * multi-file deploy prefix (ordered `-f` files + `-p ` + + * `--project-directory`) for a Git source with an applied multi-file spec, then + * the Sencho Mesh override file last (highest `-f` precedence) when the stack is + * opted into the mesh, then the action. Single-file / non-git stacks get no file + * prefix, so docker compose's built-in discovery resolves the root compose.yaml, + * byte-identical to the pre-multi-file behavior. The user's source files are + * never mutated. Lifecycle commands (deploy, update, stop/start/restart/down) + * route through this method, so they share one file prefix plus the mesh override. + * Image scans (listStackImages) and the Compose Doctor (renderConfig) reuse the + * same `authoredComposeFileArgs` prefix directly but intentionally omit the mesh + * override, rendering the user's authored model without mesh injection. + */ + /** Public wrapper for dual-arg assembly and recovery Compose invocations. */ + public async buildAuthoredComposeArgs(stackName: string, action: string[]): Promise { + return this.authoredComposeArgs(stackName, action); + } + + public async validateStackForMutation(stackName: string): Promise { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + } + + /** + * Render/validate the exact Compose invocation used by mutating operations + * (authored files, env pins, and generated Mesh override) before capture. + */ + public async validateExactComposeInvocation(stackName: string): Promise { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack path'); + } + const baseResolved = path.resolve(this.baseDir); + const stackDir = path.resolve(baseResolved, stackName); + if (!stackDir.startsWith(baseResolved + path.sep)) { + throw new Error('Invalid stack path'); + } + const args = await this.authoredComposeArgs(stackName, ['config', '--quiet']); + await this.execute('docker', args, stackDir, undefined, true); + } + + private async authoredComposeArgs(stackName: string, action: string[]): Promise { + const args: string[] = ['compose']; + const filePrefix = authoredComposeFileArgs(stackName, this.nodeId); + args.push(...filePrefix); + // Pin env resolution to the root .env when a context dir shifts the project + // directory, so deploy/update resolve the same effective config the validator did. + args.push(...await authoredComposeEnvFileArgs(stackName, this.nodeId)); + + const meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(this.nodeId, stackName); + let overridePath: string | null = null; + try { + overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName); + } catch (err) { + if (meshEnabled) { + throw err instanceof Error + ? err + : new Error(`Mesh override generation failed: ${String(err)}`); + } + console.warn('[ComposeService] mesh override skipped:', sanitizeForLog((err as Error).message)); + } + if (meshEnabled && !overridePath) { + throw new Error( + `Mesh override is required for stack "${stackName}" but could not be generated`, + ); + } + if (overridePath) { + if (filePrefix.length === 0) { + // Single-file stack: passing any -f disables compose's auto-discovery, so name + // the base file explicitly, then re-add the user's implicit override (if any) so + // it is not silently dropped, before layering the mesh override on top. + const fsSvc = FileSystemService.getInstance(this.nodeId); + const baseFilename = await fsSvc.getComposeFilename(stackName); + args.push('-f', baseFilename); + let userOverride: string | null = null; + try { + userOverride = await fsSvc.getOverrideFilename(stackName); + } catch (err) { + // Containment-guard rejections (bad stack name / symlink escape) are hard errors: + // abort the deploy rather than degrade. The "no override" case returns null rather + // than throwing, so any other throw is transient I/O: drop the override and proceed + // (logging the consequence) instead of failing the deploy. + const code = (err as { code?: string }).code; + if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') { + throw err; + } + console.warn('[ComposeService] could not resolve user compose override; deploying without it:', sanitizeForLog((err as Error).message)); + } + if (userOverride) { + args.push('-f', userOverride); + } + } + args.push('-f', overridePath); + } + args.push(...action); + return args; + } + + private execute( + command: string, + args: string[], + cwd: string, + ws?: WebSocket, + throwOnError = true, + env?: Record, + // When set, terminate the child if it emits no output for this long while + // still running (idle stall backstop). Appended last so the existing + // registry-auth call sites that pass `env` are unaffected. + idleTimeoutMs?: number + ): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd, + env: env ?? { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + } + }); + + let errorLog = ''; + let settled = false; + let exited = false; + let pendingTerminationError: Error | null = null; + const timeoutMs = getComposeCommandTimeoutMs(); + let timeout: ReturnType | null = null; + let forceKillTimeout: ReturnType | null = null; + let idleTimeout: ReturnType | null = null; + + const sendOutput = (text: string) => { + if (ws && ws.readyState === WebSocket.OPEN) { + ws.send(text); + } + }; + + const cleanup = () => { + if (timeout) { + clearTimeout(timeout); + timeout = null; + } + if (forceKillTimeout) { + clearTimeout(forceKillTimeout); + forceKillTimeout = null; + } + if (idleTimeout) { + clearTimeout(idleTimeout); + idleTimeout = null; + } + }; + + const finish = (complete: () => void) => { + if (settled) return; + settled = true; + cleanup(); + complete(); + }; + + const terminateChild = (error: Error) => { + pendingTerminationError = pendingTerminationError ?? error; + if (exited) return; + try { + child.kill('SIGTERM'); + } catch (error) { + console.warn('[ComposeService] Failed to terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + } + forceKillTimeout = setTimeout(() => { + if (exited) return; + try { + child.kill('SIGKILL'); + } catch (error) { + console.warn('[ComposeService] Failed to force terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + } + }, 5000); + }; + + // Idle stall backstop. Armed once below and reset on every output chunk; + // if it ever fires, the step has been silent for idleTimeoutMs while still + // running, so terminate it. Never rearmed after a termination is pending or + // the child has exited, so it cannot re-fire during the SIGTERM grace. + const armIdleTimeout = () => { + if (idleTimeoutMs === undefined) return; + if (exited || settled || pendingTerminationError) return; + if (idleTimeout) clearTimeout(idleTimeout); + idleTimeout = setTimeout(() => { + const seconds = Math.round(idleTimeoutMs / 1000); + sendOutput(`=== No output for ${seconds}s; the operation appears stalled and was stopped ===\n`); + terminateChild(new Error(`STACK_STALLED_OUTPUT: no output for ${seconds}s`)); + }, idleTimeoutMs); + }; + + // The progress socket is output-only: a deploy/update/down is owned by the + // HTTP request that started it, so closing or losing the socket (the user + // minimizes the panel, navigates away, or the connection blips) must not + // terminate the compose process. Termination is driven solely by the + // command timeout here and the optional idle stall backstop above. + timeout = setTimeout(() => { + const message = `Command timed out after ${Math.round(timeoutMs / 1000)}s`; + sendOutput(`${message}\n`); + terminateChild(new Error(message)); + }, timeoutMs); + + armIdleTimeout(); + + const onData = (data: Buffer) => { + const text = data.toString(); + errorLog += text; + sendOutput(text); + armIdleTimeout(); + }; + + child.stdout.on('data', onData); + child.stderr.on('data', onData); + + child.on('close', (code: number | null) => { + exited = true; + finish(() => { + sendOutput(`Command exited with code ${code}\n`); + if (pendingTerminationError) { + if (throwOnError) reject(pendingTerminationError); + else resolve(); + return; + } + if (code === 0) resolve(); + else if (throwOnError) reject(new Error(redactSensitiveText(errorLog.trim()) || `Command failed with code ${code}`)); + else resolve(); + }); + }); + + child.on('error', (error: Error & { code?: string }) => { + exited = true; + finish(() => { + const mapped = describeSpawnError(error as NodeJS.ErrnoException, { command }); + const message = redactSensitiveText(mapped.message); + sendOutput(`Error: ${message}\n`); + if (mapped.isLowMemory) { + console.warn('[ComposeService] spawn failed under memory pressure:', message); + } + if (pendingTerminationError) { + if (throwOnError) reject(pendingTerminationError); + else resolve(); + return; + } + if (throwOnError) reject(new Error(message)); + else resolve(); + }); + }); + }); + } + + private async withRegistryAuth( + fn: (env: Record) => Promise, + sendOutput?: (data: string) => void, + ): Promise { + const registries = DatabaseService.getInstance().getRegistries(); + if (registries.length === 0) { + return fn({ + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + }); + } + + const { config, warnings } = await RegistryService.getInstance().resolveDockerConfig(); + if (warnings.length > 0 && sendOutput) { + for (const warning of warnings) { + sendOutput(`[Sencho] Warning: ${warning}\n`); + } + } + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-docker-')); + const configPath = path.join(tmpDir, 'config.json'); + + try { + fs.writeFileSync(configPath, JSON.stringify(config), { mode: 0o600 }); + return await fn({ + ...process.env, + DOCKER_CONFIG: tmpDir, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + }); + } finally { + // Best-effort cleanup; each step runs independently so a file that was never + // written (e.g., writeFileSync threw) does not prevent the directory removal. + try { fs.unlinkSync(configPath); } catch { /* file may not exist */ } + try { fs.rmdirSync(tmpDir); } catch (e) { + console.warn('[ComposeService] Could not remove temp Docker config dir:', (e as Error).message); + } + } + } + + private async createAtomicBackup( + stackName: string, + operation: 'deployment' | 'update', + sendOutput: (data: string) => void, + ): Promise { + try { + const fsSvc = FileSystemService.getInstance(this.nodeId); + await fsSvc.backupStackFiles(stackName); + sendOutput(`=== Backup created for atomic ${operation} ===\n`); + } catch (error) { + console.error('Atomic backup failed for %s:', sanitizeForLog(stackName), getErrorMessage(error, 'unknown error')); + sendOutput(`=== Atomic ${operation} backup failed. Operation aborted ===\n`); + throw new Error(`Atomic ${operation} backup failed: ${getErrorMessage(error, 'unknown error')}`); + } + } + + private async restoreAtomicBackup( + stackName: string, + stackDir: string, + ws: WebSocket | undefined, + sendOutput: (data: string) => void, + ): Promise { + try { + const fsSvc = FileSystemService.getInstance(this.nodeId); + await fsSvc.restoreStackFiles(stackName); + await this.withRegistryAuth(async (env) => { + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env); + }, sendOutput); + sendOutput('=== Restored previous compose and env files ===\n'); + return true; + } catch (rollbackError) { + console.error('Rollback failed for %s:', sanitizeForLog(stackName), getErrorMessage(rollbackError, 'unknown error')); + sendOutput('=== Rollback failed. Manual intervention may be required ===\n'); + return false; + } + } + + private createContainerCrashError(exitCode: number): Error { + return new Error( + `CONTAINER_CRASHED\nExit Code: ${exitCode}\nContainer exited after deployment. Check container logs for details.` + ); + } + + async runCommand(stackName: string, action: 'down' | 'start' | 'stop' | 'restart', ws?: WebSocket): Promise { + const stackDir = path.join(this.baseDir, stackName); + await this.execute('docker', await this.authoredComposeArgs(stackName, [action]), stackDir, ws); + } + + /** Interactive compose down (Take down UI / POST /down). Plain `down` by default. */ + async runDown(stackName: string, options?: { removeVolumes?: boolean }, ws?: WebSocket): Promise { + const stackDir = path.join(this.baseDir, stackName); + const args = options?.removeVolumes ? ['down', '--volumes'] : ['down']; + await this.execute('docker', await this.authoredComposeArgs(stackName, args), stackDir, ws); + } + + /** + * Opt-in guard: when `env_block_deploy_on_missing_required` is enabled, refuse a + * deploy whose required `${VAR:?err}` variables are unset OR empty, before any + * backup, cleanup, pull, or `up` runs. Compose's own resolution is authoritative + * (it passes process.env), and on the failing path it emits no rendered model, so + * no env value is materialized. Default off and any settings-read failure both + * fall through without blocking. + */ + private async assertRequiredEnvPresent(stackName: string): Promise { + let enabled = false; + try { + enabled = DatabaseService.getInstance().getGlobalSettings()['env_block_deploy_on_missing_required'] === '1'; + } catch { + return; // safe default: a settings-read failure never blocks a deploy + } + if (!enabled) return; + const result = await this.renderConfig(stackName); + const missing = parseMissingRequiredVars(result.stderr); + if (missing.length === 0) return; + const plural = missing.length > 1; + throw new Error( + `Deploy blocked: required environment variable${plural ? 's' : ''} ${missing.join(', ')} ` + + `${plural ? 'are' : 'is'} missing. Define ${plural ? 'them' : 'it'} in a .env or env_file, then deploy again.`, + ); + } + + private async assertSafePilotBindMapping(stackName: string): Promise { + if (process.env.SENCHO_MODE !== 'pilot') return; + + let mounts: Array<{ source: string; destination: string }> | null; + try { + mounts = await SelfIdentityService.getInstance().getBindMounts(); + } catch (error) { + console.warn('[ComposeService] Could not verify pilot compose path mapping:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + return; + } + if (mounts === null) return; + + const composeDir = path.resolve(this.baseDir); + const hostComposeDir = resolveHostBindPath(composeDir, mounts); + if (!hostComposeDir || pathsMatch(hostComposeDir, composeDir)) return; + + const rendered = await this.renderConfig(stackName); + if (rendered.rendered === null) return; + + let parsed: unknown; + try { + parsed = JSON.parse(rendered.rendered); + } catch (error) { + console.warn('[ComposeService] Could not inspect rendered binds for pilot path safety:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + return; + } + const model = parseEffectiveModel(parsed, stackName); + const unsafeBind = model.services + .flatMap((service) => service.binds) + .find((bind) => isPathWithinBase(path.resolve(bind.source), composeDir)); + if (!unsafeBind) return; + + throw new Error( + `Deploy blocked: relative bind mounts resolve under ${composeDir}, but the host path is ${hostComposeDir}. ` + + `Use a 1:1 mount with the same absolute path on the host and in the Pilot Agent, then retry.`, + ); + } + + /** + * Missing-external gate: after env/Pilot asserts, before atomic backup. + * Creates safe bridge networks only when the opt-in setting is on. + */ + private async ensureExternalNetworksForDeploy( + stackName: string, + ctx?: DeployInvocationContext, + ): Promise { + const resolved = await resolveMissingExternalNetworks(this.nodeId, stackName); + if (resolved.status === 'render_unavailable') { + throw new MissingExternalNetworksError({ + kind: 'unavailable', + message: 'Sencho could not render this stack\'s Compose model to check external networks.', + }); + } + if (resolved.status === 'runtime_unavailable') { + // No declared externals: nothing to verify; proceed. + if (resolved.declaredExternalCount === 0) return; + throw new MissingExternalNetworksError({ + kind: 'unavailable', + message: 'Sencho could not read Docker networking state to check external networks.', + }); + } + + if (resolved.networks.length === 0) return; + + const unsafe = resolved.networks.filter((n) => !n.safe); + if (unsafe.length > 0) { + throw new MissingExternalNetworksError({ + kind: 'unsupported', + message: 'One or more missing external networks cannot be created safely by Sencho.', + networks: resolved.networks, + }); + } + + if (!resolved.autoCreateEnabled) { + throw new MissingExternalNetworksError({ + kind: 'prompt', + message: 'One or more external networks required by this stack are missing on this node.', + networks: resolved.networks, + }); + } + + const docker = DockerController.getInstance(this.nodeId); + const createdNames: string[] = []; + const recordCreatedNetworks = (level: 'info' | 'warning') => { + if (createdNames.length === 0) return; + invalidateNodeCaches(this.nodeId); + recordNetworkAutoCreatedActivity(this.nodeId, stackName, createdNames, level, ctx); + }; + + for (const network of resolved.networks) { + try { + await docker.createNetwork({ Name: network.name, Driver: 'bridge' }); + createdNames.push(network.name); + } catch (createErr) { + // Authoritative re-check: continue only if the network now exists. + let exists = false; + try { + const knownStacks = await FileSystemService.getInstance(this.nodeId).getStacks(); + const snapshot = await docker.getDependencySnapshot(knownStacks); + exists = snapshot.networks.some((n) => n.name === network.name); + } catch (snapErr) { + console.warn( + '[ComposeService] Post-create snapshot failed for %s:', + sanitizeForLog(network.name), + sanitizeForLog(getErrorMessage(snapErr, 'unknown')), + ); + } + if (!exists) { + recordCreatedNetworks('warning'); + throw new MissingExternalNetworksError({ + kind: 'create_failed', + message: `Failed to create external network "${network.name}".`, + networks: resolved.networks, + createdNames, + remainingNames: resolved.networks + .map((missingNetwork) => missingNetwork.name) + .filter((name) => !createdNames.includes(name)), + }); + } + // Race-existing: do not record in createdNames. + } + } + + // Re-resolve before Compose. + const recheck = await resolveMissingExternalNetworks(this.nodeId, stackName); + if (recheck.status !== 'ok' || recheck.networks.length > 0) { + recordCreatedNetworks('warning'); + throw new MissingExternalNetworksError({ + kind: recheck.status === 'ok' ? 'create_failed' : 'unavailable', + message: 'External networks were still missing after automatic creation.', + networks: recheck.networks, + createdNames, + remainingNames: recheck.networks.map((n) => n.name), + }); + } + + recordCreatedNetworks('info'); + } + + async deployStack( + stackName: string, + ws?: WebSocket, + atomic?: boolean, + ctx?: DeployInvocationContext, + ): Promise<{ recoveryId: string | null }> { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + await this.ensureExternalNetworksForDeploy(stackName, ctx); + + const stackDir = path.join(this.baseDir, stackName); + const debug = isDebugEnabled(); + const t0 = Date.now(); + if (debug) console.debug('[ComposeService:debug] deployStack', { stackName, stackDir, atomic }); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + const recoverySvc = atomic ? StackUpdateRecoveryService.getInstance() : null; + let recoveryId: string | null = null; + let handedOff = false; + + if (atomic && recoverySvc) { + sendOutput('=== Capturing rollback generation for atomic deploy ===\n'); + const candidate = await recoverySvc.captureCandidate({ + nodeId: this.nodeId, + stackName, + createdBy: 'atomic-deploy', + operationKind: 'deployment', + }); + recoveryId = candidate.id; + if (!recoverySvc.markAcquired(candidate.id)) { + await recoverySvc.abandon(candidate.id); + throw new Error('Failed to mark recovery generation as acquired'); + } + if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) { + await recoverySvc.abandon(candidate.id); + throw new Error('Failed to hand off recovery generation'); + } + handedOff = true; + if (!recoverySvc.markReconciling(candidate.id)) { + throw new Error('Failed to mark recovery generation as reconciling after handoff'); + } + } + + try { + try { + const dockerController = DockerController.getInstance(this.nodeId); + const legacyOrphans = await dockerController.getLegacyOrphanContainersByStack(stackName); + if (legacyOrphans.length > 0) { + sendOutput(`=== Cleaning up legacy orphan containers before deployment ===\n`); + await dockerController.removeContainers(legacyOrphans.map((c) => c.Id)); + } + } catch (e) { + console.warn('Failed to clean up legacy containers for %s:', sanitizeForLog(stackName), e); + } + + await this.withRegistryAuth(async (env) => { + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs()); + }, sendOutput); + + // Post-Deploy Health Probe + await new Promise(resolve => setTimeout(resolve, 3000)); + + const dockerController = DockerController.getInstance(this.nodeId); + const containers = await dockerController.getDocker().listContainers({ + all: true, + filters: { label: [`com.docker.compose.project=${stackName}`] } + }); + + for (const containerInfo of containers) { + if (containerInfo.State === 'exited') { + const container = dockerController.getDocker().getContainer(containerInfo.Id); + const inspectData = await container.inspect(); + const exitCode = inspectData.State.ExitCode; + + if (exitCode !== 0) { + throw this.createContainerCrashError(exitCode); + } + } + } + + if (atomic && recoverySvc && recoveryId) { + if (!recoverySvc.markImmediateVerified(recoveryId)) { + console.warn( + '[ComposeService] Could not CAS immediate_verified for recovery %s', + sanitizeForLog(recoveryId), + ); + } + } + if (debug) console.debug(`[ComposeService:debug] deployStack completed in ${Date.now() - t0}ms`, { stackName }); + } catch (deployError) { + if (atomic && recoverySvc && handedOff && recoveryId) { + sendOutput('\n=== Deployment failed - restoring previous runtime from recovery generation ===\n'); + const generationId = recoveryId; + const rolledBack = await compensateOrSwallow(() => + recoverySvc.compensateWithCandidate( + generationId, + (overridePath, invocation) => this.composeUpWithRecoveryOverride( + stackName, + overridePath, + ws, + invocation, + ), + ), + ); + throw new ComposeRollbackError(deployError, true, rolledBack); + } + if (atomic && recoverySvc && recoveryId && !handedOff) { + await recoverySvc.abandon(recoveryId); + } + throw deployError; + } + // Reached only on a successful deploy (the catch above always rethrows). Record + // the drift baseline here so every deploy path gets one, not just the manual + // route: bulk, Git-source, App Store, scheduler, and webhook deploys all funnel + // through this method. Internally guarded; awaited so it cannot race later work. + await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName); + // Reconcile the ledger against the just-deployed runtime: findings this deploy + // fixed are resolved and any it left are recorded (and surfaced in the activity + // feed) now, instead of waiting for someone to open the Drift tab. The rollback + // route re-deploys through this method, so it is covered; a failed atomic deploy + // instead restores the previous files and throws above, so that recovery path + // reconciles on its next deploy or scan, not here. Best-effort internally. + await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName); + // Refresh the exposure cache so posture reflects the just-deployed model. + // Best-effort: a refresh failure logs a warning but never fails the deploy. + try { + await this.refreshExposureCache(stackName); + } catch (err) { + console.warn('[ComposeService] Exposure refresh failed after deploy for %s:', + sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown'))); + } + return { recoveryId }; + } + + streamLogs(stackName: string, ws: WebSocket) { + let isClosed = false; + let isFirstRun = true; + let isWaitingForActivity = false; + + ws.on('close', () => { isClosed = true; }); + + const startStream = async () => { + if (isClosed || ws.readyState !== WebSocket.OPEN) return; + + // Canonical js/path-injection barrier at the fs.stat sink. + const baseResolved = path.resolve(this.baseDir); + const stackDir = path.resolve(baseResolved, stackName); + let stackDirGone = !stackDir.startsWith(baseResolved + path.sep); + if (!stackDirGone) { + try { + const st = await fs.promises.stat(stackDir); + stackDirGone = !st.isDirectory(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + stackDirGone = true; + } + // Other stat failures: keep the existing compose-ps path. + } + } + if (stackDirGone) { + if (ws.readyState === WebSocket.OPEN) { + ws.send(`\r\n\x1b[33m[Sencho] Stack directory is gone. Log stream idle.\x1b[0m\r\n`); + } + return; + } + + try { + const dockerController = DockerController.getInstance(this.nodeId); + const containers = await dockerController.getContainersByStack(stackName); + + if (!containers || containers.length === 0) { + if (!isWaitingForActivity) { + ws.send(`\r\n\x1b[33m[Sencho] No containers found. Waiting for activity...\x1b[0m\r\n`); + isWaitingForActivity = true; + } + setTimeout(startStream, 2000); + return; + } + + const runningContainers = containers.filter((c: any) => c.State === 'running'); + + if (!isFirstRun && runningContainers.length === 0) { + if (!isWaitingForActivity) { + ws.send(`\r\n\x1b[33m[Sencho] Log stream ended. Waiting for container activity...\x1b[0m\r\n`); + isWaitingForActivity = true; + } + setTimeout(startStream, 2000); + return; + } + + const containersToLog = isFirstRun ? containers : runningContainers; + isFirstRun = false; + isWaitingForActivity = false; + + let activeProcesses = 0; + let streamEndedHandled = false; + const localProcesses: ReturnType[] = []; + + const onWsClose = () => { + localProcesses.forEach(cp => { try { cp.kill(); } catch { } }); + }; + + ws.on('close', onWsClose); + + const handleProcessEnd = () => { + activeProcesses--; + if (activeProcesses <= 0 && !streamEndedHandled) { + streamEndedHandled = true; + ws.removeListener('close', onWsClose); + if (!isClosed && ws.readyState === WebSocket.OPEN) { + setTimeout(startStream, 1000); + } + } + }; + + for (const container of containersToLog) { + const rawName = container.Names?.[0]?.replace(/^\//, '') || container.Id; + const displayName = normalizeContainerName(rawName, stackName); + activeProcesses++; + let lineBuffer = ''; + + const sendOutput = (data: Buffer) => { + if (ws.readyState === WebSocket.OPEN) { + lineBuffer += data.toString(); + const lines = lineBuffer.split(/\r?\n/); + lineBuffer = lines.pop() || ''; + for (const line of lines) { + ws.send(LogFormatter.process(`${displayName} | ${line}`) + '\r\n'); + } + } + }; + + const flushBuffer = () => { + if (lineBuffer && ws.readyState === WebSocket.OPEN) { + ws.send(LogFormatter.process(`${displayName} | ${lineBuffer}`) + '\r\n'); + lineBuffer = ''; + } + }; + + const child = spawn('docker', ['logs', '-f', '-t', '--tail', '100', rawName], { + env: { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + } + }); + localProcesses.push(child); + child.stdout.on('data', sendOutput); + child.stderr.on('data', sendOutput); + child.on('error', handleProcessEnd); + child.on('close', () => { + flushBuffer(); + handleProcessEnd(); + }); + } + } catch (err) { + if (!isClosed && ws.readyState === WebSocket.OPEN) { + if (!isWaitingForActivity) { + ws.send(`\r\n\x1b[31m[Sencho] Error tracking containers. Retrying...\x1b[0m\r\n`); + isWaitingForActivity = true; + } + setTimeout(startStream, 2000); + } + } + }; + + startStream(); + } + + + /** + * Pinned compose-up with a recovery override layered last + * (`--pull never --no-build`). Used by manual rollback and deploy/update + * compensation. When `invocation` is set, Compose args come from the + * generation capture instead of the live database-derived invocation. + */ + async composeUpWithRecoveryOverride( + stackName: string, + overridePath: string, + ws?: WebSocket, + invocation?: RollbackInvocationRecord | null, + ): Promise { + const stackDir = path.join(this.baseDir, stackName); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + await this.withRegistryAuth(async (env) => { + await this.execute( + 'docker', + await this.buildComposeArgsWithRecoveryOverride( + stackName, + ['up', '-d', '--remove-orphans', '--pull', 'never', '--no-build'], + overridePath, + invocation ?? null, + ), + stackDir, + ws, + true, + env, + getComposeStallTimeoutMs(), + ); + }, sendOutput); + } + + /** + * Authored (+ mesh) compose args with an optional recovery override layered LAST. + * When `invocation` is provided, use its validated captured prefix so recovery + * does not mix restored files with the live deploy-spec / project-env selection. + */ + public async buildComposeArgsWithRecoveryOverride( + stackName: string, + action: string[], + recoveryOverridePath: string | null, + invocation?: RollbackInvocationRecord | null, + ): Promise { + const useCaptured = hasUsableCapturedInvocation(invocation); + const out = useCaptured + ? ['compose', ...this.composePrefixFromCapturedInvocation(stackName, invocation)] + : await this.authoredComposeArgsPrefix(stackName); + + if (useCaptured && invocation.meshEnabled) { + await this.appendCapturedMeshLayer(stackName, out); + } + + if (recoveryOverridePath) { + // Captured invocations already encode their file set; live args may need + // an explicit base (+ user override) so docker compose accepts a trailing -f. + await this.ensureExplicitComposeFiles(stackName, out, !useCaptured); + out.push('-f', recoveryOverridePath); + } + out.push(...action); + return out; + } + + /** + * Re-apply the Mesh override layer when the generation captured Mesh as enabled. + * Uses live ensureStackOverride (same absolute override path semantics as deploy). + */ + private async appendCapturedMeshLayer(stackName: string, args: string[]): Promise { + const overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName); + if (!overridePath) { + throw new Error( + `Captured Mesh-enabled invocation cannot regenerate mesh override for stack "${stackName}"`, + ); + } + await this.ensureExplicitComposeFiles(stackName, args, true); + args.push('-f', overridePath); + } + + /** Slice the global-flag prefix from authoredComposeArgs (no action tokens). */ + private async authoredComposeArgsPrefix(stackName: string): Promise { + const withSentinel = await this.authoredComposeArgs(stackName, ['__SENCHO_ACTION_SENTINEL__']); + const idx = withSentinel.indexOf('__SENCHO_ACTION_SENTINEL__'); + const prefix = idx >= 0 ? withSentinel.slice(0, idx) : withSentinel; + return [...prefix]; + } + + /** + * When args lack `-f`, pin the stack compose file (and optionally the user + * override) so a trailing override layer is accepted by docker compose. + */ + private async ensureExplicitComposeFiles( + stackName: string, + args: string[], + includeUserOverride: boolean, + ): Promise { + if (args.includes('-f')) return; + const fsSvc = FileSystemService.getInstance(this.nodeId); + args.push('-f', await fsSvc.getComposeFilename(stackName)); + if (!includeUserOverride) return; + try { + const userOverride = await fsSvc.getOverrideFilename(stackName); + if (userOverride) args.push('-f', userOverride); + } catch (err) { + const code = (err as { code?: string }).code; + if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') { + throw err; + } + console.warn( + '[ComposeService] could not resolve user compose override while pinning compose files:', + sanitizeForLog((err as Error).message), + ); + } + } + + private resolveValidatedStackDir(stackName: string): string { + const stackDir = path.resolve(this.baseDir, stackName); + if (!isPathWithinBase(stackDir, this.baseDir) || path.resolve(this.baseDir) === stackDir) { + throw new Error('Invalid stack path'); + } + return stackDir; + } + + /** + * Rebuild a spawn-safe compose global-flag prefix from a generation's + * captured invocation. Relative -f / --project-directory paths and absolute + * --env-file paths must stay inside the stack directory. + */ + private composePrefixFromCapturedInvocation( + stackName: string, + invocation: RollbackInvocationRecord, + ): string[] { + const stackDir = this.resolveValidatedStackDir(stackName); + // Empty prefix is valid (single-file auto-discovery at capture time). + const raw = [...invocation.composeArgsPrefix]; + const out: string[] = []; + for (let i = 0; i < raw.length; i++) { + const token = raw[i]; + if (token === '-f' || token === '--file') { + const file = raw[++i]; + if (!file || !isValidRelativeStackPath(file)) { + throw new Error(`Invalid captured compose file path for stack "${stackName}"`); + } + if (!isPathWithinBase(path.resolve(stackDir, file), stackDir)) { + throw new Error(`Captured compose file path escapes stack directory for "${stackName}"`); + } + out.push('-f', file); + continue; + } + if (token === '--env-file') { + const envPath = raw[++i]; + if (!envPath) { + throw new Error(`Invalid captured --env-file for stack "${stackName}"`); + } + const abs = path.resolve(envPath); + if (!isPathWithinBase(abs, stackDir)) { + throw new Error(`Captured env-file path escapes stack directory for "${stackName}"`); + } + out.push('--env-file', abs); + continue; + } + if (token === '--project-directory') { + const dir = raw[++i]; + if (!dir) { + throw new Error(`Invalid captured --project-directory for stack "${stackName}"`); + } + const abs = path.isAbsolute(dir) ? path.resolve(dir) : path.resolve(stackDir, dir); + if (!isPathWithinBase(abs, stackDir)) { + throw new Error(`Captured project-directory escapes stack directory for "${stackName}"`); + } + out.push('--project-directory', abs); + continue; + } + if (token === '-p' || token === '--project-name') { + const name = raw[++i]; + if (!name || name !== (invocation.projectName || stackName)) { + throw new Error(`Captured project name mismatch for stack "${stackName}"`); + } + out.push('-p', stackName); + continue; + } + throw new Error(`Unsupported captured compose flag "${token}" for stack "${stackName}"`); + } + return out; + } + + async updateStack( + stackName: string, + ws?: WebSocket, + atomic?: boolean, + ): Promise<{ recoveryId: string | null }> { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + const stackDir = path.join(this.baseDir, stackName); + const debug = isDebugEnabled(); + const t0 = Date.now(); + if (debug) console.debug('[ComposeService:debug] updateStack', { stackName, stackDir, atomic }); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + + // Dynamic import avoids a static cycle (recovery imports getComposeCommandTimeoutMs). + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + const recoverySvc = StackUpdateRecoveryService.getInstance(); + let recoveryId: string | null = null; + let handedOff = false; + + try { + sendOutput('=== Validating stack for update ===\n'); + sendOutput('=== Capturing rollback generation ===\n'); + const candidate = await recoverySvc.captureCandidate({ + nodeId: this.nodeId, + stackName, + createdBy: null, + operationKind: 'update', + }); + recoveryId = candidate.id; + + const buildServices = await loadStackBuildServices(this.nodeId, stackName); + const buildAware = buildServices.length > 0; + + try { + await this.withRegistryAuth(async (env) => { + if (buildAware) { + sendOutput('=== Building images ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['build', '--pull']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + sendOutput('=== Pulling registry images ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + } else { + sendOutput('=== Pulling latest images ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['pull']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + } + }, sendOutput); + } catch (acquireError) { + // Acquisition failure: abandon candidate; leave runtime untouched. + await recoverySvc.abandon(candidate.id); + recoveryId = null; + throw acquireError; + } + + if (!recoverySvc.markAcquired(candidate.id)) { + await recoverySvc.abandon(candidate.id); + throw new Error('Failed to mark recovery generation as acquired'); + } + + const dockerController = DockerController.getInstance(this.nodeId); + sendOutput('=== Classifying legacy orphans ===\n'); + const classified = await dockerController.classifyLegacyOrphansForUpdate(stackName); + if (classified.status === 'classification_failed') { + await recoverySvc.abandon(candidate.id); + recoveryId = null; + throw new Error(`Legacy orphan classification failed: ${classified.error}`); + } + + if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) { + await recoverySvc.abandon(candidate.id); + recoveryId = null; + throw new Error('Failed to hand off recovery generation'); + } + handedOff = true; + if (!recoverySvc.markReconciling(candidate.id)) { + throw new Error('Failed to mark recovery generation as reconciling after handoff'); + } + + if (classified.status === 'orphans') { + sendOutput(`=== Removing ${classified.ids.length} legacy orphan container(s) ===\n`); + const results = await dockerController.removeContainers(classified.ids); + const failed = results.filter((r) => !r.success); + if (failed.length > 0) { + throw new Error( + `Failed to remove ${failed.length} legacy orphan container(s) after handoff`, + ); + } + } + + await this.withRegistryAuth(async (env) => { + sendOutput('=== Recreating containers ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + }, sendOutput); + + // Immediate verification probe + await new Promise((resolve) => setTimeout(resolve, 3000)); + + const containers = await dockerController.getDocker().listContainers({ + all: true, + filters: { label: [`com.docker.compose.project=${stackName}`] }, + }); + + for (const containerInfo of containers) { + if (containerInfo.State === 'exited') { + const container = dockerController.getDocker().getContainer(containerInfo.Id); + const inspectData = await container.inspect(); + const exitCode = inspectData.State.ExitCode; + if (exitCode !== 0) { + throw this.createContainerCrashError(exitCode); + } + } + } + + if (!recoverySvc.markImmediateVerified(candidate.id)) { + console.warn( + '[ComposeService] Could not CAS immediate_verified for recovery %s', + sanitizeForLog(candidate.id), + ); + } + + sendOutput('=== Stack updated successfully ===\n'); + + // Defer prune until gate retention / gate link: only prune when no active holds + // would be violated. Still honor prune_on_update, but use unified holds so + // candidate/current rollback images are retained. + try { + const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1'; + if (pruneOnUpdate) { + const isImageHeld = buildUnifiedHeldImagePredicate(this.nodeId); + const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld); + const reclaimed = result.reclaimedBytes > 0 + ? ` · reclaimed ${(result.reclaimedBytes / (1024 * 1024)).toFixed(1)} MB` + : ''; + sendOutput(`=== Pruned dangling images${reclaimed} ===\n`); + } + } catch (pruneError) { + console.warn( + 'Failed to prune dangling images after update for %s:', + sanitizeForLog(stackName), + pruneError, + ); + } + + if (debug) { + console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName }); + } + } catch (updateError) { + if (!handedOff && recoveryId) { + await recoverySvc.abandon(recoveryId); + recoveryId = null; + } + if (handedOff && recoveryId) { + sendOutput('\n=== Update failed - restoring previous runtime from recovery generation ===\n'); + const generationId = recoveryId; + const rolledBack = await compensateOrSwallow(() => + recoverySvc.compensateWithCandidate( + generationId, + (overridePath, invocation) => this.composeUpWithRecoveryOverride( + stackName, + overridePath, + ws, + invocation, + ), + ), + ); + throw new ComposeRollbackError(updateError, true, rolledBack); + } + // Pre-handoff failure: abandon already handled on acquire/classify; runtime untouched. + throw updateError; + } + + await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName); + await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName); + try { + await this.refreshExposureCache(stackName); + } catch (err) { + console.warn( + '[ComposeService] Exposure refresh failed after update for %s:', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(err, 'unknown')), + ); + } + return { recoveryId }; + } + + /** + * Service-scoped update: pull (or `build --pull` for a build-backed service) + * and recreate a single service's replicas in place. Always + * `--no-deps --force-recreate`, never `--remove-orphans`, so sibling services + * keep their container ids and StartedAt. No drift re-baseline and no dangling + * prune here; the orchestrator owns per-service post-update reconciliation. + */ + async updateService(stackName: string, serviceName: string, hasBuild: boolean, ws?: WebSocket): Promise { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + const stackDir = path.join(this.baseDir, stackName); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + await this.withRegistryAuth(async (env) => { + if (hasBuild) { + sendOutput(`=== Building ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['build', '--pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + } else { + sendOutput(`=== Pulling ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + } + sendOutput(`=== Recreating ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + }, sendOutput); + } + + /** + * Recreate a single service from the image already present locally, without + * pulling or building. Used by service restore after the recovery image id has + * been retagged onto the declared ref (`--pull never --no-build` so Compose + * uses the just-retagged local image). Always `--no-deps --force-recreate`, + * never `--remove-orphans`. + */ + async recreateServiceFromLocal(stackName: string, serviceName: string, ws?: WebSocket): Promise { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + const stackDir = path.join(this.baseDir, stackName); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + await this.withRegistryAuth(async (env) => { + sendOutput(`=== Restoring ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', '--pull', 'never', '--no-build', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + }, sendOutput); + } + + public async downStack(stackName: string, options?: { removeVolumes?: boolean }): Promise { + const stackPath = path.join(this.baseDir, stackName); + try { + const args = options?.removeVolumes + ? ['down', '--volumes', '--remove-orphans'] + : ['down', '--remove-orphans']; + await this.execute('docker', await this.authoredComposeArgs(stackName, args), stackPath, undefined, false); + } catch (error) { + console.warn(`[Teardown] Docker down failed or nothing to clean up for ${sanitizeForLog(stackName)}`); + } + } + + /** + * Enumerate image references declared in a stack's compose file. + * + * Used by the pre-deploy policy gate to decide which images to scan before + * `docker compose up` runs. Path traversal is guarded against the node's + * compose base directory; missing / unreadable compose files or `.env` + * interpolation failures surface as a rejected Promise so the gate can + * block the deploy rather than silently allow it. + */ + public async listStackImages( + stackName: string, + invocation?: RollbackInvocationRecord | null, + ): Promise { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack path'); + } + const stackDir = this.resolveValidatedStackDir(stackName); + // Prefer a generation-captured invocation so restored-target policy does not + // scan with the live deploy-spec / project-env selection. + const useCaptured = hasUsableCapturedInvocation(invocation); + const fileAndEnvPrefix = useCaptured + ? [...this.composePrefixFromCapturedInvocation(stackName, invocation)] + : [ + ...authoredComposeFileArgs(stackName, this.nodeId), + ...(await authoredComposeEnvFileArgs(stackName, this.nodeId)), + ]; + if (useCaptured && invocation.meshEnabled) { + await this.appendCapturedMeshLayer(stackName, fileAndEnvPrefix); + } + const stdout = await this.captureCompose([...fileAndEnvPrefix, 'config', '--images'], stackDir); + const seen = new Set(); + const images: string[] = []; + for (const raw of stdout.split(/\r?\n/)) { + const line = raw.trim(); + if (!line) continue; + if (line.startsWith('sha256:')) continue; + if (seen.has(line)) continue; + seen.add(line); + images.push(line); + } + return images; + } + + /** Render the effective Compose model and cache the per-stack exposure + * descriptor so the Security posture can join exposed images against + * vulnerability findings without re-rendering config on every poll. + * Best-effort: render or parse failure logs a warning and keeps the + * prior cached descriptor, never failing the deploy. */ + private async refreshExposureCache(stackName: string): Promise { + const result = await this.renderConfig(stackName); + if (result.rendered === null) { + console.warn('[ComposeService] Exposure cache skipped for %s: model not renderable', + sanitizeForLog(stackName)); + return; + } + let parsed: unknown; + try { + parsed = JSON.parse(result.rendered); + } catch { + console.warn('[ComposeService] Exposure cache skipped for %s: unparseable model JSON', + sanitizeForLog(stackName)); + return; + } + const model = parseEffectiveModel(parsed, stackName); + const descriptor = deriveStackExposure(model, stackName, Date.now()); + DatabaseService.getInstance().upsertStackExposure( + this.nodeId, + stackName, + JSON.stringify(descriptor), + descriptor.computedAt, + ); + } + + private captureCompose(args: string[], cwd: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn('docker', ['compose', ...args], { + cwd, + env: { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + }, + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (data: Buffer) => { stdout += data.toString(); }); + child.stderr.on('data', (data: Buffer) => { stderr += data.toString(); }); + child.on('error', (err: NodeJS.ErrnoException) => { + const mapped = describeSpawnError(err, { command: 'docker compose' }); + if (mapped.isLowMemory) { + console.warn('[ComposeService] captureCompose spawn failed under memory pressure:', mapped.message); + } + reject(new Error(mapped.message)); + }); + child.on('close', (code) => { + if (code === 0) resolve(stdout); + else reject(new Error(stderr.trim() || `docker compose ${args.join(' ')} failed with code ${code}`)); + }); + }); + } + + /** + * Render the effective compose model as YAML (the default `docker compose + * config` output) with the exact authored invocation and NO mesh override. + * Used by the Git source detach/export contract: the rendered model becomes + * the stack's single compose.yaml. Throws when the render fails or times + * out, so the detach transaction aborts before anything changes. + */ + public async renderComposeYaml(stackName: string): Promise { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack path'); + } + const baseResolved = path.resolve(this.baseDir); + const stackDir = path.resolve(baseResolved, stackName); + if (!stackDir.startsWith(baseResolved + path.sep)) { + throw new Error('Invalid stack path'); + } + let filePrefix: string[]; + try { + filePrefix = authoredComposeFileArgs(stackName, this.nodeId); + } catch (err) { + throw err instanceof Error ? err : new Error(String(err)); + } + const envFileArgs = await authoredComposeEnvFileArgs(stackName, this.nodeId); + const child = spawn('docker', ['compose', ...filePrefix, ...envFileArgs, 'config'], { + cwd: stackDir, + env: { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + }, + }); + return new Promise((resolve, reject) => { + const MAX_OUTPUT = 5 * 1024 * 1024; // 5 MiB cap on each stream + const TIMEOUT_MS = 30_000; + // Accumulate Buffer chunks and decode ONCE at the end: chunk-wise + // toString() can split a multi-byte UTF-8 sequence across a chunk + // boundary and mangle non-ASCII values. + const outChunks: Buffer[] = []; + let outBytes = 0; + let stderr = ''; + let capped = false; + let settled = false; + const timer = setTimeout(() => { + settled = true; + clearTimeout(timer); + try { + child.kill('SIGKILL'); + } catch { + // best effort + } + reject(new Error(`docker compose config timed out after ${TIMEOUT_MS / 1000}s`)); + }, TIMEOUT_MS); + const finish = (error: Error | null) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (error) reject(error); + else resolve(Buffer.concat(outChunks).toString('utf8')); + }; + child.stdout.on('data', (data: Buffer) => { + if (capped) return; + outBytes += data.length; + if (outBytes > MAX_OUTPUT) { + // A truncated model frequently still parses as YAML; overwriting a + // working compose.yaml with it would be silent corruption. The cap is + // an error, not a truncation. + capped = true; + settled = true; + clearTimeout(timer); + try { + child.kill('SIGKILL'); + } catch { + // best effort + } + reject(new Error(`docker compose config output exceeded ${MAX_OUTPUT} bytes`)); + return; + } + outChunks.push(data); + }); + child.stderr.on('data', (data: Buffer) => { + if (stderr.length < MAX_OUTPUT) stderr += data.toString(); + }); + child.on('close', (code) => { + if (capped) return; + if (code === 0) finish(null); + else finish(new Error(stderr.trim() || `docker compose config exited with code ${code}`)); + }); + child.on('error', (err) => finish(err)); + }); + } + + /** + * Render the fully-resolved effective Compose model via `docker compose + * config --format json`. This is the AUTHORED model: it does NOT splice in + * the Sencho Mesh override, so it stays read-only (the override is + * write-generated) and reflects what the user actually edits. The override + * would also add the managed `sencho_mesh` external network and per-service + * mesh attachments, which would make preflight emit a false "external network + * not found" finding, so rendering the authored model is both safer and more + * accurate here. + * Captures stderr (where Compose reports unset variables) and never rejects + * on a non-zero exit, so the Compose Doctor can turn a failed render into a + * finding rather than an exception. Bounded by a timeout and an output cap. + * Rejects only when the docker binary cannot be spawned. + */ + public async renderConfig( + stackName: string, + ): Promise<{ rendered: string | null; stderr: string; code: number | null; timedOut: boolean }> { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack path'); + } + // Canonical inline js/path-injection barrier, kept in the same scope as the + // spawn cwd sink below. CodeQL credits neither the wrapped isPathWithinBase + // helper nor a barrier separated from the sink by the Promise-executor + // closure, so the spawn is hoisted out of the executor. startsWith already + // rejects the base dir itself, since base does not start with base + sep. + const baseResolved = path.resolve(this.baseDir); + const stackDir = path.resolve(baseResolved, stackName); + if (!stackDir.startsWith(baseResolved + path.sep)) { + throw new Error('Invalid stack path'); + } + // Render the authored multi-file model (no mesh override) so the Compose Doctor + // sees every override file; single-file stacks get an empty prefix. The env-file + // flag keeps render resolving the same root .env the validator and deploy use. + let filePrefix: string[]; + try { + filePrefix = authoredComposeFileArgs(stackName, this.nodeId); + } catch (err) { + throw err instanceof Error ? err : new Error(String(err)); + } + const envFileArgs = await authoredComposeEnvFileArgs(stackName, this.nodeId); + const child = spawn('docker', ['compose', ...filePrefix, ...envFileArgs, 'config', '--format', 'json'], { + cwd: stackDir, + env: { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + }, + }); + return new Promise((resolve, reject) => { + const MAX_OUTPUT = 5 * 1024 * 1024; // 5 MiB cap on each stream + const TIMEOUT_MS = 20_000; + let stdout = ''; + let stderr = ''; + let timedOut = false; + let capped = false; + let settled = false; + const timer = setTimeout(() => { timedOut = true; child.kill('SIGKILL'); }, TIMEOUT_MS); + const finish = (result: { rendered: string | null; stderr: string; code: number | null; timedOut: boolean }) => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(result); + }; + child.stdout.on('data', (data: Buffer) => { + stdout += data.toString(); + if (stdout.length > MAX_OUTPUT && !capped) { capped = true; child.kill('SIGKILL'); } + }); + child.stderr.on('data', (data: Buffer) => { + if (stderr.length < MAX_OUTPUT) stderr += data.toString(); + }); + child.on('error', (err: NodeJS.ErrnoException) => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(new Error(describeSpawnError(err, { command: 'docker compose' }).message)); + }); + child.on('close', (code) => { + if (timedOut) finish({ rendered: null, stderr: stderr.trim() || 'docker compose config timed out', code, timedOut: true }); + else if (capped) finish({ rendered: null, stderr: 'Rendered model exceeded the size limit', code, timedOut: false }); + else if (code === 0) finish({ rendered: stdout, stderr, code, timedOut: false }); + else finish({ rendered: null, stderr: stderr.trim() || `docker compose config failed with code ${code}`, code, timedOut: false }); + }); + }); + } +} diff --git a/backend/src/services/DockerController.ts b/backend/src/services/DockerController.ts index 7dfbabec..506e861c 100644 --- a/backend/src/services/DockerController.ts +++ b/backend/src/services/DockerController.ts @@ -2480,6 +2480,25 @@ class DockerController { return result; } + /** + * True when the stack directory is missing (ENOENT) or is not a directory. + * Other stat failures return false so callers keep the existing compose-ps path. + */ + private async isStackDirectoryAbsent(stackName: string): Promise { + // Canonical js/path-injection barrier at the fs.stat sink. + const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(this.nodeId)); + const stackDir = path.resolve(baseResolved, stackName); + if (!stackDir.startsWith(baseResolved + path.sep)) { + return true; + } + try { + const st = await fs.stat(stackDir); + return !st.isDirectory(); + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'ENOENT'; + } + } + /** * Containers visible to `docker compose ps` for this stack. Empty when Compose * does not manage any containers (including first deploy or mislabeled legacy). @@ -2530,6 +2549,7 @@ class DockerController { */ public async getLegacyOrphanContainersByStack(stackName: string): Promise> { const stackDir = path.join(NodeRegistry.getInstance().getComposeDir(this.nodeId), stackName); + if (await this.isStackDirectoryAbsent(stackName)) return []; const toIds = (list: Array<{ Id?: string }>) => list.filter((c): c is { Id: string } => typeof c.Id === 'string' && c.Id.length > 0) .map((c) => ({ Id: c.Id })); @@ -2573,6 +2593,9 @@ class DockerController { | { status: 'classification_failed'; error: string } > { const stackDir = path.join(NodeRegistry.getInstance().getComposeDir(this.nodeId), stackName); + if (await this.isStackDirectoryAbsent(stackName)) { + return { status: 'classification_failed', error: 'Stack directory is gone' }; + } const toIds = (list: Array<{ Id?: string }>) => list.filter((c): c is { Id: string } => typeof c.Id === 'string' && c.Id.length > 0) .map((c) => c.Id); @@ -2614,6 +2637,9 @@ class DockerController { // not the process default, so a non-default local node sees its own stack dir // and deploy spec. const stackDir = path.join(NodeRegistry.getInstance().getComposeDir(this.nodeId), stackName); + if (await this.isStackDirectoryAbsent(stackName)) { + return this.enrichContainers([]); + } try { const containers = await this.fetchComposePsContainers(stackName, stackDir);