diff --git a/backend/src/__tests__/compose-service.test.ts b/backend/src/__tests__/compose-service.test.ts index fd012439..ea2bba1e 100644 --- a/backend/src/__tests__/compose-service.test.ts +++ b/backend/src/__tests__/compose-service.test.ts @@ -35,6 +35,7 @@ const { mockCompensateWithCandidate, mockBuildUnifiedHeldImagePredicate, mockGetRecovery, + mockFsStat, } = vi.hoisted(() => ({ mockSpawn: vi.fn(), mockGetContainersByStack: vi.fn().mockResolvedValue([]), @@ -87,6 +88,7 @@ const { mockCompensateWithCandidate: vi.fn().mockResolvedValue(true), mockBuildUnifiedHeldImagePredicate: vi.fn().mockReturnValue(() => false), mockGetRecovery: vi.fn().mockReturnValue(undefined), + mockFsStat: vi.fn().mockResolvedValue({ isDirectory: () => true }), })); vi.mock('child_process', () => ({ spawn: mockSpawn, execFile: vi.fn() })); @@ -97,11 +99,13 @@ vi.mock('fs', () => ({ writeFileSync: (...args: unknown[]) => mockWriteFileSync(...args), unlinkSync: (...args: unknown[]) => mockUnlinkSync(...args), rmdirSync: (...args: unknown[]) => mockRmdirSync(...args), + promises: { stat: (...args: unknown[]) => mockFsStat(...args) }, }, mkdtempSync: (...args: unknown[]) => mockMkdtempSync(...args), writeFileSync: (...args: unknown[]) => mockWriteFileSync(...args), unlinkSync: (...args: unknown[]) => mockUnlinkSync(...args), rmdirSync: (...args: unknown[]) => mockRmdirSync(...args), + promises: { stat: (...args: unknown[]) => mockFsStat(...args) }, })); vi.mock('../services/NodeRegistry', () => ({ @@ -316,6 +320,7 @@ beforeEach(() => { mockIsMeshStackEnabled.mockReturnValue(false); mockCompensateWithCandidate.mockResolvedValue(true); mockBuildUnifiedHeldImagePredicate.mockReturnValue(() => false); + mockFsStat.mockResolvedValue({ isDirectory: () => true }); delete process.env.SENCHO_MODE; vi.useFakeTimers({ shouldAdvanceTime: true }); }); @@ -1709,6 +1714,84 @@ describe('ComposeService - streamLogs', () => { expect.anything(), ); }); + + it('does not reschedule when the stack directory is gone', async () => { + mockFsStat.mockRejectedValue(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })); + const ws = createMockWs(); + const svc = ComposeService.getInstance(1); + + svc.streamLogs('mystack', ws); + await vi.waitFor(() => expect(ws.send).toHaveBeenCalled()); + + const sent = (ws.send as ReturnType).mock.calls + .flatMap((c) => String(c[0]).split(/\r?\n/)) + .filter(Boolean); + expect(sent.some((line) => line.includes('Stack directory is gone'))).toBe(true); + expect(mockGetContainersByStack).not.toHaveBeenCalled(); + expect(ws.send).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(4000); + expect(mockGetContainersByStack).not.toHaveBeenCalled(); + expect(ws.send).toHaveBeenCalledTimes(1); + expect(ws.close).not.toHaveBeenCalled(); + }); + + it('retries every 2s when the stack dir exists and no containers are found', async () => { + mockGetContainersByStack.mockResolvedValue([]); + const ws = createMockWs(); + const svc = ComposeService.getInstance(1); + + svc.streamLogs('mystack', ws); + await vi.waitFor(() => expect(mockGetContainersByStack).toHaveBeenCalledTimes(1)); + + await vi.advanceTimersByTimeAsync(2000); + expect(mockGetContainersByStack).toHaveBeenCalledTimes(2); + + const sent = (ws.send as ReturnType).mock.calls + .flatMap((c) => String(c[0])) + .join(''); + expect(sent).toContain('No containers found. Waiting for activity'); + }); + + it('stops retrying after the websocket closes', async () => { + mockGetContainersByStack.mockResolvedValue([]); + const ws = createMockWs(); + const svc = ComposeService.getInstance(1); + + svc.streamLogs('mystack', ws); + await vi.waitFor(() => expect(mockGetContainersByStack).toHaveBeenCalledTimes(1)); + + ws.readyState = 3; + ws.emit('close'); + await vi.advanceTimersByTimeAsync(4000); + expect(mockGetContainersByStack).toHaveBeenCalledTimes(1); + }); + + it('stops the empty-container retry after the stack directory disappears', async () => { + mockGetContainersByStack.mockResolvedValue([]); + const ws = createMockWs(); + const svc = ComposeService.getInstance(1); + + svc.streamLogs('mystack', ws); + await vi.waitFor(() => expect(mockGetContainersByStack).toHaveBeenCalledTimes(1)); + + mockFsStat.mockRejectedValue(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })); + await vi.advanceTimersByTimeAsync(2000); + + expect(mockGetContainersByStack).toHaveBeenCalledTimes(1); + const sent = (ws.send as ReturnType).mock.calls + .flatMap((c) => String(c[0])) + .join(''); + expect(sent).toContain('No containers found. Waiting for activity'); + expect(sent).toContain('Stack directory is gone'); + expect(ws.close).not.toHaveBeenCalled(); + + const sendsAfterGone = (ws.send as ReturnType).mock.calls.length; + await vi.advanceTimersByTimeAsync(4000); + expect(mockGetContainersByStack).toHaveBeenCalledTimes(1); + expect(ws.send).toHaveBeenCalledTimes(sendsAfterGone); + expect(ws.close).not.toHaveBeenCalled(); + }); }); diff --git a/backend/src/__tests__/docker-controller.test.ts b/backend/src/__tests__/docker-controller.test.ts index 48c648c7..d5eee6b6 100644 --- a/backend/src/__tests__/docker-controller.test.ts +++ b/backend/src/__tests__/docker-controller.test.ts @@ -78,6 +78,7 @@ beforeEach(() => { // holds re-spy after this beforeEach runs. vi.spyOn(StackUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set()); vi.spyOn(ServiceUpdateRecoveryService.getInstance(), 'getHeldImageIds').mockReturnValue(new Set()); + vi.spyOn(fs, 'stat').mockResolvedValue({ isDirectory: () => true } as Awaited>); }); /** Hold specific images via the stack-side recovery service for one test. */ @@ -2478,3 +2479,73 @@ describe('DockerController - smartFallback stack-dir evidence', () => { }); }); +describe('DockerController - missing stack directory', () => { + const enoent = () => Object.assign(new Error('ENOENT: no such file or directory'), { code: 'ENOENT' }); + + beforeEach(() => { + vi.spyOn(fs, 'stat').mockRejectedValue(enoent()); + }); + + it('getContainersByStack returns [] without spawning compose', async () => { + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const dc = DockerController.getInstance(1); + const fetchSpy = spyOrphanDc(dc, 'fetchComposePsContainers'); + + await expect(dc.getContainersByStack('gone-stack')).resolves.toEqual([]); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(mockExecFileAsync).not.toHaveBeenCalled(); + expect(errSpy).not.toHaveBeenCalled(); + fetchSpy.mockRestore(); + errSpy.mockRestore(); + }); + + it('getLegacyOrphanContainersByStack returns [] without spawning compose', async () => { + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const dc = DockerController.getInstance(1); + const fetchSpy = spyOrphanDc(dc, 'fetchComposePsContainers'); + const fallbackSpy = spyOrphanDc(dc, 'smartFallback'); + + await expect(dc.getLegacyOrphanContainersByStack('gone-stack')).resolves.toEqual([]); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(fallbackSpy).not.toHaveBeenCalled(); + expect(mockExecFileAsync).not.toHaveBeenCalled(); + expect(errSpy).not.toHaveBeenCalled(); + fetchSpy.mockRestore(); + fallbackSpy.mockRestore(); + errSpy.mockRestore(); + }); + + it('classifyLegacyOrphansForUpdate returns classification_failed without spawning compose', async () => { + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const dc = DockerController.getInstance(1); + const fetchSpy = spyOrphanDc(dc, 'fetchComposePsContainers'); + + await expect(dc.classifyLegacyOrphansForUpdate('gone-stack')).resolves.toEqual({ + status: 'classification_failed', + error: 'Stack directory is gone', + }); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(mockExecFileAsync).not.toHaveBeenCalled(); + expect(errSpy).not.toHaveBeenCalled(); + fetchSpy.mockRestore(); + errSpy.mockRestore(); + }); + + it('still logs Docker CLI unavailable when the stack dir exists and compose ENOENT fires', async () => { + vi.spyOn(fs, 'stat').mockResolvedValue({ isDirectory: () => true } as Awaited>); + vi.spyOn(os, 'freemem').mockReturnValue(2 * 1024 * 1024 * 1024); + vi.spyOn(os, 'totalmem').mockReturnValue(8 * 1024 * 1024 * 1024); + mockExecFileAsync.mockRejectedValue(Object.assign(new Error('spawn docker ENOENT'), { code: 'ENOENT' })); + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + const dc = DockerController.getInstance(1); + await dc.getContainersByStack('my-stack'); + + expect(mockExecFileAsync).toHaveBeenCalled(); + const logged = errSpy.mock.calls.map((c) => c.map(String).join(' ')).join('\n'); + expect(logged).toMatch(/Docker Compose Error for/); + expect(logged).toContain('Docker CLI unavailable on this node'); + errSpy.mockRestore(); + }); +}); + diff --git a/backend/src/services/ComposeService.ts b/backend/src/services/ComposeService.ts index 30853e37..b425e94f 100644 --- a/backend/src/services/ComposeService.ts +++ b/backend/src/services/ComposeService.ts @@ -1,1631 +1,1653 @@ -import { spawn } from 'child_process'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; -import WebSocket from 'ws'; -import DockerController from './DockerController'; -import { DatabaseService } from './DatabaseService'; -import { FileSystemService } from './FileSystemService'; -import { MeshService } from './MeshService'; -import { LogFormatter } from './LogFormatter'; -import { NodeRegistry } from './NodeRegistry'; -import { RegistryService } from './RegistryService'; -import { DriftLedgerService } from './DriftLedgerService'; -import SelfIdentityService from './SelfIdentityService'; -import { parseEffectiveModel } from './preflight/effectiveModel'; -import { deriveStackExposure } from './preflight/exposure'; - -import { isDebugEnabled } from '../utils/debug'; -import { getErrorMessage } from '../utils/errors'; -import { normalizeContainerName } from '../utils/log-parsing'; -import { describeSpawnError } from '../utils/spawnErrors'; -import { isPathWithinBase, isValidStackName, isValidRelativeStackPath } from '../utils/validation'; -import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs'; -import type { RollbackInvocationRecord } from '../types/rollbackGeneration'; -import { parseMissingRequiredVars } from '../helpers/envVarParse'; -import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog'; -import { pathsMatch, resolveHostBindPath } from '../utils/composePathMapping'; -import { loadStackBuildServices } from './ImageUpdateService'; -import { resolveMissingExternalNetworks } from './network/resolveMissingExternalNetworks'; -import { - MissingExternalNetworksError, - type DeployInvocationContext, -} from './network/missingExternalNetworksError'; -import { buildUnifiedHeldImagePredicate } from './recoveryHeldImages'; -import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; -import type { NotificationCategory } from './NotificationService'; - -/** True when a generation capture recorded an invocation object (prefix may be empty). */ -function hasUsableCapturedInvocation( - invocation: RollbackInvocationRecord | null | undefined, -): invocation is RollbackInvocationRecord { - return invocation != null; -} - -function recordNetworkAutoCreatedActivity( - nodeId: number, - stackName: string, - createdNames: string[], - level: 'info' | 'warning', - ctx?: DeployInvocationContext, -): void { - if (createdNames.length === 0) return; - const names = [...createdNames].sort((a, b) => a.localeCompare(b)).join(', '); - const source = ctx?.source ?? 'manual'; - try { - DatabaseService.getInstance().addNotificationHistory(nodeId, { - level, - category: 'network_auto_created' as NotificationCategory, - message: `Auto-created external network(s) for ${stackName}: ${names} (source: ${source})`, - timestamp: Date.now(), - stack_name: stackName, - actor_username: ctx?.actor ?? null, - }); - } catch (error) { - console.error( - '[ComposeService] Failed to record network_auto_created activity for %s:', - sanitizeForLog(stackName), - sanitizeForLog(getErrorMessage(error, 'unknown')), - ); - } -} - -export class ComposeRollbackError extends Error { - public readonly rollbackAttempted: boolean; - public readonly rolledBack: boolean; - public readonly originalError: unknown; - - constructor(originalError: unknown, rollbackAttempted: boolean, rolledBack: boolean) { - super(getErrorMessage(originalError, 'Compose operation failed')); - this.name = 'ComposeRollbackError'; - this.rollbackAttempted = rollbackAttempted; - this.rolledBack = rolledBack; - this.originalError = originalError; - Object.setPrototypeOf(this, ComposeRollbackError.prototype); - } -} - -export function getComposeRollbackInfo(error: unknown): { attempted: boolean; rolledBack: boolean } | null { - if (!(error instanceof ComposeRollbackError)) { - return null; - } - return { attempted: error.rollbackAttempted, rolledBack: error.rolledBack }; -} - -function isNonFatalCompensationError(error: unknown): boolean { - const code = (error as { code?: string }).code; - return code === 'HELD_IMAGE_MISSING' || code === 'RECOVERY_PROBE_FAILED'; -} - -async function compensateOrSwallow(compensate: () => Promise): Promise { - try { - return await compensate(); - } catch (compError) { - if (!isNonFatalCompensationError(compError)) throw compError; - return false; - } -} - -const DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS = 30 * 60 * 1000; - -/** Public so other services (e.g. recovery claim leases) can size their own timers off the same ceiling without depending on a private module-local. */ -export function getComposeCommandTimeoutMs(): number { - const configured = Number(process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS); - if (Number.isFinite(configured) && configured > 0) { - return configured; - } - return DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS; -} - -// Idle backstop for long-running pull/recreate steps: if the child emits no -// output for this window while still running, the step is treated as stalled -// and terminated, so a hung `docker compose pull` surfaces a fast failure -// instead of spinning until the much longer command timeout above. Conservative -// by default because a working pull can be briefly silent while a large layer -// extracts; operators on slow links or heavy local builds can raise it. -const DEFAULT_COMPOSE_STALL_TIMEOUT_MS = 10 * 60 * 1000; - -function getComposeStallTimeoutMs(): number { - const configured = Number(process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS); - if (Number.isFinite(configured) && configured > 0) { - return configured; - } - return DEFAULT_COMPOSE_STALL_TIMEOUT_MS; -} - -/** - * ComposeService - local docker compose CLI execution. - * - * In the Distributed API model, remote node compose operations are handled - * by the remote Sencho instance. This service only executes commands locally. - */ -export class ComposeService { - private baseDir: string; - private nodeId: number; - - constructor(nodeId?: number) { - this.nodeId = nodeId ?? NodeRegistry.getInstance().getDefaultNodeId(); - this.baseDir = NodeRegistry.getInstance().getComposeDir(this.nodeId); - } - - public static getInstance(nodeId?: number): ComposeService { - return new ComposeService(nodeId); - } - - /** - * Build the authored `docker compose` argument list for a stack: the validated - * multi-file deploy prefix (ordered `-f` files + `-p ` + - * `--project-directory`) for a Git source with an applied multi-file spec, then - * the Sencho Mesh override file last (highest `-f` precedence) when the stack is - * opted into the mesh, then the action. Single-file / non-git stacks get no file - * prefix, so docker compose's built-in discovery resolves the root compose.yaml, - * byte-identical to the pre-multi-file behavior. The user's source files are - * never mutated. Lifecycle commands (deploy, update, stop/start/restart/down) - * route through this method, so they share one file prefix plus the mesh override. - * Image scans (listStackImages) and the Compose Doctor (renderConfig) reuse the - * same `authoredComposeFileArgs` prefix directly but intentionally omit the mesh - * override, rendering the user's authored model without mesh injection. - */ - /** Public wrapper for dual-arg assembly and recovery Compose invocations. */ - public async buildAuthoredComposeArgs(stackName: string, action: string[]): Promise { - return this.authoredComposeArgs(stackName, action); - } - - public async validateStackForMutation(stackName: string): Promise { - await this.assertRequiredEnvPresent(stackName); - await this.assertSafePilotBindMapping(stackName); - } - - /** - * Render/validate the exact Compose invocation used by mutating operations - * (authored files, env pins, and generated Mesh override) before capture. - */ - public async validateExactComposeInvocation(stackName: string): Promise { - if (!isValidStackName(stackName)) { - throw new Error('Invalid stack path'); - } - const baseResolved = path.resolve(this.baseDir); - const stackDir = path.resolve(baseResolved, stackName); - if (!stackDir.startsWith(baseResolved + path.sep)) { - throw new Error('Invalid stack path'); - } - const args = await this.authoredComposeArgs(stackName, ['config', '--quiet']); - await this.execute('docker', args, stackDir, undefined, true); - } - - private async authoredComposeArgs(stackName: string, action: string[]): Promise { - const args: string[] = ['compose']; - const filePrefix = authoredComposeFileArgs(stackName, this.nodeId); - args.push(...filePrefix); - // Pin env resolution to the root .env when a context dir shifts the project - // directory, so deploy/update resolve the same effective config the validator did. - args.push(...await authoredComposeEnvFileArgs(stackName, this.nodeId)); - - const meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(this.nodeId, stackName); - let overridePath: string | null = null; - try { - overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName); - } catch (err) { - if (meshEnabled) { - throw err instanceof Error - ? err - : new Error(`Mesh override generation failed: ${String(err)}`); - } - console.warn('[ComposeService] mesh override skipped:', sanitizeForLog((err as Error).message)); - } - if (meshEnabled && !overridePath) { - throw new Error( - `Mesh override is required for stack "${stackName}" but could not be generated`, - ); - } - if (overridePath) { - if (filePrefix.length === 0) { - // Single-file stack: passing any -f disables compose's auto-discovery, so name - // the base file explicitly, then re-add the user's implicit override (if any) so - // it is not silently dropped, before layering the mesh override on top. - const fsSvc = FileSystemService.getInstance(this.nodeId); - const baseFilename = await fsSvc.getComposeFilename(stackName); - args.push('-f', baseFilename); - let userOverride: string | null = null; - try { - userOverride = await fsSvc.getOverrideFilename(stackName); - } catch (err) { - // Containment-guard rejections (bad stack name / symlink escape) are hard errors: - // abort the deploy rather than degrade. The "no override" case returns null rather - // than throwing, so any other throw is transient I/O: drop the override and proceed - // (logging the consequence) instead of failing the deploy. - const code = (err as { code?: string }).code; - if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') { - throw err; - } - console.warn('[ComposeService] could not resolve user compose override; deploying without it:', sanitizeForLog((err as Error).message)); - } - if (userOverride) { - args.push('-f', userOverride); - } - } - args.push('-f', overridePath); - } - args.push(...action); - return args; - } - - private execute( - command: string, - args: string[], - cwd: string, - ws?: WebSocket, - throwOnError = true, - env?: Record, - // When set, terminate the child if it emits no output for this long while - // still running (idle stall backstop). Appended last so the existing - // registry-auth call sites that pass `env` are unaffected. - idleTimeoutMs?: number - ): Promise { - return new Promise((resolve, reject) => { - const child = spawn(command, args, { - cwd, - env: env ?? { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' - } - }); - - let errorLog = ''; - let settled = false; - let exited = false; - let pendingTerminationError: Error | null = null; - const timeoutMs = getComposeCommandTimeoutMs(); - let timeout: ReturnType | null = null; - let forceKillTimeout: ReturnType | null = null; - let idleTimeout: ReturnType | null = null; - - const sendOutput = (text: string) => { - if (ws && ws.readyState === WebSocket.OPEN) { - ws.send(text); - } - }; - - const cleanup = () => { - if (timeout) { - clearTimeout(timeout); - timeout = null; - } - if (forceKillTimeout) { - clearTimeout(forceKillTimeout); - forceKillTimeout = null; - } - if (idleTimeout) { - clearTimeout(idleTimeout); - idleTimeout = null; - } - }; - - const finish = (complete: () => void) => { - if (settled) return; - settled = true; - cleanup(); - complete(); - }; - - const terminateChild = (error: Error) => { - pendingTerminationError = pendingTerminationError ?? error; - if (exited) return; - try { - child.kill('SIGTERM'); - } catch (error) { - console.warn('[ComposeService] Failed to terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); - } - forceKillTimeout = setTimeout(() => { - if (exited) return; - try { - child.kill('SIGKILL'); - } catch (error) { - console.warn('[ComposeService] Failed to force terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); - } - }, 5000); - }; - - // Idle stall backstop. Armed once below and reset on every output chunk; - // if it ever fires, the step has been silent for idleTimeoutMs while still - // running, so terminate it. Never rearmed after a termination is pending or - // the child has exited, so it cannot re-fire during the SIGTERM grace. - const armIdleTimeout = () => { - if (idleTimeoutMs === undefined) return; - if (exited || settled || pendingTerminationError) return; - if (idleTimeout) clearTimeout(idleTimeout); - idleTimeout = setTimeout(() => { - const seconds = Math.round(idleTimeoutMs / 1000); - sendOutput(`=== No output for ${seconds}s; the operation appears stalled and was stopped ===\n`); - terminateChild(new Error(`STACK_STALLED_OUTPUT: no output for ${seconds}s`)); - }, idleTimeoutMs); - }; - - // The progress socket is output-only: a deploy/update/down is owned by the - // HTTP request that started it, so closing or losing the socket (the user - // minimizes the panel, navigates away, or the connection blips) must not - // terminate the compose process. Termination is driven solely by the - // command timeout here and the optional idle stall backstop above. - timeout = setTimeout(() => { - const message = `Command timed out after ${Math.round(timeoutMs / 1000)}s`; - sendOutput(`${message}\n`); - terminateChild(new Error(message)); - }, timeoutMs); - - armIdleTimeout(); - - const onData = (data: Buffer) => { - const text = data.toString(); - errorLog += text; - sendOutput(text); - armIdleTimeout(); - }; - - child.stdout.on('data', onData); - child.stderr.on('data', onData); - - child.on('close', (code: number | null) => { - exited = true; - finish(() => { - sendOutput(`Command exited with code ${code}\n`); - if (pendingTerminationError) { - if (throwOnError) reject(pendingTerminationError); - else resolve(); - return; - } - if (code === 0) resolve(); - else if (throwOnError) reject(new Error(redactSensitiveText(errorLog.trim()) || `Command failed with code ${code}`)); - else resolve(); - }); - }); - - child.on('error', (error: Error & { code?: string }) => { - exited = true; - finish(() => { - const mapped = describeSpawnError(error as NodeJS.ErrnoException, { command }); - const message = redactSensitiveText(mapped.message); - sendOutput(`Error: ${message}\n`); - if (mapped.isLowMemory) { - console.warn('[ComposeService] spawn failed under memory pressure:', message); - } - if (pendingTerminationError) { - if (throwOnError) reject(pendingTerminationError); - else resolve(); - return; - } - if (throwOnError) reject(new Error(message)); - else resolve(); - }); - }); - }); - } - - private async withRegistryAuth( - fn: (env: Record) => Promise, - sendOutput?: (data: string) => void, - ): Promise { - const registries = DatabaseService.getInstance().getRegistries(); - if (registries.length === 0) { - return fn({ - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }); - } - - const { config, warnings } = await RegistryService.getInstance().resolveDockerConfig(); - if (warnings.length > 0 && sendOutput) { - for (const warning of warnings) { - sendOutput(`[Sencho] Warning: ${warning}\n`); - } - } - - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-docker-')); - const configPath = path.join(tmpDir, 'config.json'); - - try { - fs.writeFileSync(configPath, JSON.stringify(config), { mode: 0o600 }); - return await fn({ - ...process.env, - DOCKER_CONFIG: tmpDir, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }); - } finally { - // Best-effort cleanup; each step runs independently so a file that was never - // written (e.g., writeFileSync threw) does not prevent the directory removal. - try { fs.unlinkSync(configPath); } catch { /* file may not exist */ } - try { fs.rmdirSync(tmpDir); } catch (e) { - console.warn('[ComposeService] Could not remove temp Docker config dir:', (e as Error).message); - } - } - } - - private async createAtomicBackup( - stackName: string, - operation: 'deployment' | 'update', - sendOutput: (data: string) => void, - ): Promise { - try { - const fsSvc = FileSystemService.getInstance(this.nodeId); - await fsSvc.backupStackFiles(stackName); - sendOutput(`=== Backup created for atomic ${operation} ===\n`); - } catch (error) { - console.error('Atomic backup failed for %s:', sanitizeForLog(stackName), getErrorMessage(error, 'unknown error')); - sendOutput(`=== Atomic ${operation} backup failed. Operation aborted ===\n`); - throw new Error(`Atomic ${operation} backup failed: ${getErrorMessage(error, 'unknown error')}`); - } - } - - private async restoreAtomicBackup( - stackName: string, - stackDir: string, - ws: WebSocket | undefined, - sendOutput: (data: string) => void, - ): Promise { - try { - const fsSvc = FileSystemService.getInstance(this.nodeId); - await fsSvc.restoreStackFiles(stackName); - await this.withRegistryAuth(async (env) => { - await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env); - }, sendOutput); - sendOutput('=== Restored previous compose and env files ===\n'); - return true; - } catch (rollbackError) { - console.error('Rollback failed for %s:', sanitizeForLog(stackName), getErrorMessage(rollbackError, 'unknown error')); - sendOutput('=== Rollback failed. Manual intervention may be required ===\n'); - return false; - } - } - - private createContainerCrashError(exitCode: number): Error { - return new Error( - `CONTAINER_CRASHED\nExit Code: ${exitCode}\nContainer exited after deployment. Check container logs for details.` - ); - } - - async runCommand(stackName: string, action: 'down' | 'start' | 'stop' | 'restart', ws?: WebSocket): Promise { - const stackDir = path.join(this.baseDir, stackName); - await this.execute('docker', await this.authoredComposeArgs(stackName, [action]), stackDir, ws); - } - - /** Interactive compose down (Take down UI / POST /down). Plain `down` by default. */ - async runDown(stackName: string, options?: { removeVolumes?: boolean }, ws?: WebSocket): Promise { - const stackDir = path.join(this.baseDir, stackName); - const args = options?.removeVolumes ? ['down', '--volumes'] : ['down']; - await this.execute('docker', await this.authoredComposeArgs(stackName, args), stackDir, ws); - } - - /** - * Opt-in guard: when `env_block_deploy_on_missing_required` is enabled, refuse a - * deploy whose required `${VAR:?err}` variables are unset OR empty, before any - * backup, cleanup, pull, or `up` runs. Compose's own resolution is authoritative - * (it passes process.env), and on the failing path it emits no rendered model, so - * no env value is materialized. Default off and any settings-read failure both - * fall through without blocking. - */ - private async assertRequiredEnvPresent(stackName: string): Promise { - let enabled = false; - try { - enabled = DatabaseService.getInstance().getGlobalSettings()['env_block_deploy_on_missing_required'] === '1'; - } catch { - return; // safe default: a settings-read failure never blocks a deploy - } - if (!enabled) return; - const result = await this.renderConfig(stackName); - const missing = parseMissingRequiredVars(result.stderr); - if (missing.length === 0) return; - const plural = missing.length > 1; - throw new Error( - `Deploy blocked: required environment variable${plural ? 's' : ''} ${missing.join(', ')} ` + - `${plural ? 'are' : 'is'} missing. Define ${plural ? 'them' : 'it'} in a .env or env_file, then deploy again.`, - ); - } - - private async assertSafePilotBindMapping(stackName: string): Promise { - if (process.env.SENCHO_MODE !== 'pilot') return; - - let mounts: Array<{ source: string; destination: string }> | null; - try { - mounts = await SelfIdentityService.getInstance().getBindMounts(); - } catch (error) { - console.warn('[ComposeService] Could not verify pilot compose path mapping:', sanitizeForLog(getErrorMessage(error, 'unknown'))); - return; - } - if (mounts === null) return; - - const composeDir = path.resolve(this.baseDir); - const hostComposeDir = resolveHostBindPath(composeDir, mounts); - if (!hostComposeDir || pathsMatch(hostComposeDir, composeDir)) return; - - const rendered = await this.renderConfig(stackName); - if (rendered.rendered === null) return; - - let parsed: unknown; - try { - parsed = JSON.parse(rendered.rendered); - } catch (error) { - console.warn('[ComposeService] Could not inspect rendered binds for pilot path safety:', sanitizeForLog(getErrorMessage(error, 'unknown'))); - return; - } - const model = parseEffectiveModel(parsed, stackName); - const unsafeBind = model.services - .flatMap((service) => service.binds) - .find((bind) => isPathWithinBase(path.resolve(bind.source), composeDir)); - if (!unsafeBind) return; - - throw new Error( - `Deploy blocked: relative bind mounts resolve under ${composeDir}, but the host path is ${hostComposeDir}. ` + - `Use a 1:1 mount with the same absolute path on the host and in the Pilot Agent, then retry.`, - ); - } - - /** - * Missing-external gate: after env/Pilot asserts, before atomic backup. - * Creates safe bridge networks only when the opt-in setting is on. - */ - private async ensureExternalNetworksForDeploy( - stackName: string, - ctx?: DeployInvocationContext, - ): Promise { - const resolved = await resolveMissingExternalNetworks(this.nodeId, stackName); - if (resolved.status === 'render_unavailable') { - throw new MissingExternalNetworksError({ - kind: 'unavailable', - message: 'Sencho could not render this stack\'s Compose model to check external networks.', - }); - } - if (resolved.status === 'runtime_unavailable') { - // No declared externals: nothing to verify; proceed. - if (resolved.declaredExternalCount === 0) return; - throw new MissingExternalNetworksError({ - kind: 'unavailable', - message: 'Sencho could not read Docker networking state to check external networks.', - }); - } - - if (resolved.networks.length === 0) return; - - const unsafe = resolved.networks.filter((n) => !n.safe); - if (unsafe.length > 0) { - throw new MissingExternalNetworksError({ - kind: 'unsupported', - message: 'One or more missing external networks cannot be created safely by Sencho.', - networks: resolved.networks, - }); - } - - if (!resolved.autoCreateEnabled) { - throw new MissingExternalNetworksError({ - kind: 'prompt', - message: 'One or more external networks required by this stack are missing on this node.', - networks: resolved.networks, - }); - } - - const docker = DockerController.getInstance(this.nodeId); - const createdNames: string[] = []; - const recordCreatedNetworks = (level: 'info' | 'warning') => { - if (createdNames.length === 0) return; - invalidateNodeCaches(this.nodeId); - recordNetworkAutoCreatedActivity(this.nodeId, stackName, createdNames, level, ctx); - }; - - for (const network of resolved.networks) { - try { - await docker.createNetwork({ Name: network.name, Driver: 'bridge' }); - createdNames.push(network.name); - } catch (createErr) { - // Authoritative re-check: continue only if the network now exists. - let exists = false; - try { - const knownStacks = await FileSystemService.getInstance(this.nodeId).getStacks(); - const snapshot = await docker.getDependencySnapshot(knownStacks); - exists = snapshot.networks.some((n) => n.name === network.name); - } catch (snapErr) { - console.warn( - '[ComposeService] Post-create snapshot failed for %s:', - sanitizeForLog(network.name), - sanitizeForLog(getErrorMessage(snapErr, 'unknown')), - ); - } - if (!exists) { - recordCreatedNetworks('warning'); - throw new MissingExternalNetworksError({ - kind: 'create_failed', - message: `Failed to create external network "${network.name}".`, - networks: resolved.networks, - createdNames, - remainingNames: resolved.networks - .map((missingNetwork) => missingNetwork.name) - .filter((name) => !createdNames.includes(name)), - }); - } - // Race-existing: do not record in createdNames. - } - } - - // Re-resolve before Compose. - const recheck = await resolveMissingExternalNetworks(this.nodeId, stackName); - if (recheck.status !== 'ok' || recheck.networks.length > 0) { - recordCreatedNetworks('warning'); - throw new MissingExternalNetworksError({ - kind: recheck.status === 'ok' ? 'create_failed' : 'unavailable', - message: 'External networks were still missing after automatic creation.', - networks: recheck.networks, - createdNames, - remainingNames: recheck.networks.map((n) => n.name), - }); - } - - recordCreatedNetworks('info'); - } - - async deployStack( - stackName: string, - ws?: WebSocket, - atomic?: boolean, - ctx?: DeployInvocationContext, - ): Promise<{ recoveryId: string | null }> { - await this.assertRequiredEnvPresent(stackName); - await this.assertSafePilotBindMapping(stackName); - await this.ensureExternalNetworksForDeploy(stackName, ctx); - - const stackDir = path.join(this.baseDir, stackName); - const debug = isDebugEnabled(); - const t0 = Date.now(); - if (debug) console.debug('[ComposeService:debug] deployStack', { stackName, stackDir, atomic }); - const sendOutput = (data: string) => { - if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); - }; - - const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); - const recoverySvc = atomic ? StackUpdateRecoveryService.getInstance() : null; - let recoveryId: string | null = null; - let handedOff = false; - - if (atomic && recoverySvc) { - sendOutput('=== Capturing rollback generation for atomic deploy ===\n'); - const candidate = await recoverySvc.captureCandidate({ - nodeId: this.nodeId, - stackName, - createdBy: 'atomic-deploy', - operationKind: 'deployment', - }); - recoveryId = candidate.id; - if (!recoverySvc.markAcquired(candidate.id)) { - await recoverySvc.abandon(candidate.id); - throw new Error('Failed to mark recovery generation as acquired'); - } - if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) { - await recoverySvc.abandon(candidate.id); - throw new Error('Failed to hand off recovery generation'); - } - handedOff = true; - if (!recoverySvc.markReconciling(candidate.id)) { - throw new Error('Failed to mark recovery generation as reconciling after handoff'); - } - } - - try { - try { - const dockerController = DockerController.getInstance(this.nodeId); - const legacyOrphans = await dockerController.getLegacyOrphanContainersByStack(stackName); - if (legacyOrphans.length > 0) { - sendOutput(`=== Cleaning up legacy orphan containers before deployment ===\n`); - await dockerController.removeContainers(legacyOrphans.map((c) => c.Id)); - } - } catch (e) { - console.warn('Failed to clean up legacy containers for %s:', sanitizeForLog(stackName), e); - } - - await this.withRegistryAuth(async (env) => { - await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs()); - }, sendOutput); - - // Post-Deploy Health Probe - await new Promise(resolve => setTimeout(resolve, 3000)); - - const dockerController = DockerController.getInstance(this.nodeId); - const containers = await dockerController.getDocker().listContainers({ - all: true, - filters: { label: [`com.docker.compose.project=${stackName}`] } - }); - - for (const containerInfo of containers) { - if (containerInfo.State === 'exited') { - const container = dockerController.getDocker().getContainer(containerInfo.Id); - const inspectData = await container.inspect(); - const exitCode = inspectData.State.ExitCode; - - if (exitCode !== 0) { - throw this.createContainerCrashError(exitCode); - } - } - } - - if (atomic && recoverySvc && recoveryId) { - if (!recoverySvc.markImmediateVerified(recoveryId)) { - console.warn( - '[ComposeService] Could not CAS immediate_verified for recovery %s', - sanitizeForLog(recoveryId), - ); - } - } - if (debug) console.debug(`[ComposeService:debug] deployStack completed in ${Date.now() - t0}ms`, { stackName }); - } catch (deployError) { - if (atomic && recoverySvc && handedOff && recoveryId) { - sendOutput('\n=== Deployment failed - restoring previous runtime from recovery generation ===\n'); - const generationId = recoveryId; - const rolledBack = await compensateOrSwallow(() => - recoverySvc.compensateWithCandidate( - generationId, - (overridePath, invocation) => this.composeUpWithRecoveryOverride( - stackName, - overridePath, - ws, - invocation, - ), - ), - ); - throw new ComposeRollbackError(deployError, true, rolledBack); - } - if (atomic && recoverySvc && recoveryId && !handedOff) { - await recoverySvc.abandon(recoveryId); - } - throw deployError; - } - // Reached only on a successful deploy (the catch above always rethrows). Record - // the drift baseline here so every deploy path gets one, not just the manual - // route: bulk, Git-source, App Store, scheduler, and webhook deploys all funnel - // through this method. Internally guarded; awaited so it cannot race later work. - await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName); - // Reconcile the ledger against the just-deployed runtime: findings this deploy - // fixed are resolved and any it left are recorded (and surfaced in the activity - // feed) now, instead of waiting for someone to open the Drift tab. The rollback - // route re-deploys through this method, so it is covered; a failed atomic deploy - // instead restores the previous files and throws above, so that recovery path - // reconciles on its next deploy or scan, not here. Best-effort internally. - await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName); - // Refresh the exposure cache so posture reflects the just-deployed model. - // Best-effort: a refresh failure logs a warning but never fails the deploy. - try { - await this.refreshExposureCache(stackName); - } catch (err) { - console.warn('[ComposeService] Exposure refresh failed after deploy for %s:', - sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown'))); - } - return { recoveryId }; - } - - streamLogs(stackName: string, ws: WebSocket) { - let isClosed = false; - let isFirstRun = true; - let isWaitingForActivity = false; - - ws.on('close', () => { isClosed = true; }); - - const startStream = async () => { - if (isClosed || ws.readyState !== WebSocket.OPEN) return; - - try { - const dockerController = DockerController.getInstance(this.nodeId); - const containers = await dockerController.getContainersByStack(stackName); - - if (!containers || containers.length === 0) { - if (!isWaitingForActivity) { - ws.send(`\r\n\x1b[33m[Sencho] No containers found. Waiting for activity...\x1b[0m\r\n`); - isWaitingForActivity = true; - } - setTimeout(startStream, 2000); - return; - } - - const runningContainers = containers.filter((c: any) => c.State === 'running'); - - if (!isFirstRun && runningContainers.length === 0) { - if (!isWaitingForActivity) { - ws.send(`\r\n\x1b[33m[Sencho] Log stream ended. Waiting for container activity...\x1b[0m\r\n`); - isWaitingForActivity = true; - } - setTimeout(startStream, 2000); - return; - } - - const containersToLog = isFirstRun ? containers : runningContainers; - isFirstRun = false; - isWaitingForActivity = false; - - let activeProcesses = 0; - let streamEndedHandled = false; - const localProcesses: ReturnType[] = []; - - const onWsClose = () => { - localProcesses.forEach(cp => { try { cp.kill(); } catch { } }); - }; - - ws.on('close', onWsClose); - - const handleProcessEnd = () => { - activeProcesses--; - if (activeProcesses <= 0 && !streamEndedHandled) { - streamEndedHandled = true; - ws.removeListener('close', onWsClose); - if (!isClosed && ws.readyState === WebSocket.OPEN) { - setTimeout(startStream, 1000); - } - } - }; - - for (const container of containersToLog) { - const rawName = container.Names?.[0]?.replace(/^\//, '') || container.Id; - const displayName = normalizeContainerName(rawName, stackName); - activeProcesses++; - let lineBuffer = ''; - - const sendOutput = (data: Buffer) => { - if (ws.readyState === WebSocket.OPEN) { - lineBuffer += data.toString(); - const lines = lineBuffer.split(/\r?\n/); - lineBuffer = lines.pop() || ''; - for (const line of lines) { - ws.send(LogFormatter.process(`${displayName} | ${line}`) + '\r\n'); - } - } - }; - - const flushBuffer = () => { - if (lineBuffer && ws.readyState === WebSocket.OPEN) { - ws.send(LogFormatter.process(`${displayName} | ${lineBuffer}`) + '\r\n'); - lineBuffer = ''; - } - }; - - const child = spawn('docker', ['logs', '-f', '-t', '--tail', '100', rawName], { - env: { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' - } - }); - localProcesses.push(child); - child.stdout.on('data', sendOutput); - child.stderr.on('data', sendOutput); - child.on('error', handleProcessEnd); - child.on('close', () => { - flushBuffer(); - handleProcessEnd(); - }); - } - } catch (err) { - if (!isClosed && ws.readyState === WebSocket.OPEN) { - if (!isWaitingForActivity) { - ws.send(`\r\n\x1b[31m[Sencho] Error tracking containers. Retrying...\x1b[0m\r\n`); - isWaitingForActivity = true; - } - setTimeout(startStream, 2000); - } - } - }; - - startStream(); - } - - - /** - * Pinned compose-up with a recovery override layered last - * (`--pull never --no-build`). Used by manual rollback and deploy/update - * compensation. When `invocation` is set, Compose args come from the - * generation capture instead of the live database-derived invocation. - */ - async composeUpWithRecoveryOverride( - stackName: string, - overridePath: string, - ws?: WebSocket, - invocation?: RollbackInvocationRecord | null, - ): Promise { - const stackDir = path.join(this.baseDir, stackName); - const sendOutput = (data: string) => { - if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); - }; - await this.withRegistryAuth(async (env) => { - await this.execute( - 'docker', - await this.buildComposeArgsWithRecoveryOverride( - stackName, - ['up', '-d', '--remove-orphans', '--pull', 'never', '--no-build'], - overridePath, - invocation ?? null, - ), - stackDir, - ws, - true, - env, - getComposeStallTimeoutMs(), - ); - }, sendOutput); - } - - /** - * Authored (+ mesh) compose args with an optional recovery override layered LAST. - * When `invocation` is provided, use its validated captured prefix so recovery - * does not mix restored files with the live deploy-spec / project-env selection. - */ - public async buildComposeArgsWithRecoveryOverride( - stackName: string, - action: string[], - recoveryOverridePath: string | null, - invocation?: RollbackInvocationRecord | null, - ): Promise { - const useCaptured = hasUsableCapturedInvocation(invocation); - const out = useCaptured - ? ['compose', ...this.composePrefixFromCapturedInvocation(stackName, invocation)] - : await this.authoredComposeArgsPrefix(stackName); - - if (useCaptured && invocation.meshEnabled) { - await this.appendCapturedMeshLayer(stackName, out); - } - - if (recoveryOverridePath) { - // Captured invocations already encode their file set; live args may need - // an explicit base (+ user override) so docker compose accepts a trailing -f. - await this.ensureExplicitComposeFiles(stackName, out, !useCaptured); - out.push('-f', recoveryOverridePath); - } - out.push(...action); - return out; - } - - /** - * Re-apply the Mesh override layer when the generation captured Mesh as enabled. - * Uses live ensureStackOverride (same absolute override path semantics as deploy). - */ - private async appendCapturedMeshLayer(stackName: string, args: string[]): Promise { - const overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName); - if (!overridePath) { - throw new Error( - `Captured Mesh-enabled invocation cannot regenerate mesh override for stack "${stackName}"`, - ); - } - await this.ensureExplicitComposeFiles(stackName, args, true); - args.push('-f', overridePath); - } - - /** Slice the global-flag prefix from authoredComposeArgs (no action tokens). */ - private async authoredComposeArgsPrefix(stackName: string): Promise { - const withSentinel = await this.authoredComposeArgs(stackName, ['__SENCHO_ACTION_SENTINEL__']); - const idx = withSentinel.indexOf('__SENCHO_ACTION_SENTINEL__'); - const prefix = idx >= 0 ? withSentinel.slice(0, idx) : withSentinel; - return [...prefix]; - } - - /** - * When args lack `-f`, pin the stack compose file (and optionally the user - * override) so a trailing override layer is accepted by docker compose. - */ - private async ensureExplicitComposeFiles( - stackName: string, - args: string[], - includeUserOverride: boolean, - ): Promise { - if (args.includes('-f')) return; - const fsSvc = FileSystemService.getInstance(this.nodeId); - args.push('-f', await fsSvc.getComposeFilename(stackName)); - if (!includeUserOverride) return; - try { - const userOverride = await fsSvc.getOverrideFilename(stackName); - if (userOverride) args.push('-f', userOverride); - } catch (err) { - const code = (err as { code?: string }).code; - if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') { - throw err; - } - console.warn( - '[ComposeService] could not resolve user compose override while pinning compose files:', - sanitizeForLog((err as Error).message), - ); - } - } - - private resolveValidatedStackDir(stackName: string): string { - const stackDir = path.resolve(this.baseDir, stackName); - if (!isPathWithinBase(stackDir, this.baseDir) || path.resolve(this.baseDir) === stackDir) { - throw new Error('Invalid stack path'); - } - return stackDir; - } - - /** - * Rebuild a spawn-safe compose global-flag prefix from a generation's - * captured invocation. Relative -f / --project-directory paths and absolute - * --env-file paths must stay inside the stack directory. - */ - private composePrefixFromCapturedInvocation( - stackName: string, - invocation: RollbackInvocationRecord, - ): string[] { - const stackDir = this.resolveValidatedStackDir(stackName); - // Empty prefix is valid (single-file auto-discovery at capture time). - const raw = [...invocation.composeArgsPrefix]; - const out: string[] = []; - for (let i = 0; i < raw.length; i++) { - const token = raw[i]; - if (token === '-f' || token === '--file') { - const file = raw[++i]; - if (!file || !isValidRelativeStackPath(file)) { - throw new Error(`Invalid captured compose file path for stack "${stackName}"`); - } - if (!isPathWithinBase(path.resolve(stackDir, file), stackDir)) { - throw new Error(`Captured compose file path escapes stack directory for "${stackName}"`); - } - out.push('-f', file); - continue; - } - if (token === '--env-file') { - const envPath = raw[++i]; - if (!envPath) { - throw new Error(`Invalid captured --env-file for stack "${stackName}"`); - } - const abs = path.resolve(envPath); - if (!isPathWithinBase(abs, stackDir)) { - throw new Error(`Captured env-file path escapes stack directory for "${stackName}"`); - } - out.push('--env-file', abs); - continue; - } - if (token === '--project-directory') { - const dir = raw[++i]; - if (!dir) { - throw new Error(`Invalid captured --project-directory for stack "${stackName}"`); - } - const abs = path.isAbsolute(dir) ? path.resolve(dir) : path.resolve(stackDir, dir); - if (!isPathWithinBase(abs, stackDir)) { - throw new Error(`Captured project-directory escapes stack directory for "${stackName}"`); - } - out.push('--project-directory', abs); - continue; - } - if (token === '-p' || token === '--project-name') { - const name = raw[++i]; - if (!name || name !== (invocation.projectName || stackName)) { - throw new Error(`Captured project name mismatch for stack "${stackName}"`); - } - out.push('-p', stackName); - continue; - } - throw new Error(`Unsupported captured compose flag "${token}" for stack "${stackName}"`); - } - return out; - } - - async updateStack( - stackName: string, - ws?: WebSocket, - atomic?: boolean, - ): Promise<{ recoveryId: string | null }> { - await this.assertRequiredEnvPresent(stackName); - await this.assertSafePilotBindMapping(stackName); - const stackDir = path.join(this.baseDir, stackName); - const debug = isDebugEnabled(); - const t0 = Date.now(); - if (debug) console.debug('[ComposeService:debug] updateStack', { stackName, stackDir, atomic }); - const sendOutput = (data: string) => { - if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); - }; - - // Dynamic import avoids a static cycle (recovery imports getComposeCommandTimeoutMs). - const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); - const recoverySvc = StackUpdateRecoveryService.getInstance(); - let recoveryId: string | null = null; - let handedOff = false; - - try { - sendOutput('=== Validating stack for update ===\n'); - sendOutput('=== Capturing rollback generation ===\n'); - const candidate = await recoverySvc.captureCandidate({ - nodeId: this.nodeId, - stackName, - createdBy: null, - operationKind: 'update', - }); - recoveryId = candidate.id; - - const buildServices = await loadStackBuildServices(this.nodeId, stackName); - const buildAware = buildServices.length > 0; - - try { - await this.withRegistryAuth(async (env) => { - if (buildAware) { - sendOutput('=== Building images ===\n'); - await this.execute( - 'docker', - await this.authoredComposeArgs(stackName, ['build', '--pull']), - stackDir, ws, true, env, getComposeStallTimeoutMs(), - ); - sendOutput('=== Pulling registry images ===\n'); - await this.execute( - 'docker', - await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']), - stackDir, ws, true, env, getComposeStallTimeoutMs(), - ); - } else { - sendOutput('=== Pulling latest images ===\n'); - await this.execute( - 'docker', - await this.authoredComposeArgs(stackName, ['pull']), - stackDir, ws, true, env, getComposeStallTimeoutMs(), - ); - } - }, sendOutput); - } catch (acquireError) { - // Acquisition failure: abandon candidate; leave runtime untouched. - await recoverySvc.abandon(candidate.id); - recoveryId = null; - throw acquireError; - } - - if (!recoverySvc.markAcquired(candidate.id)) { - await recoverySvc.abandon(candidate.id); - throw new Error('Failed to mark recovery generation as acquired'); - } - - const dockerController = DockerController.getInstance(this.nodeId); - sendOutput('=== Classifying legacy orphans ===\n'); - const classified = await dockerController.classifyLegacyOrphansForUpdate(stackName); - if (classified.status === 'classification_failed') { - await recoverySvc.abandon(candidate.id); - recoveryId = null; - throw new Error(`Legacy orphan classification failed: ${classified.error}`); - } - - if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) { - await recoverySvc.abandon(candidate.id); - recoveryId = null; - throw new Error('Failed to hand off recovery generation'); - } - handedOff = true; - if (!recoverySvc.markReconciling(candidate.id)) { - throw new Error('Failed to mark recovery generation as reconciling after handoff'); - } - - if (classified.status === 'orphans') { - sendOutput(`=== Removing ${classified.ids.length} legacy orphan container(s) ===\n`); - const results = await dockerController.removeContainers(classified.ids); - const failed = results.filter((r) => !r.success); - if (failed.length > 0) { - throw new Error( - `Failed to remove ${failed.length} legacy orphan container(s) after handoff`, - ); - } - } - - await this.withRegistryAuth(async (env) => { - sendOutput('=== Recreating containers ===\n'); - await this.execute( - 'docker', - await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), - stackDir, ws, true, env, getComposeStallTimeoutMs(), - ); - }, sendOutput); - - // Immediate verification probe - await new Promise((resolve) => setTimeout(resolve, 3000)); - - const containers = await dockerController.getDocker().listContainers({ - all: true, - filters: { label: [`com.docker.compose.project=${stackName}`] }, - }); - - for (const containerInfo of containers) { - if (containerInfo.State === 'exited') { - const container = dockerController.getDocker().getContainer(containerInfo.Id); - const inspectData = await container.inspect(); - const exitCode = inspectData.State.ExitCode; - if (exitCode !== 0) { - throw this.createContainerCrashError(exitCode); - } - } - } - - if (!recoverySvc.markImmediateVerified(candidate.id)) { - console.warn( - '[ComposeService] Could not CAS immediate_verified for recovery %s', - sanitizeForLog(candidate.id), - ); - } - - sendOutput('=== Stack updated successfully ===\n'); - - // Defer prune until gate retention / gate link: only prune when no active holds - // would be violated. Still honor prune_on_update, but use unified holds so - // candidate/current rollback images are retained. - try { - const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1'; - if (pruneOnUpdate) { - const isImageHeld = buildUnifiedHeldImagePredicate(this.nodeId); - const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld); - const reclaimed = result.reclaimedBytes > 0 - ? ` · reclaimed ${(result.reclaimedBytes / (1024 * 1024)).toFixed(1)} MB` - : ''; - sendOutput(`=== Pruned dangling images${reclaimed} ===\n`); - } - } catch (pruneError) { - console.warn( - 'Failed to prune dangling images after update for %s:', - sanitizeForLog(stackName), - pruneError, - ); - } - - if (debug) { - console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName }); - } - } catch (updateError) { - if (!handedOff && recoveryId) { - await recoverySvc.abandon(recoveryId); - recoveryId = null; - } - if (handedOff && recoveryId) { - sendOutput('\n=== Update failed - restoring previous runtime from recovery generation ===\n'); - const generationId = recoveryId; - const rolledBack = await compensateOrSwallow(() => - recoverySvc.compensateWithCandidate( - generationId, - (overridePath, invocation) => this.composeUpWithRecoveryOverride( - stackName, - overridePath, - ws, - invocation, - ), - ), - ); - throw new ComposeRollbackError(updateError, true, rolledBack); - } - // Pre-handoff failure: abandon already handled on acquire/classify; runtime untouched. - throw updateError; - } - - await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName); - await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName); - try { - await this.refreshExposureCache(stackName); - } catch (err) { - console.warn( - '[ComposeService] Exposure refresh failed after update for %s:', - sanitizeForLog(stackName), - sanitizeForLog(getErrorMessage(err, 'unknown')), - ); - } - return { recoveryId }; - } - - /** - * Service-scoped update: pull (or `build --pull` for a build-backed service) - * and recreate a single service's replicas in place. Always - * `--no-deps --force-recreate`, never `--remove-orphans`, so sibling services - * keep their container ids and StartedAt. No drift re-baseline and no dangling - * prune here; the orchestrator owns per-service post-update reconciliation. - */ - async updateService(stackName: string, serviceName: string, hasBuild: boolean, ws?: WebSocket): Promise { - await this.assertRequiredEnvPresent(stackName); - await this.assertSafePilotBindMapping(stackName); - const stackDir = path.join(this.baseDir, stackName); - const sendOutput = (data: string) => { - if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); - }; - await this.withRegistryAuth(async (env) => { - if (hasBuild) { - sendOutput(`=== Building ${serviceName} ===\n`); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['build', '--pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); - } else { - sendOutput(`=== Pulling ${serviceName} ===\n`); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); - } - sendOutput(`=== Recreating ${serviceName} ===\n`); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); - }, sendOutput); - } - - /** - * Recreate a single service from the image already present locally, without - * pulling or building. Used by service restore after the recovery image id has - * been retagged onto the declared ref (`--pull never --no-build` so Compose - * uses the just-retagged local image). Always `--no-deps --force-recreate`, - * never `--remove-orphans`. - */ - async recreateServiceFromLocal(stackName: string, serviceName: string, ws?: WebSocket): Promise { - await this.assertRequiredEnvPresent(stackName); - await this.assertSafePilotBindMapping(stackName); - const stackDir = path.join(this.baseDir, stackName); - const sendOutput = (data: string) => { - if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); - }; - await this.withRegistryAuth(async (env) => { - sendOutput(`=== Restoring ${serviceName} ===\n`); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', '--pull', 'never', '--no-build', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); - }, sendOutput); - } - - public async downStack(stackName: string, options?: { removeVolumes?: boolean }): Promise { - const stackPath = path.join(this.baseDir, stackName); - try { - const args = options?.removeVolumes - ? ['down', '--volumes', '--remove-orphans'] - : ['down', '--remove-orphans']; - await this.execute('docker', await this.authoredComposeArgs(stackName, args), stackPath, undefined, false); - } catch (error) { - console.warn(`[Teardown] Docker down failed or nothing to clean up for ${sanitizeForLog(stackName)}`); - } - } - - /** - * Enumerate image references declared in a stack's compose file. - * - * Used by the pre-deploy policy gate to decide which images to scan before - * `docker compose up` runs. Path traversal is guarded against the node's - * compose base directory; missing / unreadable compose files or `.env` - * interpolation failures surface as a rejected Promise so the gate can - * block the deploy rather than silently allow it. - */ - public async listStackImages( - stackName: string, - invocation?: RollbackInvocationRecord | null, - ): Promise { - if (!isValidStackName(stackName)) { - throw new Error('Invalid stack path'); - } - const stackDir = this.resolveValidatedStackDir(stackName); - // Prefer a generation-captured invocation so restored-target policy does not - // scan with the live deploy-spec / project-env selection. - const useCaptured = hasUsableCapturedInvocation(invocation); - const fileAndEnvPrefix = useCaptured - ? [...this.composePrefixFromCapturedInvocation(stackName, invocation)] - : [ - ...authoredComposeFileArgs(stackName, this.nodeId), - ...(await authoredComposeEnvFileArgs(stackName, this.nodeId)), - ]; - if (useCaptured && invocation.meshEnabled) { - await this.appendCapturedMeshLayer(stackName, fileAndEnvPrefix); - } - const stdout = await this.captureCompose([...fileAndEnvPrefix, 'config', '--images'], stackDir); - const seen = new Set(); - const images: string[] = []; - for (const raw of stdout.split(/\r?\n/)) { - const line = raw.trim(); - if (!line) continue; - if (line.startsWith('sha256:')) continue; - if (seen.has(line)) continue; - seen.add(line); - images.push(line); - } - return images; - } - - /** Render the effective Compose model and cache the per-stack exposure - * descriptor so the Security posture can join exposed images against - * vulnerability findings without re-rendering config on every poll. - * Best-effort: render or parse failure logs a warning and keeps the - * prior cached descriptor, never failing the deploy. */ - private async refreshExposureCache(stackName: string): Promise { - const result = await this.renderConfig(stackName); - if (result.rendered === null) { - console.warn('[ComposeService] Exposure cache skipped for %s: model not renderable', - sanitizeForLog(stackName)); - return; - } - let parsed: unknown; - try { - parsed = JSON.parse(result.rendered); - } catch { - console.warn('[ComposeService] Exposure cache skipped for %s: unparseable model JSON', - sanitizeForLog(stackName)); - return; - } - const model = parseEffectiveModel(parsed, stackName); - const descriptor = deriveStackExposure(model, stackName, Date.now()); - DatabaseService.getInstance().upsertStackExposure( - this.nodeId, - stackName, - JSON.stringify(descriptor), - descriptor.computedAt, - ); - } - - private captureCompose(args: string[], cwd: string): Promise { - return new Promise((resolve, reject) => { - const child = spawn('docker', ['compose', ...args], { - cwd, - env: { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }, - }); - let stdout = ''; - let stderr = ''; - child.stdout.on('data', (data: Buffer) => { stdout += data.toString(); }); - child.stderr.on('data', (data: Buffer) => { stderr += data.toString(); }); - child.on('error', (err: NodeJS.ErrnoException) => { - const mapped = describeSpawnError(err, { command: 'docker compose' }); - if (mapped.isLowMemory) { - console.warn('[ComposeService] captureCompose spawn failed under memory pressure:', mapped.message); - } - reject(new Error(mapped.message)); - }); - child.on('close', (code) => { - if (code === 0) resolve(stdout); - else reject(new Error(stderr.trim() || `docker compose ${args.join(' ')} failed with code ${code}`)); - }); - }); - } - - /** - * Render the effective compose model as YAML (the default `docker compose - * config` output) with the exact authored invocation and NO mesh override. - * Used by the Git source detach/export contract: the rendered model becomes - * the stack's single compose.yaml. Throws when the render fails or times - * out, so the detach transaction aborts before anything changes. - */ - public async renderComposeYaml(stackName: string): Promise { - if (!isValidStackName(stackName)) { - throw new Error('Invalid stack path'); - } - const baseResolved = path.resolve(this.baseDir); - const stackDir = path.resolve(baseResolved, stackName); - if (!stackDir.startsWith(baseResolved + path.sep)) { - throw new Error('Invalid stack path'); - } - let filePrefix: string[]; - try { - filePrefix = authoredComposeFileArgs(stackName, this.nodeId); - } catch (err) { - throw err instanceof Error ? err : new Error(String(err)); - } - const envFileArgs = await authoredComposeEnvFileArgs(stackName, this.nodeId); - const child = spawn('docker', ['compose', ...filePrefix, ...envFileArgs, 'config'], { - cwd: stackDir, - env: { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }, - }); - return new Promise((resolve, reject) => { - const MAX_OUTPUT = 5 * 1024 * 1024; // 5 MiB cap on each stream - const TIMEOUT_MS = 30_000; - // Accumulate Buffer chunks and decode ONCE at the end: chunk-wise - // toString() can split a multi-byte UTF-8 sequence across a chunk - // boundary and mangle non-ASCII values. - const outChunks: Buffer[] = []; - let outBytes = 0; - let stderr = ''; - let capped = false; - let settled = false; - const timer = setTimeout(() => { - settled = true; - clearTimeout(timer); - try { - child.kill('SIGKILL'); - } catch { - // best effort - } - reject(new Error(`docker compose config timed out after ${TIMEOUT_MS / 1000}s`)); - }, TIMEOUT_MS); - const finish = (error: Error | null) => { - if (settled) return; - settled = true; - clearTimeout(timer); - if (error) reject(error); - else resolve(Buffer.concat(outChunks).toString('utf8')); - }; - child.stdout.on('data', (data: Buffer) => { - if (capped) return; - outBytes += data.length; - if (outBytes > MAX_OUTPUT) { - // A truncated model frequently still parses as YAML; overwriting a - // working compose.yaml with it would be silent corruption. The cap is - // an error, not a truncation. - capped = true; - settled = true; - clearTimeout(timer); - try { - child.kill('SIGKILL'); - } catch { - // best effort - } - reject(new Error(`docker compose config output exceeded ${MAX_OUTPUT} bytes`)); - return; - } - outChunks.push(data); - }); - child.stderr.on('data', (data: Buffer) => { - if (stderr.length < MAX_OUTPUT) stderr += data.toString(); - }); - child.on('close', (code) => { - if (capped) return; - if (code === 0) finish(null); - else finish(new Error(stderr.trim() || `docker compose config exited with code ${code}`)); - }); - child.on('error', (err) => finish(err)); - }); - } - - /** - * Render the fully-resolved effective Compose model via `docker compose - * config --format json`. This is the AUTHORED model: it does NOT splice in - * the Sencho Mesh override, so it stays read-only (the override is - * write-generated) and reflects what the user actually edits. The override - * would also add the managed `sencho_mesh` external network and per-service - * mesh attachments, which would make preflight emit a false "external network - * not found" finding, so rendering the authored model is both safer and more - * accurate here. - * Captures stderr (where Compose reports unset variables) and never rejects - * on a non-zero exit, so the Compose Doctor can turn a failed render into a - * finding rather than an exception. Bounded by a timeout and an output cap. - * Rejects only when the docker binary cannot be spawned. - */ - public async renderConfig( - stackName: string, - ): Promise<{ rendered: string | null; stderr: string; code: number | null; timedOut: boolean }> { - if (!isValidStackName(stackName)) { - throw new Error('Invalid stack path'); - } - // Canonical inline js/path-injection barrier, kept in the same scope as the - // spawn cwd sink below. CodeQL credits neither the wrapped isPathWithinBase - // helper nor a barrier separated from the sink by the Promise-executor - // closure, so the spawn is hoisted out of the executor. startsWith already - // rejects the base dir itself, since base does not start with base + sep. - const baseResolved = path.resolve(this.baseDir); - const stackDir = path.resolve(baseResolved, stackName); - if (!stackDir.startsWith(baseResolved + path.sep)) { - throw new Error('Invalid stack path'); - } - // Render the authored multi-file model (no mesh override) so the Compose Doctor - // sees every override file; single-file stacks get an empty prefix. The env-file - // flag keeps render resolving the same root .env the validator and deploy use. - let filePrefix: string[]; - try { - filePrefix = authoredComposeFileArgs(stackName, this.nodeId); - } catch (err) { - throw err instanceof Error ? err : new Error(String(err)); - } - const envFileArgs = await authoredComposeEnvFileArgs(stackName, this.nodeId); - const child = spawn('docker', ['compose', ...filePrefix, ...envFileArgs, 'config', '--format', 'json'], { - cwd: stackDir, - env: { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }, - }); - return new Promise((resolve, reject) => { - const MAX_OUTPUT = 5 * 1024 * 1024; // 5 MiB cap on each stream - const TIMEOUT_MS = 20_000; - let stdout = ''; - let stderr = ''; - let timedOut = false; - let capped = false; - let settled = false; - const timer = setTimeout(() => { timedOut = true; child.kill('SIGKILL'); }, TIMEOUT_MS); - const finish = (result: { rendered: string | null; stderr: string; code: number | null; timedOut: boolean }) => { - if (settled) return; - settled = true; - clearTimeout(timer); - resolve(result); - }; - child.stdout.on('data', (data: Buffer) => { - stdout += data.toString(); - if (stdout.length > MAX_OUTPUT && !capped) { capped = true; child.kill('SIGKILL'); } - }); - child.stderr.on('data', (data: Buffer) => { - if (stderr.length < MAX_OUTPUT) stderr += data.toString(); - }); - child.on('error', (err: NodeJS.ErrnoException) => { - if (settled) return; - settled = true; - clearTimeout(timer); - reject(new Error(describeSpawnError(err, { command: 'docker compose' }).message)); - }); - child.on('close', (code) => { - if (timedOut) finish({ rendered: null, stderr: stderr.trim() || 'docker compose config timed out', code, timedOut: true }); - else if (capped) finish({ rendered: null, stderr: 'Rendered model exceeded the size limit', code, timedOut: false }); - else if (code === 0) finish({ rendered: stdout, stderr, code, timedOut: false }); - else finish({ rendered: null, stderr: stderr.trim() || `docker compose config failed with code ${code}`, code, timedOut: false }); - }); - }); - } -} +import { spawn } from 'child_process'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import WebSocket from 'ws'; +import DockerController from './DockerController'; +import { DatabaseService } from './DatabaseService'; +import { FileSystemService } from './FileSystemService'; +import { MeshService } from './MeshService'; +import { LogFormatter } from './LogFormatter'; +import { NodeRegistry } from './NodeRegistry'; +import { RegistryService } from './RegistryService'; +import { DriftLedgerService } from './DriftLedgerService'; +import SelfIdentityService from './SelfIdentityService'; +import { parseEffectiveModel } from './preflight/effectiveModel'; +import { deriveStackExposure } from './preflight/exposure'; + +import { isDebugEnabled } from '../utils/debug'; +import { getErrorMessage } from '../utils/errors'; +import { normalizeContainerName } from '../utils/log-parsing'; +import { describeSpawnError } from '../utils/spawnErrors'; +import { isPathWithinBase, isValidStackName, isValidRelativeStackPath } from '../utils/validation'; +import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs'; +import type { RollbackInvocationRecord } from '../types/rollbackGeneration'; +import { parseMissingRequiredVars } from '../helpers/envVarParse'; +import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog'; +import { pathsMatch, resolveHostBindPath } from '../utils/composePathMapping'; +import { loadStackBuildServices } from './ImageUpdateService'; +import { resolveMissingExternalNetworks } from './network/resolveMissingExternalNetworks'; +import { + MissingExternalNetworksError, + type DeployInvocationContext, +} from './network/missingExternalNetworksError'; +import { buildUnifiedHeldImagePredicate } from './recoveryHeldImages'; +import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; +import type { NotificationCategory } from './NotificationService'; + +/** True when a generation capture recorded an invocation object (prefix may be empty). */ +function hasUsableCapturedInvocation( + invocation: RollbackInvocationRecord | null | undefined, +): invocation is RollbackInvocationRecord { + return invocation != null; +} + +function recordNetworkAutoCreatedActivity( + nodeId: number, + stackName: string, + createdNames: string[], + level: 'info' | 'warning', + ctx?: DeployInvocationContext, +): void { + if (createdNames.length === 0) return; + const names = [...createdNames].sort((a, b) => a.localeCompare(b)).join(', '); + const source = ctx?.source ?? 'manual'; + try { + DatabaseService.getInstance().addNotificationHistory(nodeId, { + level, + category: 'network_auto_created' as NotificationCategory, + message: `Auto-created external network(s) for ${stackName}: ${names} (source: ${source})`, + timestamp: Date.now(), + stack_name: stackName, + actor_username: ctx?.actor ?? null, + }); + } catch (error) { + console.error( + '[ComposeService] Failed to record network_auto_created activity for %s:', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + } +} + +export class ComposeRollbackError extends Error { + public readonly rollbackAttempted: boolean; + public readonly rolledBack: boolean; + public readonly originalError: unknown; + + constructor(originalError: unknown, rollbackAttempted: boolean, rolledBack: boolean) { + super(getErrorMessage(originalError, 'Compose operation failed')); + this.name = 'ComposeRollbackError'; + this.rollbackAttempted = rollbackAttempted; + this.rolledBack = rolledBack; + this.originalError = originalError; + Object.setPrototypeOf(this, ComposeRollbackError.prototype); + } +} + +export function getComposeRollbackInfo(error: unknown): { attempted: boolean; rolledBack: boolean } | null { + if (!(error instanceof ComposeRollbackError)) { + return null; + } + return { attempted: error.rollbackAttempted, rolledBack: error.rolledBack }; +} + +function isNonFatalCompensationError(error: unknown): boolean { + const code = (error as { code?: string }).code; + return code === 'HELD_IMAGE_MISSING' || code === 'RECOVERY_PROBE_FAILED'; +} + +async function compensateOrSwallow(compensate: () => Promise): Promise { + try { + return await compensate(); + } catch (compError) { + if (!isNonFatalCompensationError(compError)) throw compError; + return false; + } +} + +const DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS = 30 * 60 * 1000; + +/** Public so other services (e.g. recovery claim leases) can size their own timers off the same ceiling without depending on a private module-local. */ +export function getComposeCommandTimeoutMs(): number { + const configured = Number(process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS); + if (Number.isFinite(configured) && configured > 0) { + return configured; + } + return DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS; +} + +// Idle backstop for long-running pull/recreate steps: if the child emits no +// output for this window while still running, the step is treated as stalled +// and terminated, so a hung `docker compose pull` surfaces a fast failure +// instead of spinning until the much longer command timeout above. Conservative +// by default because a working pull can be briefly silent while a large layer +// extracts; operators on slow links or heavy local builds can raise it. +const DEFAULT_COMPOSE_STALL_TIMEOUT_MS = 10 * 60 * 1000; + +function getComposeStallTimeoutMs(): number { + const configured = Number(process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS); + if (Number.isFinite(configured) && configured > 0) { + return configured; + } + return DEFAULT_COMPOSE_STALL_TIMEOUT_MS; +} + +/** + * ComposeService - local docker compose CLI execution. + * + * In the Distributed API model, remote node compose operations are handled + * by the remote Sencho instance. This service only executes commands locally. + */ +export class ComposeService { + private baseDir: string; + private nodeId: number; + + constructor(nodeId?: number) { + this.nodeId = nodeId ?? NodeRegistry.getInstance().getDefaultNodeId(); + this.baseDir = NodeRegistry.getInstance().getComposeDir(this.nodeId); + } + + public static getInstance(nodeId?: number): ComposeService { + return new ComposeService(nodeId); + } + + /** + * Build the authored `docker compose` argument list for a stack: the validated + * multi-file deploy prefix (ordered `-f` files + `-p ` + + * `--project-directory`) for a Git source with an applied multi-file spec, then + * the Sencho Mesh override file last (highest `-f` precedence) when the stack is + * opted into the mesh, then the action. Single-file / non-git stacks get no file + * prefix, so docker compose's built-in discovery resolves the root compose.yaml, + * byte-identical to the pre-multi-file behavior. The user's source files are + * never mutated. Lifecycle commands (deploy, update, stop/start/restart/down) + * route through this method, so they share one file prefix plus the mesh override. + * Image scans (listStackImages) and the Compose Doctor (renderConfig) reuse the + * same `authoredComposeFileArgs` prefix directly but intentionally omit the mesh + * override, rendering the user's authored model without mesh injection. + */ + /** Public wrapper for dual-arg assembly and recovery Compose invocations. */ + public async buildAuthoredComposeArgs(stackName: string, action: string[]): Promise { + return this.authoredComposeArgs(stackName, action); + } + + public async validateStackForMutation(stackName: string): Promise { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + } + + /** + * Render/validate the exact Compose invocation used by mutating operations + * (authored files, env pins, and generated Mesh override) before capture. + */ + public async validateExactComposeInvocation(stackName: string): Promise { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack path'); + } + const baseResolved = path.resolve(this.baseDir); + const stackDir = path.resolve(baseResolved, stackName); + if (!stackDir.startsWith(baseResolved + path.sep)) { + throw new Error('Invalid stack path'); + } + const args = await this.authoredComposeArgs(stackName, ['config', '--quiet']); + await this.execute('docker', args, stackDir, undefined, true); + } + + private async authoredComposeArgs(stackName: string, action: string[]): Promise { + const args: string[] = ['compose']; + const filePrefix = authoredComposeFileArgs(stackName, this.nodeId); + args.push(...filePrefix); + // Pin env resolution to the root .env when a context dir shifts the project + // directory, so deploy/update resolve the same effective config the validator did. + args.push(...await authoredComposeEnvFileArgs(stackName, this.nodeId)); + + const meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(this.nodeId, stackName); + let overridePath: string | null = null; + try { + overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName); + } catch (err) { + if (meshEnabled) { + throw err instanceof Error + ? err + : new Error(`Mesh override generation failed: ${String(err)}`); + } + console.warn('[ComposeService] mesh override skipped:', sanitizeForLog((err as Error).message)); + } + if (meshEnabled && !overridePath) { + throw new Error( + `Mesh override is required for stack "${stackName}" but could not be generated`, + ); + } + if (overridePath) { + if (filePrefix.length === 0) { + // Single-file stack: passing any -f disables compose's auto-discovery, so name + // the base file explicitly, then re-add the user's implicit override (if any) so + // it is not silently dropped, before layering the mesh override on top. + const fsSvc = FileSystemService.getInstance(this.nodeId); + const baseFilename = await fsSvc.getComposeFilename(stackName); + args.push('-f', baseFilename); + let userOverride: string | null = null; + try { + userOverride = await fsSvc.getOverrideFilename(stackName); + } catch (err) { + // Containment-guard rejections (bad stack name / symlink escape) are hard errors: + // abort the deploy rather than degrade. The "no override" case returns null rather + // than throwing, so any other throw is transient I/O: drop the override and proceed + // (logging the consequence) instead of failing the deploy. + const code = (err as { code?: string }).code; + if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') { + throw err; + } + console.warn('[ComposeService] could not resolve user compose override; deploying without it:', sanitizeForLog((err as Error).message)); + } + if (userOverride) { + args.push('-f', userOverride); + } + } + args.push('-f', overridePath); + } + args.push(...action); + return args; + } + + private execute( + command: string, + args: string[], + cwd: string, + ws?: WebSocket, + throwOnError = true, + env?: Record, + // When set, terminate the child if it emits no output for this long while + // still running (idle stall backstop). Appended last so the existing + // registry-auth call sites that pass `env` are unaffected. + idleTimeoutMs?: number + ): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd, + env: env ?? { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + } + }); + + let errorLog = ''; + let settled = false; + let exited = false; + let pendingTerminationError: Error | null = null; + const timeoutMs = getComposeCommandTimeoutMs(); + let timeout: ReturnType | null = null; + let forceKillTimeout: ReturnType | null = null; + let idleTimeout: ReturnType | null = null; + + const sendOutput = (text: string) => { + if (ws && ws.readyState === WebSocket.OPEN) { + ws.send(text); + } + }; + + const cleanup = () => { + if (timeout) { + clearTimeout(timeout); + timeout = null; + } + if (forceKillTimeout) { + clearTimeout(forceKillTimeout); + forceKillTimeout = null; + } + if (idleTimeout) { + clearTimeout(idleTimeout); + idleTimeout = null; + } + }; + + const finish = (complete: () => void) => { + if (settled) return; + settled = true; + cleanup(); + complete(); + }; + + const terminateChild = (error: Error) => { + pendingTerminationError = pendingTerminationError ?? error; + if (exited) return; + try { + child.kill('SIGTERM'); + } catch (error) { + console.warn('[ComposeService] Failed to terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + } + forceKillTimeout = setTimeout(() => { + if (exited) return; + try { + child.kill('SIGKILL'); + } catch (error) { + console.warn('[ComposeService] Failed to force terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + } + }, 5000); + }; + + // Idle stall backstop. Armed once below and reset on every output chunk; + // if it ever fires, the step has been silent for idleTimeoutMs while still + // running, so terminate it. Never rearmed after a termination is pending or + // the child has exited, so it cannot re-fire during the SIGTERM grace. + const armIdleTimeout = () => { + if (idleTimeoutMs === undefined) return; + if (exited || settled || pendingTerminationError) return; + if (idleTimeout) clearTimeout(idleTimeout); + idleTimeout = setTimeout(() => { + const seconds = Math.round(idleTimeoutMs / 1000); + sendOutput(`=== No output for ${seconds}s; the operation appears stalled and was stopped ===\n`); + terminateChild(new Error(`STACK_STALLED_OUTPUT: no output for ${seconds}s`)); + }, idleTimeoutMs); + }; + + // The progress socket is output-only: a deploy/update/down is owned by the + // HTTP request that started it, so closing or losing the socket (the user + // minimizes the panel, navigates away, or the connection blips) must not + // terminate the compose process. Termination is driven solely by the + // command timeout here and the optional idle stall backstop above. + timeout = setTimeout(() => { + const message = `Command timed out after ${Math.round(timeoutMs / 1000)}s`; + sendOutput(`${message}\n`); + terminateChild(new Error(message)); + }, timeoutMs); + + armIdleTimeout(); + + const onData = (data: Buffer) => { + const text = data.toString(); + errorLog += text; + sendOutput(text); + armIdleTimeout(); + }; + + child.stdout.on('data', onData); + child.stderr.on('data', onData); + + child.on('close', (code: number | null) => { + exited = true; + finish(() => { + sendOutput(`Command exited with code ${code}\n`); + if (pendingTerminationError) { + if (throwOnError) reject(pendingTerminationError); + else resolve(); + return; + } + if (code === 0) resolve(); + else if (throwOnError) reject(new Error(redactSensitiveText(errorLog.trim()) || `Command failed with code ${code}`)); + else resolve(); + }); + }); + + child.on('error', (error: Error & { code?: string }) => { + exited = true; + finish(() => { + const mapped = describeSpawnError(error as NodeJS.ErrnoException, { command }); + const message = redactSensitiveText(mapped.message); + sendOutput(`Error: ${message}\n`); + if (mapped.isLowMemory) { + console.warn('[ComposeService] spawn failed under memory pressure:', message); + } + if (pendingTerminationError) { + if (throwOnError) reject(pendingTerminationError); + else resolve(); + return; + } + if (throwOnError) reject(new Error(message)); + else resolve(); + }); + }); + }); + } + + private async withRegistryAuth( + fn: (env: Record) => Promise, + sendOutput?: (data: string) => void, + ): Promise { + const registries = DatabaseService.getInstance().getRegistries(); + if (registries.length === 0) { + return fn({ + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + }); + } + + const { config, warnings } = await RegistryService.getInstance().resolveDockerConfig(); + if (warnings.length > 0 && sendOutput) { + for (const warning of warnings) { + sendOutput(`[Sencho] Warning: ${warning}\n`); + } + } + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-docker-')); + const configPath = path.join(tmpDir, 'config.json'); + + try { + fs.writeFileSync(configPath, JSON.stringify(config), { mode: 0o600 }); + return await fn({ + ...process.env, + DOCKER_CONFIG: tmpDir, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + }); + } finally { + // Best-effort cleanup; each step runs independently so a file that was never + // written (e.g., writeFileSync threw) does not prevent the directory removal. + try { fs.unlinkSync(configPath); } catch { /* file may not exist */ } + try { fs.rmdirSync(tmpDir); } catch (e) { + console.warn('[ComposeService] Could not remove temp Docker config dir:', (e as Error).message); + } + } + } + + private async createAtomicBackup( + stackName: string, + operation: 'deployment' | 'update', + sendOutput: (data: string) => void, + ): Promise { + try { + const fsSvc = FileSystemService.getInstance(this.nodeId); + await fsSvc.backupStackFiles(stackName); + sendOutput(`=== Backup created for atomic ${operation} ===\n`); + } catch (error) { + console.error('Atomic backup failed for %s:', sanitizeForLog(stackName), getErrorMessage(error, 'unknown error')); + sendOutput(`=== Atomic ${operation} backup failed. Operation aborted ===\n`); + throw new Error(`Atomic ${operation} backup failed: ${getErrorMessage(error, 'unknown error')}`); + } + } + + private async restoreAtomicBackup( + stackName: string, + stackDir: string, + ws: WebSocket | undefined, + sendOutput: (data: string) => void, + ): Promise { + try { + const fsSvc = FileSystemService.getInstance(this.nodeId); + await fsSvc.restoreStackFiles(stackName); + await this.withRegistryAuth(async (env) => { + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env); + }, sendOutput); + sendOutput('=== Restored previous compose and env files ===\n'); + return true; + } catch (rollbackError) { + console.error('Rollback failed for %s:', sanitizeForLog(stackName), getErrorMessage(rollbackError, 'unknown error')); + sendOutput('=== Rollback failed. Manual intervention may be required ===\n'); + return false; + } + } + + private createContainerCrashError(exitCode: number): Error { + return new Error( + `CONTAINER_CRASHED\nExit Code: ${exitCode}\nContainer exited after deployment. Check container logs for details.` + ); + } + + async runCommand(stackName: string, action: 'down' | 'start' | 'stop' | 'restart', ws?: WebSocket): Promise { + const stackDir = path.join(this.baseDir, stackName); + await this.execute('docker', await this.authoredComposeArgs(stackName, [action]), stackDir, ws); + } + + /** Interactive compose down (Take down UI / POST /down). Plain `down` by default. */ + async runDown(stackName: string, options?: { removeVolumes?: boolean }, ws?: WebSocket): Promise { + const stackDir = path.join(this.baseDir, stackName); + const args = options?.removeVolumes ? ['down', '--volumes'] : ['down']; + await this.execute('docker', await this.authoredComposeArgs(stackName, args), stackDir, ws); + } + + /** + * Opt-in guard: when `env_block_deploy_on_missing_required` is enabled, refuse a + * deploy whose required `${VAR:?err}` variables are unset OR empty, before any + * backup, cleanup, pull, or `up` runs. Compose's own resolution is authoritative + * (it passes process.env), and on the failing path it emits no rendered model, so + * no env value is materialized. Default off and any settings-read failure both + * fall through without blocking. + */ + private async assertRequiredEnvPresent(stackName: string): Promise { + let enabled = false; + try { + enabled = DatabaseService.getInstance().getGlobalSettings()['env_block_deploy_on_missing_required'] === '1'; + } catch { + return; // safe default: a settings-read failure never blocks a deploy + } + if (!enabled) return; + const result = await this.renderConfig(stackName); + const missing = parseMissingRequiredVars(result.stderr); + if (missing.length === 0) return; + const plural = missing.length > 1; + throw new Error( + `Deploy blocked: required environment variable${plural ? 's' : ''} ${missing.join(', ')} ` + + `${plural ? 'are' : 'is'} missing. Define ${plural ? 'them' : 'it'} in a .env or env_file, then deploy again.`, + ); + } + + private async assertSafePilotBindMapping(stackName: string): Promise { + if (process.env.SENCHO_MODE !== 'pilot') return; + + let mounts: Array<{ source: string; destination: string }> | null; + try { + mounts = await SelfIdentityService.getInstance().getBindMounts(); + } catch (error) { + console.warn('[ComposeService] Could not verify pilot compose path mapping:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + return; + } + if (mounts === null) return; + + const composeDir = path.resolve(this.baseDir); + const hostComposeDir = resolveHostBindPath(composeDir, mounts); + if (!hostComposeDir || pathsMatch(hostComposeDir, composeDir)) return; + + const rendered = await this.renderConfig(stackName); + if (rendered.rendered === null) return; + + let parsed: unknown; + try { + parsed = JSON.parse(rendered.rendered); + } catch (error) { + console.warn('[ComposeService] Could not inspect rendered binds for pilot path safety:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + return; + } + const model = parseEffectiveModel(parsed, stackName); + const unsafeBind = model.services + .flatMap((service) => service.binds) + .find((bind) => isPathWithinBase(path.resolve(bind.source), composeDir)); + if (!unsafeBind) return; + + throw new Error( + `Deploy blocked: relative bind mounts resolve under ${composeDir}, but the host path is ${hostComposeDir}. ` + + `Use a 1:1 mount with the same absolute path on the host and in the Pilot Agent, then retry.`, + ); + } + + /** + * Missing-external gate: after env/Pilot asserts, before atomic backup. + * Creates safe bridge networks only when the opt-in setting is on. + */ + private async ensureExternalNetworksForDeploy( + stackName: string, + ctx?: DeployInvocationContext, + ): Promise { + const resolved = await resolveMissingExternalNetworks(this.nodeId, stackName); + if (resolved.status === 'render_unavailable') { + throw new MissingExternalNetworksError({ + kind: 'unavailable', + message: 'Sencho could not render this stack\'s Compose model to check external networks.', + }); + } + if (resolved.status === 'runtime_unavailable') { + // No declared externals: nothing to verify; proceed. + if (resolved.declaredExternalCount === 0) return; + throw new MissingExternalNetworksError({ + kind: 'unavailable', + message: 'Sencho could not read Docker networking state to check external networks.', + }); + } + + if (resolved.networks.length === 0) return; + + const unsafe = resolved.networks.filter((n) => !n.safe); + if (unsafe.length > 0) { + throw new MissingExternalNetworksError({ + kind: 'unsupported', + message: 'One or more missing external networks cannot be created safely by Sencho.', + networks: resolved.networks, + }); + } + + if (!resolved.autoCreateEnabled) { + throw new MissingExternalNetworksError({ + kind: 'prompt', + message: 'One or more external networks required by this stack are missing on this node.', + networks: resolved.networks, + }); + } + + const docker = DockerController.getInstance(this.nodeId); + const createdNames: string[] = []; + const recordCreatedNetworks = (level: 'info' | 'warning') => { + if (createdNames.length === 0) return; + invalidateNodeCaches(this.nodeId); + recordNetworkAutoCreatedActivity(this.nodeId, stackName, createdNames, level, ctx); + }; + + for (const network of resolved.networks) { + try { + await docker.createNetwork({ Name: network.name, Driver: 'bridge' }); + createdNames.push(network.name); + } catch (createErr) { + // Authoritative re-check: continue only if the network now exists. + let exists = false; + try { + const knownStacks = await FileSystemService.getInstance(this.nodeId).getStacks(); + const snapshot = await docker.getDependencySnapshot(knownStacks); + exists = snapshot.networks.some((n) => n.name === network.name); + } catch (snapErr) { + console.warn( + '[ComposeService] Post-create snapshot failed for %s:', + sanitizeForLog(network.name), + sanitizeForLog(getErrorMessage(snapErr, 'unknown')), + ); + } + if (!exists) { + recordCreatedNetworks('warning'); + throw new MissingExternalNetworksError({ + kind: 'create_failed', + message: `Failed to create external network "${network.name}".`, + networks: resolved.networks, + createdNames, + remainingNames: resolved.networks + .map((missingNetwork) => missingNetwork.name) + .filter((name) => !createdNames.includes(name)), + }); + } + // Race-existing: do not record in createdNames. + } + } + + // Re-resolve before Compose. + const recheck = await resolveMissingExternalNetworks(this.nodeId, stackName); + if (recheck.status !== 'ok' || recheck.networks.length > 0) { + recordCreatedNetworks('warning'); + throw new MissingExternalNetworksError({ + kind: recheck.status === 'ok' ? 'create_failed' : 'unavailable', + message: 'External networks were still missing after automatic creation.', + networks: recheck.networks, + createdNames, + remainingNames: recheck.networks.map((n) => n.name), + }); + } + + recordCreatedNetworks('info'); + } + + async deployStack( + stackName: string, + ws?: WebSocket, + atomic?: boolean, + ctx?: DeployInvocationContext, + ): Promise<{ recoveryId: string | null }> { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + await this.ensureExternalNetworksForDeploy(stackName, ctx); + + const stackDir = path.join(this.baseDir, stackName); + const debug = isDebugEnabled(); + const t0 = Date.now(); + if (debug) console.debug('[ComposeService:debug] deployStack', { stackName, stackDir, atomic }); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + const recoverySvc = atomic ? StackUpdateRecoveryService.getInstance() : null; + let recoveryId: string | null = null; + let handedOff = false; + + if (atomic && recoverySvc) { + sendOutput('=== Capturing rollback generation for atomic deploy ===\n'); + const candidate = await recoverySvc.captureCandidate({ + nodeId: this.nodeId, + stackName, + createdBy: 'atomic-deploy', + operationKind: 'deployment', + }); + recoveryId = candidate.id; + if (!recoverySvc.markAcquired(candidate.id)) { + await recoverySvc.abandon(candidate.id); + throw new Error('Failed to mark recovery generation as acquired'); + } + if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) { + await recoverySvc.abandon(candidate.id); + throw new Error('Failed to hand off recovery generation'); + } + handedOff = true; + if (!recoverySvc.markReconciling(candidate.id)) { + throw new Error('Failed to mark recovery generation as reconciling after handoff'); + } + } + + try { + try { + const dockerController = DockerController.getInstance(this.nodeId); + const legacyOrphans = await dockerController.getLegacyOrphanContainersByStack(stackName); + if (legacyOrphans.length > 0) { + sendOutput(`=== Cleaning up legacy orphan containers before deployment ===\n`); + await dockerController.removeContainers(legacyOrphans.map((c) => c.Id)); + } + } catch (e) { + console.warn('Failed to clean up legacy containers for %s:', sanitizeForLog(stackName), e); + } + + await this.withRegistryAuth(async (env) => { + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs()); + }, sendOutput); + + // Post-Deploy Health Probe + await new Promise(resolve => setTimeout(resolve, 3000)); + + const dockerController = DockerController.getInstance(this.nodeId); + const containers = await dockerController.getDocker().listContainers({ + all: true, + filters: { label: [`com.docker.compose.project=${stackName}`] } + }); + + for (const containerInfo of containers) { + if (containerInfo.State === 'exited') { + const container = dockerController.getDocker().getContainer(containerInfo.Id); + const inspectData = await container.inspect(); + const exitCode = inspectData.State.ExitCode; + + if (exitCode !== 0) { + throw this.createContainerCrashError(exitCode); + } + } + } + + if (atomic && recoverySvc && recoveryId) { + if (!recoverySvc.markImmediateVerified(recoveryId)) { + console.warn( + '[ComposeService] Could not CAS immediate_verified for recovery %s', + sanitizeForLog(recoveryId), + ); + } + } + if (debug) console.debug(`[ComposeService:debug] deployStack completed in ${Date.now() - t0}ms`, { stackName }); + } catch (deployError) { + if (atomic && recoverySvc && handedOff && recoveryId) { + sendOutput('\n=== Deployment failed - restoring previous runtime from recovery generation ===\n'); + const generationId = recoveryId; + const rolledBack = await compensateOrSwallow(() => + recoverySvc.compensateWithCandidate( + generationId, + (overridePath, invocation) => this.composeUpWithRecoveryOverride( + stackName, + overridePath, + ws, + invocation, + ), + ), + ); + throw new ComposeRollbackError(deployError, true, rolledBack); + } + if (atomic && recoverySvc && recoveryId && !handedOff) { + await recoverySvc.abandon(recoveryId); + } + throw deployError; + } + // Reached only on a successful deploy (the catch above always rethrows). Record + // the drift baseline here so every deploy path gets one, not just the manual + // route: bulk, Git-source, App Store, scheduler, and webhook deploys all funnel + // through this method. Internally guarded; awaited so it cannot race later work. + await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName); + // Reconcile the ledger against the just-deployed runtime: findings this deploy + // fixed are resolved and any it left are recorded (and surfaced in the activity + // feed) now, instead of waiting for someone to open the Drift tab. The rollback + // route re-deploys through this method, so it is covered; a failed atomic deploy + // instead restores the previous files and throws above, so that recovery path + // reconciles on its next deploy or scan, not here. Best-effort internally. + await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName); + // Refresh the exposure cache so posture reflects the just-deployed model. + // Best-effort: a refresh failure logs a warning but never fails the deploy. + try { + await this.refreshExposureCache(stackName); + } catch (err) { + console.warn('[ComposeService] Exposure refresh failed after deploy for %s:', + sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown'))); + } + return { recoveryId }; + } + + streamLogs(stackName: string, ws: WebSocket) { + let isClosed = false; + let isFirstRun = true; + let isWaitingForActivity = false; + + ws.on('close', () => { isClosed = true; }); + + const startStream = async () => { + if (isClosed || ws.readyState !== WebSocket.OPEN) return; + + // Canonical js/path-injection barrier at the fs.stat sink. + const baseResolved = path.resolve(this.baseDir); + const stackDir = path.resolve(baseResolved, stackName); + let stackDirGone = !stackDir.startsWith(baseResolved + path.sep); + if (!stackDirGone) { + try { + const st = await fs.promises.stat(stackDir); + stackDirGone = !st.isDirectory(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + stackDirGone = true; + } + // Other stat failures: keep the existing compose-ps path. + } + } + if (stackDirGone) { + if (ws.readyState === WebSocket.OPEN) { + ws.send(`\r\n\x1b[33m[Sencho] Stack directory is gone. Log stream idle.\x1b[0m\r\n`); + } + return; + } + + try { + const dockerController = DockerController.getInstance(this.nodeId); + const containers = await dockerController.getContainersByStack(stackName); + + if (!containers || containers.length === 0) { + if (!isWaitingForActivity) { + ws.send(`\r\n\x1b[33m[Sencho] No containers found. Waiting for activity...\x1b[0m\r\n`); + isWaitingForActivity = true; + } + setTimeout(startStream, 2000); + return; + } + + const runningContainers = containers.filter((c: any) => c.State === 'running'); + + if (!isFirstRun && runningContainers.length === 0) { + if (!isWaitingForActivity) { + ws.send(`\r\n\x1b[33m[Sencho] Log stream ended. Waiting for container activity...\x1b[0m\r\n`); + isWaitingForActivity = true; + } + setTimeout(startStream, 2000); + return; + } + + const containersToLog = isFirstRun ? containers : runningContainers; + isFirstRun = false; + isWaitingForActivity = false; + + let activeProcesses = 0; + let streamEndedHandled = false; + const localProcesses: ReturnType[] = []; + + const onWsClose = () => { + localProcesses.forEach(cp => { try { cp.kill(); } catch { } }); + }; + + ws.on('close', onWsClose); + + const handleProcessEnd = () => { + activeProcesses--; + if (activeProcesses <= 0 && !streamEndedHandled) { + streamEndedHandled = true; + ws.removeListener('close', onWsClose); + if (!isClosed && ws.readyState === WebSocket.OPEN) { + setTimeout(startStream, 1000); + } + } + }; + + for (const container of containersToLog) { + const rawName = container.Names?.[0]?.replace(/^\//, '') || container.Id; + const displayName = normalizeContainerName(rawName, stackName); + activeProcesses++; + let lineBuffer = ''; + + const sendOutput = (data: Buffer) => { + if (ws.readyState === WebSocket.OPEN) { + lineBuffer += data.toString(); + const lines = lineBuffer.split(/\r?\n/); + lineBuffer = lines.pop() || ''; + for (const line of lines) { + ws.send(LogFormatter.process(`${displayName} | ${line}`) + '\r\n'); + } + } + }; + + const flushBuffer = () => { + if (lineBuffer && ws.readyState === WebSocket.OPEN) { + ws.send(LogFormatter.process(`${displayName} | ${lineBuffer}`) + '\r\n'); + lineBuffer = ''; + } + }; + + const child = spawn('docker', ['logs', '-f', '-t', '--tail', '100', rawName], { + env: { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + } + }); + localProcesses.push(child); + child.stdout.on('data', sendOutput); + child.stderr.on('data', sendOutput); + child.on('error', handleProcessEnd); + child.on('close', () => { + flushBuffer(); + handleProcessEnd(); + }); + } + } catch (err) { + if (!isClosed && ws.readyState === WebSocket.OPEN) { + if (!isWaitingForActivity) { + ws.send(`\r\n\x1b[31m[Sencho] Error tracking containers. Retrying...\x1b[0m\r\n`); + isWaitingForActivity = true; + } + setTimeout(startStream, 2000); + } + } + }; + + startStream(); + } + + + /** + * Pinned compose-up with a recovery override layered last + * (`--pull never --no-build`). Used by manual rollback and deploy/update + * compensation. When `invocation` is set, Compose args come from the + * generation capture instead of the live database-derived invocation. + */ + async composeUpWithRecoveryOverride( + stackName: string, + overridePath: string, + ws?: WebSocket, + invocation?: RollbackInvocationRecord | null, + ): Promise { + const stackDir = path.join(this.baseDir, stackName); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + await this.withRegistryAuth(async (env) => { + await this.execute( + 'docker', + await this.buildComposeArgsWithRecoveryOverride( + stackName, + ['up', '-d', '--remove-orphans', '--pull', 'never', '--no-build'], + overridePath, + invocation ?? null, + ), + stackDir, + ws, + true, + env, + getComposeStallTimeoutMs(), + ); + }, sendOutput); + } + + /** + * Authored (+ mesh) compose args with an optional recovery override layered LAST. + * When `invocation` is provided, use its validated captured prefix so recovery + * does not mix restored files with the live deploy-spec / project-env selection. + */ + public async buildComposeArgsWithRecoveryOverride( + stackName: string, + action: string[], + recoveryOverridePath: string | null, + invocation?: RollbackInvocationRecord | null, + ): Promise { + const useCaptured = hasUsableCapturedInvocation(invocation); + const out = useCaptured + ? ['compose', ...this.composePrefixFromCapturedInvocation(stackName, invocation)] + : await this.authoredComposeArgsPrefix(stackName); + + if (useCaptured && invocation.meshEnabled) { + await this.appendCapturedMeshLayer(stackName, out); + } + + if (recoveryOverridePath) { + // Captured invocations already encode their file set; live args may need + // an explicit base (+ user override) so docker compose accepts a trailing -f. + await this.ensureExplicitComposeFiles(stackName, out, !useCaptured); + out.push('-f', recoveryOverridePath); + } + out.push(...action); + return out; + } + + /** + * Re-apply the Mesh override layer when the generation captured Mesh as enabled. + * Uses live ensureStackOverride (same absolute override path semantics as deploy). + */ + private async appendCapturedMeshLayer(stackName: string, args: string[]): Promise { + const overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName); + if (!overridePath) { + throw new Error( + `Captured Mesh-enabled invocation cannot regenerate mesh override for stack "${stackName}"`, + ); + } + await this.ensureExplicitComposeFiles(stackName, args, true); + args.push('-f', overridePath); + } + + /** Slice the global-flag prefix from authoredComposeArgs (no action tokens). */ + private async authoredComposeArgsPrefix(stackName: string): Promise { + const withSentinel = await this.authoredComposeArgs(stackName, ['__SENCHO_ACTION_SENTINEL__']); + const idx = withSentinel.indexOf('__SENCHO_ACTION_SENTINEL__'); + const prefix = idx >= 0 ? withSentinel.slice(0, idx) : withSentinel; + return [...prefix]; + } + + /** + * When args lack `-f`, pin the stack compose file (and optionally the user + * override) so a trailing override layer is accepted by docker compose. + */ + private async ensureExplicitComposeFiles( + stackName: string, + args: string[], + includeUserOverride: boolean, + ): Promise { + if (args.includes('-f')) return; + const fsSvc = FileSystemService.getInstance(this.nodeId); + args.push('-f', await fsSvc.getComposeFilename(stackName)); + if (!includeUserOverride) return; + try { + const userOverride = await fsSvc.getOverrideFilename(stackName); + if (userOverride) args.push('-f', userOverride); + } catch (err) { + const code = (err as { code?: string }).code; + if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') { + throw err; + } + console.warn( + '[ComposeService] could not resolve user compose override while pinning compose files:', + sanitizeForLog((err as Error).message), + ); + } + } + + private resolveValidatedStackDir(stackName: string): string { + const stackDir = path.resolve(this.baseDir, stackName); + if (!isPathWithinBase(stackDir, this.baseDir) || path.resolve(this.baseDir) === stackDir) { + throw new Error('Invalid stack path'); + } + return stackDir; + } + + /** + * Rebuild a spawn-safe compose global-flag prefix from a generation's + * captured invocation. Relative -f / --project-directory paths and absolute + * --env-file paths must stay inside the stack directory. + */ + private composePrefixFromCapturedInvocation( + stackName: string, + invocation: RollbackInvocationRecord, + ): string[] { + const stackDir = this.resolveValidatedStackDir(stackName); + // Empty prefix is valid (single-file auto-discovery at capture time). + const raw = [...invocation.composeArgsPrefix]; + const out: string[] = []; + for (let i = 0; i < raw.length; i++) { + const token = raw[i]; + if (token === '-f' || token === '--file') { + const file = raw[++i]; + if (!file || !isValidRelativeStackPath(file)) { + throw new Error(`Invalid captured compose file path for stack "${stackName}"`); + } + if (!isPathWithinBase(path.resolve(stackDir, file), stackDir)) { + throw new Error(`Captured compose file path escapes stack directory for "${stackName}"`); + } + out.push('-f', file); + continue; + } + if (token === '--env-file') { + const envPath = raw[++i]; + if (!envPath) { + throw new Error(`Invalid captured --env-file for stack "${stackName}"`); + } + const abs = path.resolve(envPath); + if (!isPathWithinBase(abs, stackDir)) { + throw new Error(`Captured env-file path escapes stack directory for "${stackName}"`); + } + out.push('--env-file', abs); + continue; + } + if (token === '--project-directory') { + const dir = raw[++i]; + if (!dir) { + throw new Error(`Invalid captured --project-directory for stack "${stackName}"`); + } + const abs = path.isAbsolute(dir) ? path.resolve(dir) : path.resolve(stackDir, dir); + if (!isPathWithinBase(abs, stackDir)) { + throw new Error(`Captured project-directory escapes stack directory for "${stackName}"`); + } + out.push('--project-directory', abs); + continue; + } + if (token === '-p' || token === '--project-name') { + const name = raw[++i]; + if (!name || name !== (invocation.projectName || stackName)) { + throw new Error(`Captured project name mismatch for stack "${stackName}"`); + } + out.push('-p', stackName); + continue; + } + throw new Error(`Unsupported captured compose flag "${token}" for stack "${stackName}"`); + } + return out; + } + + async updateStack( + stackName: string, + ws?: WebSocket, + atomic?: boolean, + ): Promise<{ recoveryId: string | null }> { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + const stackDir = path.join(this.baseDir, stackName); + const debug = isDebugEnabled(); + const t0 = Date.now(); + if (debug) console.debug('[ComposeService:debug] updateStack', { stackName, stackDir, atomic }); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + + // Dynamic import avoids a static cycle (recovery imports getComposeCommandTimeoutMs). + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + const recoverySvc = StackUpdateRecoveryService.getInstance(); + let recoveryId: string | null = null; + let handedOff = false; + + try { + sendOutput('=== Validating stack for update ===\n'); + sendOutput('=== Capturing rollback generation ===\n'); + const candidate = await recoverySvc.captureCandidate({ + nodeId: this.nodeId, + stackName, + createdBy: null, + operationKind: 'update', + }); + recoveryId = candidate.id; + + const buildServices = await loadStackBuildServices(this.nodeId, stackName); + const buildAware = buildServices.length > 0; + + try { + await this.withRegistryAuth(async (env) => { + if (buildAware) { + sendOutput('=== Building images ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['build', '--pull']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + sendOutput('=== Pulling registry images ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + } else { + sendOutput('=== Pulling latest images ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['pull']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + } + }, sendOutput); + } catch (acquireError) { + // Acquisition failure: abandon candidate; leave runtime untouched. + await recoverySvc.abandon(candidate.id); + recoveryId = null; + throw acquireError; + } + + if (!recoverySvc.markAcquired(candidate.id)) { + await recoverySvc.abandon(candidate.id); + throw new Error('Failed to mark recovery generation as acquired'); + } + + const dockerController = DockerController.getInstance(this.nodeId); + sendOutput('=== Classifying legacy orphans ===\n'); + const classified = await dockerController.classifyLegacyOrphansForUpdate(stackName); + if (classified.status === 'classification_failed') { + await recoverySvc.abandon(candidate.id); + recoveryId = null; + throw new Error(`Legacy orphan classification failed: ${classified.error}`); + } + + if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) { + await recoverySvc.abandon(candidate.id); + recoveryId = null; + throw new Error('Failed to hand off recovery generation'); + } + handedOff = true; + if (!recoverySvc.markReconciling(candidate.id)) { + throw new Error('Failed to mark recovery generation as reconciling after handoff'); + } + + if (classified.status === 'orphans') { + sendOutput(`=== Removing ${classified.ids.length} legacy orphan container(s) ===\n`); + const results = await dockerController.removeContainers(classified.ids); + const failed = results.filter((r) => !r.success); + if (failed.length > 0) { + throw new Error( + `Failed to remove ${failed.length} legacy orphan container(s) after handoff`, + ); + } + } + + await this.withRegistryAuth(async (env) => { + sendOutput('=== Recreating containers ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + }, sendOutput); + + // Immediate verification probe + await new Promise((resolve) => setTimeout(resolve, 3000)); + + const containers = await dockerController.getDocker().listContainers({ + all: true, + filters: { label: [`com.docker.compose.project=${stackName}`] }, + }); + + for (const containerInfo of containers) { + if (containerInfo.State === 'exited') { + const container = dockerController.getDocker().getContainer(containerInfo.Id); + const inspectData = await container.inspect(); + const exitCode = inspectData.State.ExitCode; + if (exitCode !== 0) { + throw this.createContainerCrashError(exitCode); + } + } + } + + if (!recoverySvc.markImmediateVerified(candidate.id)) { + console.warn( + '[ComposeService] Could not CAS immediate_verified for recovery %s', + sanitizeForLog(candidate.id), + ); + } + + sendOutput('=== Stack updated successfully ===\n'); + + // Defer prune until gate retention / gate link: only prune when no active holds + // would be violated. Still honor prune_on_update, but use unified holds so + // candidate/current rollback images are retained. + try { + const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1'; + if (pruneOnUpdate) { + const isImageHeld = buildUnifiedHeldImagePredicate(this.nodeId); + const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld); + const reclaimed = result.reclaimedBytes > 0 + ? ` · reclaimed ${(result.reclaimedBytes / (1024 * 1024)).toFixed(1)} MB` + : ''; + sendOutput(`=== Pruned dangling images${reclaimed} ===\n`); + } + } catch (pruneError) { + console.warn( + 'Failed to prune dangling images after update for %s:', + sanitizeForLog(stackName), + pruneError, + ); + } + + if (debug) { + console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName }); + } + } catch (updateError) { + if (!handedOff && recoveryId) { + await recoverySvc.abandon(recoveryId); + recoveryId = null; + } + if (handedOff && recoveryId) { + sendOutput('\n=== Update failed - restoring previous runtime from recovery generation ===\n'); + const generationId = recoveryId; + const rolledBack = await compensateOrSwallow(() => + recoverySvc.compensateWithCandidate( + generationId, + (overridePath, invocation) => this.composeUpWithRecoveryOverride( + stackName, + overridePath, + ws, + invocation, + ), + ), + ); + throw new ComposeRollbackError(updateError, true, rolledBack); + } + // Pre-handoff failure: abandon already handled on acquire/classify; runtime untouched. + throw updateError; + } + + await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName); + await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName); + try { + await this.refreshExposureCache(stackName); + } catch (err) { + console.warn( + '[ComposeService] Exposure refresh failed after update for %s:', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(err, 'unknown')), + ); + } + return { recoveryId }; + } + + /** + * Service-scoped update: pull (or `build --pull` for a build-backed service) + * and recreate a single service's replicas in place. Always + * `--no-deps --force-recreate`, never `--remove-orphans`, so sibling services + * keep their container ids and StartedAt. No drift re-baseline and no dangling + * prune here; the orchestrator owns per-service post-update reconciliation. + */ + async updateService(stackName: string, serviceName: string, hasBuild: boolean, ws?: WebSocket): Promise { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + const stackDir = path.join(this.baseDir, stackName); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + await this.withRegistryAuth(async (env) => { + if (hasBuild) { + sendOutput(`=== Building ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['build', '--pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + } else { + sendOutput(`=== Pulling ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + } + sendOutput(`=== Recreating ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + }, sendOutput); + } + + /** + * Recreate a single service from the image already present locally, without + * pulling or building. Used by service restore after the recovery image id has + * been retagged onto the declared ref (`--pull never --no-build` so Compose + * uses the just-retagged local image). Always `--no-deps --force-recreate`, + * never `--remove-orphans`. + */ + async recreateServiceFromLocal(stackName: string, serviceName: string, ws?: WebSocket): Promise { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + const stackDir = path.join(this.baseDir, stackName); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + await this.withRegistryAuth(async (env) => { + sendOutput(`=== Restoring ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', '--pull', 'never', '--no-build', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + }, sendOutput); + } + + public async downStack(stackName: string, options?: { removeVolumes?: boolean }): Promise { + const stackPath = path.join(this.baseDir, stackName); + try { + const args = options?.removeVolumes + ? ['down', '--volumes', '--remove-orphans'] + : ['down', '--remove-orphans']; + await this.execute('docker', await this.authoredComposeArgs(stackName, args), stackPath, undefined, false); + } catch (error) { + console.warn(`[Teardown] Docker down failed or nothing to clean up for ${sanitizeForLog(stackName)}`); + } + } + + /** + * Enumerate image references declared in a stack's compose file. + * + * Used by the pre-deploy policy gate to decide which images to scan before + * `docker compose up` runs. Path traversal is guarded against the node's + * compose base directory; missing / unreadable compose files or `.env` + * interpolation failures surface as a rejected Promise so the gate can + * block the deploy rather than silently allow it. + */ + public async listStackImages( + stackName: string, + invocation?: RollbackInvocationRecord | null, + ): Promise { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack path'); + } + const stackDir = this.resolveValidatedStackDir(stackName); + // Prefer a generation-captured invocation so restored-target policy does not + // scan with the live deploy-spec / project-env selection. + const useCaptured = hasUsableCapturedInvocation(invocation); + const fileAndEnvPrefix = useCaptured + ? [...this.composePrefixFromCapturedInvocation(stackName, invocation)] + : [ + ...authoredComposeFileArgs(stackName, this.nodeId), + ...(await authoredComposeEnvFileArgs(stackName, this.nodeId)), + ]; + if (useCaptured && invocation.meshEnabled) { + await this.appendCapturedMeshLayer(stackName, fileAndEnvPrefix); + } + const stdout = await this.captureCompose([...fileAndEnvPrefix, 'config', '--images'], stackDir); + const seen = new Set(); + const images: string[] = []; + for (const raw of stdout.split(/\r?\n/)) { + const line = raw.trim(); + if (!line) continue; + if (line.startsWith('sha256:')) continue; + if (seen.has(line)) continue; + seen.add(line); + images.push(line); + } + return images; + } + + /** Render the effective Compose model and cache the per-stack exposure + * descriptor so the Security posture can join exposed images against + * vulnerability findings without re-rendering config on every poll. + * Best-effort: render or parse failure logs a warning and keeps the + * prior cached descriptor, never failing the deploy. */ + private async refreshExposureCache(stackName: string): Promise { + const result = await this.renderConfig(stackName); + if (result.rendered === null) { + console.warn('[ComposeService] Exposure cache skipped for %s: model not renderable', + sanitizeForLog(stackName)); + return; + } + let parsed: unknown; + try { + parsed = JSON.parse(result.rendered); + } catch { + console.warn('[ComposeService] Exposure cache skipped for %s: unparseable model JSON', + sanitizeForLog(stackName)); + return; + } + const model = parseEffectiveModel(parsed, stackName); + const descriptor = deriveStackExposure(model, stackName, Date.now()); + DatabaseService.getInstance().upsertStackExposure( + this.nodeId, + stackName, + JSON.stringify(descriptor), + descriptor.computedAt, + ); + } + + private captureCompose(args: string[], cwd: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn('docker', ['compose', ...args], { + cwd, + env: { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + }, + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (data: Buffer) => { stdout += data.toString(); }); + child.stderr.on('data', (data: Buffer) => { stderr += data.toString(); }); + child.on('error', (err: NodeJS.ErrnoException) => { + const mapped = describeSpawnError(err, { command: 'docker compose' }); + if (mapped.isLowMemory) { + console.warn('[ComposeService] captureCompose spawn failed under memory pressure:', mapped.message); + } + reject(new Error(mapped.message)); + }); + child.on('close', (code) => { + if (code === 0) resolve(stdout); + else reject(new Error(stderr.trim() || `docker compose ${args.join(' ')} failed with code ${code}`)); + }); + }); + } + + /** + * Render the effective compose model as YAML (the default `docker compose + * config` output) with the exact authored invocation and NO mesh override. + * Used by the Git source detach/export contract: the rendered model becomes + * the stack's single compose.yaml. Throws when the render fails or times + * out, so the detach transaction aborts before anything changes. + */ + public async renderComposeYaml(stackName: string): Promise { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack path'); + } + const baseResolved = path.resolve(this.baseDir); + const stackDir = path.resolve(baseResolved, stackName); + if (!stackDir.startsWith(baseResolved + path.sep)) { + throw new Error('Invalid stack path'); + } + let filePrefix: string[]; + try { + filePrefix = authoredComposeFileArgs(stackName, this.nodeId); + } catch (err) { + throw err instanceof Error ? err : new Error(String(err)); + } + const envFileArgs = await authoredComposeEnvFileArgs(stackName, this.nodeId); + const child = spawn('docker', ['compose', ...filePrefix, ...envFileArgs, 'config'], { + cwd: stackDir, + env: { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + }, + }); + return new Promise((resolve, reject) => { + const MAX_OUTPUT = 5 * 1024 * 1024; // 5 MiB cap on each stream + const TIMEOUT_MS = 30_000; + // Accumulate Buffer chunks and decode ONCE at the end: chunk-wise + // toString() can split a multi-byte UTF-8 sequence across a chunk + // boundary and mangle non-ASCII values. + const outChunks: Buffer[] = []; + let outBytes = 0; + let stderr = ''; + let capped = false; + let settled = false; + const timer = setTimeout(() => { + settled = true; + clearTimeout(timer); + try { + child.kill('SIGKILL'); + } catch { + // best effort + } + reject(new Error(`docker compose config timed out after ${TIMEOUT_MS / 1000}s`)); + }, TIMEOUT_MS); + const finish = (error: Error | null) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (error) reject(error); + else resolve(Buffer.concat(outChunks).toString('utf8')); + }; + child.stdout.on('data', (data: Buffer) => { + if (capped) return; + outBytes += data.length; + if (outBytes > MAX_OUTPUT) { + // A truncated model frequently still parses as YAML; overwriting a + // working compose.yaml with it would be silent corruption. The cap is + // an error, not a truncation. + capped = true; + settled = true; + clearTimeout(timer); + try { + child.kill('SIGKILL'); + } catch { + // best effort + } + reject(new Error(`docker compose config output exceeded ${MAX_OUTPUT} bytes`)); + return; + } + outChunks.push(data); + }); + child.stderr.on('data', (data: Buffer) => { + if (stderr.length < MAX_OUTPUT) stderr += data.toString(); + }); + child.on('close', (code) => { + if (capped) return; + if (code === 0) finish(null); + else finish(new Error(stderr.trim() || `docker compose config exited with code ${code}`)); + }); + child.on('error', (err) => finish(err)); + }); + } + + /** + * Render the fully-resolved effective Compose model via `docker compose + * config --format json`. This is the AUTHORED model: it does NOT splice in + * the Sencho Mesh override, so it stays read-only (the override is + * write-generated) and reflects what the user actually edits. The override + * would also add the managed `sencho_mesh` external network and per-service + * mesh attachments, which would make preflight emit a false "external network + * not found" finding, so rendering the authored model is both safer and more + * accurate here. + * Captures stderr (where Compose reports unset variables) and never rejects + * on a non-zero exit, so the Compose Doctor can turn a failed render into a + * finding rather than an exception. Bounded by a timeout and an output cap. + * Rejects only when the docker binary cannot be spawned. + */ + public async renderConfig( + stackName: string, + ): Promise<{ rendered: string | null; stderr: string; code: number | null; timedOut: boolean }> { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack path'); + } + // Canonical inline js/path-injection barrier, kept in the same scope as the + // spawn cwd sink below. CodeQL credits neither the wrapped isPathWithinBase + // helper nor a barrier separated from the sink by the Promise-executor + // closure, so the spawn is hoisted out of the executor. startsWith already + // rejects the base dir itself, since base does not start with base + sep. + const baseResolved = path.resolve(this.baseDir); + const stackDir = path.resolve(baseResolved, stackName); + if (!stackDir.startsWith(baseResolved + path.sep)) { + throw new Error('Invalid stack path'); + } + // Render the authored multi-file model (no mesh override) so the Compose Doctor + // sees every override file; single-file stacks get an empty prefix. The env-file + // flag keeps render resolving the same root .env the validator and deploy use. + let filePrefix: string[]; + try { + filePrefix = authoredComposeFileArgs(stackName, this.nodeId); + } catch (err) { + throw err instanceof Error ? err : new Error(String(err)); + } + const envFileArgs = await authoredComposeEnvFileArgs(stackName, this.nodeId); + const child = spawn('docker', ['compose', ...filePrefix, ...envFileArgs, 'config', '--format', 'json'], { + cwd: stackDir, + env: { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + }, + }); + return new Promise((resolve, reject) => { + const MAX_OUTPUT = 5 * 1024 * 1024; // 5 MiB cap on each stream + const TIMEOUT_MS = 20_000; + let stdout = ''; + let stderr = ''; + let timedOut = false; + let capped = false; + let settled = false; + const timer = setTimeout(() => { timedOut = true; child.kill('SIGKILL'); }, TIMEOUT_MS); + const finish = (result: { rendered: string | null; stderr: string; code: number | null; timedOut: boolean }) => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(result); + }; + child.stdout.on('data', (data: Buffer) => { + stdout += data.toString(); + if (stdout.length > MAX_OUTPUT && !capped) { capped = true; child.kill('SIGKILL'); } + }); + child.stderr.on('data', (data: Buffer) => { + if (stderr.length < MAX_OUTPUT) stderr += data.toString(); + }); + child.on('error', (err: NodeJS.ErrnoException) => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(new Error(describeSpawnError(err, { command: 'docker compose' }).message)); + }); + child.on('close', (code) => { + if (timedOut) finish({ rendered: null, stderr: stderr.trim() || 'docker compose config timed out', code, timedOut: true }); + else if (capped) finish({ rendered: null, stderr: 'Rendered model exceeded the size limit', code, timedOut: false }); + else if (code === 0) finish({ rendered: stdout, stderr, code, timedOut: false }); + else finish({ rendered: null, stderr: stderr.trim() || `docker compose config failed with code ${code}`, code, timedOut: false }); + }); + }); + } +} diff --git a/backend/src/services/DockerController.ts b/backend/src/services/DockerController.ts index 7dfbabec..506e861c 100644 --- a/backend/src/services/DockerController.ts +++ b/backend/src/services/DockerController.ts @@ -2480,6 +2480,25 @@ class DockerController { return result; } + /** + * True when the stack directory is missing (ENOENT) or is not a directory. + * Other stat failures return false so callers keep the existing compose-ps path. + */ + private async isStackDirectoryAbsent(stackName: string): Promise { + // Canonical js/path-injection barrier at the fs.stat sink. + const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(this.nodeId)); + const stackDir = path.resolve(baseResolved, stackName); + if (!stackDir.startsWith(baseResolved + path.sep)) { + return true; + } + try { + const st = await fs.stat(stackDir); + return !st.isDirectory(); + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'ENOENT'; + } + } + /** * Containers visible to `docker compose ps` for this stack. Empty when Compose * does not manage any containers (including first deploy or mislabeled legacy). @@ -2530,6 +2549,7 @@ class DockerController { */ public async getLegacyOrphanContainersByStack(stackName: string): Promise> { const stackDir = path.join(NodeRegistry.getInstance().getComposeDir(this.nodeId), stackName); + if (await this.isStackDirectoryAbsent(stackName)) return []; const toIds = (list: Array<{ Id?: string }>) => list.filter((c): c is { Id: string } => typeof c.Id === 'string' && c.Id.length > 0) .map((c) => ({ Id: c.Id })); @@ -2573,6 +2593,9 @@ class DockerController { | { status: 'classification_failed'; error: string } > { const stackDir = path.join(NodeRegistry.getInstance().getComposeDir(this.nodeId), stackName); + if (await this.isStackDirectoryAbsent(stackName)) { + return { status: 'classification_failed', error: 'Stack directory is gone' }; + } const toIds = (list: Array<{ Id?: string }>) => list.filter((c): c is { Id: string } => typeof c.Id === 'string' && c.Id.length > 0) .map((c) => c.Id); @@ -2614,6 +2637,9 @@ class DockerController { // not the process default, so a non-default local node sees its own stack dir // and deploy spec. const stackDir = path.join(NodeRegistry.getInstance().getComposeDir(this.nodeId), stackName); + if (await this.isStackDirectoryAbsent(stackName)) { + return this.enrichContainers([]); + } try { const containers = await this.fetchComposePsContainers(stackName, stackDir);