diff --git a/backend/src/__tests__/compose-service.test.ts b/backend/src/__tests__/compose-service.test.ts index 0aefd2c4..299eac11 100644 --- a/backend/src/__tests__/compose-service.test.ts +++ b/backend/src/__tests__/compose-service.test.ts @@ -4,6 +4,7 @@ */ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { EventEmitter } from 'events'; +import os from 'os'; import type WebSocket from 'ws'; // ── Hoisted mocks ────────────────────────────────────────────────────── @@ -270,6 +271,69 @@ describe('ComposeService - runCommand', () => { proc.emit('close', null); await expectation; }); + + it('rewrites ENOMEM spawn failures as host out-of-memory', async () => { + const freememSpy = vi.spyOn(os, 'freemem').mockReturnValue(32 * 1024 * 1024); + const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(6612 * 1024 * 1024); + try { + const proc = createMockProcess(); + mockSpawn.mockReturnValue(proc); + const ws = createMockWs(); + + const svc = ComposeService.getInstance(1); + const promise = svc.runCommand('my-stack', 'restart', ws); + const err = Object.assign(new Error('spawn docker ENOMEM'), { code: 'ENOMEM' }); + proc.emit('error', err); + + await expect(promise).rejects.toThrow(/Out of memory while launching docker/); + const sendCalls = ws.send.mock.calls.map(c => c[0] as string); + expect(sendCalls.some(msg => msg.includes('Out of memory while launching docker'))).toBe(true); + } finally { + freememSpy.mockRestore(); + totalmemSpy.mockRestore(); + } + }); + + it('rewrites ENOENT spawn failures as host OOM when free memory is below the floor', async () => { + const freememSpy = vi.spyOn(os, 'freemem').mockReturnValue(32 * 1024 * 1024); + const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(6612 * 1024 * 1024); + try { + const proc = createMockProcess(); + mockSpawn.mockReturnValue(proc); + const ws = createMockWs(); + + const svc = ComposeService.getInstance(1); + const promise = svc.runCommand('my-stack', 'restart', ws); + const err = Object.assign(new Error('spawn docker ENOENT'), { code: 'ENOENT' }); + proc.emit('error', err); + + await expect(promise).rejects.toThrow(/Out of memory while launching docker/); + const sendCalls = ws.send.mock.calls.map(c => c[0] as string); + expect(sendCalls.some(msg => msg.includes('reported as ENOENT under memory pressure'))).toBe(true); + } finally { + freememSpy.mockRestore(); + totalmemSpy.mockRestore(); + } + }); + + it('preserves "Docker CLI unavailable" wording on healthy-memory ENOENT for docker', async () => { + const freememSpy = vi.spyOn(os, 'freemem').mockReturnValue(2 * 1024 * 1024 * 1024); + const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(6612 * 1024 * 1024); + try { + const proc = createMockProcess(); + mockSpawn.mockReturnValue(proc); + + const svc = ComposeService.getInstance(1); + const promise = svc.runCommand('my-stack', 'restart'); + const err = Object.assign(new Error('spawn docker ENOENT'), { code: 'ENOENT' }); + proc.emit('error', err); + + await expect(promise).rejects.toThrow('Docker CLI unavailable on this node'); + } finally { + freememSpy.mockRestore(); + totalmemSpy.mockRestore(); + } + }); }); // ── deployStack ──────────────────────────────────────────────────────── diff --git a/backend/src/__tests__/filesystem.test.ts b/backend/src/__tests__/filesystem.test.ts index 30fcdfcc..123b08f6 100644 --- a/backend/src/__tests__/filesystem.test.ts +++ b/backend/src/__tests__/filesystem.test.ts @@ -6,18 +6,20 @@ * Permission errors are surfaced to the caller like any other failure * (no Docker-helper fallback). */ -import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import os from 'os'; import path from 'path'; -const { mockRm } = vi.hoisted(() => ({ +const { mockRm, mockReaddir } = vi.hoisted(() => ({ mockRm: vi.fn(), + mockReaddir: vi.fn(), })); vi.mock('fs', () => ({ promises: { rm: mockRm, mkdir: vi.fn(), - readdir: vi.fn(), + readdir: mockReaddir, readFile: vi.fn(), writeFile: vi.fn(), access: vi.fn(), @@ -79,3 +81,49 @@ describe('FileSystemService.deleteStack', () => { await expect(service.deleteStack('io-error-stack')).rejects.toThrow(/disk I\/O error/); }); }); + +describe('FileSystemService.getStacks', () => { + let service: FileSystemService; + let warnSpy: ReturnType; + + beforeEach(() => { + vi.clearAllMocks(); + service = FileSystemService.getInstance(); + warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + }); + + afterEach(() => { + warnSpy.mockRestore(); + }); + + it('returns [] and logs ENOMEM-specific message with host free memory', async () => { + const err = Object.assign( + new Error("ENOMEM: not enough memory, scandir '/test/compose'"), + { code: 'ENOMEM' }, + ); + mockReaddir.mockRejectedValueOnce(err); + const freememSpy = vi.spyOn(os, 'freemem').mockReturnValue(37 * 1024 * 1024); + + try { + const result = await service.getStacks(); + expect(result).toEqual([]); + const warning = warnSpy.mock.calls[0]?.[0] as string; + expect(warning).toContain('ENOMEM'); + expect(warning).toContain('host free memory: 37 MiB'); + expect(warning).toContain('Returning empty list'); + } finally { + freememSpy.mockRestore(); + } + }); + + it('returns [] and logs the raw error message for non-ENOMEM errors', async () => { + const err = Object.assign(new Error('EACCES: permission denied'), { code: 'EACCES' }); + mockReaddir.mockRejectedValueOnce(err); + + const result = await service.getStacks(); + expect(result).toEqual([]); + const warning = warnSpy.mock.calls[0]?.[0] as string; + expect(warning).toContain('EACCES: permission denied'); + expect(warning).not.toContain('host free memory'); + }); +}); diff --git a/backend/src/__tests__/spawn-error-mapping.test.ts b/backend/src/__tests__/spawn-error-mapping.test.ts new file mode 100644 index 00000000..346a39f2 --- /dev/null +++ b/backend/src/__tests__/spawn-error-mapping.test.ts @@ -0,0 +1,96 @@ +/** + * Unit tests for describeSpawnError. Rewrites misleading spawn ENOENT errors + * to attribute to host memory pressure when free memory is below the floor, + * while preserving the existing "Docker CLI unavailable on this node" mapping + * on healthy hosts. Covers the F-15 ENOMEM-masquerading-as-ENOENT behavior. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import os from 'os'; + +import { describeSpawnError, LOW_MEMORY_FLOOR_BYTES } from '../utils/spawnErrors'; + +const HIGH_FREE = 2 * 1024 * 1024 * 1024; // 2 GiB free (healthy) +const LOW_FREE = 32 * 1024 * 1024; // 32 MiB free (the F-15 repro point) +const TOTAL = 6612 * 1024 * 1024; // workstation total in the repro + +let freememSpy: ReturnType; +let totalmemSpy: ReturnType; + +function setMemory(free: number): void { + freememSpy.mockReturnValue(free); + totalmemSpy.mockReturnValue(TOTAL); +} + +beforeEach(() => { + freememSpy = vi.spyOn(os, 'freemem'); + totalmemSpy = vi.spyOn(os, 'totalmem'); +}); + +afterEach(() => { + freememSpy.mockRestore(); + totalmemSpy.mockRestore(); +}); + +describe('describeSpawnError', () => { + it('rewrites explicit ENOMEM regardless of free memory', () => { + setMemory(HIGH_FREE); + const err = Object.assign(new Error('ENOMEM: not enough memory'), { code: 'ENOMEM' }); + const mapped = describeSpawnError(err, { command: 'docker' }); + expect(mapped.isLowMemory).toBe(true); + expect(mapped.message).toContain('Out of memory while launching docker'); + expect(mapped.message).toMatch(/host free memory: \d+ MiB of \d+ MiB/); + }); + + it('rewrites ENOENT-for-docker as OOM when free memory is below the floor', () => { + setMemory(LOW_FREE); + const err = Object.assign(new Error('spawn docker ENOENT'), { code: 'ENOENT' }); + const mapped = describeSpawnError(err, { command: 'docker' }); + expect(mapped.isLowMemory).toBe(true); + expect(mapped.message).toContain('Out of memory while launching docker'); + expect(mapped.message).toContain('reported as ENOENT under memory pressure'); + }); + + it('preserves the existing "Docker CLI unavailable" wording on healthy memory', () => { + setMemory(HIGH_FREE); + const err = Object.assign(new Error('spawn docker ENOENT'), { code: 'ENOENT' }); + const mapped = describeSpawnError(err, { command: 'docker' }); + expect(mapped.isLowMemory).toBe(false); + expect(mapped.message).toBe('Docker CLI unavailable on this node'); + }); + + it('rewrites ENOENT for non-docker commands (e.g. /bin/sh) under memory pressure', () => { + setMemory(LOW_FREE); + const err = Object.assign(new Error('spawn /bin/sh ENOENT'), { code: 'ENOENT' }); + const mapped = describeSpawnError(err, { command: '/bin/sh' }); + expect(mapped.isLowMemory).toBe(true); + expect(mapped.message).toContain('Out of memory while launching /bin/sh'); + }); + + it('passes through ENOENT for non-docker commands on healthy memory', () => { + setMemory(HIGH_FREE); + const err = Object.assign(new Error('spawn /bin/sh ENOENT'), { code: 'ENOENT' }); + const mapped = describeSpawnError(err, { command: '/bin/sh' }); + expect(mapped.isLowMemory).toBe(false); + expect(mapped.message).toBe('spawn /bin/sh ENOENT'); + }); + + it('passes through unrelated errors unchanged', () => { + setMemory(HIGH_FREE); + const err = Object.assign(new Error('permission denied'), { code: 'EACCES' }); + const mapped = describeSpawnError(err, { command: 'docker' }); + expect(mapped.isLowMemory).toBe(false); + expect(mapped.message).toBe('permission denied'); + }); + + it('crosses the threshold exactly at LOW_MEMORY_FLOOR_BYTES', () => { + setMemory(LOW_MEMORY_FLOOR_BYTES); + const err = Object.assign(new Error('spawn docker ENOENT'), { code: 'ENOENT' }); + const mapped = describeSpawnError(err, { command: 'docker' }); + expect(mapped.isLowMemory).toBe(false); + expect(mapped.message).toBe('Docker CLI unavailable on this node'); + + setMemory(LOW_MEMORY_FLOOR_BYTES - 1); + const mapped2 = describeSpawnError(err, { command: 'docker' }); + expect(mapped2.isLowMemory).toBe(true); + }); +}); diff --git a/backend/src/bootstrap/startup.ts b/backend/src/bootstrap/startup.ts index 765eac1b..2dba69ab 100644 --- a/backend/src/bootstrap/startup.ts +++ b/backend/src/bootstrap/startup.ts @@ -1,5 +1,6 @@ import type { Server } from 'http'; import crypto from 'crypto'; +import os from 'os'; import { FileSystemService } from '../services/FileSystemService'; import { NodeRegistry } from '../services/NodeRegistry'; import { DatabaseService } from '../services/DatabaseService'; @@ -21,6 +22,7 @@ import { PilotMetrics } from '../services/PilotMetrics'; import { invalidateRemoteMetaCache } from '../helpers/cacheInvalidation'; import { sweepStaleTempDirs as sweepStaleGitTempDirs } from '../services/GitSourceService'; import { PORT } from '../helpers/constants'; +import { LOW_MEMORY_FLOOR_BYTES } from '../utils/spawnErrors'; function isPilotMode(): boolean { return process.env.SENCHO_MODE === 'pilot'; @@ -56,6 +58,19 @@ export function ensurePilotJwtSecret(): boolean { * port. */ export async function startServer(server: Server): Promise { + const freeBytes = os.freemem(); + const freeMiB = Math.round(freeBytes / (1024 * 1024)); + const totalMiB = Math.round(os.totalmem() / (1024 * 1024)); + const floorMiB = Math.round(LOW_MEMORY_FLOOR_BYTES / (1024 * 1024)); + console.log(`[Startup] Host memory: ${freeMiB} MiB free of ${totalMiB} MiB`); + if (freeBytes < LOW_MEMORY_FLOOR_BYTES) { + console.warn( + `[Startup] Free host memory is ${freeMiB} MiB (below ${floorMiB} MiB floor). ` + + 'Sencho operations that spawn child processes (docker, /bin/sh) may fail ' + + 'with misleading ENOENT errors under memory pressure.' + ); + } + try { console.log('Running stack migration check...'); const defaultFsService = FileSystemService.getInstance(NodeRegistry.getInstance().getDefaultNodeId()); diff --git a/backend/src/services/ComposeService.ts b/backend/src/services/ComposeService.ts index ba596bd2..acb64d63 100644 --- a/backend/src/services/ComposeService.ts +++ b/backend/src/services/ComposeService.ts @@ -13,6 +13,7 @@ import { RegistryService } from './RegistryService'; import { isDebugEnabled } from '../utils/debug'; import { getErrorMessage } from '../utils/errors'; +import { describeSpawnError } from '../utils/spawnErrors'; import { isPathWithinBase, isValidStackName } from '../utils/validation'; import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog'; @@ -31,22 +32,22 @@ export class ComposeRollbackError extends Error { } } -export function getComposeRollbackInfo(error: unknown): { attempted: boolean; rolledBack: boolean } | null { - if (!(error instanceof ComposeRollbackError)) { - return null; - } - return { attempted: error.rollbackAttempted, rolledBack: error.rolledBack }; -} - -const DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS = 30 * 60 * 1000; - -function getComposeCommandTimeoutMs(): number { - const configured = Number(process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS); - if (Number.isFinite(configured) && configured > 0) { - return configured; - } - return DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS; -} +export function getComposeRollbackInfo(error: unknown): { attempted: boolean; rolledBack: boolean } | null { + if (!(error instanceof ComposeRollbackError)) { + return null; + } + return { attempted: error.rollbackAttempted, rolledBack: error.rolledBack }; +} + +const DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS = 30 * 60 * 1000; + +function getComposeCommandTimeoutMs(): number { + const configured = Number(process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS); + if (Number.isFinite(configured) && configured > 0) { + return configured; + } + return DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS; +} /** * ComposeService - local docker compose CLI execution. @@ -97,127 +98,128 @@ export class ComposeService { ws?: WebSocket, throwOnError = true, env?: Record - ): Promise { - return new Promise((resolve, reject) => { - const child = spawn(command, args, { - cwd, - env: env ?? { + ): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd, + env: env ?? { ...process.env, PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' } - }); - - let errorLog = ''; - let settled = false; - let exited = false; - let pendingTerminationError: Error | null = null; - const timeoutMs = getComposeCommandTimeoutMs(); - let timeout: ReturnType | null = null; - let forceKillTimeout: ReturnType | null = null; - - const sendOutput = (text: string) => { - if (ws && ws.readyState === WebSocket.OPEN) { - ws.send(text); - } - }; - - const cleanup = () => { - if (timeout) { - clearTimeout(timeout); - timeout = null; - } - if (forceKillTimeout) { - clearTimeout(forceKillTimeout); - forceKillTimeout = null; - } - if (ws) { - ws.removeListener('close', onClientDisconnect); - } - }; - - const finish = (complete: () => void) => { - if (settled) return; - settled = true; - cleanup(); - complete(); - }; - - const terminateChild = (error: Error) => { - pendingTerminationError = pendingTerminationError ?? error; - if (exited) return; - try { - child.kill('SIGTERM'); - } catch (error) { - console.warn('[ComposeService] Failed to terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); - } - forceKillTimeout = setTimeout(() => { - if (exited) return; - try { - child.kill('SIGKILL'); - } catch (error) { - console.warn('[ComposeService] Failed to force terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); - } - }, 5000); - }; - - const onClientDisconnect = () => { - const message = 'Command cancelled because the client disconnected'; - terminateChild(new Error(message)); - }; - - timeout = setTimeout(() => { - const message = `Command timed out after ${Math.round(timeoutMs / 1000)}s`; - sendOutput(`${message}\n`); - terminateChild(new Error(message)); - }, timeoutMs); - - if (ws) { - ws.once('close', onClientDisconnect); - } - - const onData = (data: Buffer) => { - const text = data.toString(); - errorLog += text; - sendOutput(text); - }; + }); + + let errorLog = ''; + let settled = false; + let exited = false; + let pendingTerminationError: Error | null = null; + const timeoutMs = getComposeCommandTimeoutMs(); + let timeout: ReturnType | null = null; + let forceKillTimeout: ReturnType | null = null; + + const sendOutput = (text: string) => { + if (ws && ws.readyState === WebSocket.OPEN) { + ws.send(text); + } + }; + + const cleanup = () => { + if (timeout) { + clearTimeout(timeout); + timeout = null; + } + if (forceKillTimeout) { + clearTimeout(forceKillTimeout); + forceKillTimeout = null; + } + if (ws) { + ws.removeListener('close', onClientDisconnect); + } + }; + + const finish = (complete: () => void) => { + if (settled) return; + settled = true; + cleanup(); + complete(); + }; + + const terminateChild = (error: Error) => { + pendingTerminationError = pendingTerminationError ?? error; + if (exited) return; + try { + child.kill('SIGTERM'); + } catch (error) { + console.warn('[ComposeService] Failed to terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + } + forceKillTimeout = setTimeout(() => { + if (exited) return; + try { + child.kill('SIGKILL'); + } catch (error) { + console.warn('[ComposeService] Failed to force terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + } + }, 5000); + }; + + const onClientDisconnect = () => { + const message = 'Command cancelled because the client disconnected'; + terminateChild(new Error(message)); + }; + + timeout = setTimeout(() => { + const message = `Command timed out after ${Math.round(timeoutMs / 1000)}s`; + sendOutput(`${message}\n`); + terminateChild(new Error(message)); + }, timeoutMs); + + if (ws) { + ws.once('close', onClientDisconnect); + } + + const onData = (data: Buffer) => { + const text = data.toString(); + errorLog += text; + sendOutput(text); + }; child.stdout.on('data', onData); - child.stderr.on('data', onData); - - child.on('close', (code: number | null) => { - exited = true; - finish(() => { - sendOutput(`Command exited with code ${code}\n`); - if (pendingTerminationError) { - if (throwOnError) reject(pendingTerminationError); - else resolve(); - return; - } - if (code === 0) resolve(); - else if (throwOnError) reject(new Error(redactSensitiveText(errorLog.trim()) || `Command failed with code ${code}`)); - else resolve(); - }); - }); - - child.on('error', (error: Error & { code?: string }) => { - exited = true; - finish(() => { - let message = redactSensitiveText(error.message); - if (error.code === 'ENOENT' && /^spawn docker(?:$| )/.test(error.message)) { - message = 'Docker CLI unavailable on this node'; - } - sendOutput(`Error: ${message}\n`); - if (pendingTerminationError) { - if (throwOnError) reject(pendingTerminationError); - else resolve(); - return; - } - if (throwOnError) reject(new Error(message)); - else resolve(); - }); - }); - }); - } + child.stderr.on('data', onData); + + child.on('close', (code: number | null) => { + exited = true; + finish(() => { + sendOutput(`Command exited with code ${code}\n`); + if (pendingTerminationError) { + if (throwOnError) reject(pendingTerminationError); + else resolve(); + return; + } + if (code === 0) resolve(); + else if (throwOnError) reject(new Error(redactSensitiveText(errorLog.trim()) || `Command failed with code ${code}`)); + else resolve(); + }); + }); + + child.on('error', (error: Error & { code?: string }) => { + exited = true; + finish(() => { + const mapped = describeSpawnError(error as NodeJS.ErrnoException, { command }); + const message = redactSensitiveText(mapped.message); + sendOutput(`Error: ${message}\n`); + if (mapped.isLowMemory) { + console.warn('[ComposeService] spawn failed under memory pressure:', message); + } + if (pendingTerminationError) { + if (throwOnError) reject(pendingTerminationError); + else resolve(); + return; + } + if (throwOnError) reject(new Error(message)); + else resolve(); + }); + }); + }); + } private async withRegistryAuth( fn: (env: Record) => Promise, @@ -596,7 +598,13 @@ export class ComposeService { let stderr = ''; child.stdout.on('data', (data: Buffer) => { stdout += data.toString(); }); child.stderr.on('data', (data: Buffer) => { stderr += data.toString(); }); - child.on('error', (err) => reject(err)); + child.on('error', (err: NodeJS.ErrnoException) => { + const mapped = describeSpawnError(err, { command: 'docker compose' }); + if (mapped.isLowMemory) { + console.warn('[ComposeService] captureCompose spawn failed under memory pressure:', mapped.message); + } + reject(new Error(mapped.message)); + }); child.on('close', (code) => { if (code === 0) resolve(stdout); else reject(new Error(stderr.trim() || `docker compose ${args.join(' ')} failed with code ${code}`)); diff --git a/backend/src/services/DockerController.ts b/backend/src/services/DockerController.ts index ed0facdb..ff9c1ff1 100644 --- a/backend/src/services/DockerController.ts +++ b/backend/src/services/DockerController.ts @@ -11,6 +11,7 @@ import { CacheService } from './CacheService'; import { isPathWithinBase } from '../utils/validation'; import { isDebugEnabled } from '../utils/debug'; import { sanitizeForLog } from '../utils/safeLog'; +import { describeSpawnError } from '../utils/spawnErrors'; const execAsync = promisify(exec); const COMPOSE_DIR = process.env.COMPOSE_DIR || '/app/compose'; @@ -909,9 +910,13 @@ class DockerController { return await this.enrichContainers(await this.smartFallback(stackName, stackDir)); } catch (error) { - // If command fails (e.g., stack not deployed, invalid YAML, missing env_file) - const execError = error as { stderr?: string; message?: string }; - console.error('Docker Compose Error for %s:', sanitizeForLog(stackName), sanitizeForLog(execError.stderr || execError.message || 'unknown')); + // If command fails (e.g., stack not deployed, invalid YAML, missing env_file, + // or host under memory pressure causing posix_spawn to fail with ENOMEM, + // which Linux libuv can surface as ENOENT). + const execError = error as NodeJS.ErrnoException & { stderr?: string }; + const mapped = describeSpawnError(execError, { command: 'docker compose ps' }); + const detail = execError.stderr || mapped.message; + console.error('Docker Compose Error for %s:', sanitizeForLog(stackName), sanitizeForLog(detail)); // Try smart fallback even on error return await this.enrichContainers(await this.smartFallback(stackName, stackDir)); diff --git a/backend/src/services/FileSystemService.ts b/backend/src/services/FileSystemService.ts index bba060c8..a9b5c7da 100644 --- a/backend/src/services/FileSystemService.ts +++ b/backend/src/services/FileSystemService.ts @@ -1,4 +1,5 @@ import path from 'path'; +import os from 'os'; import { promises as fsPromises, createReadStream } from 'fs'; import type { Readable } from 'stream'; import { NodeRegistry } from './NodeRegistry'; @@ -135,7 +136,12 @@ export class FileSystemService { return stackNames; } catch (error: any) { - console.warn(`[FileSystemService] Failed to list stacks: ${error.message}`); + if (error?.code === 'ENOMEM') { + const freeMiB = Math.round(os.freemem() / (1024 * 1024)); + console.warn(`[FileSystemService] Failed to list stacks: ENOMEM (host free memory: ${freeMiB} MiB). Returning empty list.`); + } else { + console.warn(`[FileSystemService] Failed to list stacks: ${error.message}`); + } return []; } } diff --git a/backend/src/utils/spawnErrors.ts b/backend/src/utils/spawnErrors.ts new file mode 100644 index 00000000..1eccd198 --- /dev/null +++ b/backend/src/utils/spawnErrors.ts @@ -0,0 +1,63 @@ +import os from 'os'; + +/** + * Below this floor of free host memory, treat an ENOENT or ENOMEM + * spawn failure as a memory-pressure event rather than a missing binary. + * Linux libuv's posix_spawn can fail to allocate its argv/path-search arena + * under memory pressure and surface the underlying ENOMEM as ENOENT, sending + * operators down a "missing binary" debugging path when the real cause is + * host OOM. 128 MiB is well above the point where posix_spawn starts dropping + * arenas yet low enough that a healthy homelab host never trips it. + */ +export const LOW_MEMORY_FLOOR_BYTES = 128 * 1024 * 1024; + +export interface SpawnErrorContext { + /** Literal first argument passed to spawn/exec (e.g. "docker", "/bin/sh"). */ + command: string; +} + +export interface MappedSpawnError { + /** Operator-facing message. Safe to send through WS / log / throw. */ + message: string; + /** True when the failure was attributed to host memory pressure. */ + isLowMemory: boolean; +} + +/** + * Rewrite a spawn / exec error into an operator-facing message. + * + * Ordering matters: an explicit ENOMEM always wins, then an ENOENT under low + * free memory is treated as the libuv masquerade case, and only after that + * does the genuine "docker CLI missing" mapping fire. Other errors pass + * through unchanged. + */ +export function describeSpawnError( + error: NodeJS.ErrnoException, + ctx: SpawnErrorContext, +): MappedSpawnError { + const free = os.freemem(); + const total = os.totalmem(); + const freeMiB = Math.round(free / (1024 * 1024)); + const totalMiB = Math.round(total / (1024 * 1024)); + const lowMem = free < LOW_MEMORY_FLOOR_BYTES; + + if (error.code === 'ENOMEM') { + return { + message: `Out of memory while launching ${ctx.command} (host free memory: ${freeMiB} MiB of ${totalMiB} MiB)`, + isLowMemory: true, + }; + } + + if (error.code === 'ENOENT' && lowMem) { + return { + message: `Out of memory while launching ${ctx.command} (host free memory: ${freeMiB} MiB of ${totalMiB} MiB; reported as ENOENT under memory pressure)`, + isLowMemory: true, + }; + } + + if (error.code === 'ENOENT' && /^spawn docker(?:$| )/.test(error.message ?? '')) { + return { message: 'Docker CLI unavailable on this node', isLowMemory: false }; + } + + return { message: error.message ?? String(error), isLowMemory: false }; +}