fix(spawn): attribute ENOMEM and ENOENT-under-memory-pressure spawn failures to host OOM (#1111)

Operators previously saw "spawn docker ENOENT" or "spawn /bin/sh ENOENT" when
the host was under memory pressure, which sent them down a missing-binary
debugging path. Linux libuv's posix_spawn can fail to allocate its argv /
path-search arena under low free memory and surface the underlying ENOMEM
as ENOENT.

Centralizes spawn-error mapping in a new utils/spawnErrors.ts helper:
- Explicit ENOMEM is rewritten to "Out of memory while launching <command>
  (host free memory: X MiB of Y MiB)".
- ENOENT under the 128 MiB free-memory floor is rewritten with the same
  wording plus a "reported as ENOENT under memory pressure" hint.
- ENOENT for docker on a healthy host preserves the existing
  "Docker CLI unavailable on this node" mapping.
- Other errors pass through unchanged.

Applied at the four named offenders: ComposeService.execute(),
ComposeService.captureCompose(), DockerController.getContainersByStack(),
and FileSystemService.getStacks() (which gets an ENOMEM-aware log line
for the scandir failure).

Startup also logs host free/total MiB once and warns when free memory is
below the 128 MiB floor, so the diagnostic surfaces before the first
spawn attempt rather than after it fails.

37 tests cover the mapping function directly and the ComposeService /
FileSystemService integration paths.
This commit is contained in:
Anso
2026-05-19 07:27:12 -04:00
committed by GitHub
parent 5a2aed22fd
commit 9dbce9c3c7
8 changed files with 445 additions and 140 deletions
@@ -4,6 +4,7 @@
*/
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { EventEmitter } from 'events';
import os from 'os';
import type WebSocket from 'ws';
// ── Hoisted mocks ──────────────────────────────────────────────────────
@@ -270,6 +271,69 @@ describe('ComposeService - runCommand', () => {
proc.emit('close', null);
await expectation;
});
it('rewrites ENOMEM spawn failures as host out-of-memory', async () => {
const freememSpy = vi.spyOn(os, 'freemem').mockReturnValue(32 * 1024 * 1024);
const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(6612 * 1024 * 1024);
try {
const proc = createMockProcess();
mockSpawn.mockReturnValue(proc);
const ws = createMockWs();
const svc = ComposeService.getInstance(1);
const promise = svc.runCommand('my-stack', 'restart', ws);
const err = Object.assign(new Error('spawn docker ENOMEM'), { code: 'ENOMEM' });
proc.emit('error', err);
await expect(promise).rejects.toThrow(/Out of memory while launching docker/);
const sendCalls = ws.send.mock.calls.map(c => c[0] as string);
expect(sendCalls.some(msg => msg.includes('Out of memory while launching docker'))).toBe(true);
} finally {
freememSpy.mockRestore();
totalmemSpy.mockRestore();
}
});
it('rewrites ENOENT spawn failures as host OOM when free memory is below the floor', async () => {
const freememSpy = vi.spyOn(os, 'freemem').mockReturnValue(32 * 1024 * 1024);
const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(6612 * 1024 * 1024);
try {
const proc = createMockProcess();
mockSpawn.mockReturnValue(proc);
const ws = createMockWs();
const svc = ComposeService.getInstance(1);
const promise = svc.runCommand('my-stack', 'restart', ws);
const err = Object.assign(new Error('spawn docker ENOENT'), { code: 'ENOENT' });
proc.emit('error', err);
await expect(promise).rejects.toThrow(/Out of memory while launching docker/);
const sendCalls = ws.send.mock.calls.map(c => c[0] as string);
expect(sendCalls.some(msg => msg.includes('reported as ENOENT under memory pressure'))).toBe(true);
} finally {
freememSpy.mockRestore();
totalmemSpy.mockRestore();
}
});
it('preserves "Docker CLI unavailable" wording on healthy-memory ENOENT for docker', async () => {
const freememSpy = vi.spyOn(os, 'freemem').mockReturnValue(2 * 1024 * 1024 * 1024);
const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(6612 * 1024 * 1024);
try {
const proc = createMockProcess();
mockSpawn.mockReturnValue(proc);
const svc = ComposeService.getInstance(1);
const promise = svc.runCommand('my-stack', 'restart');
const err = Object.assign(new Error('spawn docker ENOENT'), { code: 'ENOENT' });
proc.emit('error', err);
await expect(promise).rejects.toThrow('Docker CLI unavailable on this node');
} finally {
freememSpy.mockRestore();
totalmemSpy.mockRestore();
}
});
});
// ── deployStack ────────────────────────────────────────────────────────
+51 -3
View File
@@ -6,18 +6,20 @@
* Permission errors are surfaced to the caller like any other failure
* (no Docker-helper fallback).
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import os from 'os';
import path from 'path';
const { mockRm } = vi.hoisted(() => ({
const { mockRm, mockReaddir } = vi.hoisted(() => ({
mockRm: vi.fn(),
mockReaddir: vi.fn(),
}));
vi.mock('fs', () => ({
promises: {
rm: mockRm,
mkdir: vi.fn(),
readdir: vi.fn(),
readdir: mockReaddir,
readFile: vi.fn(),
writeFile: vi.fn(),
access: vi.fn(),
@@ -79,3 +81,49 @@ describe('FileSystemService.deleteStack', () => {
await expect(service.deleteStack('io-error-stack')).rejects.toThrow(/disk I\/O error/);
});
});
describe('FileSystemService.getStacks', () => {
let service: FileSystemService;
let warnSpy: ReturnType<typeof vi.spyOn>;
beforeEach(() => {
vi.clearAllMocks();
service = FileSystemService.getInstance();
warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
});
afterEach(() => {
warnSpy.mockRestore();
});
it('returns [] and logs ENOMEM-specific message with host free memory', async () => {
const err = Object.assign(
new Error("ENOMEM: not enough memory, scandir '/test/compose'"),
{ code: 'ENOMEM' },
);
mockReaddir.mockRejectedValueOnce(err);
const freememSpy = vi.spyOn(os, 'freemem').mockReturnValue(37 * 1024 * 1024);
try {
const result = await service.getStacks();
expect(result).toEqual([]);
const warning = warnSpy.mock.calls[0]?.[0] as string;
expect(warning).toContain('ENOMEM');
expect(warning).toContain('host free memory: 37 MiB');
expect(warning).toContain('Returning empty list');
} finally {
freememSpy.mockRestore();
}
});
it('returns [] and logs the raw error message for non-ENOMEM errors', async () => {
const err = Object.assign(new Error('EACCES: permission denied'), { code: 'EACCES' });
mockReaddir.mockRejectedValueOnce(err);
const result = await service.getStacks();
expect(result).toEqual([]);
const warning = warnSpy.mock.calls[0]?.[0] as string;
expect(warning).toContain('EACCES: permission denied');
expect(warning).not.toContain('host free memory');
});
});
@@ -0,0 +1,96 @@
/**
* Unit tests for describeSpawnError. Rewrites misleading spawn ENOENT errors
* to attribute to host memory pressure when free memory is below the floor,
* while preserving the existing "Docker CLI unavailable on this node" mapping
* on healthy hosts. Covers the F-15 ENOMEM-masquerading-as-ENOENT behavior.
*/
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import os from 'os';
import { describeSpawnError, LOW_MEMORY_FLOOR_BYTES } from '../utils/spawnErrors';
const HIGH_FREE = 2 * 1024 * 1024 * 1024; // 2 GiB free (healthy)
const LOW_FREE = 32 * 1024 * 1024; // 32 MiB free (the F-15 repro point)
const TOTAL = 6612 * 1024 * 1024; // workstation total in the repro
let freememSpy: ReturnType<typeof vi.spyOn>;
let totalmemSpy: ReturnType<typeof vi.spyOn>;
function setMemory(free: number): void {
freememSpy.mockReturnValue(free);
totalmemSpy.mockReturnValue(TOTAL);
}
beforeEach(() => {
freememSpy = vi.spyOn(os, 'freemem');
totalmemSpy = vi.spyOn(os, 'totalmem');
});
afterEach(() => {
freememSpy.mockRestore();
totalmemSpy.mockRestore();
});
describe('describeSpawnError', () => {
it('rewrites explicit ENOMEM regardless of free memory', () => {
setMemory(HIGH_FREE);
const err = Object.assign(new Error('ENOMEM: not enough memory'), { code: 'ENOMEM' });
const mapped = describeSpawnError(err, { command: 'docker' });
expect(mapped.isLowMemory).toBe(true);
expect(mapped.message).toContain('Out of memory while launching docker');
expect(mapped.message).toMatch(/host free memory: \d+ MiB of \d+ MiB/);
});
it('rewrites ENOENT-for-docker as OOM when free memory is below the floor', () => {
setMemory(LOW_FREE);
const err = Object.assign(new Error('spawn docker ENOENT'), { code: 'ENOENT' });
const mapped = describeSpawnError(err, { command: 'docker' });
expect(mapped.isLowMemory).toBe(true);
expect(mapped.message).toContain('Out of memory while launching docker');
expect(mapped.message).toContain('reported as ENOENT under memory pressure');
});
it('preserves the existing "Docker CLI unavailable" wording on healthy memory', () => {
setMemory(HIGH_FREE);
const err = Object.assign(new Error('spawn docker ENOENT'), { code: 'ENOENT' });
const mapped = describeSpawnError(err, { command: 'docker' });
expect(mapped.isLowMemory).toBe(false);
expect(mapped.message).toBe('Docker CLI unavailable on this node');
});
it('rewrites ENOENT for non-docker commands (e.g. /bin/sh) under memory pressure', () => {
setMemory(LOW_FREE);
const err = Object.assign(new Error('spawn /bin/sh ENOENT'), { code: 'ENOENT' });
const mapped = describeSpawnError(err, { command: '/bin/sh' });
expect(mapped.isLowMemory).toBe(true);
expect(mapped.message).toContain('Out of memory while launching /bin/sh');
});
it('passes through ENOENT for non-docker commands on healthy memory', () => {
setMemory(HIGH_FREE);
const err = Object.assign(new Error('spawn /bin/sh ENOENT'), { code: 'ENOENT' });
const mapped = describeSpawnError(err, { command: '/bin/sh' });
expect(mapped.isLowMemory).toBe(false);
expect(mapped.message).toBe('spawn /bin/sh ENOENT');
});
it('passes through unrelated errors unchanged', () => {
setMemory(HIGH_FREE);
const err = Object.assign(new Error('permission denied'), { code: 'EACCES' });
const mapped = describeSpawnError(err, { command: 'docker' });
expect(mapped.isLowMemory).toBe(false);
expect(mapped.message).toBe('permission denied');
});
it('crosses the threshold exactly at LOW_MEMORY_FLOOR_BYTES', () => {
setMemory(LOW_MEMORY_FLOOR_BYTES);
const err = Object.assign(new Error('spawn docker ENOENT'), { code: 'ENOENT' });
const mapped = describeSpawnError(err, { command: 'docker' });
expect(mapped.isLowMemory).toBe(false);
expect(mapped.message).toBe('Docker CLI unavailable on this node');
setMemory(LOW_MEMORY_FLOOR_BYTES - 1);
const mapped2 = describeSpawnError(err, { command: 'docker' });
expect(mapped2.isLowMemory).toBe(true);
});
});
+15
View File
@@ -1,5 +1,6 @@
import type { Server } from 'http';
import crypto from 'crypto';
import os from 'os';
import { FileSystemService } from '../services/FileSystemService';
import { NodeRegistry } from '../services/NodeRegistry';
import { DatabaseService } from '../services/DatabaseService';
@@ -21,6 +22,7 @@ import { PilotMetrics } from '../services/PilotMetrics';
import { invalidateRemoteMetaCache } from '../helpers/cacheInvalidation';
import { sweepStaleTempDirs as sweepStaleGitTempDirs } from '../services/GitSourceService';
import { PORT } from '../helpers/constants';
import { LOW_MEMORY_FLOOR_BYTES } from '../utils/spawnErrors';
function isPilotMode(): boolean {
return process.env.SENCHO_MODE === 'pilot';
@@ -56,6 +58,19 @@ export function ensurePilotJwtSecret(): boolean {
* port.
*/
export async function startServer(server: Server): Promise<void> {
const freeBytes = os.freemem();
const freeMiB = Math.round(freeBytes / (1024 * 1024));
const totalMiB = Math.round(os.totalmem() / (1024 * 1024));
const floorMiB = Math.round(LOW_MEMORY_FLOOR_BYTES / (1024 * 1024));
console.log(`[Startup] Host memory: ${freeMiB} MiB free of ${totalMiB} MiB`);
if (freeBytes < LOW_MEMORY_FLOOR_BYTES) {
console.warn(
`[Startup] Free host memory is ${freeMiB} MiB (below ${floorMiB} MiB floor). ` +
'Sencho operations that spawn child processes (docker, /bin/sh) may fail ' +
'with misleading ENOENT errors under memory pressure.'
);
}
try {
console.log('Running stack migration check...');
const defaultFsService = FileSystemService.getInstance(NodeRegistry.getInstance().getDefaultNodeId());
+141 -133
View File
@@ -13,6 +13,7 @@ import { RegistryService } from './RegistryService';
import { isDebugEnabled } from '../utils/debug';
import { getErrorMessage } from '../utils/errors';
import { describeSpawnError } from '../utils/spawnErrors';
import { isPathWithinBase, isValidStackName } from '../utils/validation';
import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog';
@@ -31,22 +32,22 @@ export class ComposeRollbackError extends Error {
}
}
export function getComposeRollbackInfo(error: unknown): { attempted: boolean; rolledBack: boolean } | null {
if (!(error instanceof ComposeRollbackError)) {
return null;
}
return { attempted: error.rollbackAttempted, rolledBack: error.rolledBack };
}
const DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS = 30 * 60 * 1000;
function getComposeCommandTimeoutMs(): number {
const configured = Number(process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS);
if (Number.isFinite(configured) && configured > 0) {
return configured;
}
return DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS;
}
export function getComposeRollbackInfo(error: unknown): { attempted: boolean; rolledBack: boolean } | null {
if (!(error instanceof ComposeRollbackError)) {
return null;
}
return { attempted: error.rollbackAttempted, rolledBack: error.rolledBack };
}
const DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS = 30 * 60 * 1000;
function getComposeCommandTimeoutMs(): number {
const configured = Number(process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS);
if (Number.isFinite(configured) && configured > 0) {
return configured;
}
return DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS;
}
/**
* ComposeService - local docker compose CLI execution.
@@ -97,127 +98,128 @@ export class ComposeService {
ws?: WebSocket,
throwOnError = true,
env?: Record<string, string | undefined>
): Promise<void> {
return new Promise((resolve, reject) => {
const child = spawn(command, args, {
cwd,
env: env ?? {
): Promise<void> {
return new Promise((resolve, reject) => {
const child = spawn(command, args, {
cwd,
env: env ?? {
...process.env,
PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
}
});
let errorLog = '';
let settled = false;
let exited = false;
let pendingTerminationError: Error | null = null;
const timeoutMs = getComposeCommandTimeoutMs();
let timeout: ReturnType<typeof setTimeout> | null = null;
let forceKillTimeout: ReturnType<typeof setTimeout> | null = null;
const sendOutput = (text: string) => {
if (ws && ws.readyState === WebSocket.OPEN) {
ws.send(text);
}
};
const cleanup = () => {
if (timeout) {
clearTimeout(timeout);
timeout = null;
}
if (forceKillTimeout) {
clearTimeout(forceKillTimeout);
forceKillTimeout = null;
}
if (ws) {
ws.removeListener('close', onClientDisconnect);
}
};
const finish = (complete: () => void) => {
if (settled) return;
settled = true;
cleanup();
complete();
};
const terminateChild = (error: Error) => {
pendingTerminationError = pendingTerminationError ?? error;
if (exited) return;
try {
child.kill('SIGTERM');
} catch (error) {
console.warn('[ComposeService] Failed to terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown')));
}
forceKillTimeout = setTimeout(() => {
if (exited) return;
try {
child.kill('SIGKILL');
} catch (error) {
console.warn('[ComposeService] Failed to force terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown')));
}
}, 5000);
};
const onClientDisconnect = () => {
const message = 'Command cancelled because the client disconnected';
terminateChild(new Error(message));
};
timeout = setTimeout(() => {
const message = `Command timed out after ${Math.round(timeoutMs / 1000)}s`;
sendOutput(`${message}\n`);
terminateChild(new Error(message));
}, timeoutMs);
if (ws) {
ws.once('close', onClientDisconnect);
}
const onData = (data: Buffer) => {
const text = data.toString();
errorLog += text;
sendOutput(text);
};
});
let errorLog = '';
let settled = false;
let exited = false;
let pendingTerminationError: Error | null = null;
const timeoutMs = getComposeCommandTimeoutMs();
let timeout: ReturnType<typeof setTimeout> | null = null;
let forceKillTimeout: ReturnType<typeof setTimeout> | null = null;
const sendOutput = (text: string) => {
if (ws && ws.readyState === WebSocket.OPEN) {
ws.send(text);
}
};
const cleanup = () => {
if (timeout) {
clearTimeout(timeout);
timeout = null;
}
if (forceKillTimeout) {
clearTimeout(forceKillTimeout);
forceKillTimeout = null;
}
if (ws) {
ws.removeListener('close', onClientDisconnect);
}
};
const finish = (complete: () => void) => {
if (settled) return;
settled = true;
cleanup();
complete();
};
const terminateChild = (error: Error) => {
pendingTerminationError = pendingTerminationError ?? error;
if (exited) return;
try {
child.kill('SIGTERM');
} catch (error) {
console.warn('[ComposeService] Failed to terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown')));
}
forceKillTimeout = setTimeout(() => {
if (exited) return;
try {
child.kill('SIGKILL');
} catch (error) {
console.warn('[ComposeService] Failed to force terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown')));
}
}, 5000);
};
const onClientDisconnect = () => {
const message = 'Command cancelled because the client disconnected';
terminateChild(new Error(message));
};
timeout = setTimeout(() => {
const message = `Command timed out after ${Math.round(timeoutMs / 1000)}s`;
sendOutput(`${message}\n`);
terminateChild(new Error(message));
}, timeoutMs);
if (ws) {
ws.once('close', onClientDisconnect);
}
const onData = (data: Buffer) => {
const text = data.toString();
errorLog += text;
sendOutput(text);
};
child.stdout.on('data', onData);
child.stderr.on('data', onData);
child.on('close', (code: number | null) => {
exited = true;
finish(() => {
sendOutput(`Command exited with code ${code}\n`);
if (pendingTerminationError) {
if (throwOnError) reject(pendingTerminationError);
else resolve();
return;
}
if (code === 0) resolve();
else if (throwOnError) reject(new Error(redactSensitiveText(errorLog.trim()) || `Command failed with code ${code}`));
else resolve();
});
});
child.on('error', (error: Error & { code?: string }) => {
exited = true;
finish(() => {
let message = redactSensitiveText(error.message);
if (error.code === 'ENOENT' && /^spawn docker(?:$| )/.test(error.message)) {
message = 'Docker CLI unavailable on this node';
}
sendOutput(`Error: ${message}\n`);
if (pendingTerminationError) {
if (throwOnError) reject(pendingTerminationError);
else resolve();
return;
}
if (throwOnError) reject(new Error(message));
else resolve();
});
});
});
}
child.stderr.on('data', onData);
child.on('close', (code: number | null) => {
exited = true;
finish(() => {
sendOutput(`Command exited with code ${code}\n`);
if (pendingTerminationError) {
if (throwOnError) reject(pendingTerminationError);
else resolve();
return;
}
if (code === 0) resolve();
else if (throwOnError) reject(new Error(redactSensitiveText(errorLog.trim()) || `Command failed with code ${code}`));
else resolve();
});
});
child.on('error', (error: Error & { code?: string }) => {
exited = true;
finish(() => {
const mapped = describeSpawnError(error as NodeJS.ErrnoException, { command });
const message = redactSensitiveText(mapped.message);
sendOutput(`Error: ${message}\n`);
if (mapped.isLowMemory) {
console.warn('[ComposeService] spawn failed under memory pressure:', message);
}
if (pendingTerminationError) {
if (throwOnError) reject(pendingTerminationError);
else resolve();
return;
}
if (throwOnError) reject(new Error(message));
else resolve();
});
});
});
}
private async withRegistryAuth<T>(
fn: (env: Record<string, string | undefined>) => Promise<T>,
@@ -596,7 +598,13 @@ export class ComposeService {
let stderr = '';
child.stdout.on('data', (data: Buffer) => { stdout += data.toString(); });
child.stderr.on('data', (data: Buffer) => { stderr += data.toString(); });
child.on('error', (err) => reject(err));
child.on('error', (err: NodeJS.ErrnoException) => {
const mapped = describeSpawnError(err, { command: 'docker compose' });
if (mapped.isLowMemory) {
console.warn('[ComposeService] captureCompose spawn failed under memory pressure:', mapped.message);
}
reject(new Error(mapped.message));
});
child.on('close', (code) => {
if (code === 0) resolve(stdout);
else reject(new Error(stderr.trim() || `docker compose ${args.join(' ')} failed with code ${code}`));
+8 -3
View File
@@ -11,6 +11,7 @@ import { CacheService } from './CacheService';
import { isPathWithinBase } from '../utils/validation';
import { isDebugEnabled } from '../utils/debug';
import { sanitizeForLog } from '../utils/safeLog';
import { describeSpawnError } from '../utils/spawnErrors';
const execAsync = promisify(exec);
const COMPOSE_DIR = process.env.COMPOSE_DIR || '/app/compose';
@@ -909,9 +910,13 @@ class DockerController {
return await this.enrichContainers(await this.smartFallback(stackName, stackDir));
} catch (error) {
// If command fails (e.g., stack not deployed, invalid YAML, missing env_file)
const execError = error as { stderr?: string; message?: string };
console.error('Docker Compose Error for %s:', sanitizeForLog(stackName), sanitizeForLog(execError.stderr || execError.message || 'unknown'));
// If command fails (e.g., stack not deployed, invalid YAML, missing env_file,
// or host under memory pressure causing posix_spawn to fail with ENOMEM,
// which Linux libuv can surface as ENOENT).
const execError = error as NodeJS.ErrnoException & { stderr?: string };
const mapped = describeSpawnError(execError, { command: 'docker compose ps' });
const detail = execError.stderr || mapped.message;
console.error('Docker Compose Error for %s:', sanitizeForLog(stackName), sanitizeForLog(detail));
// Try smart fallback even on error
return await this.enrichContainers(await this.smartFallback(stackName, stackDir));
+7 -1
View File
@@ -1,4 +1,5 @@
import path from 'path';
import os from 'os';
import { promises as fsPromises, createReadStream } from 'fs';
import type { Readable } from 'stream';
import { NodeRegistry } from './NodeRegistry';
@@ -135,7 +136,12 @@ export class FileSystemService {
return stackNames;
} catch (error: any) {
console.warn(`[FileSystemService] Failed to list stacks: ${error.message}`);
if (error?.code === 'ENOMEM') {
const freeMiB = Math.round(os.freemem() / (1024 * 1024));
console.warn(`[FileSystemService] Failed to list stacks: ENOMEM (host free memory: ${freeMiB} MiB). Returning empty list.`);
} else {
console.warn(`[FileSystemService] Failed to list stacks: ${error.message}`);
}
return [];
}
}
+63
View File
@@ -0,0 +1,63 @@
import os from 'os';
/**
* Below this floor of free host memory, treat an ENOENT or ENOMEM
* spawn failure as a memory-pressure event rather than a missing binary.
* Linux libuv's posix_spawn can fail to allocate its argv/path-search arena
* under memory pressure and surface the underlying ENOMEM as ENOENT, sending
* operators down a "missing binary" debugging path when the real cause is
* host OOM. 128 MiB is well above the point where posix_spawn starts dropping
* arenas yet low enough that a healthy homelab host never trips it.
*/
export const LOW_MEMORY_FLOOR_BYTES = 128 * 1024 * 1024;
export interface SpawnErrorContext {
/** Literal first argument passed to spawn/exec (e.g. "docker", "/bin/sh"). */
command: string;
}
export interface MappedSpawnError {
/** Operator-facing message. Safe to send through WS / log / throw. */
message: string;
/** True when the failure was attributed to host memory pressure. */
isLowMemory: boolean;
}
/**
* Rewrite a spawn / exec error into an operator-facing message.
*
* Ordering matters: an explicit ENOMEM always wins, then an ENOENT under low
* free memory is treated as the libuv masquerade case, and only after that
* does the genuine "docker CLI missing" mapping fire. Other errors pass
* through unchanged.
*/
export function describeSpawnError(
error: NodeJS.ErrnoException,
ctx: SpawnErrorContext,
): MappedSpawnError {
const free = os.freemem();
const total = os.totalmem();
const freeMiB = Math.round(free / (1024 * 1024));
const totalMiB = Math.round(total / (1024 * 1024));
const lowMem = free < LOW_MEMORY_FLOOR_BYTES;
if (error.code === 'ENOMEM') {
return {
message: `Out of memory while launching ${ctx.command} (host free memory: ${freeMiB} MiB of ${totalMiB} MiB)`,
isLowMemory: true,
};
}
if (error.code === 'ENOENT' && lowMem) {
return {
message: `Out of memory while launching ${ctx.command} (host free memory: ${freeMiB} MiB of ${totalMiB} MiB; reported as ENOENT under memory pressure)`,
isLowMemory: true,
};
}
if (error.code === 'ENOENT' && /^spawn docker(?:$| )/.test(error.message ?? '')) {
return { message: 'Docker CLI unavailable on this node', isLowMemory: false };
}
return { message: error.message ?? String(error), isLowMemory: false };
}