mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-21 15:46:43 +00:00
fix(spawn): attribute ENOMEM and ENOENT-under-memory-pressure spawn failures to host OOM (#1111)
Operators previously saw "spawn docker ENOENT" or "spawn /bin/sh ENOENT" when the host was under memory pressure, which sent them down a missing-binary debugging path. Linux libuv's posix_spawn can fail to allocate its argv / path-search arena under low free memory and surface the underlying ENOMEM as ENOENT. Centralizes spawn-error mapping in a new utils/spawnErrors.ts helper: - Explicit ENOMEM is rewritten to "Out of memory while launching <command> (host free memory: X MiB of Y MiB)". - ENOENT under the 128 MiB free-memory floor is rewritten with the same wording plus a "reported as ENOENT under memory pressure" hint. - ENOENT for docker on a healthy host preserves the existing "Docker CLI unavailable on this node" mapping. - Other errors pass through unchanged. Applied at the four named offenders: ComposeService.execute(), ComposeService.captureCompose(), DockerController.getContainersByStack(), and FileSystemService.getStacks() (which gets an ENOMEM-aware log line for the scandir failure). Startup also logs host free/total MiB once and warns when free memory is below the 128 MiB floor, so the diagnostic surfaces before the first spawn attempt rather than after it fails. 37 tests cover the mapping function directly and the ComposeService / FileSystemService integration paths.
This commit is contained in:
Anso
@@ -13,6 +13,7 @@ import { RegistryService } from './RegistryService';
|
||||
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { describeSpawnError } from '../utils/spawnErrors';
|
||||
import { isPathWithinBase, isValidStackName } from '../utils/validation';
|
||||
import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog';
|
||||
|
||||
@@ -31,22 +32,22 @@ export class ComposeRollbackError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
export function getComposeRollbackInfo(error: unknown): { attempted: boolean; rolledBack: boolean } | null {
|
||||
if (!(error instanceof ComposeRollbackError)) {
|
||||
return null;
|
||||
}
|
||||
return { attempted: error.rollbackAttempted, rolledBack: error.rolledBack };
|
||||
}
|
||||
|
||||
const DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS = 30 * 60 * 1000;
|
||||
|
||||
function getComposeCommandTimeoutMs(): number {
|
||||
const configured = Number(process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS);
|
||||
if (Number.isFinite(configured) && configured > 0) {
|
||||
return configured;
|
||||
}
|
||||
return DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS;
|
||||
}
|
||||
export function getComposeRollbackInfo(error: unknown): { attempted: boolean; rolledBack: boolean } | null {
|
||||
if (!(error instanceof ComposeRollbackError)) {
|
||||
return null;
|
||||
}
|
||||
return { attempted: error.rollbackAttempted, rolledBack: error.rolledBack };
|
||||
}
|
||||
|
||||
const DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS = 30 * 60 * 1000;
|
||||
|
||||
function getComposeCommandTimeoutMs(): number {
|
||||
const configured = Number(process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS);
|
||||
if (Number.isFinite(configured) && configured > 0) {
|
||||
return configured;
|
||||
}
|
||||
return DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
/**
|
||||
* ComposeService - local docker compose CLI execution.
|
||||
@@ -97,127 +98,128 @@ export class ComposeService {
|
||||
ws?: WebSocket,
|
||||
throwOnError = true,
|
||||
env?: Record<string, string | undefined>
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, {
|
||||
cwd,
|
||||
env: env ?? {
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, {
|
||||
cwd,
|
||||
env: env ?? {
|
||||
...process.env,
|
||||
PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
|
||||
}
|
||||
});
|
||||
|
||||
let errorLog = '';
|
||||
let settled = false;
|
||||
let exited = false;
|
||||
let pendingTerminationError: Error | null = null;
|
||||
const timeoutMs = getComposeCommandTimeoutMs();
|
||||
let timeout: ReturnType<typeof setTimeout> | null = null;
|
||||
let forceKillTimeout: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
const sendOutput = (text: string) => {
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(text);
|
||||
}
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
timeout = null;
|
||||
}
|
||||
if (forceKillTimeout) {
|
||||
clearTimeout(forceKillTimeout);
|
||||
forceKillTimeout = null;
|
||||
}
|
||||
if (ws) {
|
||||
ws.removeListener('close', onClientDisconnect);
|
||||
}
|
||||
};
|
||||
|
||||
const finish = (complete: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
complete();
|
||||
};
|
||||
|
||||
const terminateChild = (error: Error) => {
|
||||
pendingTerminationError = pendingTerminationError ?? error;
|
||||
if (exited) return;
|
||||
try {
|
||||
child.kill('SIGTERM');
|
||||
} catch (error) {
|
||||
console.warn('[ComposeService] Failed to terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
forceKillTimeout = setTimeout(() => {
|
||||
if (exited) return;
|
||||
try {
|
||||
child.kill('SIGKILL');
|
||||
} catch (error) {
|
||||
console.warn('[ComposeService] Failed to force terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
}, 5000);
|
||||
};
|
||||
|
||||
const onClientDisconnect = () => {
|
||||
const message = 'Command cancelled because the client disconnected';
|
||||
terminateChild(new Error(message));
|
||||
};
|
||||
|
||||
timeout = setTimeout(() => {
|
||||
const message = `Command timed out after ${Math.round(timeoutMs / 1000)}s`;
|
||||
sendOutput(`${message}\n`);
|
||||
terminateChild(new Error(message));
|
||||
}, timeoutMs);
|
||||
|
||||
if (ws) {
|
||||
ws.once('close', onClientDisconnect);
|
||||
}
|
||||
|
||||
const onData = (data: Buffer) => {
|
||||
const text = data.toString();
|
||||
errorLog += text;
|
||||
sendOutput(text);
|
||||
};
|
||||
});
|
||||
|
||||
let errorLog = '';
|
||||
let settled = false;
|
||||
let exited = false;
|
||||
let pendingTerminationError: Error | null = null;
|
||||
const timeoutMs = getComposeCommandTimeoutMs();
|
||||
let timeout: ReturnType<typeof setTimeout> | null = null;
|
||||
let forceKillTimeout: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
const sendOutput = (text: string) => {
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(text);
|
||||
}
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
timeout = null;
|
||||
}
|
||||
if (forceKillTimeout) {
|
||||
clearTimeout(forceKillTimeout);
|
||||
forceKillTimeout = null;
|
||||
}
|
||||
if (ws) {
|
||||
ws.removeListener('close', onClientDisconnect);
|
||||
}
|
||||
};
|
||||
|
||||
const finish = (complete: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
complete();
|
||||
};
|
||||
|
||||
const terminateChild = (error: Error) => {
|
||||
pendingTerminationError = pendingTerminationError ?? error;
|
||||
if (exited) return;
|
||||
try {
|
||||
child.kill('SIGTERM');
|
||||
} catch (error) {
|
||||
console.warn('[ComposeService] Failed to terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
forceKillTimeout = setTimeout(() => {
|
||||
if (exited) return;
|
||||
try {
|
||||
child.kill('SIGKILL');
|
||||
} catch (error) {
|
||||
console.warn('[ComposeService] Failed to force terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
}, 5000);
|
||||
};
|
||||
|
||||
const onClientDisconnect = () => {
|
||||
const message = 'Command cancelled because the client disconnected';
|
||||
terminateChild(new Error(message));
|
||||
};
|
||||
|
||||
timeout = setTimeout(() => {
|
||||
const message = `Command timed out after ${Math.round(timeoutMs / 1000)}s`;
|
||||
sendOutput(`${message}\n`);
|
||||
terminateChild(new Error(message));
|
||||
}, timeoutMs);
|
||||
|
||||
if (ws) {
|
||||
ws.once('close', onClientDisconnect);
|
||||
}
|
||||
|
||||
const onData = (data: Buffer) => {
|
||||
const text = data.toString();
|
||||
errorLog += text;
|
||||
sendOutput(text);
|
||||
};
|
||||
|
||||
child.stdout.on('data', onData);
|
||||
child.stderr.on('data', onData);
|
||||
|
||||
child.on('close', (code: number | null) => {
|
||||
exited = true;
|
||||
finish(() => {
|
||||
sendOutput(`Command exited with code ${code}\n`);
|
||||
if (pendingTerminationError) {
|
||||
if (throwOnError) reject(pendingTerminationError);
|
||||
else resolve();
|
||||
return;
|
||||
}
|
||||
if (code === 0) resolve();
|
||||
else if (throwOnError) reject(new Error(redactSensitiveText(errorLog.trim()) || `Command failed with code ${code}`));
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
|
||||
child.on('error', (error: Error & { code?: string }) => {
|
||||
exited = true;
|
||||
finish(() => {
|
||||
let message = redactSensitiveText(error.message);
|
||||
if (error.code === 'ENOENT' && /^spawn docker(?:$| )/.test(error.message)) {
|
||||
message = 'Docker CLI unavailable on this node';
|
||||
}
|
||||
sendOutput(`Error: ${message}\n`);
|
||||
if (pendingTerminationError) {
|
||||
if (throwOnError) reject(pendingTerminationError);
|
||||
else resolve();
|
||||
return;
|
||||
}
|
||||
if (throwOnError) reject(new Error(message));
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
child.stderr.on('data', onData);
|
||||
|
||||
child.on('close', (code: number | null) => {
|
||||
exited = true;
|
||||
finish(() => {
|
||||
sendOutput(`Command exited with code ${code}\n`);
|
||||
if (pendingTerminationError) {
|
||||
if (throwOnError) reject(pendingTerminationError);
|
||||
else resolve();
|
||||
return;
|
||||
}
|
||||
if (code === 0) resolve();
|
||||
else if (throwOnError) reject(new Error(redactSensitiveText(errorLog.trim()) || `Command failed with code ${code}`));
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
|
||||
child.on('error', (error: Error & { code?: string }) => {
|
||||
exited = true;
|
||||
finish(() => {
|
||||
const mapped = describeSpawnError(error as NodeJS.ErrnoException, { command });
|
||||
const message = redactSensitiveText(mapped.message);
|
||||
sendOutput(`Error: ${message}\n`);
|
||||
if (mapped.isLowMemory) {
|
||||
console.warn('[ComposeService] spawn failed under memory pressure:', message);
|
||||
}
|
||||
if (pendingTerminationError) {
|
||||
if (throwOnError) reject(pendingTerminationError);
|
||||
else resolve();
|
||||
return;
|
||||
}
|
||||
if (throwOnError) reject(new Error(message));
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private async withRegistryAuth<T>(
|
||||
fn: (env: Record<string, string | undefined>) => Promise<T>,
|
||||
@@ -596,7 +598,13 @@ export class ComposeService {
|
||||
let stderr = '';
|
||||
child.stdout.on('data', (data: Buffer) => { stdout += data.toString(); });
|
||||
child.stderr.on('data', (data: Buffer) => { stderr += data.toString(); });
|
||||
child.on('error', (err) => reject(err));
|
||||
child.on('error', (err: NodeJS.ErrnoException) => {
|
||||
const mapped = describeSpawnError(err, { command: 'docker compose' });
|
||||
if (mapped.isLowMemory) {
|
||||
console.warn('[ComposeService] captureCompose spawn failed under memory pressure:', mapped.message);
|
||||
}
|
||||
reject(new Error(mapped.message));
|
||||
});
|
||||
child.on('close', (code) => {
|
||||
if (code === 0) resolve(stdout);
|
||||
else reject(new Error(stderr.trim() || `docker compose ${args.join(' ')} failed with code ${code}`));
|
||||
|
||||
@@ -11,6 +11,7 @@ import { CacheService } from './CacheService';
|
||||
import { isPathWithinBase } from '../utils/validation';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { describeSpawnError } from '../utils/spawnErrors';
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
const COMPOSE_DIR = process.env.COMPOSE_DIR || '/app/compose';
|
||||
@@ -909,9 +910,13 @@ class DockerController {
|
||||
return await this.enrichContainers(await this.smartFallback(stackName, stackDir));
|
||||
|
||||
} catch (error) {
|
||||
// If command fails (e.g., stack not deployed, invalid YAML, missing env_file)
|
||||
const execError = error as { stderr?: string; message?: string };
|
||||
console.error('Docker Compose Error for %s:', sanitizeForLog(stackName), sanitizeForLog(execError.stderr || execError.message || 'unknown'));
|
||||
// If command fails (e.g., stack not deployed, invalid YAML, missing env_file,
|
||||
// or host under memory pressure causing posix_spawn to fail with ENOMEM,
|
||||
// which Linux libuv can surface as ENOENT).
|
||||
const execError = error as NodeJS.ErrnoException & { stderr?: string };
|
||||
const mapped = describeSpawnError(execError, { command: 'docker compose ps' });
|
||||
const detail = execError.stderr || mapped.message;
|
||||
console.error('Docker Compose Error for %s:', sanitizeForLog(stackName), sanitizeForLog(detail));
|
||||
|
||||
// Try smart fallback even on error
|
||||
return await this.enrichContainers(await this.smartFallback(stackName, stackDir));
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
import { promises as fsPromises, createReadStream } from 'fs';
|
||||
import type { Readable } from 'stream';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
@@ -135,7 +136,12 @@ export class FileSystemService {
|
||||
|
||||
return stackNames;
|
||||
} catch (error: any) {
|
||||
console.warn(`[FileSystemService] Failed to list stacks: ${error.message}`);
|
||||
if (error?.code === 'ENOMEM') {
|
||||
const freeMiB = Math.round(os.freemem() / (1024 * 1024));
|
||||
console.warn(`[FileSystemService] Failed to list stacks: ENOMEM (host free memory: ${freeMiB} MiB). Returning empty list.`);
|
||||
} else {
|
||||
console.warn(`[FileSystemService] Failed to list stacks: ${error.message}`);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user