fix(spawn): attribute ENOMEM and ENOENT-under-memory-pressure spawn failures to host OOM (#1111)

Operators previously saw "spawn docker ENOENT" or "spawn /bin/sh ENOENT" when
the host was under memory pressure, which sent them down a missing-binary
debugging path. Linux libuv's posix_spawn can fail to allocate its argv /
path-search arena under low free memory and surface the underlying ENOMEM
as ENOENT.

Centralizes spawn-error mapping in a new utils/spawnErrors.ts helper:
- Explicit ENOMEM is rewritten to "Out of memory while launching <command>
  (host free memory: X MiB of Y MiB)".
- ENOENT under the 128 MiB free-memory floor is rewritten with the same
  wording plus a "reported as ENOENT under memory pressure" hint.
- ENOENT for docker on a healthy host preserves the existing
  "Docker CLI unavailable on this node" mapping.
- Other errors pass through unchanged.

Applied at the four named offenders: ComposeService.execute(),
ComposeService.captureCompose(), DockerController.getContainersByStack(),
and FileSystemService.getStacks() (which gets an ENOMEM-aware log line
for the scandir failure).

Startup also logs host free/total MiB once and warns when free memory is
below the 128 MiB floor, so the diagnostic surfaces before the first
spawn attempt rather than after it fails.

37 tests cover the mapping function directly and the ComposeService /
FileSystemService integration paths.
This commit is contained in:
Anso
2026-05-19 07:27:12 -04:00
committed by GitHub
parent 5a2aed22fd
commit 9dbce9c3c7
8 changed files with 445 additions and 140 deletions
+51 -3
View File
@@ -6,18 +6,20 @@
* Permission errors are surfaced to the caller like any other failure
* (no Docker-helper fallback).
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import os from 'os';
import path from 'path';
const { mockRm } = vi.hoisted(() => ({
const { mockRm, mockReaddir } = vi.hoisted(() => ({
mockRm: vi.fn(),
mockReaddir: vi.fn(),
}));
vi.mock('fs', () => ({
promises: {
rm: mockRm,
mkdir: vi.fn(),
readdir: vi.fn(),
readdir: mockReaddir,
readFile: vi.fn(),
writeFile: vi.fn(),
access: vi.fn(),
@@ -79,3 +81,49 @@ describe('FileSystemService.deleteStack', () => {
await expect(service.deleteStack('io-error-stack')).rejects.toThrow(/disk I\/O error/);
});
});
describe('FileSystemService.getStacks', () => {
let service: FileSystemService;
let warnSpy: ReturnType<typeof vi.spyOn>;
beforeEach(() => {
vi.clearAllMocks();
service = FileSystemService.getInstance();
warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
});
afterEach(() => {
warnSpy.mockRestore();
});
it('returns [] and logs ENOMEM-specific message with host free memory', async () => {
const err = Object.assign(
new Error("ENOMEM: not enough memory, scandir '/test/compose'"),
{ code: 'ENOMEM' },
);
mockReaddir.mockRejectedValueOnce(err);
const freememSpy = vi.spyOn(os, 'freemem').mockReturnValue(37 * 1024 * 1024);
try {
const result = await service.getStacks();
expect(result).toEqual([]);
const warning = warnSpy.mock.calls[0]?.[0] as string;
expect(warning).toContain('ENOMEM');
expect(warning).toContain('host free memory: 37 MiB');
expect(warning).toContain('Returning empty list');
} finally {
freememSpy.mockRestore();
}
});
it('returns [] and logs the raw error message for non-ENOMEM errors', async () => {
const err = Object.assign(new Error('EACCES: permission denied'), { code: 'EACCES' });
mockReaddir.mockRejectedValueOnce(err);
const result = await service.getStacks();
expect(result).toEqual([]);
const warning = warnSpy.mock.calls[0]?.[0] as string;
expect(warning).toContain('EACCES: permission denied');
expect(warning).not.toContain('host free memory');
});
});