Files
sencho/backend/src/__tests__/container-lifecycle-classifier.test.ts
T
Anso 44a89d9d2e fix(docker-events): harden crash detection against edge cases (#591)
* fix(docker-events): harden crash detection against edge cases

- Isolate per-container failures in the reconcile path so one failed
  container inspect cannot abort classification for the rest of the
  batch after a reconnect.
- Fall back to inspecting State.OOMKilled when a container exits with
  code 137 and no oom event preceded the die, so cgroup OOM kills that
  lose the oom event are still classified correctly. The dedup check
  runs before the fallback so crashloops do not hammer the daemon.
- Guard the intentional-kill window against wildly future-dated Docker
  timestamps by switching to a signed age comparison with a bounded
  negative-skew tolerance, so clock skew cannot flip a genuine crash
  into an intentional stop.
- Align diagnostic logging with the codebase [Service:diag] convention
  and add one informational lifecycle log on boot and shutdown so
  operators running in developer mode can confirm the watcher started.
- New tests cover gap-inspect isolation, the OOM inspect fallback (both
  success and failure paths), duplicate die events collapsing within
  the grace window, and clock-skew bounds on the classifier.

* docs(alerts): add troubleshooting entries for crash detection toggle and rate limits
2026-04-14 14:37:28 -04:00

136 lines
4.7 KiB
TypeScript

/**
* Unit tests for ContainerLifecycleClassifier.
*
* Pure function tests: no I/O, no mocks, no timers. The classifier is the
* single source of truth for crash/intentional/clean/oom classification,
* so these tests pin its behaviour contract.
*/
import { describe, it, expect } from 'vitest';
import {
classifyDie,
classifyGapExit,
INTENTIONAL_KILL_WINDOW_MS,
MAX_NEGATIVE_SKEW_MS,
} from '../services/ContainerLifecycleClassifier';
describe('classifyDie', () => {
const now = 1_700_000_000_000;
it('classifies as intentional when kill was recent', () => {
const result = classifyDie(
{ at: now, exitCode: 1 },
{ lastKillAt: now - 1_000 },
);
expect(result).toBe('intentional');
});
it('classifies as intentional at exactly the boundary', () => {
const result = classifyDie(
{ at: now, exitCode: 137 },
{ lastKillAt: now - INTENTIONAL_KILL_WINDOW_MS },
);
expect(result).toBe('intentional');
});
it('classifies as crash when kill was outside the window', () => {
const result = classifyDie(
{ at: now, exitCode: 1 },
{ lastKillAt: now - (INTENTIONAL_KILL_WINDOW_MS + 1) },
);
expect(result).toBe('crash');
});
it('classifies as clean when exit code is 0 with no kill', () => {
const result = classifyDie({ at: now, exitCode: 0 }, {});
expect(result).toBe('clean');
});
it('classifies as crash when exit code is non-zero with no kill', () => {
const result = classifyDie({ at: now, exitCode: 1 }, {});
expect(result).toBe('crash');
});
it('classifies as crash when exit code is undefined (malformed)', () => {
const result = classifyDie({ at: now, exitCode: undefined }, {});
expect(result).toBe('crash');
});
it('classifies as oom when oomPending is true, overriding exit code', () => {
const result = classifyDie(
{ at: now, exitCode: 0 },
{ oomPending: true },
);
expect(result).toBe('oom');
});
it('oom takes priority over a recent kill', () => {
const result = classifyDie(
{ at: now, exitCode: 137 },
{ oomPending: true, lastKillAt: now - 1_000 },
);
expect(result).toBe('oom');
});
it('accepts a kill arriving slightly after the die (out-of-order delivery)', () => {
// DockerEventService's 500ms grace window means the kill can land
// after the die. The classifier treats that as intentional.
const result = classifyDie(
{ at: now, exitCode: 1 },
{ lastKillAt: now + 200 },
);
expect(result).toBe('intentional');
});
it('treats a kill far outside the window (either direction) as crash', () => {
const result = classifyDie(
{ at: now, exitCode: 1 },
{ lastKillAt: now + (INTENTIONAL_KILL_WINDOW_MS + 1) },
);
expect(result).toBe('crash');
});
it('rejects a die with a wildly future-dated timestamp (clock skew)', () => {
// Die timestamp 120s in the future while the kill happened "now":
// a real out-of-order delivery is bounded by the 500ms grace window,
// so 120s of future skew must not match an earlier intentional kill.
const result = classifyDie(
{ at: now + 120_000, exitCode: 1 },
{ lastKillAt: now },
);
expect(result).toBe('crash');
});
it('accepts a die with modest negative skew within the tolerance window', () => {
// Kill happened slightly after the die (bounded out-of-order delivery
// plus normal clock skew, under MAX_NEGATIVE_SKEW_MS). Still intentional.
const skew = MAX_NEGATIVE_SKEW_MS - 1_000;
const result = classifyDie(
{ at: now, exitCode: 1 },
{ lastKillAt: now + skew },
);
expect(result).toBe('intentional');
});
});
describe('classifyGapExit', () => {
it('classifies OOMKilled containers as oom', () => {
expect(classifyGapExit({ State: { OOMKilled: true, ExitCode: 137 } })).toBe('oom');
});
it('classifies exit code 0 as clean', () => {
expect(classifyGapExit({ State: { OOMKilled: false, ExitCode: 0 } })).toBe('clean');
});
it('classifies non-zero exit code as crash', () => {
expect(classifyGapExit({ State: { OOMKilled: false, ExitCode: 1 } })).toBe('crash');
});
it('handles missing State gracefully', () => {
expect(classifyGapExit({})).toBe('crash');
});
it('handles undefined exit code as crash', () => {
expect(classifyGapExit({ State: {} })).toBe('crash');
});
});