fix(notifications): replace polling with Docker event stream for container lifecycle detection (#588)

* fix(notifications): replace polling with Docker event stream for container lifecycle detection

Replaces the 30-second MonitorService crash-detection poll with a causal,
per-node Docker events stream. Eliminates false crash alerts on intentional
stops (docker stop, compose down, stack restart/update), detects OOM kills
as a distinct alert category, and surfaces real crashes in real time.

A new DockerEventManager spawns one DockerEventService per local node. Each
service consumes the filtered container event stream, classifies die events
against recent kill/oom state, and reconciles container state via snapshot
diffing on connect and reconnect. Rate limiting, exponential backoff with
jitter, and parse-error tolerance keep the stream resilient under load and
during daemon interruptions.

MonitorService retains host limits, janitor, version check, and stack metric
alerts; crash and healthcheck detection move out entirely.

* fix(tests): silence require-imports lint in hoisted mock factory
This commit is contained in:
Anso
2026-04-14 13:47:48 -04:00
committed by GitHub
parent 6ffac2a0db
commit ad9a6859e6
11 changed files with 1720 additions and 124 deletions
@@ -0,0 +1,112 @@
/**
* Unit tests for ContainerLifecycleClassifier.
*
* Pure function tests: no I/O, no mocks, no timers. The classifier is the
* single source of truth for crash/intentional/clean/oom classification,
* so these tests pin its behaviour contract.
*/
import { describe, it, expect } from 'vitest';
import {
classifyDie,
classifyGapExit,
INTENTIONAL_KILL_WINDOW_MS,
} from '../services/ContainerLifecycleClassifier';
describe('classifyDie', () => {
const now = 1_700_000_000_000;
it('classifies as intentional when kill was recent', () => {
const result = classifyDie(
{ at: now, exitCode: 1 },
{ lastKillAt: now - 1_000 },
);
expect(result).toBe('intentional');
});
it('classifies as intentional at exactly the boundary', () => {
const result = classifyDie(
{ at: now, exitCode: 137 },
{ lastKillAt: now - INTENTIONAL_KILL_WINDOW_MS },
);
expect(result).toBe('intentional');
});
it('classifies as crash when kill was outside the window', () => {
const result = classifyDie(
{ at: now, exitCode: 1 },
{ lastKillAt: now - (INTENTIONAL_KILL_WINDOW_MS + 1) },
);
expect(result).toBe('crash');
});
it('classifies as clean when exit code is 0 with no kill', () => {
const result = classifyDie({ at: now, exitCode: 0 }, {});
expect(result).toBe('clean');
});
it('classifies as crash when exit code is non-zero with no kill', () => {
const result = classifyDie({ at: now, exitCode: 1 }, {});
expect(result).toBe('crash');
});
it('classifies as crash when exit code is undefined (malformed)', () => {
const result = classifyDie({ at: now, exitCode: undefined }, {});
expect(result).toBe('crash');
});
it('classifies as oom when oomPending is true, overriding exit code', () => {
const result = classifyDie(
{ at: now, exitCode: 0 },
{ oomPending: true },
);
expect(result).toBe('oom');
});
it('oom takes priority over a recent kill', () => {
const result = classifyDie(
{ at: now, exitCode: 137 },
{ oomPending: true, lastKillAt: now - 1_000 },
);
expect(result).toBe('oom');
});
it('accepts a kill arriving slightly after the die (out-of-order delivery)', () => {
// DockerEventService's 500ms grace window means the kill can land
// after the die. The classifier treats that as intentional.
const result = classifyDie(
{ at: now, exitCode: 1 },
{ lastKillAt: now + 200 },
);
expect(result).toBe('intentional');
});
it('treats a kill far outside the window (either direction) as crash', () => {
const result = classifyDie(
{ at: now, exitCode: 1 },
{ lastKillAt: now + (INTENTIONAL_KILL_WINDOW_MS + 1) },
);
expect(result).toBe('crash');
});
});
describe('classifyGapExit', () => {
it('classifies OOMKilled containers as oom', () => {
expect(classifyGapExit({ State: { OOMKilled: true, ExitCode: 137 } })).toBe('oom');
});
it('classifies exit code 0 as clean', () => {
expect(classifyGapExit({ State: { OOMKilled: false, ExitCode: 0 } })).toBe('clean');
});
it('classifies non-zero exit code as crash', () => {
expect(classifyGapExit({ State: { OOMKilled: false, ExitCode: 1 } })).toBe('crash');
});
it('handles missing State gracefully', () => {
expect(classifyGapExit({})).toBe('crash');
});
it('handles undefined exit code as crash', () => {
expect(classifyGapExit({ State: {} })).toBe('crash');
});
});