mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-31 04:38:11 +00:00
fix(notifications): replace polling with Docker event stream for container lifecycle detection (#588)
* fix(notifications): replace polling with Docker event stream for container lifecycle detection Replaces the 30-second MonitorService crash-detection poll with a causal, per-node Docker events stream. Eliminates false crash alerts on intentional stops (docker stop, compose down, stack restart/update), detects OOM kills as a distinct alert category, and surfaces real crashes in real time. A new DockerEventManager spawns one DockerEventService per local node. Each service consumes the filtered container event stream, classifies die events against recent kill/oom state, and reconciles container state via snapshot diffing on connect and reconnect. Rate limiting, exponential backoff with jitter, and parse-error tolerance keep the stream resilient under load and during daemon interruptions. MonitorService retains host limits, janitor, version check, and stack metric alerts; crash and healthcheck detection move out entirely. * fix(tests): silence require-imports lint in hoisted mock factory
This commit is contained in:
Anso
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* Unit tests for ContainerLifecycleClassifier.
|
||||
*
|
||||
* Pure function tests: no I/O, no mocks, no timers. The classifier is the
|
||||
* single source of truth for crash/intentional/clean/oom classification,
|
||||
* so these tests pin its behaviour contract.
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
classifyDie,
|
||||
classifyGapExit,
|
||||
INTENTIONAL_KILL_WINDOW_MS,
|
||||
} from '../services/ContainerLifecycleClassifier';
|
||||
|
||||
describe('classifyDie', () => {
|
||||
const now = 1_700_000_000_000;
|
||||
|
||||
it('classifies as intentional when kill was recent', () => {
|
||||
const result = classifyDie(
|
||||
{ at: now, exitCode: 1 },
|
||||
{ lastKillAt: now - 1_000 },
|
||||
);
|
||||
expect(result).toBe('intentional');
|
||||
});
|
||||
|
||||
it('classifies as intentional at exactly the boundary', () => {
|
||||
const result = classifyDie(
|
||||
{ at: now, exitCode: 137 },
|
||||
{ lastKillAt: now - INTENTIONAL_KILL_WINDOW_MS },
|
||||
);
|
||||
expect(result).toBe('intentional');
|
||||
});
|
||||
|
||||
it('classifies as crash when kill was outside the window', () => {
|
||||
const result = classifyDie(
|
||||
{ at: now, exitCode: 1 },
|
||||
{ lastKillAt: now - (INTENTIONAL_KILL_WINDOW_MS + 1) },
|
||||
);
|
||||
expect(result).toBe('crash');
|
||||
});
|
||||
|
||||
it('classifies as clean when exit code is 0 with no kill', () => {
|
||||
const result = classifyDie({ at: now, exitCode: 0 }, {});
|
||||
expect(result).toBe('clean');
|
||||
});
|
||||
|
||||
it('classifies as crash when exit code is non-zero with no kill', () => {
|
||||
const result = classifyDie({ at: now, exitCode: 1 }, {});
|
||||
expect(result).toBe('crash');
|
||||
});
|
||||
|
||||
it('classifies as crash when exit code is undefined (malformed)', () => {
|
||||
const result = classifyDie({ at: now, exitCode: undefined }, {});
|
||||
expect(result).toBe('crash');
|
||||
});
|
||||
|
||||
it('classifies as oom when oomPending is true, overriding exit code', () => {
|
||||
const result = classifyDie(
|
||||
{ at: now, exitCode: 0 },
|
||||
{ oomPending: true },
|
||||
);
|
||||
expect(result).toBe('oom');
|
||||
});
|
||||
|
||||
it('oom takes priority over a recent kill', () => {
|
||||
const result = classifyDie(
|
||||
{ at: now, exitCode: 137 },
|
||||
{ oomPending: true, lastKillAt: now - 1_000 },
|
||||
);
|
||||
expect(result).toBe('oom');
|
||||
});
|
||||
|
||||
it('accepts a kill arriving slightly after the die (out-of-order delivery)', () => {
|
||||
// DockerEventService's 500ms grace window means the kill can land
|
||||
// after the die. The classifier treats that as intentional.
|
||||
const result = classifyDie(
|
||||
{ at: now, exitCode: 1 },
|
||||
{ lastKillAt: now + 200 },
|
||||
);
|
||||
expect(result).toBe('intentional');
|
||||
});
|
||||
|
||||
it('treats a kill far outside the window (either direction) as crash', () => {
|
||||
const result = classifyDie(
|
||||
{ at: now, exitCode: 1 },
|
||||
{ lastKillAt: now + (INTENTIONAL_KILL_WINDOW_MS + 1) },
|
||||
);
|
||||
expect(result).toBe('crash');
|
||||
});
|
||||
});
|
||||
|
||||
describe('classifyGapExit', () => {
|
||||
it('classifies OOMKilled containers as oom', () => {
|
||||
expect(classifyGapExit({ State: { OOMKilled: true, ExitCode: 137 } })).toBe('oom');
|
||||
});
|
||||
|
||||
it('classifies exit code 0 as clean', () => {
|
||||
expect(classifyGapExit({ State: { OOMKilled: false, ExitCode: 0 } })).toBe('clean');
|
||||
});
|
||||
|
||||
it('classifies non-zero exit code as crash', () => {
|
||||
expect(classifyGapExit({ State: { OOMKilled: false, ExitCode: 1 } })).toBe('crash');
|
||||
});
|
||||
|
||||
it('handles missing State gracefully', () => {
|
||||
expect(classifyGapExit({})).toBe('crash');
|
||||
});
|
||||
|
||||
it('handles undefined exit code as crash', () => {
|
||||
expect(classifyGapExit({ State: {} })).toBe('crash');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,210 @@
|
||||
/**
|
||||
* Unit tests for DockerEventManager.
|
||||
*
|
||||
* Verifies that:
|
||||
* - Boot enumerates only local nodes and spawns one service each.
|
||||
* - 'node-added' for a local node spawns a service.
|
||||
* - 'node-added' for a remote node does nothing.
|
||||
* - 'node-removed' tears down the matching service.
|
||||
* - 'node-updated' respawns when type flips remote <-> local.
|
||||
* - Stop unsubscribes and tears down all services.
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { EventEmitter } from 'events';
|
||||
|
||||
// ── Hoisted mocks ──────────────────────────────────────────────────────
|
||||
// Note: the hoisted factory runs before top-level imports resolve, so we
|
||||
// load `events` via require() here rather than referencing the top-level
|
||||
// EventEmitter import (which is not yet initialised at hoist time). A
|
||||
// top-level `await import()` would trigger TS1378 under the current
|
||||
// tsconfig, so require is the pragmatic choice for this hoisted factory.
|
||||
const {
|
||||
mockGetNodes,
|
||||
mockGetNode,
|
||||
serviceStart,
|
||||
serviceShutdown,
|
||||
DockerEventServiceCtor,
|
||||
registryInstance,
|
||||
} = vi.hoisted(() => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const { EventEmitter: HoistedEE } = require('events');
|
||||
const start = vi.fn().mockResolvedValue(undefined);
|
||||
const shutdown = vi.fn();
|
||||
// A real class (not a vi.fn) so `new` works reliably across vitest versions.
|
||||
class FakeDockerEventService {
|
||||
public readonly nodeId: number;
|
||||
public readonly nodeName: string;
|
||||
constructor(nodeId: number, nodeName: string) {
|
||||
this.nodeId = nodeId;
|
||||
this.nodeName = nodeName;
|
||||
}
|
||||
start() { return start(); }
|
||||
shutdown() { shutdown(); }
|
||||
getStatus() {
|
||||
return {
|
||||
nodeId: this.nodeId,
|
||||
nodeName: this.nodeName,
|
||||
status: 'connected' as const,
|
||||
reconnectAttempts: 0,
|
||||
trackedContainers: 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
const ctorSpy = vi.fn((nodeId: number, nodeName: string) =>
|
||||
new FakeDockerEventService(nodeId, nodeName),
|
||||
);
|
||||
// Wrap in a Proxy so `new Ctor(...)` both constructs via the real class
|
||||
// and records the call on the spy for assertions.
|
||||
const Ctor = new Proxy(FakeDockerEventService, {
|
||||
construct(_target, args: [number, string]) {
|
||||
ctorSpy(...args);
|
||||
return new FakeDockerEventService(...args);
|
||||
},
|
||||
});
|
||||
return {
|
||||
mockGetNodes: vi.fn(),
|
||||
mockGetNode: vi.fn(),
|
||||
serviceStart: start,
|
||||
serviceShutdown: shutdown,
|
||||
DockerEventServiceCtor: Object.assign(Ctor, { _spy: ctorSpy }),
|
||||
registryInstance: new HoistedEE() as EventEmitter,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('../services/DatabaseService', () => ({
|
||||
DatabaseService: {
|
||||
getInstance: () => ({
|
||||
getNodes: mockGetNodes,
|
||||
getNode: mockGetNode,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/NodeRegistry', () => ({
|
||||
NodeRegistry: {
|
||||
getInstance: () => registryInstance,
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/DockerEventService', () => ({
|
||||
DockerEventService: DockerEventServiceCtor,
|
||||
}));
|
||||
|
||||
import { DockerEventManager } from '../services/DockerEventManager';
|
||||
|
||||
// Proxy records constructor calls on _spy for assertions.
|
||||
const ctorSpy = (DockerEventServiceCtor as unknown as { _spy: ReturnType<typeof vi.fn> })._spy;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
// Reset singleton between tests.
|
||||
(DockerEventManager as unknown as { instance: DockerEventManager | undefined }).instance = undefined;
|
||||
registryInstance.removeAllListeners();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
DockerEventManager.getInstance().stop();
|
||||
});
|
||||
|
||||
describe('DockerEventManager - boot', () => {
|
||||
it('spawns a service for each local node and skips remote nodes', async () => {
|
||||
mockGetNodes.mockReturnValue([
|
||||
{ id: 1, name: 'local-a', type: 'local' },
|
||||
{ id: 2, name: 'remote-b', type: 'remote' },
|
||||
{ id: 3, name: 'local-c', type: 'local' },
|
||||
]);
|
||||
|
||||
await DockerEventManager.getInstance().start();
|
||||
|
||||
expect(ctorSpy).toHaveBeenCalledTimes(2);
|
||||
expect(ctorSpy).toHaveBeenCalledWith(1, 'local-a');
|
||||
expect(ctorSpy).toHaveBeenCalledWith(3, 'local-c');
|
||||
expect(serviceStart).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('second start is a no-op', async () => {
|
||||
mockGetNodes.mockReturnValue([{ id: 1, name: 'a', type: 'local' }]);
|
||||
|
||||
const mgr = DockerEventManager.getInstance();
|
||||
await mgr.start();
|
||||
await mgr.start();
|
||||
|
||||
expect(ctorSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DockerEventManager - node lifecycle events', () => {
|
||||
it('spawns a service when a local node is added', async () => {
|
||||
mockGetNodes.mockReturnValue([]);
|
||||
await DockerEventManager.getInstance().start();
|
||||
|
||||
mockGetNode.mockReturnValue({ id: 5, name: 'new-local', type: 'local' });
|
||||
registryInstance.emit('node-added', 5);
|
||||
await vi.waitFor(() => expect(serviceStart).toHaveBeenCalled());
|
||||
|
||||
expect(ctorSpy).toHaveBeenCalledWith(5, 'new-local');
|
||||
});
|
||||
|
||||
it('does nothing when a remote node is added', async () => {
|
||||
mockGetNodes.mockReturnValue([]);
|
||||
await DockerEventManager.getInstance().start();
|
||||
|
||||
mockGetNode.mockReturnValue({ id: 6, name: 'new-remote', type: 'remote' });
|
||||
registryInstance.emit('node-added', 6);
|
||||
// Give the async handler a tick to run.
|
||||
await new Promise(r => setTimeout(r, 0));
|
||||
|
||||
expect(ctorSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('shuts down the service when a node is removed', async () => {
|
||||
mockGetNodes.mockReturnValue([{ id: 7, name: 'to-remove', type: 'local' }]);
|
||||
await DockerEventManager.getInstance().start();
|
||||
expect(ctorSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
registryInstance.emit('node-removed', 7);
|
||||
expect(serviceShutdown).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('respawns when a remote node becomes local', async () => {
|
||||
mockGetNodes.mockReturnValue([]); // starts with nothing
|
||||
await DockerEventManager.getInstance().start();
|
||||
|
||||
// Now the node exists and is local.
|
||||
mockGetNode.mockReturnValue({ id: 9, name: 'flipped', type: 'local' });
|
||||
registryInstance.emit('node-updated', 9);
|
||||
await vi.waitFor(() => expect(serviceStart).toHaveBeenCalled());
|
||||
|
||||
expect(ctorSpy).toHaveBeenCalledWith(9, 'flipped');
|
||||
});
|
||||
|
||||
it('tears down when a local node becomes remote', async () => {
|
||||
mockGetNodes.mockReturnValue([{ id: 10, name: 'was-local', type: 'local' }]);
|
||||
await DockerEventManager.getInstance().start();
|
||||
expect(ctorSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
mockGetNode.mockReturnValue({ id: 10, name: 'was-local', type: 'remote' });
|
||||
registryInstance.emit('node-updated', 10);
|
||||
await new Promise(r => setTimeout(r, 0));
|
||||
|
||||
expect(serviceShutdown).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DockerEventManager - shutdown', () => {
|
||||
it('stops every service and removes listeners', async () => {
|
||||
mockGetNodes.mockReturnValue([
|
||||
{ id: 1, name: 'a', type: 'local' },
|
||||
{ id: 2, name: 'b', type: 'local' },
|
||||
]);
|
||||
const mgr = DockerEventManager.getInstance();
|
||||
await mgr.start();
|
||||
|
||||
mgr.stop();
|
||||
|
||||
expect(serviceShutdown).toHaveBeenCalledTimes(2);
|
||||
expect(registryInstance.listenerCount('node-added')).toBe(0);
|
||||
expect(registryInstance.listenerCount('node-removed')).toBe(0);
|
||||
expect(registryInstance.listenerCount('node-updated')).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,467 @@
|
||||
/**
|
||||
* Unit tests for DockerEventService.
|
||||
*
|
||||
* Mocks the Docker client stream via a small helper that exposes push() and
|
||||
* error() hooks so tests can drive the stream deterministically. Focuses on:
|
||||
* - classification of kill / die / oom / health_status
|
||||
* - the 500ms grace window for out-of-order die events
|
||||
* - rate limiting and overflow summary
|
||||
* - reconciliation on connect (baseline vs gap exits)
|
||||
* - mass-event detection on reconnect
|
||||
* - reconnect backoff + one-time warning/info alerts
|
||||
* - malformed payload tolerance
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { EventEmitter } from 'events';
|
||||
|
||||
// ── Hoisted mocks ──────────────────────────────────────────────────────
|
||||
|
||||
const {
|
||||
mockDispatchAlert,
|
||||
mockGetGlobalSettings,
|
||||
mockGetEvents,
|
||||
mockListContainers,
|
||||
mockInspect,
|
||||
mockGetContainer,
|
||||
mockGetDocker,
|
||||
} = vi.hoisted(() => ({
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
|
||||
mockGetGlobalSettings: vi.fn().mockReturnValue({ global_crash: '1' }),
|
||||
mockGetEvents: vi.fn(),
|
||||
mockListContainers: vi.fn().mockResolvedValue([]),
|
||||
mockInspect: vi.fn().mockResolvedValue({}),
|
||||
mockGetContainer: vi.fn(),
|
||||
mockGetDocker: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../services/NotificationService', () => ({
|
||||
NotificationService: {
|
||||
getInstance: () => ({ dispatchAlert: mockDispatchAlert }),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/DatabaseService', () => ({
|
||||
DatabaseService: {
|
||||
getInstance: () => ({ getGlobalSettings: mockGetGlobalSettings }),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/NodeRegistry', () => ({
|
||||
NodeRegistry: {
|
||||
getInstance: () => ({ getDocker: mockGetDocker }),
|
||||
},
|
||||
}));
|
||||
|
||||
// ── Fake Docker stream helper ──────────────────────────────────────────
|
||||
|
||||
interface FakeStream extends EventEmitter {
|
||||
destroyed: boolean;
|
||||
destroy: () => void;
|
||||
push: (event: Record<string, unknown>) => void;
|
||||
pushRaw: (raw: string) => void;
|
||||
error: (err: Error) => void;
|
||||
}
|
||||
|
||||
function makeStream(): FakeStream {
|
||||
const ee = new EventEmitter() as FakeStream;
|
||||
ee.destroyed = false;
|
||||
ee.destroy = () => { ee.destroyed = true; };
|
||||
ee.push = (event) => ee.emit('data', Buffer.from(JSON.stringify(event) + '\n'));
|
||||
ee.pushRaw = (raw) => ee.emit('data', Buffer.from(raw));
|
||||
ee.error = (err) => ee.emit('error', err);
|
||||
return ee;
|
||||
}
|
||||
|
||||
// ── Setup ──────────────────────────────────────────────────────────────
|
||||
|
||||
import { DockerEventService } from '../services/DockerEventService';
|
||||
|
||||
let stream: FakeStream;
|
||||
let service: DockerEventService;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.useFakeTimers();
|
||||
mockGetGlobalSettings.mockReturnValue({ global_crash: '1' });
|
||||
|
||||
stream = makeStream();
|
||||
mockGetEvents.mockImplementation(async () => stream);
|
||||
mockListContainers.mockResolvedValue([]);
|
||||
mockGetContainer.mockImplementation((id: string) => ({
|
||||
inspect: () => mockInspect(id),
|
||||
}));
|
||||
mockGetDocker.mockReturnValue({
|
||||
getEvents: mockGetEvents,
|
||||
listContainers: mockListContainers,
|
||||
getContainer: mockGetContainer,
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
service?.shutdown();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
// ── Classification via event stream ────────────────────────────────────
|
||||
|
||||
describe('DockerEventService - die classification', () => {
|
||||
it('emits crash alert on die with non-zero exit code and no prior kill', async () => {
|
||||
service = new DockerEventService(1, 'local');
|
||||
await service.start();
|
||||
|
||||
stream.push({
|
||||
Type: 'container',
|
||||
Action: 'die',
|
||||
Actor: { ID: 'c1', Attributes: { exitCode: '1', name: 'web' } },
|
||||
time: 1700000000,
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(600); // past the 500ms grace window
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
expect.stringContaining('Container Crash Detected'),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('does not emit when die follows a recent kill (intentional)', async () => {
|
||||
service = new DockerEventService(1, 'local');
|
||||
await service.start();
|
||||
|
||||
stream.push({
|
||||
Type: 'container',
|
||||
Action: 'kill',
|
||||
Actor: { ID: 'c1', Attributes: { signal: '15' } },
|
||||
time: 1700000000,
|
||||
});
|
||||
stream.push({
|
||||
Type: 'container',
|
||||
Action: 'die',
|
||||
Actor: { ID: 'c1', Attributes: { exitCode: '1', name: 'web' } },
|
||||
time: 1700000001,
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(600);
|
||||
|
||||
const crashCall = mockDispatchAlert.mock.calls.find(c =>
|
||||
typeof c[1] === 'string' && c[1].includes('Crash'));
|
||||
expect(crashCall).toBeUndefined();
|
||||
});
|
||||
|
||||
it('reclassifies as intentional when kill arrives within the 500ms grace window', async () => {
|
||||
service = new DockerEventService(1, 'local');
|
||||
await service.start();
|
||||
|
||||
// die first, kill 200ms later
|
||||
stream.push({
|
||||
Type: 'container',
|
||||
Action: 'die',
|
||||
Actor: { ID: 'c2', Attributes: { exitCode: '1', name: 'svc' } },
|
||||
time: 1700000000,
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
stream.push({
|
||||
Type: 'container',
|
||||
Action: 'kill',
|
||||
Actor: { ID: 'c2', Attributes: { signal: '15' } },
|
||||
time: 1700000001,
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(400); // total > 500ms
|
||||
|
||||
const crashCall = mockDispatchAlert.mock.calls.find(c =>
|
||||
typeof c[1] === 'string' && c[1].includes('Crash'));
|
||||
expect(crashCall).toBeUndefined();
|
||||
});
|
||||
|
||||
it('emits OOM alert when oom precedes die', async () => {
|
||||
service = new DockerEventService(1, 'local');
|
||||
await service.start();
|
||||
|
||||
stream.push({ Type: 'container', Action: 'oom', Actor: { ID: 'c3', Attributes: { name: 'hog' } } });
|
||||
stream.push({
|
||||
Type: 'container',
|
||||
Action: 'die',
|
||||
Actor: { ID: 'c3', Attributes: { exitCode: '137', name: 'hog' } },
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(600);
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
expect.stringContaining('OOM Kill'),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('does not emit when exit code is 0 (clean exit)', async () => {
|
||||
service = new DockerEventService(1, 'local');
|
||||
await service.start();
|
||||
|
||||
stream.push({
|
||||
Type: 'container',
|
||||
Action: 'die',
|
||||
Actor: { ID: 'c4', Attributes: { exitCode: '0', name: 'oneshot' } },
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(600);
|
||||
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('emits unhealthy alert on health_status: unhealthy', async () => {
|
||||
service = new DockerEventService(1, 'local');
|
||||
await service.start();
|
||||
|
||||
stream.push({
|
||||
Type: 'container',
|
||||
Action: 'health_status: unhealthy',
|
||||
Actor: { ID: 'c5', Attributes: { name: 'api' } },
|
||||
});
|
||||
await vi.runOnlyPendingTimersAsync();
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
expect.stringContaining('Healthcheck failed'),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('does not emit when global_crash is disabled', async () => {
|
||||
mockGetGlobalSettings.mockReturnValue({ global_crash: '0' });
|
||||
service = new DockerEventService(1, 'local');
|
||||
await service.start();
|
||||
|
||||
stream.push({
|
||||
Type: 'container',
|
||||
Action: 'die',
|
||||
Actor: { ID: 'c6', Attributes: { exitCode: '1', name: 'web' } },
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(600);
|
||||
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('clears crash dedup when container starts again', async () => {
|
||||
service = new DockerEventService(1, 'local');
|
||||
await service.start();
|
||||
|
||||
// First crash
|
||||
stream.push({
|
||||
Type: 'container',
|
||||
Action: 'die',
|
||||
Actor: { ID: 'c7', Attributes: { exitCode: '1', name: 'web' } },
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(600);
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Start event clears dedup
|
||||
stream.push({ Type: 'container', Action: 'start', Actor: { ID: 'c7' } });
|
||||
|
||||
// Second crash should fire again
|
||||
stream.push({
|
||||
Type: 'container',
|
||||
Action: 'die',
|
||||
Actor: { ID: 'c7', Attributes: { exitCode: '2', name: 'web' } },
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(600);
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Rate limiting ──────────────────────────────────────────────────────
|
||||
|
||||
describe('DockerEventService - rate limiting', () => {
|
||||
it('batches overflow crashes into a single summary alert', async () => {
|
||||
service = new DockerEventService(1, 'local');
|
||||
await service.start();
|
||||
|
||||
// Push 22 die events (limit is 20).
|
||||
for (let i = 0; i < 22; i++) {
|
||||
stream.push({
|
||||
Type: 'container',
|
||||
Action: 'die',
|
||||
Actor: { ID: `c-${i}`, Attributes: { exitCode: '1', name: `n-${i}` } },
|
||||
});
|
||||
}
|
||||
await vi.advanceTimersByTimeAsync(600);
|
||||
|
||||
const crashCalls = mockDispatchAlert.mock.calls.filter(c =>
|
||||
typeof c[1] === 'string' && c[1].includes('Crash'));
|
||||
expect(crashCalls).toHaveLength(20);
|
||||
|
||||
// After the rate window, a summary warning fires.
|
||||
await vi.advanceTimersByTimeAsync(61_000);
|
||||
const summaryCalls = mockDispatchAlert.mock.calls.filter(c =>
|
||||
typeof c[1] === 'string' && c[1].includes('additional containers crashed'));
|
||||
expect(summaryCalls).toHaveLength(1);
|
||||
expect(summaryCalls[0][1]).toContain('2 additional');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Malformed payloads ─────────────────────────────────────────────────
|
||||
|
||||
describe('DockerEventService - malformed payloads', () => {
|
||||
it('tolerates a bad JSON line without tearing down the stream', async () => {
|
||||
service = new DockerEventService(1, 'local');
|
||||
await service.start();
|
||||
|
||||
stream.pushRaw('not json\n');
|
||||
stream.push({
|
||||
Type: 'container',
|
||||
Action: 'die',
|
||||
Actor: { ID: 'ok', Attributes: { exitCode: '1', name: 'ok' } },
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(600);
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
expect.stringContaining('Container Crash Detected'),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Reconciliation ─────────────────────────────────────────────────────
|
||||
|
||||
describe('DockerEventService - reconciliation', () => {
|
||||
it('on first boot, records pre-existing exited containers as baseline and does not alert', async () => {
|
||||
mockListContainers.mockResolvedValue([
|
||||
{ Id: 'pre-1', State: 'exited' },
|
||||
{ Id: 'pre-2', State: 'exited' },
|
||||
{ Id: 'run-1', State: 'running' },
|
||||
]);
|
||||
|
||||
service = new DockerEventService(1, 'local');
|
||||
await service.start();
|
||||
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('emits mass-event summary on reconnect when >20% of containers newly exited', async () => {
|
||||
// First connect: 10 running containers as baseline.
|
||||
const running = Array.from({ length: 10 }, (_, i) => ({
|
||||
Id: `c-${i}`,
|
||||
State: 'running',
|
||||
}));
|
||||
mockListContainers.mockResolvedValueOnce(running);
|
||||
|
||||
service = new DockerEventService(1, 'local');
|
||||
await service.start();
|
||||
|
||||
// Simulate stream drop.
|
||||
stream.error(new Error('connection reset'));
|
||||
stream = makeStream();
|
||||
mockGetEvents.mockImplementation(async () => stream);
|
||||
|
||||
// On reconnect: 5 of them now exited (>20%).
|
||||
const mixed = running.map((c, i) => ({
|
||||
Id: c.Id,
|
||||
State: i < 5 ? 'exited' : 'running',
|
||||
}));
|
||||
mockListContainers.mockResolvedValueOnce(mixed);
|
||||
|
||||
// Drain reconnect backoff + reconciliation.
|
||||
await vi.advanceTimersByTimeAsync(2_000);
|
||||
// Flush microtasks spawned by the async reconnect + reconcile chain
|
||||
// without running the recurring prune interval.
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
const massCall = mockDispatchAlert.mock.calls.find(c =>
|
||||
typeof c[1] === 'string' && c[1].includes('daemon interruption'));
|
||||
expect(massCall).toBeDefined();
|
||||
});
|
||||
|
||||
it('classifies individual gap exits on reconnect when below mass threshold', async () => {
|
||||
// Baseline: 10 containers running.
|
||||
const baseline = Array.from({ length: 10 }, (_, i) => ({
|
||||
Id: `c-${i}`,
|
||||
State: 'running',
|
||||
}));
|
||||
mockListContainers.mockResolvedValueOnce(baseline);
|
||||
|
||||
service = new DockerEventService(1, 'local');
|
||||
await service.start();
|
||||
|
||||
// Drop + one new exit (10%, below 20% threshold).
|
||||
stream.error(new Error('bad'));
|
||||
stream = makeStream();
|
||||
mockGetEvents.mockImplementation(async () => stream);
|
||||
|
||||
const postReconnect = baseline.map((c, i) => ({
|
||||
Id: c.Id,
|
||||
State: i === 0 ? 'exited' : 'running',
|
||||
}));
|
||||
mockListContainers.mockResolvedValueOnce(postReconnect);
|
||||
|
||||
mockInspect.mockResolvedValueOnce({
|
||||
Name: '/crashed-app',
|
||||
State: { ExitCode: 9, OOMKilled: false },
|
||||
Config: { Labels: { 'com.docker.compose.project': 'my-stack' } },
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(2_000);
|
||||
// Flush microtasks spawned by the async reconnect + reconcile chain
|
||||
// without running the recurring prune interval.
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
const crashCall = mockDispatchAlert.mock.calls.find(c =>
|
||||
typeof c[1] === 'string' && c[1].includes('crashed-app'));
|
||||
expect(crashCall).toBeDefined();
|
||||
expect(crashCall?.[2]).toBe('my-stack');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Reconnect lifecycle ────────────────────────────────────────────────
|
||||
|
||||
describe('DockerEventService - reconnect', () => {
|
||||
it('emits one-time warning on first disconnect and info on reconnect', async () => {
|
||||
service = new DockerEventService(1, 'local');
|
||||
await service.start();
|
||||
|
||||
stream.error(new Error('connection reset'));
|
||||
|
||||
// Reconnect succeeds immediately after backoff.
|
||||
stream = makeStream();
|
||||
mockGetEvents.mockImplementation(async () => stream);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(2_000);
|
||||
// Flush microtasks spawned by the async reconnect + reconcile chain
|
||||
// without running the recurring prune interval.
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
const warn = mockDispatchAlert.mock.calls.find(c => c[0] === 'warning');
|
||||
const info = mockDispatchAlert.mock.calls.find(c => c[0] === 'info');
|
||||
expect(warn?.[1]).toContain('Lost connection');
|
||||
expect(info?.[1]).toContain('Reconnected');
|
||||
});
|
||||
|
||||
it('shutdown cancels pending reconnect', async () => {
|
||||
service = new DockerEventService(1, 'local');
|
||||
await service.start();
|
||||
|
||||
stream.error(new Error('broken'));
|
||||
mockGetEvents.mockClear();
|
||||
service.shutdown();
|
||||
|
||||
// Advance time well past the first backoff window; no new connect should run.
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
expect(mockGetEvents).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Diagnostics ────────────────────────────────────────────────────────
|
||||
|
||||
describe('DockerEventService - getStatus', () => {
|
||||
it('reports connected after start', async () => {
|
||||
service = new DockerEventService(42, 'my-node');
|
||||
await service.start();
|
||||
|
||||
const status = service.getStatus();
|
||||
expect(status.nodeId).toBe(42);
|
||||
expect(status.nodeName).toBe('my-node');
|
||||
expect(status.status).toBe('connected');
|
||||
});
|
||||
});
|
||||
@@ -313,68 +313,8 @@ describe('MonitorService - evaluateGlobalSettings', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── Global crash detection ─────────────────────────────────────────────
|
||||
|
||||
describe('MonitorService - global crash detection', () => {
|
||||
it('detects exited containers with non-intentional exit codes', async () => {
|
||||
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
||||
mockGetAllContainers.mockResolvedValue([{
|
||||
Id: 'crash-1',
|
||||
State: 'exited',
|
||||
Status: 'Exited (1) 5 seconds ago',
|
||||
Names: ['/my-container'],
|
||||
}]);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ global_crash: '1' });
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('error', expect.stringContaining('Crash'), undefined);
|
||||
});
|
||||
|
||||
it('ignores exit codes 0, 137, 143, 255', async () => {
|
||||
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
||||
const intentionalExits = [0, 137, 143, 255];
|
||||
|
||||
for (const code of intentionalExits) {
|
||||
mockDispatchAlert.mockClear();
|
||||
mockGetAllContainers.mockResolvedValue([{
|
||||
Id: `safe-${code}`,
|
||||
State: 'exited',
|
||||
Status: `Exited (${code}) 5 seconds ago`,
|
||||
Names: ['/safe-container'],
|
||||
}]);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
(MonitorService as any).instance = undefined;
|
||||
await (svc as any).evaluateGlobalSettings({ global_crash: '1' });
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('error', expect.stringContaining('Crash'));
|
||||
}
|
||||
});
|
||||
|
||||
it('detects unhealthy containers', async () => {
|
||||
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
||||
mockGetAllContainers.mockResolvedValue([{
|
||||
Id: 'sick-1',
|
||||
State: 'unhealthy',
|
||||
Status: 'Up 2 hours (unhealthy)',
|
||||
Names: ['/sick-container'],
|
||||
}]);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ global_crash: '1' });
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('error', expect.stringContaining('unhealthy'), undefined);
|
||||
});
|
||||
|
||||
it('skips remote nodes', async () => {
|
||||
mockGetNodes.mockReturnValue([{ id: 2, name: 'remote-node', type: 'remote' }]);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ global_crash: '1' });
|
||||
|
||||
expect(mockGetAllContainers).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
// Crash + healthcheck detection now lives in DockerEventService (event-driven).
|
||||
// Tests for those flows live in docker-event-service.test.ts.
|
||||
|
||||
// ── Alert breach state machine ─────────────────────────────────────────
|
||||
|
||||
|
||||
Reference in New Issue
Block a user