diff --git a/backend/src/__tests__/container-lifecycle-classifier.test.ts b/backend/src/__tests__/container-lifecycle-classifier.test.ts new file mode 100644 index 00000000..a4a51e32 --- /dev/null +++ b/backend/src/__tests__/container-lifecycle-classifier.test.ts @@ -0,0 +1,112 @@ +/** + * Unit tests for ContainerLifecycleClassifier. + * + * Pure function tests: no I/O, no mocks, no timers. The classifier is the + * single source of truth for crash/intentional/clean/oom classification, + * so these tests pin its behaviour contract. + */ +import { describe, it, expect } from 'vitest'; +import { + classifyDie, + classifyGapExit, + INTENTIONAL_KILL_WINDOW_MS, +} from '../services/ContainerLifecycleClassifier'; + +describe('classifyDie', () => { + const now = 1_700_000_000_000; + + it('classifies as intentional when kill was recent', () => { + const result = classifyDie( + { at: now, exitCode: 1 }, + { lastKillAt: now - 1_000 }, + ); + expect(result).toBe('intentional'); + }); + + it('classifies as intentional at exactly the boundary', () => { + const result = classifyDie( + { at: now, exitCode: 137 }, + { lastKillAt: now - INTENTIONAL_KILL_WINDOW_MS }, + ); + expect(result).toBe('intentional'); + }); + + it('classifies as crash when kill was outside the window', () => { + const result = classifyDie( + { at: now, exitCode: 1 }, + { lastKillAt: now - (INTENTIONAL_KILL_WINDOW_MS + 1) }, + ); + expect(result).toBe('crash'); + }); + + it('classifies as clean when exit code is 0 with no kill', () => { + const result = classifyDie({ at: now, exitCode: 0 }, {}); + expect(result).toBe('clean'); + }); + + it('classifies as crash when exit code is non-zero with no kill', () => { + const result = classifyDie({ at: now, exitCode: 1 }, {}); + expect(result).toBe('crash'); + }); + + it('classifies as crash when exit code is undefined (malformed)', () => { + const result = classifyDie({ at: now, exitCode: undefined }, {}); + expect(result).toBe('crash'); + }); + + it('classifies as oom when oomPending is true, overriding exit code', () => { + const result = classifyDie( + { at: now, exitCode: 0 }, + { oomPending: true }, + ); + expect(result).toBe('oom'); + }); + + it('oom takes priority over a recent kill', () => { + const result = classifyDie( + { at: now, exitCode: 137 }, + { oomPending: true, lastKillAt: now - 1_000 }, + ); + expect(result).toBe('oom'); + }); + + it('accepts a kill arriving slightly after the die (out-of-order delivery)', () => { + // DockerEventService's 500ms grace window means the kill can land + // after the die. The classifier treats that as intentional. + const result = classifyDie( + { at: now, exitCode: 1 }, + { lastKillAt: now + 200 }, + ); + expect(result).toBe('intentional'); + }); + + it('treats a kill far outside the window (either direction) as crash', () => { + const result = classifyDie( + { at: now, exitCode: 1 }, + { lastKillAt: now + (INTENTIONAL_KILL_WINDOW_MS + 1) }, + ); + expect(result).toBe('crash'); + }); +}); + +describe('classifyGapExit', () => { + it('classifies OOMKilled containers as oom', () => { + expect(classifyGapExit({ State: { OOMKilled: true, ExitCode: 137 } })).toBe('oom'); + }); + + it('classifies exit code 0 as clean', () => { + expect(classifyGapExit({ State: { OOMKilled: false, ExitCode: 0 } })).toBe('clean'); + }); + + it('classifies non-zero exit code as crash', () => { + expect(classifyGapExit({ State: { OOMKilled: false, ExitCode: 1 } })).toBe('crash'); + }); + + it('handles missing State gracefully', () => { + expect(classifyGapExit({})).toBe('crash'); + }); + + it('handles undefined exit code as crash', () => { + expect(classifyGapExit({ State: {} })).toBe('crash'); + }); +}); diff --git a/backend/src/__tests__/docker-event-manager.test.ts b/backend/src/__tests__/docker-event-manager.test.ts new file mode 100644 index 00000000..49829e37 --- /dev/null +++ b/backend/src/__tests__/docker-event-manager.test.ts @@ -0,0 +1,210 @@ +/** + * Unit tests for DockerEventManager. + * + * Verifies that: + * - Boot enumerates only local nodes and spawns one service each. + * - 'node-added' for a local node spawns a service. + * - 'node-added' for a remote node does nothing. + * - 'node-removed' tears down the matching service. + * - 'node-updated' respawns when type flips remote <-> local. + * - Stop unsubscribes and tears down all services. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { EventEmitter } from 'events'; + +// ── Hoisted mocks ────────────────────────────────────────────────────── +// Note: the hoisted factory runs before top-level imports resolve, so we +// load `events` via require() here rather than referencing the top-level +// EventEmitter import (which is not yet initialised at hoist time). A +// top-level `await import()` would trigger TS1378 under the current +// tsconfig, so require is the pragmatic choice for this hoisted factory. +const { + mockGetNodes, + mockGetNode, + serviceStart, + serviceShutdown, + DockerEventServiceCtor, + registryInstance, +} = vi.hoisted(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { EventEmitter: HoistedEE } = require('events'); + const start = vi.fn().mockResolvedValue(undefined); + const shutdown = vi.fn(); + // A real class (not a vi.fn) so `new` works reliably across vitest versions. + class FakeDockerEventService { + public readonly nodeId: number; + public readonly nodeName: string; + constructor(nodeId: number, nodeName: string) { + this.nodeId = nodeId; + this.nodeName = nodeName; + } + start() { return start(); } + shutdown() { shutdown(); } + getStatus() { + return { + nodeId: this.nodeId, + nodeName: this.nodeName, + status: 'connected' as const, + reconnectAttempts: 0, + trackedContainers: 0, + }; + } + } + const ctorSpy = vi.fn((nodeId: number, nodeName: string) => + new FakeDockerEventService(nodeId, nodeName), + ); + // Wrap in a Proxy so `new Ctor(...)` both constructs via the real class + // and records the call on the spy for assertions. + const Ctor = new Proxy(FakeDockerEventService, { + construct(_target, args: [number, string]) { + ctorSpy(...args); + return new FakeDockerEventService(...args); + }, + }); + return { + mockGetNodes: vi.fn(), + mockGetNode: vi.fn(), + serviceStart: start, + serviceShutdown: shutdown, + DockerEventServiceCtor: Object.assign(Ctor, { _spy: ctorSpy }), + registryInstance: new HoistedEE() as EventEmitter, + }; +}); + +vi.mock('../services/DatabaseService', () => ({ + DatabaseService: { + getInstance: () => ({ + getNodes: mockGetNodes, + getNode: mockGetNode, + }), + }, +})); + +vi.mock('../services/NodeRegistry', () => ({ + NodeRegistry: { + getInstance: () => registryInstance, + }, +})); + +vi.mock('../services/DockerEventService', () => ({ + DockerEventService: DockerEventServiceCtor, +})); + +import { DockerEventManager } from '../services/DockerEventManager'; + +// Proxy records constructor calls on _spy for assertions. +const ctorSpy = (DockerEventServiceCtor as unknown as { _spy: ReturnType })._spy; + +beforeEach(() => { + vi.clearAllMocks(); + // Reset singleton between tests. + (DockerEventManager as unknown as { instance: DockerEventManager | undefined }).instance = undefined; + registryInstance.removeAllListeners(); +}); + +afterEach(() => { + DockerEventManager.getInstance().stop(); +}); + +describe('DockerEventManager - boot', () => { + it('spawns a service for each local node and skips remote nodes', async () => { + mockGetNodes.mockReturnValue([ + { id: 1, name: 'local-a', type: 'local' }, + { id: 2, name: 'remote-b', type: 'remote' }, + { id: 3, name: 'local-c', type: 'local' }, + ]); + + await DockerEventManager.getInstance().start(); + + expect(ctorSpy).toHaveBeenCalledTimes(2); + expect(ctorSpy).toHaveBeenCalledWith(1, 'local-a'); + expect(ctorSpy).toHaveBeenCalledWith(3, 'local-c'); + expect(serviceStart).toHaveBeenCalledTimes(2); + }); + + it('second start is a no-op', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'a', type: 'local' }]); + + const mgr = DockerEventManager.getInstance(); + await mgr.start(); + await mgr.start(); + + expect(ctorSpy).toHaveBeenCalledTimes(1); + }); +}); + +describe('DockerEventManager - node lifecycle events', () => { + it('spawns a service when a local node is added', async () => { + mockGetNodes.mockReturnValue([]); + await DockerEventManager.getInstance().start(); + + mockGetNode.mockReturnValue({ id: 5, name: 'new-local', type: 'local' }); + registryInstance.emit('node-added', 5); + await vi.waitFor(() => expect(serviceStart).toHaveBeenCalled()); + + expect(ctorSpy).toHaveBeenCalledWith(5, 'new-local'); + }); + + it('does nothing when a remote node is added', async () => { + mockGetNodes.mockReturnValue([]); + await DockerEventManager.getInstance().start(); + + mockGetNode.mockReturnValue({ id: 6, name: 'new-remote', type: 'remote' }); + registryInstance.emit('node-added', 6); + // Give the async handler a tick to run. + await new Promise(r => setTimeout(r, 0)); + + expect(ctorSpy).not.toHaveBeenCalled(); + }); + + it('shuts down the service when a node is removed', async () => { + mockGetNodes.mockReturnValue([{ id: 7, name: 'to-remove', type: 'local' }]); + await DockerEventManager.getInstance().start(); + expect(ctorSpy).toHaveBeenCalledTimes(1); + + registryInstance.emit('node-removed', 7); + expect(serviceShutdown).toHaveBeenCalledTimes(1); + }); + + it('respawns when a remote node becomes local', async () => { + mockGetNodes.mockReturnValue([]); // starts with nothing + await DockerEventManager.getInstance().start(); + + // Now the node exists and is local. + mockGetNode.mockReturnValue({ id: 9, name: 'flipped', type: 'local' }); + registryInstance.emit('node-updated', 9); + await vi.waitFor(() => expect(serviceStart).toHaveBeenCalled()); + + expect(ctorSpy).toHaveBeenCalledWith(9, 'flipped'); + }); + + it('tears down when a local node becomes remote', async () => { + mockGetNodes.mockReturnValue([{ id: 10, name: 'was-local', type: 'local' }]); + await DockerEventManager.getInstance().start(); + expect(ctorSpy).toHaveBeenCalledTimes(1); + + mockGetNode.mockReturnValue({ id: 10, name: 'was-local', type: 'remote' }); + registryInstance.emit('node-updated', 10); + await new Promise(r => setTimeout(r, 0)); + + expect(serviceShutdown).toHaveBeenCalledTimes(1); + }); +}); + +describe('DockerEventManager - shutdown', () => { + it('stops every service and removes listeners', async () => { + mockGetNodes.mockReturnValue([ + { id: 1, name: 'a', type: 'local' }, + { id: 2, name: 'b', type: 'local' }, + ]); + const mgr = DockerEventManager.getInstance(); + await mgr.start(); + + mgr.stop(); + + expect(serviceShutdown).toHaveBeenCalledTimes(2); + expect(registryInstance.listenerCount('node-added')).toBe(0); + expect(registryInstance.listenerCount('node-removed')).toBe(0); + expect(registryInstance.listenerCount('node-updated')).toBe(0); + }); +}); diff --git a/backend/src/__tests__/docker-event-service.test.ts b/backend/src/__tests__/docker-event-service.test.ts new file mode 100644 index 00000000..5cdd1de2 --- /dev/null +++ b/backend/src/__tests__/docker-event-service.test.ts @@ -0,0 +1,467 @@ +/** + * Unit tests for DockerEventService. + * + * Mocks the Docker client stream via a small helper that exposes push() and + * error() hooks so tests can drive the stream deterministically. Focuses on: + * - classification of kill / die / oom / health_status + * - the 500ms grace window for out-of-order die events + * - rate limiting and overflow summary + * - reconciliation on connect (baseline vs gap exits) + * - mass-event detection on reconnect + * - reconnect backoff + one-time warning/info alerts + * - malformed payload tolerance + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { EventEmitter } from 'events'; + +// ── Hoisted mocks ────────────────────────────────────────────────────── + +const { + mockDispatchAlert, + mockGetGlobalSettings, + mockGetEvents, + mockListContainers, + mockInspect, + mockGetContainer, + mockGetDocker, +} = vi.hoisted(() => ({ + mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockGetGlobalSettings: vi.fn().mockReturnValue({ global_crash: '1' }), + mockGetEvents: vi.fn(), + mockListContainers: vi.fn().mockResolvedValue([]), + mockInspect: vi.fn().mockResolvedValue({}), + mockGetContainer: vi.fn(), + mockGetDocker: vi.fn(), +})); + +vi.mock('../services/NotificationService', () => ({ + NotificationService: { + getInstance: () => ({ dispatchAlert: mockDispatchAlert }), + }, +})); + +vi.mock('../services/DatabaseService', () => ({ + DatabaseService: { + getInstance: () => ({ getGlobalSettings: mockGetGlobalSettings }), + }, +})); + +vi.mock('../services/NodeRegistry', () => ({ + NodeRegistry: { + getInstance: () => ({ getDocker: mockGetDocker }), + }, +})); + +// ── Fake Docker stream helper ────────────────────────────────────────── + +interface FakeStream extends EventEmitter { + destroyed: boolean; + destroy: () => void; + push: (event: Record) => void; + pushRaw: (raw: string) => void; + error: (err: Error) => void; +} + +function makeStream(): FakeStream { + const ee = new EventEmitter() as FakeStream; + ee.destroyed = false; + ee.destroy = () => { ee.destroyed = true; }; + ee.push = (event) => ee.emit('data', Buffer.from(JSON.stringify(event) + '\n')); + ee.pushRaw = (raw) => ee.emit('data', Buffer.from(raw)); + ee.error = (err) => ee.emit('error', err); + return ee; +} + +// ── Setup ────────────────────────────────────────────────────────────── + +import { DockerEventService } from '../services/DockerEventService'; + +let stream: FakeStream; +let service: DockerEventService; + +beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers(); + mockGetGlobalSettings.mockReturnValue({ global_crash: '1' }); + + stream = makeStream(); + mockGetEvents.mockImplementation(async () => stream); + mockListContainers.mockResolvedValue([]); + mockGetContainer.mockImplementation((id: string) => ({ + inspect: () => mockInspect(id), + })); + mockGetDocker.mockReturnValue({ + getEvents: mockGetEvents, + listContainers: mockListContainers, + getContainer: mockGetContainer, + }); +}); + +afterEach(() => { + service?.shutdown(); + vi.useRealTimers(); +}); + +// ── Classification via event stream ──────────────────────────────────── + +describe('DockerEventService - die classification', () => { + it('emits crash alert on die with non-zero exit code and no prior kill', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: 'c1', Attributes: { exitCode: '1', name: 'web' } }, + time: 1700000000, + }); + await vi.advanceTimersByTimeAsync(600); // past the 500ms grace window + + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'error', + expect.stringContaining('Container Crash Detected'), + undefined, + ); + }); + + it('does not emit when die follows a recent kill (intentional)', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'kill', + Actor: { ID: 'c1', Attributes: { signal: '15' } }, + time: 1700000000, + }); + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: 'c1', Attributes: { exitCode: '1', name: 'web' } }, + time: 1700000001, + }); + await vi.advanceTimersByTimeAsync(600); + + const crashCall = mockDispatchAlert.mock.calls.find(c => + typeof c[1] === 'string' && c[1].includes('Crash')); + expect(crashCall).toBeUndefined(); + }); + + it('reclassifies as intentional when kill arrives within the 500ms grace window', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + // die first, kill 200ms later + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: 'c2', Attributes: { exitCode: '1', name: 'svc' } }, + time: 1700000000, + }); + await vi.advanceTimersByTimeAsync(200); + stream.push({ + Type: 'container', + Action: 'kill', + Actor: { ID: 'c2', Attributes: { signal: '15' } }, + time: 1700000001, + }); + await vi.advanceTimersByTimeAsync(400); // total > 500ms + + const crashCall = mockDispatchAlert.mock.calls.find(c => + typeof c[1] === 'string' && c[1].includes('Crash')); + expect(crashCall).toBeUndefined(); + }); + + it('emits OOM alert when oom precedes die', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ Type: 'container', Action: 'oom', Actor: { ID: 'c3', Attributes: { name: 'hog' } } }); + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: 'c3', Attributes: { exitCode: '137', name: 'hog' } }, + }); + await vi.advanceTimersByTimeAsync(600); + + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'error', + expect.stringContaining('OOM Kill'), + undefined, + ); + }); + + it('does not emit when exit code is 0 (clean exit)', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: 'c4', Attributes: { exitCode: '0', name: 'oneshot' } }, + }); + await vi.advanceTimersByTimeAsync(600); + + expect(mockDispatchAlert).not.toHaveBeenCalled(); + }); + + it('emits unhealthy alert on health_status: unhealthy', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'c5', Attributes: { name: 'api' } }, + }); + await vi.runOnlyPendingTimersAsync(); + + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'error', + expect.stringContaining('Healthcheck failed'), + undefined, + ); + }); + + it('does not emit when global_crash is disabled', async () => { + mockGetGlobalSettings.mockReturnValue({ global_crash: '0' }); + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: 'c6', Attributes: { exitCode: '1', name: 'web' } }, + }); + await vi.advanceTimersByTimeAsync(600); + + expect(mockDispatchAlert).not.toHaveBeenCalled(); + }); + + it('clears crash dedup when container starts again', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + // First crash + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: 'c7', Attributes: { exitCode: '1', name: 'web' } }, + }); + await vi.advanceTimersByTimeAsync(600); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + // Start event clears dedup + stream.push({ Type: 'container', Action: 'start', Actor: { ID: 'c7' } }); + + // Second crash should fire again + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: 'c7', Attributes: { exitCode: '2', name: 'web' } }, + }); + await vi.advanceTimersByTimeAsync(600); + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + }); +}); + +// ── Rate limiting ────────────────────────────────────────────────────── + +describe('DockerEventService - rate limiting', () => { + it('batches overflow crashes into a single summary alert', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + // Push 22 die events (limit is 20). + for (let i = 0; i < 22; i++) { + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: `c-${i}`, Attributes: { exitCode: '1', name: `n-${i}` } }, + }); + } + await vi.advanceTimersByTimeAsync(600); + + const crashCalls = mockDispatchAlert.mock.calls.filter(c => + typeof c[1] === 'string' && c[1].includes('Crash')); + expect(crashCalls).toHaveLength(20); + + // After the rate window, a summary warning fires. + await vi.advanceTimersByTimeAsync(61_000); + const summaryCalls = mockDispatchAlert.mock.calls.filter(c => + typeof c[1] === 'string' && c[1].includes('additional containers crashed')); + expect(summaryCalls).toHaveLength(1); + expect(summaryCalls[0][1]).toContain('2 additional'); + }); +}); + +// ── Malformed payloads ───────────────────────────────────────────────── + +describe('DockerEventService - malformed payloads', () => { + it('tolerates a bad JSON line without tearing down the stream', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.pushRaw('not json\n'); + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: 'ok', Attributes: { exitCode: '1', name: 'ok' } }, + }); + await vi.advanceTimersByTimeAsync(600); + + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'error', + expect.stringContaining('Container Crash Detected'), + undefined, + ); + }); +}); + +// ── Reconciliation ───────────────────────────────────────────────────── + +describe('DockerEventService - reconciliation', () => { + it('on first boot, records pre-existing exited containers as baseline and does not alert', async () => { + mockListContainers.mockResolvedValue([ + { Id: 'pre-1', State: 'exited' }, + { Id: 'pre-2', State: 'exited' }, + { Id: 'run-1', State: 'running' }, + ]); + + service = new DockerEventService(1, 'local'); + await service.start(); + + expect(mockDispatchAlert).not.toHaveBeenCalled(); + }); + + it('emits mass-event summary on reconnect when >20% of containers newly exited', async () => { + // First connect: 10 running containers as baseline. + const running = Array.from({ length: 10 }, (_, i) => ({ + Id: `c-${i}`, + State: 'running', + })); + mockListContainers.mockResolvedValueOnce(running); + + service = new DockerEventService(1, 'local'); + await service.start(); + + // Simulate stream drop. + stream.error(new Error('connection reset')); + stream = makeStream(); + mockGetEvents.mockImplementation(async () => stream); + + // On reconnect: 5 of them now exited (>20%). + const mixed = running.map((c, i) => ({ + Id: c.Id, + State: i < 5 ? 'exited' : 'running', + })); + mockListContainers.mockResolvedValueOnce(mixed); + + // Drain reconnect backoff + reconciliation. + await vi.advanceTimersByTimeAsync(2_000); + // Flush microtasks spawned by the async reconnect + reconcile chain + // without running the recurring prune interval. + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + + const massCall = mockDispatchAlert.mock.calls.find(c => + typeof c[1] === 'string' && c[1].includes('daemon interruption')); + expect(massCall).toBeDefined(); + }); + + it('classifies individual gap exits on reconnect when below mass threshold', async () => { + // Baseline: 10 containers running. + const baseline = Array.from({ length: 10 }, (_, i) => ({ + Id: `c-${i}`, + State: 'running', + })); + mockListContainers.mockResolvedValueOnce(baseline); + + service = new DockerEventService(1, 'local'); + await service.start(); + + // Drop + one new exit (10%, below 20% threshold). + stream.error(new Error('bad')); + stream = makeStream(); + mockGetEvents.mockImplementation(async () => stream); + + const postReconnect = baseline.map((c, i) => ({ + Id: c.Id, + State: i === 0 ? 'exited' : 'running', + })); + mockListContainers.mockResolvedValueOnce(postReconnect); + + mockInspect.mockResolvedValueOnce({ + Name: '/crashed-app', + State: { ExitCode: 9, OOMKilled: false }, + Config: { Labels: { 'com.docker.compose.project': 'my-stack' } }, + }); + + await vi.advanceTimersByTimeAsync(2_000); + // Flush microtasks spawned by the async reconnect + reconcile chain + // without running the recurring prune interval. + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + + const crashCall = mockDispatchAlert.mock.calls.find(c => + typeof c[1] === 'string' && c[1].includes('crashed-app')); + expect(crashCall).toBeDefined(); + expect(crashCall?.[2]).toBe('my-stack'); + }); +}); + +// ── Reconnect lifecycle ──────────────────────────────────────────────── + +describe('DockerEventService - reconnect', () => { + it('emits one-time warning on first disconnect and info on reconnect', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.error(new Error('connection reset')); + + // Reconnect succeeds immediately after backoff. + stream = makeStream(); + mockGetEvents.mockImplementation(async () => stream); + + await vi.advanceTimersByTimeAsync(2_000); + // Flush microtasks spawned by the async reconnect + reconcile chain + // without running the recurring prune interval. + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + + const warn = mockDispatchAlert.mock.calls.find(c => c[0] === 'warning'); + const info = mockDispatchAlert.mock.calls.find(c => c[0] === 'info'); + expect(warn?.[1]).toContain('Lost connection'); + expect(info?.[1]).toContain('Reconnected'); + }); + + it('shutdown cancels pending reconnect', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.error(new Error('broken')); + mockGetEvents.mockClear(); + service.shutdown(); + + // Advance time well past the first backoff window; no new connect should run. + await vi.advanceTimersByTimeAsync(5_000); + expect(mockGetEvents).not.toHaveBeenCalled(); + }); +}); + +// ── Diagnostics ──────────────────────────────────────────────────────── + +describe('DockerEventService - getStatus', () => { + it('reports connected after start', async () => { + service = new DockerEventService(42, 'my-node'); + await service.start(); + + const status = service.getStatus(); + expect(status.nodeId).toBe(42); + expect(status.nodeName).toBe('my-node'); + expect(status.status).toBe('connected'); + }); +}); diff --git a/backend/src/__tests__/monitor-service.test.ts b/backend/src/__tests__/monitor-service.test.ts index 1aac13de..141c61d5 100644 --- a/backend/src/__tests__/monitor-service.test.ts +++ b/backend/src/__tests__/monitor-service.test.ts @@ -313,68 +313,8 @@ describe('MonitorService - evaluateGlobalSettings', () => { }); }); -// ── Global crash detection ───────────────────────────────────────────── - -describe('MonitorService - global crash detection', () => { - it('detects exited containers with non-intentional exit codes', async () => { - mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); - mockGetAllContainers.mockResolvedValue([{ - Id: 'crash-1', - State: 'exited', - Status: 'Exited (1) 5 seconds ago', - Names: ['/my-container'], - }]); - - const svc = MonitorService.getInstance(); - await (svc as any).evaluateGlobalSettings({ global_crash: '1' }); - - expect(mockDispatchAlert).toHaveBeenCalledWith('error', expect.stringContaining('Crash'), undefined); - }); - - it('ignores exit codes 0, 137, 143, 255', async () => { - mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); - const intentionalExits = [0, 137, 143, 255]; - - for (const code of intentionalExits) { - mockDispatchAlert.mockClear(); - mockGetAllContainers.mockResolvedValue([{ - Id: `safe-${code}`, - State: 'exited', - Status: `Exited (${code}) 5 seconds ago`, - Names: ['/safe-container'], - }]); - - const svc = MonitorService.getInstance(); - (MonitorService as any).instance = undefined; - await (svc as any).evaluateGlobalSettings({ global_crash: '1' }); - expect(mockDispatchAlert).not.toHaveBeenCalledWith('error', expect.stringContaining('Crash')); - } - }); - - it('detects unhealthy containers', async () => { - mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); - mockGetAllContainers.mockResolvedValue([{ - Id: 'sick-1', - State: 'unhealthy', - Status: 'Up 2 hours (unhealthy)', - Names: ['/sick-container'], - }]); - - const svc = MonitorService.getInstance(); - await (svc as any).evaluateGlobalSettings({ global_crash: '1' }); - - expect(mockDispatchAlert).toHaveBeenCalledWith('error', expect.stringContaining('unhealthy'), undefined); - }); - - it('skips remote nodes', async () => { - mockGetNodes.mockReturnValue([{ id: 2, name: 'remote-node', type: 'remote' }]); - - const svc = MonitorService.getInstance(); - await (svc as any).evaluateGlobalSettings({ global_crash: '1' }); - - expect(mockGetAllContainers).not.toHaveBeenCalled(); - }); -}); +// Crash + healthcheck detection now lives in DockerEventService (event-driven). +// Tests for those flows live in docker-event-service.test.ts. // ── Alert breach state machine ───────────────────────────────────────── diff --git a/backend/src/index.ts b/backend/src/index.ts index a8a58c37..74364c32 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -23,6 +23,7 @@ import { HostTerminalService } from './services/HostTerminalService'; import { DatabaseService, Node, AuthProvider, ScheduledTask, UserRole, ResourceType } from './services/DatabaseService'; import { NotificationService } from './services/NotificationService'; import { MonitorService } from './services/MonitorService'; +import { DockerEventManager } from './services/DockerEventManager'; import { ImageUpdateService } from './services/ImageUpdateService'; import { templateService } from './services/TemplateService'; import { ErrorParser } from './utils/ErrorParser'; @@ -6226,6 +6227,10 @@ app.post('/api/nodes', async (req: Request, res: Response) => { api_token: api_token || '', }); + // Notify subscribers (e.g. DockerEventManager) so a new local node gets + // its event stream spun up immediately, not on next restart. + NodeRegistry.getInstance().notifyNodeAdded(id); + const isPlainHttp = type === 'remote' && api_url && api_url.startsWith('http://'); res.json({ success: true, @@ -6266,6 +6271,7 @@ app.put('/api/nodes/:id', async (req: Request, res: Response) => { // Evict cached Docker connection so it reconnects with new config NodeRegistry.getInstance().evictConnection(id); + NodeRegistry.getInstance().notifyNodeUpdated(id); const isPlainHttp = updates.api_url && updates.api_url.startsWith('http://'); res.json({ @@ -6292,6 +6298,7 @@ app.delete('/api/nodes/:id', async (req: Request, res: Response) => { const id = parseInt(nodeIdParam); DatabaseService.getInstance().deleteNode(id); NodeRegistry.getInstance().evictConnection(id); + NodeRegistry.getInstance().notifyNodeRemoved(id); CacheService.getInstance().invalidate(`${REMOTE_META_NAMESPACE}:${id}`); updateTracker.delete(id); res.json({ success: true }); @@ -6410,6 +6417,9 @@ async function startServer() { // Start Background Watchdog MonitorService.getInstance().start(); + // Start Docker Event Stream (causal crash/OOM/health detection per local node) + await DockerEventManager.getInstance().start(); + // Start Background Image Update Checker ImageUpdateService.getInstance().start(); @@ -6443,6 +6453,9 @@ const gracefulShutdown = (signal: string) => { try { MonitorService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] MonitorService cleanup failed:', (e as Error).message); } + try { DockerEventManager.getInstance().stop(); } catch (e) { + console.warn('[Shutdown] DockerEventManager cleanup failed:', (e as Error).message); + } try { ImageUpdateService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] ImageUpdateService cleanup failed:', (e as Error).message); } diff --git a/backend/src/services/ContainerLifecycleClassifier.ts b/backend/src/services/ContainerLifecycleClassifier.ts new file mode 100644 index 00000000..9866b437 --- /dev/null +++ b/backend/src/services/ContainerLifecycleClassifier.ts @@ -0,0 +1,73 @@ +/** + * ContainerLifecycleClassifier + * + * Pure classification helpers for Docker container lifecycle events. No I/O, + * no side effects, no singletons. Consumed by DockerEventService. + * + * The classifier answers a single question: given a `die` event and the + * container's recent lifecycle state, is this exit intentional, a clean exit, + * an OOM kill, or a crash worth alerting on? + */ + +export type Classification = 'intentional' | 'clean' | 'crash' | 'oom'; + +/** Window (ms) after a `kill` event within which a subsequent `die` is considered intentional. */ +export const INTENTIONAL_KILL_WINDOW_MS = 60_000; + +export interface ContainerLifecycleState { + /** Timestamp (ms) of the most recent `kill` event for this container, if any. */ + lastKillAt?: number; + /** True when an `oom` event has been observed and the matching `die` has not yet arrived. */ + oomPending?: boolean; +} + +export interface DieEventInput { + /** Time the die event occurred (ms). Typically Date.now() when the event was parsed. */ + at: number; + /** Exit code reported by Docker. May be undefined for malformed events (treated as non-zero). */ + exitCode: number | undefined; +} + +/** + * Classify a die event against the container's current lifecycle state. + * + * Priority order: + * 1. OOM pending → 'oom' (OOM kills are meaningful even if exitCode looks clean) + * 2. Recent kill within window → 'intentional' + * 3. Exit code 0 → 'clean' + * 4. Anything else → 'crash' + */ +export function classifyDie( + input: DieEventInput, + state: ContainerLifecycleState, +): Classification { + if (state.oomPending) return 'oom'; + + if (typeof state.lastKillAt === 'number') { + // Use absolute age so out-of-order deliveries (kill arrives slightly + // after die) still classify as intentional. DockerEventService's 500ms + // die grace window makes this realistically bounded. + const age = Math.abs(input.at - state.lastKillAt); + if (age <= INTENTIONAL_KILL_WINDOW_MS) { + return 'intentional'; + } + } + + if (input.exitCode === 0) return 'clean'; + return 'crash'; +} + +/** + * Classify a gap exit discovered during reconciliation (no die event observed + * because the stream was disconnected). Uses the container inspect result + * rather than event state. + */ +export function classifyGapExit(inspect: { + State?: { OOMKilled?: boolean; ExitCode?: number }; +}): Classification { + const oom = inspect.State?.OOMKilled === true; + if (oom) return 'oom'; + const code = inspect.State?.ExitCode; + if (code === 0) return 'clean'; + return 'crash'; +} diff --git a/backend/src/services/DockerEventManager.ts b/backend/src/services/DockerEventManager.ts new file mode 100644 index 00000000..9cda97ca --- /dev/null +++ b/backend/src/services/DockerEventManager.ts @@ -0,0 +1,127 @@ +import { DatabaseService, Node } from './DatabaseService'; +import { NodeRegistry } from './NodeRegistry'; +import { DockerEventService } from './DockerEventService'; +import { isDebugEnabled } from '../utils/debug'; + +/** + * DockerEventManager + * + * Singleton coordinator that owns one DockerEventService per local node. + * Spawns services on boot for every existing local node, and reacts to + * NodeRegistry 'node-added' / 'node-removed' / 'node-updated' events to + * keep the service map in sync with the database. + * + * Remote nodes self-monitor on their own Sencho instance; this manager does + * not subscribe to remote Docker daemons. + */ +export class DockerEventManager { + private static instance: DockerEventManager; + private services: Map = new Map(); + private started = false; + + private readonly onNodeAdded = (id: number) => { void this.handleNodeAdded(id); }; + private readonly onNodeRemoved = (id: number) => { this.handleNodeRemoved(id); }; + private readonly onNodeUpdated = (id: number) => { void this.handleNodeUpdated(id); }; + + private constructor() { /* private: use getInstance */ } + + public static getInstance(): DockerEventManager { + if (!DockerEventManager.instance) { + DockerEventManager.instance = new DockerEventManager(); + } + return DockerEventManager.instance; + } + + /** Boot: spawn a DockerEventService for every existing local node. */ + public async start(): Promise { + if (this.started) return; + this.started = true; + + const registry = NodeRegistry.getInstance(); + registry.on('node-added', this.onNodeAdded); + registry.on('node-removed', this.onNodeRemoved); + registry.on('node-updated', this.onNodeUpdated); + + // Spawn in parallel so one slow node can't block boot for the others. + const nodes = DatabaseService.getInstance().getNodes() + .filter(n => n.type === 'local' && typeof n.id === 'number'); + await Promise.all(nodes.map(n => this.spawn(n))); + } + + /** Shutdown: stop every service and unsubscribe from registry events. */ + public stop(): void { + if (!this.started) return; + this.started = false; + + const registry = NodeRegistry.getInstance(); + registry.off('node-added', this.onNodeAdded); + registry.off('node-removed', this.onNodeRemoved); + registry.off('node-updated', this.onNodeUpdated); + + for (const service of this.services.values()) service.shutdown(); + this.services.clear(); + } + + /** Aggregated status for diagnostics (e.g. /api/health). */ + public getStatus(): Array> { + return Array.from(this.services.values()).map(s => s.getStatus()); + } + + // ======================================================================== + // Node lifecycle handlers + // ======================================================================== + + private async handleNodeAdded(nodeId: number): Promise { + if (this.services.has(nodeId)) return; + const node = DatabaseService.getInstance().getNode(nodeId); + if (!node || node.type !== 'local') return; + await this.spawn(node); + } + + private handleNodeRemoved(nodeId: number): void { + const service = this.services.get(nodeId); + if (!service) return; + service.shutdown(); + this.services.delete(nodeId); + } + + private async handleNodeUpdated(nodeId: number): Promise { + const node = DatabaseService.getInstance().getNode(nodeId); + const existing = this.services.get(nodeId); + + // Node became remote (or was deleted): tear down. + if (!node || node.type !== 'local') { + if (existing) { + existing.shutdown(); + this.services.delete(nodeId); + } + return; + } + + // Node is local: ensure a service exists (respawn if missing). + if (!existing) { + await this.spawn(node); + } + } + + // ======================================================================== + // Service spawning + // ======================================================================== + + private async spawn(node: Node): Promise { + if (typeof node.id !== 'number') return; + if (this.services.has(node.id)) return; + + const service = new DockerEventService(node.id, node.name); + this.services.set(node.id, service); + + try { + await service.start(); + } catch (err) { + if (isDebugEnabled()) { + console.log(`[DockerEventManager] failed to start service for node ${node.name}:`, + err instanceof Error ? err.message : err); + } + } + } +} diff --git a/backend/src/services/DockerEventService.ts b/backend/src/services/DockerEventService.ts new file mode 100644 index 00000000..013a0cef --- /dev/null +++ b/backend/src/services/DockerEventService.ts @@ -0,0 +1,621 @@ +import Docker from 'dockerode'; +import { NodeRegistry } from './NodeRegistry'; +import { NotificationService } from './NotificationService'; +import { DatabaseService } from './DatabaseService'; +import { + classifyDie, + classifyGapExit, + Classification, + ContainerLifecycleState, +} from './ContainerLifecycleClassifier'; +import { isDebugEnabled } from '../utils/debug'; + +/** + * DockerEventService + * + * Subscribes to Docker's container event stream for a single local node and + * translates causal events (kill / die / oom / health_status) into alerts. + * + * One instance is spawned per local node by DockerEventManager. Each instance + * owns a dedicated Docker client, stream, reconnect timer, and state map - no + * shared mutable state between per-node services. + * + * See docs/features/alerts-notifications.mdx for user-facing behaviour. + */ + +/** Grace window after a `die` before classifying, to absorb out-of-order kill events. */ +const DIE_GRACE_WINDOW_MS = 500; + +/** Max crash alerts emitted per node within RATE_WINDOW_MS. Overflow is batched. */ +const RATE_LIMIT_MAX = 20; +const RATE_WINDOW_MS = 60_000; + +/** Dedup window for repeat crash alerts of the same container. */ +const CRASH_DEDUP_MS = 60 * 60_000; + +/** Interval for pruning stale container state from memory. */ +const PRUNE_INTERVAL_MS = 60_000; +const STATE_STALE_AFTER_MS = 10 * 60_000; + +/** Parse-error threshold: >N errors per window triggers a single warning alert. */ +const PARSE_ERROR_THRESHOLD = 10; +const PARSE_ERROR_WINDOW_MS = 60_000; + +/** Fraction of exited containers on reconnect that triggers mass-event handling. */ +const MASS_EVENT_THRESHOLD = 0.2; + +/** Reconnect backoff bounds. */ +const RECONNECT_BASE_MS = 1_000; +const RECONNECT_MAX_MS = 60_000; +const RECONNECT_JITTER_MS = 500; + +/** Compose project label key used by docker compose on every container it creates. */ +const COMPOSE_PROJECT_LABEL = 'com.docker.compose.project'; + +/** TTL for the cached global_crash settings flag (sub-second so toggle takes effect quickly). */ +const SETTINGS_CACHE_MS = 500; + +interface InternalContainerState extends ContainerLifecycleState { + name?: string; + stackName?: string; + lastCrashAlertAt?: number; + lastActivityAt: number; +} + +interface DockerEventPayload { + Type?: string; + Action?: string; + Actor?: { + ID?: string; + Attributes?: Record; + }; + time?: number; + timeNano?: number; +} + +type LifecycleStatus = 'disconnected' | 'connecting' | 'connected' | 'stopped'; + +export class DockerEventService { + private readonly nodeId: number; + private readonly nodeName: string; + private readonly docker: Docker; + private readonly notifier: NotificationService; + + private status: LifecycleStatus = 'disconnected'; + private stream: NodeJS.ReadableStream | null = null; + private reconnectAttempts = 0; + private reconnectTimer: NodeJS.Timeout | null = null; + private pruneTimer: NodeJS.Timeout | null = null; + + /** Per-container lifecycle state, keyed by Docker container ID. */ + private containerState: Map = new Map(); + + /** Pending die timers keyed by container ID (for the 500ms grace window). */ + private pendingDieTimers: Map = new Map(); + + /** Rate-limiter bookkeeping. */ + private rateWindowStart = 0; + private rateCount = 0; + private suppressedCount = 0; + private summaryTimer: NodeJS.Timeout | null = null; + + /** Parse-error tracking for flooded bad payloads. */ + private parseErrorWindowStart = 0; + private parseErrorCount = 0; + private parseWarningEmitted = false; + + /** True once we've completed the initial boot reconciliation. */ + private bootReconciled = false; + + /** IDs of containers that were exited at the last known-good moment. */ + private exitedBaseline: Set = new Set(); + + /** Whether we've already emitted the one-time "lost connection" warning. */ + private disconnectedNoticeEmitted = false; + + /** Cache for the global_crash toggle to avoid a DB read per event. */ + private crashAlertsCache: { value: boolean; at: number } | null = null; + + constructor(nodeId: number, nodeName: string) { + this.nodeId = nodeId; + this.nodeName = nodeName; + this.docker = NodeRegistry.getInstance().getDocker(nodeId); + this.notifier = NotificationService.getInstance(); + } + + /** Open the event stream and begin consuming events. Safe to call once. */ + public async start(): Promise { + if (this.status !== 'disconnected') return; + this.pruneTimer = setInterval(() => this.pruneStaleState(), PRUNE_INTERVAL_MS); + await this.connect(); + } + + /** Close the stream, cancel timers, and clear state. */ + public shutdown(): void { + this.status = 'stopped'; + this.clearReconnectTimer(); + if (this.pruneTimer) { + clearInterval(this.pruneTimer); + this.pruneTimer = null; + } + if (this.summaryTimer) { + clearTimeout(this.summaryTimer); + this.summaryTimer = null; + } + for (const timer of this.pendingDieTimers.values()) clearTimeout(timer); + this.pendingDieTimers.clear(); + this.detachStream(); + this.containerState.clear(); + } + + // ======================================================================== + // Connection lifecycle + // ======================================================================== + + private async connect(): Promise { + if (this.status === 'stopped') return; + this.status = 'connecting'; + try { + const stream = await this.docker.getEvents({ + filters: { type: ['container'] }, + }) as unknown as NodeJS.ReadableStream; + + this.stream = stream; + this.status = 'connected'; + + if (this.disconnectedNoticeEmitted) { + await this.emitInfo(`Reconnected to Docker daemon.`); + this.disconnectedNoticeEmitted = false; + } + + this.reconnectAttempts = 0; + this.attachStreamHandlers(stream); + + await this.reconcile(); + } catch (error) { + this.handleDisconnect(error); + } + } + + private attachStreamHandlers(stream: NodeJS.ReadableStream): void { + let buffer = ''; + stream.on('data', (chunk: Buffer) => { + buffer += chunk.toString('utf8'); + const lines = buffer.split('\n'); + buffer = lines.pop() ?? ''; + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) continue; + this.handleRawEvent(trimmed); + } + }); + stream.on('error', (err) => this.handleDisconnect(err)); + stream.on('end', () => this.handleDisconnect(new Error('Event stream ended'))); + stream.on('close', () => this.handleDisconnect(new Error('Event stream closed'))); + } + + private detachStream(): void { + const s = this.stream; + this.stream = null; + if (!s) return; + try { + s.removeAllListeners(); + const destroyable = s as unknown as { destroy?: () => void }; + destroyable.destroy?.(); + } catch { + /* noop */ + } + } + + private handleDisconnect(error: unknown): void { + if (this.status === 'stopped') return; + this.detachStream(); + this.status = 'disconnected'; + + if (!this.disconnectedNoticeEmitted) { + this.disconnectedNoticeEmitted = true; + void this.emitWarning(`Lost connection to Docker daemon; monitoring paused.`); + } + + if (isDebugEnabled()) { + console.log(`[DockerEventService:${this.nodeName}] disconnected:`, + error instanceof Error ? error.message : error); + } + this.scheduleReconnect(); + } + + private scheduleReconnect(): void { + if (this.status === 'stopped') return; + this.clearReconnectTimer(); + const attempt = this.reconnectAttempts; + const base = Math.min(RECONNECT_MAX_MS, RECONNECT_BASE_MS * Math.pow(2, attempt)); + const jitter = Math.floor(Math.random() * RECONNECT_JITTER_MS); + const delay = base + jitter; + this.reconnectAttempts += 1; + this.reconnectTimer = setTimeout(() => { + this.reconnectTimer = null; + void this.connect(); + }, delay); + } + + private clearReconnectTimer(): void { + if (this.reconnectTimer) { + clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + } + } + + // ======================================================================== + // Reconciliation + // ======================================================================== + + /** + * Snapshot all containers on connect. On first boot, record the baseline + * silently. On subsequent reconnects, treat newly-exited containers as + * gap exits and classify them (or batch as a mass event). + */ + private async reconcile(): Promise { + let containers; + try { + containers = await this.docker.listContainers({ all: true }); + } catch (err) { + if (isDebugEnabled()) { + console.log(`[DockerEventService:${this.nodeName}] reconcile list failed:`, + err instanceof Error ? err.message : err); + } + return; + } + + const exitedNow = new Set( + containers.filter(c => c.State === 'exited').map(c => c.Id) + ); + + if (!this.bootReconciled) { + this.exitedBaseline = exitedNow; + this.bootReconciled = true; + return; + } + + const newlyExited = [...exitedNow].filter(id => !this.exitedBaseline.has(id)); + const totalKnown = Math.max(1, containers.length); + const exitRatio = newlyExited.length / totalKnown; + + if (newlyExited.length === 0) { + this.exitedBaseline = exitedNow; + return; + } + + if (exitRatio >= MASS_EVENT_THRESHOLD) { + await this.emitInfo( + `Docker daemon interruption detected: ${newlyExited.length} containers exited during connection gap.` + ); + } else { + // Inspect + classify in parallel. Below the mass-event threshold + // newlyExited is small by definition, so unbounded concurrency is fine. + await Promise.all(newlyExited.map(id => this.classifyGap(id))); + } + + this.exitedBaseline = exitedNow; + } + + private async classifyGap(containerId: string): Promise { + try { + const inspect = await this.docker.getContainer(containerId).inspect(); + const classification = classifyGapExit({ State: inspect.State }); + if (classification === 'clean' || classification === 'intentional') return; + + const name = inspect.Name?.replace(/^\//, '') ?? containerId.slice(0, 12); + const stackName = inspect.Config?.Labels?.[COMPOSE_PROJECT_LABEL]; + const exitCode = inspect.State?.ExitCode ?? 0; + + // Gap exits have no in-memory state, so there's no dedup to bump. + await this.emitClassification(classification, null, { name, exitCode, stackName }); + } catch (err) { + if (isDebugEnabled()) { + console.log(`[DockerEventService:${this.nodeName}] gap inspect failed:`, + err instanceof Error ? err.message : err); + } + } + } + + // ======================================================================== + // Event handling + // ======================================================================== + + private handleRawEvent(line: string): void { + let payload: DockerEventPayload; + try { + payload = JSON.parse(line); + } catch { + this.trackParseError(); + return; + } + try { + this.handleEvent(payload); + } catch (err) { + if (isDebugEnabled()) { + console.log(`[DockerEventService:${this.nodeName}] event handler threw:`, + err instanceof Error ? err.message : err); + } + } + } + + private handleEvent(event: DockerEventPayload): void { + if (event.Type !== 'container') return; + const action = event.Action ?? ''; + const id = event.Actor?.ID; + if (!id) return; + + // Normalize: `health_status: unhealthy` -> base action + const baseAction = action.startsWith('health_status') ? 'health_status' : action; + + switch (baseAction) { + case 'kill': + return this.onKill(id, event); + case 'die': + return this.onDie(id, event); + case 'oom': + return this.onOom(id); + case 'health_status': + return this.onHealthStatus(id, action, event); + case 'start': + return this.onStart(id); + case 'destroy': + return this.onDestroy(id); + } + } + + private onKill(id: string, event: DockerEventPayload): void { + const state = this.getOrCreateState(id, event); + state.lastKillAt = this.eventTimeMs(event); + state.lastActivityAt = Date.now(); + } + + private onDie(id: string, event: DockerEventPayload): void { + // Defer classification to absorb out-of-order kill events. + const existing = this.pendingDieTimers.get(id); + if (existing) clearTimeout(existing); + const timer = setTimeout(() => { + this.pendingDieTimers.delete(id); + this.classifyDie(id, event); + }, DIE_GRACE_WINDOW_MS); + this.pendingDieTimers.set(id, timer); + } + + private onOom(id: string): void { + const state = this.getOrCreateState(id); + state.oomPending = true; + state.lastActivityAt = Date.now(); + } + + private onHealthStatus(id: string, action: string, event: DockerEventPayload): void { + if (!action.includes('unhealthy')) return; + const state = this.getOrCreateState(id, event); + state.lastActivityAt = Date.now(); + if (!this.isCrashAlertsEnabled()) return; + const name = state.name ?? id.slice(0, 12); + const stackName = state.stackName; + void this.emitError( + `Healthcheck failed: ${name} is unhealthy.`, + stackName, + ); + } + + private onStart(id: string): void { + const state = this.containerState.get(id); + if (!state) return; + // Container came back: clear transient flags but keep identity. + state.lastKillAt = undefined; + state.oomPending = undefined; + state.lastCrashAlertAt = undefined; + state.lastActivityAt = Date.now(); + } + + private onDestroy(id: string): void { + this.containerState.delete(id); + const pending = this.pendingDieTimers.get(id); + if (pending) { + clearTimeout(pending); + this.pendingDieTimers.delete(id); + } + } + + private classifyDie(id: string, event: DockerEventPayload): void { + const state = this.getOrCreateState(id, event); + const exitCodeStr = event.Actor?.Attributes?.exitCode; + const parsedExit = exitCodeStr !== undefined ? parseInt(exitCodeStr, 10) : undefined; + const exitCode = Number.isFinite(parsedExit) ? (parsedExit as number) : undefined; + const now = Date.now(); + + const classification = classifyDie( + { at: this.eventTimeMs(event), exitCode }, + { lastKillAt: state.lastKillAt, oomPending: state.oomPending }, + ); + + // Die arrived: clear the oom flag regardless (we've now used it). + state.oomPending = undefined; + state.lastActivityAt = now; + + if (classification === 'intentional' || classification === 'clean') return; + + // Dedup: skip if we already alerted on this container within the window. + if (state.lastCrashAlertAt && now - state.lastCrashAlertAt < CRASH_DEDUP_MS) { + return; + } + + void this.emitClassification(classification, state, { + name: state.name ?? id.slice(0, 12), + exitCode: exitCode ?? 0, + stackName: state.stackName, + }); + } + + // ======================================================================== + // Alert emission + rate limiting + // ======================================================================== + + private async emitClassification( + classification: Classification, + state: InternalContainerState | null, + info: { name: string; exitCode: number; stackName?: string }, + ): Promise { + // Respect the existing global crash-alerts toggle so users who have + // disabled these notifications in Settings remain opted out. + if (!this.isCrashAlertsEnabled()) return; + + const message = classification === 'oom' + ? `Container OOM Kill: ${info.name} was killed by the OOM killer (out of memory).` + : `Container Crash Detected: ${info.name} exited unexpectedly (Code: ${info.exitCode}).`; + + if (!this.consumeRateToken()) { + this.suppressedCount += 1; + this.scheduleSummary(); + return; + } + + // Stamp the dedup clock only after the alert is actually dispatched, so + // rate-suppressed alerts don't silently lock out the next real crash. + if (state) state.lastCrashAlertAt = Date.now(); + + await this.emitError(message, info.stackName); + } + + private isCrashAlertsEnabled(): boolean { + const now = Date.now(); + const cached = this.crashAlertsCache; + if (cached && now - cached.at < SETTINGS_CACHE_MS) return cached.value; + let value = false; + try { + const settings = DatabaseService.getInstance().getGlobalSettings(); + value = settings['global_crash'] === '1'; + } catch (err) { + // Default-deny on settings lookup failure: don't spam users if the + // DB is temporarily unavailable. + if (isDebugEnabled()) { + console.log(`[DockerEventService:${this.nodeName}] settings lookup failed:`, + err instanceof Error ? err.message : err); + } + } + this.crashAlertsCache = { value, at: now }; + return value; + } + + /** Return true if an alert can be emitted now. Side effect: increments counters. */ + private consumeRateToken(): boolean { + const now = Date.now(); + if (now - this.rateWindowStart >= RATE_WINDOW_MS) { + this.rateWindowStart = now; + this.rateCount = 0; + } + if (this.rateCount >= RATE_LIMIT_MAX) return false; + this.rateCount += 1; + return true; + } + + private scheduleSummary(): void { + if (this.summaryTimer) return; + const remaining = RATE_WINDOW_MS - (Date.now() - this.rateWindowStart); + const delay = Math.max(1_000, remaining); + this.summaryTimer = setTimeout(() => { + const count = this.suppressedCount; + this.summaryTimer = null; + this.suppressedCount = 0; + if (count > 0) { + void this.emitWarning( + `${count} additional containers crashed in the last minute.`, + ); + } + }, delay); + } + + private trackParseError(): void { + const now = Date.now(); + if (now - this.parseErrorWindowStart >= PARSE_ERROR_WINDOW_MS) { + this.parseErrorWindowStart = now; + this.parseErrorCount = 0; + this.parseWarningEmitted = false; + } + this.parseErrorCount += 1; + if (this.parseErrorCount > PARSE_ERROR_THRESHOLD && !this.parseWarningEmitted) { + this.parseWarningEmitted = true; + void this.emitWarning( + `Received malformed Docker event payloads. Monitoring continues but some events may be skipped.`, + ); + } + } + + // ======================================================================== + // State + helpers + // ======================================================================== + + private getOrCreateState(id: string, event?: DockerEventPayload): InternalContainerState { + let state = this.containerState.get(id); + if (!state) { + state = { lastActivityAt: Date.now() }; + this.containerState.set(id, state); + } + if (event) { + const attrs = event.Actor?.Attributes ?? {}; + if (attrs.name && !state.name) state.name = attrs.name; + const project = attrs[COMPOSE_PROJECT_LABEL]; + if (project && !state.stackName) state.stackName = project; + } + return state; + } + + private eventTimeMs(event: DockerEventPayload): number { + if (typeof event.timeNano === 'number') return Math.floor(event.timeNano / 1_000_000); + if (typeof event.time === 'number') return event.time * 1000; + return Date.now(); + } + + private pruneStaleState(): void { + if (this.containerState.size === 0) return; + const cutoff = Date.now() - STATE_STALE_AFTER_MS; + for (const [id, state] of this.containerState) { + if (state.lastActivityAt < cutoff) { + this.containerState.delete(id); + } + } + } + + // ======================================================================== + // Notification wrappers (prefix with node name for multi-node clarity) + // ======================================================================== + + private async emitError(message: string, stackName?: string): Promise { + return this.notifier.dispatchAlert('error', this.prefix(message), stackName); + } + + private async emitWarning(message: string, stackName?: string): Promise { + return this.notifier.dispatchAlert('warning', this.prefix(message), stackName); + } + + private async emitInfo(message: string, stackName?: string): Promise { + return this.notifier.dispatchAlert('info', this.prefix(message), stackName); + } + + private prefix(message: string): string { + return `[Node: ${this.nodeName}] ${message}`; + } + + // ======================================================================== + // Diagnostics + // ======================================================================== + + public getStatus(): { + nodeId: number; + nodeName: string; + status: LifecycleStatus; + reconnectAttempts: number; + trackedContainers: number; + } { + return { + nodeId: this.nodeId, + nodeName: this.nodeName, + status: this.status, + reconnectAttempts: this.reconnectAttempts, + trackedContainers: this.containerState.size, + }; + } +} diff --git a/backend/src/services/MonitorService.ts b/backend/src/services/MonitorService.ts index 8baabea3..18e01609 100644 --- a/backend/src/services/MonitorService.ts +++ b/backend/src/services/MonitorService.ts @@ -69,10 +69,9 @@ export class MonitorService { // key: rule_id, value: AlertState private activeBreaches = new Map(); - // Track containers that have already been alerted as crashed to avoid - // duplicate alerts. key: containerId, value: timestamp when alerted. - private alertedCrashes = new Map(); - private static readonly CRASH_ALERT_TTL_MS = 60 * 60 * 1000; // 1 hour + // Crash and healthcheck detection live in DockerEventService (event-driven, + // causal classification). MonitorService no longer polls for container + // exits; see backend/src/services/DockerEventService.ts. // Sencho version check cooldown (6 hours between external API calls) private lastVersionCheckAt = 0; @@ -126,7 +125,6 @@ export class MonitorService { } private async evaluateGlobalSettings(settings: Record) { - const notifier = NotificationService.getInstance(); const HOST_ALERT_COOLDOWN_MS = 5 * 60 * 1000; // 5 minutes between repeat alerts // 1. Host Limits @@ -160,61 +158,9 @@ export class MonitorService { console.error('Error checking host limits in watchdog', e); } - // 2. Global Crash Detect - if (settings['global_crash'] === '1') { - // Prune expired entries from the crash tracker - const now = Date.now(); - for (const [id, ts] of this.alertedCrashes) { - if (now - ts > MonitorService.CRASH_ALERT_TTL_MS) this.alertedCrashes.delete(id); - } - - try { - const nodes = DatabaseService.getInstance().getNodes(); - const runningIds = new Set(); - - for (const node of nodes) { - if (!node.id) continue; - // Remote nodes run their own MonitorService locally - if (node.type === 'remote') continue; - try { - const docker = DockerController.getInstance(node.id); - const containers = await docker.getAllContainers(); - for (const c of containers) { - if (c.State === 'running') { - runningIds.add(c.Id); - continue; - } - // Skip containers already alerted - if (this.alertedCrashes.has(c.Id)) continue; - - const containerStack = c.Labels?.['com.docker.compose.project'] || undefined; - - if (c.State === 'exited') { - const match = c.Status.match(/Exited \((\d+)\)/i); - const exitCode = match ? parseInt(match[1], 10) : null; - const intentionalExitCodes = [0, 137, 143, 255]; - if (exitCode !== null && !intentionalExitCodes.includes(exitCode)) { - await notifier.dispatchAlert('error', `[Node: ${node.name}] Container Crash Detected: ${c.Names[0]} exited unexpectedly (Code: ${exitCode}).`, containerStack); - this.alertedCrashes.set(c.Id, now); - } - } else if (String(c.Status).includes('unhealthy')) { - await notifier.dispatchAlert('error', `[Node: ${node.name}] Healthcheck Failed: Container ${c.Names[0]} is unhealthy.`, containerStack); - this.alertedCrashes.set(c.Id, now); - } - } - } catch (err) { - console.error(`Error checking crashes on node ${node.name}`, err); - } - } - - // Clear crash tracking for containers that are running again - for (const id of this.alertedCrashes.keys()) { - if (runningIds.has(id)) this.alertedCrashes.delete(id); - } - } catch (e) { - console.error('Error checking global crashes', e); - } - } + // 2. (Removed) Container crash + healthcheck detection moved to + // DockerEventService: event-driven, causal, distinguishes + // intentional stops from real crashes, detects OOM kills. // 3. Docker Janitor Check try { diff --git a/backend/src/services/NodeRegistry.ts b/backend/src/services/NodeRegistry.ts index 4467e40b..6bf66df5 100644 --- a/backend/src/services/NodeRegistry.ts +++ b/backend/src/services/NodeRegistry.ts @@ -1,5 +1,6 @@ import Docker from 'dockerode'; import axios from 'axios'; +import { EventEmitter } from 'events'; import { DatabaseService, Node } from './DatabaseService'; import { fetchRemoteMeta } from './CapabilityRegistry'; @@ -10,12 +11,23 @@ import { fetchRemoteMeta } from './CapabilityRegistry'; * - Local nodes: direct Docker socket connection via Dockerode (unchanged) * - Remote nodes: HTTP/WS proxy to a remote Sencho instance (api_url + api_token) * No direct Docker TCP connections are made for remote nodes. + * + * Extends EventEmitter so subscribers (e.g. DockerEventManager) can react to + * node lifecycle changes. Emits: + * - 'node-added' (nodeId: number) after a node is created + * - 'node-removed' (nodeId: number) after a node is deleted + * - 'node-updated' (nodeId: number) after a node is updated (type may change) + * Route handlers in index.ts are responsible for calling the notify* helpers. */ -export class NodeRegistry { +export class NodeRegistry extends EventEmitter { private static instance: NodeRegistry; private connections: Map = new Map(); - private constructor() { } + private constructor() { + super(); + // Raise the default listener cap (10) so future subscribers do not trip a warning. + this.setMaxListeners(50); + } public static getInstance(): NodeRegistry { if (!NodeRegistry.instance) { @@ -210,6 +222,31 @@ export class NodeRegistry { this.connections.delete(nodeId); } + /** + * Emit 'node-added' for subscribers (e.g. DockerEventManager). + * Call this from the POST /api/nodes route after the DB insert succeeds. + */ + public notifyNodeAdded(nodeId: number): void { + this.emit('node-added', nodeId); + } + + /** + * Emit 'node-removed' for subscribers. + * Call this from the DELETE /api/nodes/:id route after the DB delete succeeds. + */ + public notifyNodeRemoved(nodeId: number): void { + this.emit('node-removed', nodeId); + } + + /** + * Emit 'node-updated' for subscribers. Type changes (local<->remote) are + * handled downstream by tearing down and respawning the subscription. + * Call this from the PUT /api/nodes/:id route after the DB update succeeds. + */ + public notifyNodeUpdated(nodeId: number): void { + this.emit('node-updated', nodeId); + } + /** * Flush all cached connections. */ diff --git a/docs/features/alerts-notifications.mdx b/docs/features/alerts-notifications.mdx index 973c5194..7d3b45bf 100644 --- a/docs/features/alerts-notifications.mdx +++ b/docs/features/alerts-notifications.mdx @@ -164,6 +164,44 @@ When the periodic image check (every 6 hours) detects that a stack has new upstr Both notification types use the same channel routing as alerts: if notification routes are configured for a stack, those channels receive the message; otherwise, global notification channels are used as a fallback. +## Container crash detection + +Sencho watches every container on each of your nodes in real time and notifies you when something exits unexpectedly. Detection is causal: Sencho distinguishes crashes from intentional stops, so stopping a stack, restarting it, or running `docker compose down` will not produce a false crash alert. + +### What triggers a crash alert + +| Situation | Alert | +|---|---| +| Container exits with a non-zero exit code without being asked to stop | **Crash** alert (level: error) | +| Container is killed by the kernel for exceeding its memory limit | **OOM Kill** alert (level: error) | +| Healthcheck reports the container as unhealthy | **Healthcheck failed** alert (level: error) | + +Alerts arrive within a couple of seconds of the event, not on a polling interval. + +### What does not trigger a crash alert + +- Stopping, restarting, updating, or removing a stack from Sencho +- Running `docker stop`, `docker restart`, or `docker compose down` from a terminal on the host +- A container exiting cleanly with exit code `0` +- A container being replaced during an image update + +### Global toggle + +Crash and unhealthy alerts are gated by the **Global Crash Detection** toggle in **Settings > System**. When disabled, Sencho stops dispatching crash and unhealthy notifications on all nodes. Stack metric alerts and update notifications are unaffected by this toggle. + +### Docker daemon interruptions + +If the Docker daemon becomes unreachable (daemon restart, socket lost, network issue on a remote node), Sencho sends a single **Lost connection to Docker daemon** warning and pauses crash detection on the affected node. When the connection is restored, Sencho reconciles container state against a pre-disconnect snapshot: + +- If most of your containers are still running, individual gap exits are classified and alerts are sent as normal. +- If a large share of containers exited during the outage (for example after a daemon restart), Sencho consolidates them into a single **Docker daemon interruption detected** informational notification instead of paging you for every container. + +A matching **Reconnected to Docker daemon** info notification confirms monitoring has resumed. + +### High-churn events + +When many crashes land in a short window (for example, a large stack coming down unexpectedly), Sencho dispatches an initial batch and summarizes the remainder in a single **N additional containers crashed in the last minute** entry so your notification channels are not flooded. + ## Troubleshooting ### Notifications not being delivered @@ -180,6 +218,18 @@ Both notification types use the same channel routing as alerts: if notification - Check the **cooldown** period; after an alert fires, it will not fire again until the cooldown expires - Verify the metric is being collected; container must be running for stats to be gathered +### I stopped a stack but got a crash alert + +This should not happen on current versions. If it does, confirm Sencho can reach the Docker daemon on the affected node (the **Lost connection to Docker daemon** warning is dispatched when the socket is unreachable). Previously, Sencho relied on polling and could mistake intentional stops for crashes; detection is now causal and in real time. + +### I'm seeing fewer crash alerts after upgrading + +Expected. Intentional stops, `docker stop` from a host terminal, and scheduled stack recreations no longer trigger crash alerts. Real crashes, OOM kills, and failing healthchecks still alert. If you want to opt out entirely, toggle **Global Crash Detection** off in **Settings > System**. + +### After a Docker daemon restart I only got one summary notification + +Expected. When a large share of containers exits during a Docker daemon interruption, Sencho consolidates them into a single informational notification instead of paging per container. Individual crashes that happen after the reconnect are alerted on as normal. + ### Delete confirmation dialog Deleting an alert rule now requires confirmation. Click the trash icon next to a rule, then confirm in the dialog that appears.