/** * Unit tests for DockerEventService. * * Mocks the Docker client stream via a small helper that exposes push() and * error() hooks so tests can drive the stream deterministically. Focuses on: * - classification of kill / die / oom / health_status * - the 500ms grace window for out-of-order die events * - rate limiting and overflow summary * - reconciliation on connect (baseline vs gap exits) * - mass-event detection on reconnect * - reconnect backoff + one-time warning/info alerts * - malformed payload tolerance */ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { EventEmitter } from 'events'; // ── Hoisted mocks ────────────────────────────────────────────────────── const { mockDispatchAlert, mockBroadcastEvent, mockGetGlobalSettings, mockGetEvents, mockListContainers, mockInspect, mockGetContainer, mockGetDocker, mockIsOwnContainer, } = vi.hoisted(() => ({ mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockBroadcastEvent: vi.fn(), mockGetGlobalSettings: vi.fn().mockReturnValue({ global_crash: '1' }), mockGetEvents: vi.fn(), mockListContainers: vi.fn().mockResolvedValue([]), mockInspect: vi.fn().mockResolvedValue({}), mockGetContainer: vi.fn(), mockGetDocker: vi.fn(), mockIsOwnContainer: vi.fn().mockReturnValue(false), })); vi.mock('../services/NotificationService', () => ({ NotificationService: { getInstance: () => ({ dispatchAlert: mockDispatchAlert, broadcastEvent: mockBroadcastEvent, }), }, })); vi.mock('../services/DatabaseService', () => ({ DatabaseService: { getInstance: () => ({ getGlobalSettings: mockGetGlobalSettings }), }, })); vi.mock('../services/NodeRegistry', () => ({ NodeRegistry: { getInstance: () => ({ getDocker: mockGetDocker }), }, })); vi.mock('../services/SelfIdentityService', () => ({ default: { getInstance: () => ({ isOwnContainer: mockIsOwnContainer, }), }, })); // ── Fake Docker stream helper ────────────────────────────────────────── interface FakeStream extends EventEmitter { destroyed: boolean; destroy: () => void; push: (event: Record) => void; pushRaw: (raw: string) => void; error: (err: Error) => void; } function makeStream(): FakeStream { const ee = new EventEmitter() as FakeStream; ee.destroyed = false; ee.destroy = () => { ee.destroyed = true; }; ee.push = (event) => ee.emit('data', Buffer.from(JSON.stringify(event) + '\n')); ee.pushRaw = (raw) => ee.emit('data', Buffer.from(raw)); ee.error = (err) => ee.emit('error', err); return ee; } // ── Setup ────────────────────────────────────────────────────────────── import { DockerEventService } from '../services/DockerEventService'; import { CacheService } from '../services/CacheService'; let stream: FakeStream; let service: DockerEventService; beforeEach(() => { vi.clearAllMocks(); vi.useFakeTimers(); mockDispatchAlert.mockReset(); mockDispatchAlert.mockResolvedValue({ persisted: true }); mockGetGlobalSettings.mockReturnValue({ global_crash: '1' }); mockIsOwnContainer.mockReset(); mockIsOwnContainer.mockReturnValue(false); stream = makeStream(); mockGetEvents.mockImplementation(async () => stream); mockListContainers.mockResolvedValue([]); mockGetContainer.mockImplementation((id: string) => ({ inspect: () => mockInspect(id), })); mockGetDocker.mockReturnValue({ getEvents: mockGetEvents, listContainers: mockListContainers, getContainer: mockGetContainer, }); }); afterEach(() => { service?.shutdown(); vi.useRealTimers(); }); async function flushMicrotasks(): Promise { await Promise.resolve(); await Promise.resolve(); } // ── Classification via event stream ──────────────────────────────────── describe('DockerEventService - die classification', () => { it('emits crash alert on die with non-zero exit code and no prior kill', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'c1', Attributes: { exitCode: '1', name: 'web' } }, time: 1700000000, }); await vi.advanceTimersByTimeAsync(600); // past the 500ms grace window expect(mockDispatchAlert).toHaveBeenCalledWith( 'error', 'monitor_alert', 'Container Crash Detected: web exited unexpectedly (Code: 1).', expect.objectContaining({ containerName: 'web' }), ); }); it('stamps a crash signal and clears it on a later clean exit', async () => { service = new DockerEventService(1, 'local'); await service.start(); // Crash: non-zero exit, no prior kill. stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'c-clean', Attributes: { exitCode: '1', name: 'web' } }, }); await vi.advanceTimersByTimeAsync(600); expect(service.getContainerState('c-clean')?.crashedAt).toBeTypeOf('number'); // A subsequent clean exit (code 0) must wipe the stale crash signal. stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'c-clean', Attributes: { exitCode: '0', name: 'web' } }, }); await vi.advanceTimersByTimeAsync(600); expect(service.getContainerState('c-clean')?.crashedAt).toBeUndefined(); }); it('does not stamp a crash for a die that was superseded by a start', async () => { service = new DockerEventService(1, 'local'); await service.start(); // Establish tracked state so the later start is recorded. stream.push({ Type: 'container', Action: 'health_status: healthy', Actor: { ID: 'c-race', Attributes: { name: 'web' } }, }); // Crash, then restart strictly later but still within the 500ms grace // window before the die is classified. stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'c-race', Attributes: { exitCode: '1', name: 'web' } }, }); await vi.advanceTimersByTimeAsync(100); stream.push({ Type: 'container', Action: 'start', Actor: { ID: 'c-race', Attributes: { name: 'web' } }, }); await vi.advanceTimersByTimeAsync(600); expect(service.getContainerState('c-race')?.crashedAt).toBeUndefined(); }); it('keeps stack and container routing for non-self compose crashes', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'app-id', Attributes: { exitCode: '1', name: 'web', 'com.docker.compose.project': 'web-stack', }, }, }); await vi.advanceTimersByTimeAsync(600); expect(mockDispatchAlert).toHaveBeenCalledWith( 'error', 'monitor_alert', expect.stringContaining('Container Crash Detected'), expect.objectContaining({ stackName: 'web-stack', containerName: 'web' }), ); }); it('routes self-container crashes as system-only notifications', async () => { mockIsOwnContainer.mockImplementation((idOrName: string) => idOrName === 'self-id' || idOrName === 'sencho', ); service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'self-id', Attributes: { exitCode: '1', name: 'sencho', 'com.docker.compose.project': 'sencho', }, }, }); await vi.advanceTimersByTimeAsync(600); expect(mockDispatchAlert).toHaveBeenCalledWith( 'error', 'monitor_alert', expect.stringContaining('Container Crash Detected'), { actor: 'system:docker-events' }, ); }); it('does not emit when die follows a recent kill (intentional)', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'kill', Actor: { ID: 'c1', Attributes: { signal: '15' } }, time: 1700000000, }); stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'c1', Attributes: { exitCode: '1', name: 'web' } }, time: 1700000001, }); await vi.advanceTimersByTimeAsync(600); const crashCall = mockDispatchAlert.mock.calls.find(c => typeof c[2] === 'string' && c[2].includes('Crash')); expect(crashCall).toBeUndefined(); }); it('reclassifies as intentional when kill arrives within the 500ms grace window', async () => { service = new DockerEventService(1, 'local'); await service.start(); // die first, kill 200ms later stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'c2', Attributes: { exitCode: '1', name: 'svc' } }, time: 1700000000, }); await vi.advanceTimersByTimeAsync(200); stream.push({ Type: 'container', Action: 'kill', Actor: { ID: 'c2', Attributes: { signal: '15' } }, time: 1700000001, }); await vi.advanceTimersByTimeAsync(400); // total > 500ms const crashCall = mockDispatchAlert.mock.calls.find(c => typeof c[2] === 'string' && c[2].includes('Crash')); expect(crashCall).toBeUndefined(); }); it('emits OOM alert when oom precedes die', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'oom', Actor: { ID: 'c3', Attributes: { name: 'hog' } } }); stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'c3', Attributes: { exitCode: '137', name: 'hog' } }, }); await vi.advanceTimersByTimeAsync(600); expect(mockDispatchAlert).toHaveBeenCalledWith( 'error', 'monitor_alert', expect.stringContaining('OOM Kill'), expect.objectContaining({ containerName: 'hog' }), ); }); it('does not emit when exit code is 0 (clean exit)', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'c4', Attributes: { exitCode: '0', name: 'oneshot' } }, }); await vi.advanceTimersByTimeAsync(600); expect(mockDispatchAlert).not.toHaveBeenCalled(); }); it('emits unhealthy alert on health_status: unhealthy', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'c5', Attributes: { name: 'api' } }, }); await vi.runOnlyPendingTimersAsync(); expect(mockDispatchAlert).toHaveBeenCalledWith( 'error', 'monitor_alert', expect.stringContaining('Healthcheck failed'), expect.objectContaining({ containerName: 'api' }), ); }); it('routes self-container unhealthy alerts as system-only notifications', async () => { mockIsOwnContainer.mockImplementation((idOrName: string) => idOrName === 'self-id' || idOrName === 'sencho', ); service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'self-id', Attributes: { name: 'sencho', 'com.docker.compose.project': 'sencho', }, }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledWith( 'error', 'monitor_alert', expect.stringContaining('Healthcheck failed'), { actor: 'system:docker-events' }, ); }); it('does not emit when global_crash is disabled', async () => { mockGetGlobalSettings.mockReturnValue({ global_crash: '0' }); service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'c6', Attributes: { exitCode: '1', name: 'web' } }, }); await vi.advanceTimersByTimeAsync(600); expect(mockDispatchAlert).not.toHaveBeenCalled(); }); it('clears crash dedup when container starts again', async () => { service = new DockerEventService(1, 'local'); await service.start(); // First crash stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'c7', Attributes: { exitCode: '1', name: 'web' } }, }); await vi.advanceTimersByTimeAsync(600); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); // Start event clears dedup stream.push({ Type: 'container', Action: 'start', Actor: { ID: 'c7' } }); // Second crash should fire again stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'c7', Attributes: { exitCode: '2', name: 'web' } }, }); await vi.advanceTimersByTimeAsync(600); expect(mockDispatchAlert).toHaveBeenCalledTimes(2); }); }); // ── Rate limiting ────────────────────────────────────────────────────── describe('DockerEventService - rate limiting', () => { it('batches overflow crashes into a single summary alert', async () => { service = new DockerEventService(1, 'local'); await service.start(); // Push 22 die events (limit is 20). for (let i = 0; i < 22; i++) { stream.push({ Type: 'container', Action: 'die', Actor: { ID: `c-${i}`, Attributes: { exitCode: '1', name: `n-${i}` } }, }); } await vi.advanceTimersByTimeAsync(600); const crashCalls = mockDispatchAlert.mock.calls.filter(c => typeof c[2] === 'string' && c[2].includes('Crash')); expect(crashCalls).toHaveLength(20); // After the rate window, a summary warning fires. await vi.advanceTimersByTimeAsync(61_000); const summaryCalls = mockDispatchAlert.mock.calls.filter(c => typeof c[2] === 'string' && c[2].includes('rate-limited')); expect(summaryCalls).toHaveLength(1); expect(summaryCalls[0][2]).toBe( '2 additional container crash alerts were rate-limited in the last minute.', ); }); }); // ── Health alert transition / dedup / rate sharing ───────────────────── describe('DockerEventService - health alert gates', () => { it('emits once for duplicate unhealthy while already unhealthy and keeps unhealthySince', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'h1', Attributes: { name: 'api', 'com.docker.compose.project': 'web' }, }, }); await flushMicrotasks(); const first = service.getContainerState('h1'); expect(first?.healthStatus).toBe('unhealthy'); expect(first?.unhealthySince).toBeTypeOf('number'); const since = first!.unhealthySince!; stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'h1', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); expect(service.getContainerState('h1')?.unhealthySince).toBe(since); expect(service.getContainerState('h1')?.healthStatus).toBe('unhealthy'); }); it('suppresses a second flap while the first health dispatch is still pending', async () => { let resolveFirst!: (value: { persisted: boolean }) => void; mockDispatchAlert.mockImplementationOnce( () => new Promise<{ persisted: boolean }>((resolve) => { resolveFirst = resolve; }), ); service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'h2', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); stream.push({ Type: 'container', Action: 'health_status: healthy', Actor: { ID: 'h2', Attributes: { name: 'api' } }, }); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'h2', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); resolveFirst({ persisted: true }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); }); it('retries a deferred flap when the in-flight dispatch fails to persist', async () => { let resolveFirst!: (value: { persisted: boolean }) => void; mockDispatchAlert .mockImplementationOnce( () => new Promise<{ persisted: boolean }>((resolve) => { resolveFirst = resolve; }), ) .mockResolvedValue({ persisted: true }); service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'h2b', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); stream.push({ Type: 'container', Action: 'health_status: healthy', Actor: { ID: 'h2b', Attributes: { name: 'api' } }, }); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'h2b', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); resolveFirst({ persisted: false }); await flushMicrotasks(); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(2); }); it('does not advance health dedup when persisted is false, allowing a later transition', async () => { mockDispatchAlert .mockResolvedValueOnce({ persisted: false }) .mockResolvedValue({ persisted: true }); service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'h3', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); stream.push({ Type: 'container', Action: 'health_status: healthy', Actor: { ID: 'h3', Attributes: { name: 'api' } }, }); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'h3', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(2); }); it('keeps health dedup across the 10-minute prune window and re-emits after 60 minutes', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'h4', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); // Recover so lastActivity ages without fresh unhealthy events; prune // may drop containerState after 10m but health dedup must survive. stream.push({ Type: 'container', Action: 'health_status: healthy', Actor: { ID: 'h4', Attributes: { name: 'api' } }, }); await flushMicrotasks(); await vi.advanceTimersByTimeAsync(11 * 60_000); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'h4', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); await vi.advanceTimersByTimeAsync(50 * 60_000); stream.push({ Type: 'container', Action: 'health_status: healthy', Actor: { ID: 'h4', Attributes: { name: 'api' } }, }); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'h4', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(2); }); it('does not let crash and health dedup suppress one another', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'h5', Attributes: { exitCode: '1', name: 'api' } }, }); await vi.advanceTimersByTimeAsync(600); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); stream.push({ Type: 'container', Action: 'start', Actor: { ID: 'h5', Attributes: { name: 'api' } }, }); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'h5', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(2); expect(mockDispatchAlert.mock.calls[1][2]).toContain('Healthcheck failed'); }); it('does not consume rate tokens for deduped duplicate health events', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'h6', Attributes: { name: 'api' } }, }); await flushMicrotasks(); for (let i = 0; i < 5; i++) { stream.push({ Type: 'container', Action: 'health_status: healthy', Actor: { ID: 'h6', Attributes: { name: 'api' } }, }); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'h6', Attributes: { name: 'api' } }, }); await flushMicrotasks(); } expect(mockDispatchAlert).toHaveBeenCalledTimes(1); for (let i = 0; i < 19; i++) { stream.push({ Type: 'container', Action: 'die', Actor: { ID: `crash-${i}`, Attributes: { exitCode: '1', name: `n-${i}` } }, }); } await vi.advanceTimersByTimeAsync(600); const crashCalls = mockDispatchAlert.mock.calls.filter(c => typeof c[2] === 'string' && c[2].includes('Crash')); expect(crashCalls).toHaveLength(19); }); it('emits an accurate health-only rate-limit roll-up', async () => { service = new DockerEventService(1, 'local'); await service.start(); for (let i = 0; i < 22; i++) { stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: `hh-${i}`, Attributes: { name: `svc-${i}` } }, }); await flushMicrotasks(); } const healthCalls = mockDispatchAlert.mock.calls.filter(c => typeof c[2] === 'string' && c[2].includes('Healthcheck failed')); expect(healthCalls).toHaveLength(20); await vi.advanceTimersByTimeAsync(61_000); const summaryCalls = mockDispatchAlert.mock.calls.filter(c => typeof c[2] === 'string' && c[2].includes('rate-limited')); expect(summaryCalls).toHaveLength(1); expect(summaryCalls[0][2]).toBe( '2 additional container health alerts were rate-limited in the last minute.', ); }); it('emits an accurate mixed crash and health rate-limit roll-up', async () => { service = new DockerEventService(1, 'local'); await service.start(); for (let i = 0; i < 10; i++) { stream.push({ Type: 'container', Action: 'die', Actor: { ID: `mc-${i}`, Attributes: { exitCode: '1', name: `c-${i}` } }, }); } await vi.advanceTimersByTimeAsync(600); for (let i = 0; i < 10; i++) { stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: `mh-${i}`, Attributes: { name: `h-${i}` } }, }); await flushMicrotasks(); } for (let i = 10; i < 12; i++) { stream.push({ Type: 'container', Action: 'die', Actor: { ID: `mc-${i}`, Attributes: { exitCode: '1', name: `c-${i}` } }, }); } await vi.advanceTimersByTimeAsync(600); for (let i = 10; i < 13; i++) { stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: `mh-${i}`, Attributes: { name: `h-${i}` } }, }); await flushMicrotasks(); } await vi.advanceTimersByTimeAsync(61_000); const summaryCalls = mockDispatchAlert.mock.calls.filter(c => typeof c[2] === 'string' && c[2].includes('rate-limited')); expect(summaryCalls).toHaveLength(1); expect(summaryCalls[0][2]).toBe( '5 additional container alerts were rate-limited in the last minute (2 crash, 3 health).', ); }); it('keeps Auto-Heal health state when global_crash is off', async () => { mockGetGlobalSettings.mockReturnValue({ global_crash: '0' }); service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'h7', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).not.toHaveBeenCalled(); expect(service.getContainerState('h7')).toMatchObject({ healthStatus: 'unhealthy', }); expect(service.getContainerState('h7')?.unhealthySince).toBeTypeOf('number'); }); it('does not refund a rate token into the next fixed window', async () => { let resolvePersist!: (value: { persisted: boolean }) => void; mockDispatchAlert.mockImplementation( () => new Promise<{ persisted: boolean }>((resolve) => { resolvePersist = resolve; }), ); service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'cross-window', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); // Cross the fixed window, then consume a token in the new window so // refund would inflate W2's count if it ignored the issuing window. await vi.advanceTimersByTimeAsync(60_000); mockDispatchAlert.mockResolvedValue({ persisted: true }); stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'cw-roll', Attributes: { exitCode: '1', name: 'roll' }, }, }); await vi.advanceTimersByTimeAsync(600); expect( mockDispatchAlert.mock.calls.filter( (c) => typeof c[2] === 'string' && c[2].includes('Crash Detected'), ), ).toHaveLength(1); // Fail the original health persist: old code refunds into W2 (count 1→0). resolvePersist({ persisted: false }); await flushMicrotasks(); // Fill the rest of W2 to the 20-alert cap (19 more crashes). for (let i = 0; i < 19; i++) { stream.push({ Type: 'container', Action: 'die', Actor: { ID: `cw-${i}`, Attributes: { exitCode: '1', name: `svc-${i}` }, }, }); } await vi.advanceTimersByTimeAsync(600); // Cap must still hold: a cross-window refund would have allowed a 21st. stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'cw-overflow', Attributes: { exitCode: '1', name: 'overflow' }, }, }); await vi.advanceTimersByTimeAsync(600); const crashCalls = mockDispatchAlert.mock.calls.filter( (c) => typeof c[2] === 'string' && c[2].includes('Crash Detected'), ); expect(crashCalls).toHaveLength(20); }); it('clears pending health retry when the container recovers before persist fails', async () => { let resolvePersist!: (value: { persisted: boolean }) => void; mockDispatchAlert.mockImplementation( () => new Promise<{ persisted: boolean }>((resolve) => { resolvePersist = resolve; }), ); service = new DockerEventService(1, 'local'); await service.start(); // First unhealthy starts an in-flight dispatch. stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'recover-pending', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); // Flap while in flight: recover then re-fail so a pending-retry marker is set. stream.push({ Type: 'container', Action: 'health_status: healthy', Actor: { ID: 'recover-pending', Attributes: { name: 'api' } }, }); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'recover-pending', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); // Recover again before the first persist settles; marker must be dropped. stream.push({ Type: 'container', Action: 'health_status: healthy', Actor: { ID: 'recover-pending', Attributes: { name: 'api' } }, }); await flushMicrotasks(); resolvePersist({ persisted: false }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); // Later unrelated persist failure must not spuriously retry from a stale marker. mockDispatchAlert.mockResolvedValueOnce({ persisted: false }); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'recover-pending', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(2); }); it('does not dispatch deferred health retry after shutdown', async () => { let resolvePersist!: (value: { persisted: boolean }) => void; mockDispatchAlert.mockImplementation( () => new Promise<{ persisted: boolean }>((resolve) => { resolvePersist = resolve; }), ); service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'shutdown-pending', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); service.shutdown(); resolvePersist({ persisted: false }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); }); it('clears health alert bookkeeping on destroy', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'destroy-dedup', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); stream.push({ Type: 'container', Action: 'destroy', Actor: { ID: 'destroy-dedup', Attributes: { name: 'api' } }, }); await flushMicrotasks(); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'destroy-dedup', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(2); }); it('does not retry a deferred health alert after destroy', async () => { let resolvePersist!: (value: { persisted: boolean }) => void; mockDispatchAlert.mockImplementation( () => new Promise<{ persisted: boolean }>((resolve) => { resolvePersist = resolve; }), ); service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } }, }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); // Set a pending-retry marker while the first dispatch is in flight. stream.push({ Type: 'container', Action: 'health_status: healthy', Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } }, }); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } }, }); await flushMicrotasks(); stream.push({ Type: 'container', Action: 'destroy', Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } }, }); await flushMicrotasks(); resolvePersist({ persisted: false }); await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); }); }); // ── Malformed payloads ───────────────────────────────────────────────── describe('DockerEventService - malformed payloads', () => { it('tolerates a bad JSON line without tearing down the stream', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.pushRaw('not json\n'); stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'ok', Attributes: { exitCode: '1', name: 'ok' } }, }); await vi.advanceTimersByTimeAsync(600); expect(mockDispatchAlert).toHaveBeenCalledWith( 'error', 'monitor_alert', expect.stringContaining('Container Crash Detected'), expect.objectContaining({ containerName: 'ok' }), ); }); }); // ── Reconciliation ───────────────────────────────────────────────────── describe('DockerEventService - reconciliation', () => { it('on first boot, records pre-existing exited containers as baseline and does not alert', async () => { mockListContainers.mockResolvedValue([ { Id: 'pre-1', State: 'exited' }, { Id: 'pre-2', State: 'exited' }, { Id: 'run-1', State: 'running' }, ]); service = new DockerEventService(1, 'local'); await service.start(); expect(mockDispatchAlert).not.toHaveBeenCalled(); }); it('emits mass-event summary on reconnect when >20% of containers newly exited', async () => { // First connect: 10 running containers as baseline. const running = Array.from({ length: 10 }, (_, i) => ({ Id: `c-${i}`, State: 'running', })); mockListContainers.mockResolvedValueOnce(running); service = new DockerEventService(1, 'local'); await service.start(); // Simulate stream drop. stream.error(new Error('connection reset')); stream = makeStream(); mockGetEvents.mockImplementation(async () => stream); // On reconnect: 5 of them now exited (>20%). const mixed = running.map((c, i) => ({ Id: c.Id, State: i < 5 ? 'exited' : 'running', })); mockListContainers.mockResolvedValueOnce(mixed); // Drain reconnect backoff + reconciliation. await vi.advanceTimersByTimeAsync(2_000); // Flush microtasks spawned by the async reconnect + reconcile chain // without running the recurring prune interval. await Promise.resolve(); await Promise.resolve(); await Promise.resolve(); const massCall = mockDispatchAlert.mock.calls.find(c => typeof c[2] === 'string' && c[2].includes('daemon interruption')); expect(massCall).toBeDefined(); }); it('classifies individual gap exits on reconnect when below mass threshold', async () => { // Baseline: 10 containers running. const baseline = Array.from({ length: 10 }, (_, i) => ({ Id: `c-${i}`, State: 'running', })); mockListContainers.mockResolvedValueOnce(baseline); service = new DockerEventService(1, 'local'); await service.start(); // Drop + one new exit (10%, below 20% threshold). stream.error(new Error('bad')); stream = makeStream(); mockGetEvents.mockImplementation(async () => stream); const postReconnect = baseline.map((c, i) => ({ Id: c.Id, State: i === 0 ? 'exited' : 'running', })); mockListContainers.mockResolvedValueOnce(postReconnect); mockInspect.mockResolvedValueOnce({ Name: '/crashed-app', State: { ExitCode: 9, OOMKilled: false }, Config: { Labels: { 'com.docker.compose.project': 'my-stack' } }, }); await vi.advanceTimersByTimeAsync(2_000); // Flush microtasks spawned by the async reconnect + reconcile chain // without running the recurring prune interval. await Promise.resolve(); await Promise.resolve(); await Promise.resolve(); const crashCall = mockDispatchAlert.mock.calls.find(c => typeof c[2] === 'string' && c[2].includes('crashed-app')); expect(crashCall).toBeDefined(); expect(crashCall?.[3]).toMatchObject({ stackName: 'my-stack' }); }); }); // ── Reconnect lifecycle ──────────────────────────────────────────────── describe('DockerEventService - reconnect', () => { it('emits one-time warning on first disconnect and info on reconnect', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.error(new Error('connection reset')); // Reconnect succeeds immediately after backoff. stream = makeStream(); mockGetEvents.mockImplementation(async () => stream); await vi.advanceTimersByTimeAsync(2_000); // Flush microtasks spawned by the async reconnect + reconcile chain // without running the recurring prune interval. await Promise.resolve(); await Promise.resolve(); await Promise.resolve(); const warn = mockDispatchAlert.mock.calls.find(c => c[0] === 'warning'); const info = mockDispatchAlert.mock.calls.find(c => c[0] === 'info'); expect(warn?.[2]).toBe('Lost connection to Docker daemon; monitoring paused.'); expect(info?.[2]).toBe('Reconnected to Docker daemon.'); }); it('shutdown cancels pending reconnect', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.error(new Error('broken')); mockGetEvents.mockClear(); service.shutdown(); // Advance time well past the first backoff window; no new connect should run. await vi.advanceTimersByTimeAsync(5_000); expect(mockGetEvents).not.toHaveBeenCalled(); }); }); // ── Hardening: gap isolation / OOM fallback / concurrent dies ───────── describe('DockerEventService - hardening', () => { it('isolates failures inside classifyGap so one bad inspect does not abort the batch', async () => { // 1 exit out of 10 (10%) stays below the 20% mass-event threshold, so // the gap classifier inspects the container individually. That inspect // fails; the service must isolate the failure and still classify a // subsequent die as a crash. const baseline = Array.from({ length: 10 }, (_, i) => ({ Id: `c-${i}`, State: 'running', })); mockListContainers.mockResolvedValueOnce(baseline); service = new DockerEventService(1, 'local'); await service.start(); stream.error(new Error('broken')); stream = makeStream(); mockGetEvents.mockImplementation(async () => stream); const postReconnect = baseline.map((c, i) => ({ Id: c.Id, State: i < 1 ? 'exited' : 'running', })); mockListContainers.mockResolvedValueOnce(postReconnect); mockInspect.mockRejectedValueOnce(new Error('gone')); await vi.advanceTimersByTimeAsync(2_000); await Promise.resolve(); await Promise.resolve(); await Promise.resolve(); mockDispatchAlert.mockClear(); stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'post-recovery', Attributes: { exitCode: '1', name: 'app' } }, }); await vi.advanceTimersByTimeAsync(600); expect(mockDispatchAlert).toHaveBeenCalledWith( 'error', 'monitor_alert', expect.stringContaining('Container Crash Detected'), expect.objectContaining({ containerName: 'app' }), ); }); it('classifies exit code 137 as OOM when container inspect reports OOMKilled (no oom event)', async () => { service = new DockerEventService(1, 'local'); await service.start(); // Inspect fallback: no prior `oom` event, but the container's // State.OOMKilled is true. mockInspect.mockResolvedValueOnce({ State: { OOMKilled: true, ExitCode: 137 }, }); stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'cgroup-killed', Attributes: { exitCode: '137', name: 'hog' } }, }); await vi.advanceTimersByTimeAsync(600); // Allow the awaited inspect in classifyDie to resolve. await Promise.resolve(); await Promise.resolve(); await Promise.resolve(); const oomCall = mockDispatchAlert.mock.calls.find(c => typeof c[2] === 'string' && c[2].includes('OOM Kill')); const crashCall = mockDispatchAlert.mock.calls.find(c => typeof c[2] === 'string' && c[2].includes('Crash Detected')); expect(oomCall).toBeDefined(); expect(crashCall).toBeUndefined(); }); it('falls back to crash when exit 137 die inspect fails', async () => { service = new DockerEventService(1, 'local'); await service.start(); mockInspect.mockRejectedValueOnce(new Error('no such container')); stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'gone', Attributes: { exitCode: '137', name: 'ephemeral' } }, }); await vi.advanceTimersByTimeAsync(600); await Promise.resolve(); await Promise.resolve(); await Promise.resolve(); // Inspect failed, so classification stays as the original 'crash'. expect(mockDispatchAlert).toHaveBeenCalledWith( 'error', 'monitor_alert', expect.stringContaining('Container Crash Detected'), expect.objectContaining({ containerName: 'ephemeral' }), ); }); it('collapses duplicate die events for the same container within the grace window', async () => { service = new DockerEventService(1, 'local'); await service.start(); // Two dies for the same container within 500ms: the second cancels // the first pending timer and reschedules. Exactly one crash alert // must fire, using the later exit code. stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'dup', Attributes: { exitCode: '1', name: 'dup' } }, }); await vi.advanceTimersByTimeAsync(100); stream.push({ Type: 'container', Action: 'die', Actor: { ID: 'dup', Attributes: { exitCode: '2', name: 'dup' } }, }); await vi.advanceTimersByTimeAsync(700); const crashCalls = mockDispatchAlert.mock.calls.filter(c => typeof c[2] === 'string' && c[2].includes('Crash Detected')); expect(crashCalls).toHaveLength(1); expect(crashCalls[0][2]).toContain('Code: 2'); }); }); // ── State-invalidate broadcasts ──────────────────────────────────────── describe('DockerEventService - state-invalidate broadcasts', () => { let invalidateSpy: ReturnType; beforeEach(() => { invalidateSpy = vi.spyOn(CacheService.getInstance(), 'invalidate'); }); afterEach(() => { invalidateSpy.mockRestore(); }); it('broadcasts state-invalidate and drops the statuses cache on container start', async () => { service = new DockerEventService(7, 'node-7'); await service.start(); stream.push({ Type: 'container', Action: 'start', Actor: { ID: 'aaa', Attributes: { 'com.docker.compose.project': 'web' } }, time: 1, }); await vi.advanceTimersByTimeAsync(1); expect(mockBroadcastEvent).toHaveBeenCalledWith(expect.objectContaining({ type: 'state-invalidate', scope: 'stack', nodeId: 7, stackName: 'web', containerId: 'aaa', action: 'start', })); // The UI refetch this broadcast triggers must recompute, not hit a // cache entry made stale by the event. Only the statuses key is // touched; the full invalidateNodeCaches helper is not used here. expect(invalidateSpy).toHaveBeenCalledTimes(1); expect(invalidateSpy).toHaveBeenCalledWith('stack-statuses:7'); }); it('does not broadcast stack state-invalidate for Sencho self-container events', async () => { mockIsOwnContainer.mockImplementation((idOrName: string) => idOrName === 'self-id' || idOrName === 'sencho', ); service = new DockerEventService(7, 'node-7'); await service.start(); stream.push({ Type: 'container', Action: 'start', Actor: { ID: 'self-id', Attributes: { name: 'sencho', 'com.docker.compose.project': 'sencho', }, }, time: 1, }); await vi.advanceTimersByTimeAsync(1); expect(mockBroadcastEvent).not.toHaveBeenCalled(); expect(invalidateSpy).not.toHaveBeenCalled(); }); it('broadcasts state-invalidate on health_status:unhealthy', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'health_status: unhealthy', Actor: { ID: 'bbb', Attributes: { 'com.docker.compose.project': 'api' } }, time: 1, }); await vi.advanceTimersByTimeAsync(1); const states = mockBroadcastEvent.mock.calls.filter(c => (c[0] as { type?: string }).type === 'state-invalidate'); expect(states.length).toBeGreaterThan(0); expect(states[0][0]).toMatchObject({ action: 'health_status', stackName: 'api' }); expect(invalidateSpy).toHaveBeenCalledWith('stack-statuses:1'); }); it('does not broadcast state-invalidate on non-state actions like exec_create', async () => { service = new DockerEventService(1, 'local'); await service.start(); stream.push({ Type: 'container', Action: 'exec_create: /bin/sh', Actor: { ID: 'ccc' }, time: 1, }); await vi.advanceTimersByTimeAsync(1); expect(mockBroadcastEvent).not.toHaveBeenCalled(); expect(invalidateSpy).not.toHaveBeenCalled(); }); }); // ── Diagnostics ──────────────────────────────────────────────────────── describe('DockerEventService - getStatus', () => { it('reports connected after start', async () => { service = new DockerEventService(42, 'my-node'); await service.start(); const status = service.getStatus(); expect(status.nodeId).toBe(42); expect(status.nodeName).toBe('my-node'); expect(status.status).toBe('connected'); }); });