mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-16 05:28:28 +00:00
Anso
55ca82abb2
The 3s stack-statuses cache TTL never survived the 10s dashboard poll, so every ordinary poll recomputed. Raise the TTL to 15s and move the git-source label and self-identity enrichment inside the cached payload so cache hits serve fully decorated statuses with zero per-request work. Invalidation closes the gaps the longer TTL would otherwise widen: - DockerEventService drops stack-statuses:<nodeId> on container state events so the UI's state-invalidate refetch recomputes instead of hitting a stale entry. The narrow key only: container events do not reshape stack identity or file roots, and the stats key self-refreshes on its own 2s TTL. - git-source link and unlink invalidate node caches before responding, so the source label stays fresh without waiting for the TTL. - a payload whose enrichment degraded (identity probe failure or git-source scan failure) is never cached, so a mislabeled not-self or 'local' badge cannot persist for a full TTL window. Running outside Docker is not degradation, so host installs cache normally.
1439 lines
51 KiB
TypeScript
1439 lines
51 KiB
TypeScript
/**
|
|
* Unit tests for DockerEventService.
|
|
*
|
|
* Mocks the Docker client stream via a small helper that exposes push() and
|
|
* error() hooks so tests can drive the stream deterministically. Focuses on:
|
|
* - classification of kill / die / oom / health_status
|
|
* - the 500ms grace window for out-of-order die events
|
|
* - rate limiting and overflow summary
|
|
* - reconciliation on connect (baseline vs gap exits)
|
|
* - mass-event detection on reconnect
|
|
* - reconnect backoff + one-time warning/info alerts
|
|
* - malformed payload tolerance
|
|
*/
|
|
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
import { EventEmitter } from 'events';
|
|
|
|
// ── Hoisted mocks ──────────────────────────────────────────────────────
|
|
|
|
const {
|
|
mockDispatchAlert,
|
|
mockBroadcastEvent,
|
|
mockGetGlobalSettings,
|
|
mockGetEvents,
|
|
mockListContainers,
|
|
mockInspect,
|
|
mockGetContainer,
|
|
mockGetDocker,
|
|
mockIsOwnContainer,
|
|
} = vi.hoisted(() => ({
|
|
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
|
|
mockBroadcastEvent: vi.fn(),
|
|
mockGetGlobalSettings: vi.fn().mockReturnValue({ global_crash: '1' }),
|
|
mockGetEvents: vi.fn(),
|
|
mockListContainers: vi.fn().mockResolvedValue([]),
|
|
mockInspect: vi.fn().mockResolvedValue({}),
|
|
mockGetContainer: vi.fn(),
|
|
mockGetDocker: vi.fn(),
|
|
mockIsOwnContainer: vi.fn().mockReturnValue(false),
|
|
}));
|
|
|
|
vi.mock('../services/NotificationService', () => ({
|
|
NotificationService: {
|
|
getInstance: () => ({
|
|
dispatchAlert: mockDispatchAlert,
|
|
broadcastEvent: mockBroadcastEvent,
|
|
}),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/DatabaseService', () => ({
|
|
DatabaseService: {
|
|
getInstance: () => ({ getGlobalSettings: mockGetGlobalSettings }),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/NodeRegistry', () => ({
|
|
NodeRegistry: {
|
|
getInstance: () => ({ getDocker: mockGetDocker }),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/SelfIdentityService', () => ({
|
|
default: {
|
|
getInstance: () => ({
|
|
isOwnContainer: mockIsOwnContainer,
|
|
}),
|
|
},
|
|
}));
|
|
|
|
// ── Fake Docker stream helper ──────────────────────────────────────────
|
|
|
|
interface FakeStream extends EventEmitter {
|
|
destroyed: boolean;
|
|
destroy: () => void;
|
|
push: (event: Record<string, unknown>) => void;
|
|
pushRaw: (raw: string) => void;
|
|
error: (err: Error) => void;
|
|
}
|
|
|
|
function makeStream(): FakeStream {
|
|
const ee = new EventEmitter() as FakeStream;
|
|
ee.destroyed = false;
|
|
ee.destroy = () => { ee.destroyed = true; };
|
|
ee.push = (event) => ee.emit('data', Buffer.from(JSON.stringify(event) + '\n'));
|
|
ee.pushRaw = (raw) => ee.emit('data', Buffer.from(raw));
|
|
ee.error = (err) => ee.emit('error', err);
|
|
return ee;
|
|
}
|
|
|
|
// ── Setup ──────────────────────────────────────────────────────────────
|
|
|
|
import { DockerEventService } from '../services/DockerEventService';
|
|
import { CacheService } from '../services/CacheService';
|
|
|
|
let stream: FakeStream;
|
|
let service: DockerEventService;
|
|
|
|
beforeEach(() => {
|
|
vi.clearAllMocks();
|
|
vi.useFakeTimers();
|
|
mockDispatchAlert.mockReset();
|
|
mockDispatchAlert.mockResolvedValue({ persisted: true });
|
|
mockGetGlobalSettings.mockReturnValue({ global_crash: '1' });
|
|
mockIsOwnContainer.mockReset();
|
|
mockIsOwnContainer.mockReturnValue(false);
|
|
|
|
stream = makeStream();
|
|
mockGetEvents.mockImplementation(async () => stream);
|
|
mockListContainers.mockResolvedValue([]);
|
|
mockGetContainer.mockImplementation((id: string) => ({
|
|
inspect: () => mockInspect(id),
|
|
}));
|
|
mockGetDocker.mockReturnValue({
|
|
getEvents: mockGetEvents,
|
|
listContainers: mockListContainers,
|
|
getContainer: mockGetContainer,
|
|
});
|
|
});
|
|
|
|
afterEach(() => {
|
|
service?.shutdown();
|
|
vi.useRealTimers();
|
|
});
|
|
|
|
async function flushMicrotasks(): Promise<void> {
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
}
|
|
|
|
// ── Classification via event stream ────────────────────────────────────
|
|
|
|
describe('DockerEventService - die classification', () => {
|
|
it('emits crash alert on die with non-zero exit code and no prior kill', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'c1', Attributes: { exitCode: '1', name: 'web' } },
|
|
time: 1700000000,
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600); // past the 500ms grace window
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'error',
|
|
'monitor_alert',
|
|
'Container Crash Detected: web exited unexpectedly (Code: 1).',
|
|
expect.objectContaining({ containerName: 'web' }),
|
|
);
|
|
});
|
|
|
|
it('stamps a crash signal and clears it on a later clean exit', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
// Crash: non-zero exit, no prior kill.
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'c-clean', Attributes: { exitCode: '1', name: 'web' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
expect(service.getContainerState('c-clean')?.crashedAt).toBeTypeOf('number');
|
|
|
|
// A subsequent clean exit (code 0) must wipe the stale crash signal.
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'c-clean', Attributes: { exitCode: '0', name: 'web' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
expect(service.getContainerState('c-clean')?.crashedAt).toBeUndefined();
|
|
});
|
|
|
|
it('does not stamp a crash for a die that was superseded by a start', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
// Establish tracked state so the later start is recorded.
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: healthy',
|
|
Actor: { ID: 'c-race', Attributes: { name: 'web' } },
|
|
});
|
|
// Crash, then restart strictly later but still within the 500ms grace
|
|
// window before the die is classified.
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'c-race', Attributes: { exitCode: '1', name: 'web' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(100);
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'start',
|
|
Actor: { ID: 'c-race', Attributes: { name: 'web' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
|
|
expect(service.getContainerState('c-race')?.crashedAt).toBeUndefined();
|
|
});
|
|
|
|
it('keeps stack and container routing for non-self compose crashes', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: {
|
|
ID: 'app-id',
|
|
Attributes: {
|
|
exitCode: '1',
|
|
name: 'web',
|
|
'com.docker.compose.project': 'web-stack',
|
|
},
|
|
},
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'error',
|
|
'monitor_alert',
|
|
expect.stringContaining('Container Crash Detected'),
|
|
expect.objectContaining({ stackName: 'web-stack', containerName: 'web' }),
|
|
);
|
|
});
|
|
|
|
it('routes self-container crashes as system-only notifications', async () => {
|
|
mockIsOwnContainer.mockImplementation((idOrName: string) =>
|
|
idOrName === 'self-id' || idOrName === 'sencho',
|
|
);
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: {
|
|
ID: 'self-id',
|
|
Attributes: {
|
|
exitCode: '1',
|
|
name: 'sencho',
|
|
'com.docker.compose.project': 'sencho',
|
|
},
|
|
},
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'error',
|
|
'monitor_alert',
|
|
expect.stringContaining('Container Crash Detected'),
|
|
{ actor: 'system:docker-events' },
|
|
);
|
|
});
|
|
|
|
it('does not emit when die follows a recent kill (intentional)', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'kill',
|
|
Actor: { ID: 'c1', Attributes: { signal: '15' } },
|
|
time: 1700000000,
|
|
});
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'c1', Attributes: { exitCode: '1', name: 'web' } },
|
|
time: 1700000001,
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
|
|
const crashCall = mockDispatchAlert.mock.calls.find(c =>
|
|
typeof c[2] === 'string' && c[2].includes('Crash'));
|
|
expect(crashCall).toBeUndefined();
|
|
});
|
|
|
|
it('reclassifies as intentional when kill arrives within the 500ms grace window', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
// die first, kill 200ms later
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'c2', Attributes: { exitCode: '1', name: 'svc' } },
|
|
time: 1700000000,
|
|
});
|
|
await vi.advanceTimersByTimeAsync(200);
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'kill',
|
|
Actor: { ID: 'c2', Attributes: { signal: '15' } },
|
|
time: 1700000001,
|
|
});
|
|
await vi.advanceTimersByTimeAsync(400); // total > 500ms
|
|
|
|
const crashCall = mockDispatchAlert.mock.calls.find(c =>
|
|
typeof c[2] === 'string' && c[2].includes('Crash'));
|
|
expect(crashCall).toBeUndefined();
|
|
});
|
|
|
|
it('emits OOM alert when oom precedes die', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({ Type: 'container', Action: 'oom', Actor: { ID: 'c3', Attributes: { name: 'hog' } } });
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'c3', Attributes: { exitCode: '137', name: 'hog' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'error',
|
|
'monitor_alert',
|
|
expect.stringContaining('OOM Kill'),
|
|
expect.objectContaining({ containerName: 'hog' }),
|
|
);
|
|
});
|
|
|
|
it('does not emit when exit code is 0 (clean exit)', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'c4', Attributes: { exitCode: '0', name: 'oneshot' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
|
|
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('emits unhealthy alert on health_status: unhealthy', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'c5', Attributes: { name: 'api' } },
|
|
});
|
|
await vi.runOnlyPendingTimersAsync();
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'error',
|
|
'monitor_alert',
|
|
expect.stringContaining('Healthcheck failed'),
|
|
expect.objectContaining({ containerName: 'api' }),
|
|
);
|
|
});
|
|
|
|
it('routes self-container unhealthy alerts as system-only notifications', async () => {
|
|
mockIsOwnContainer.mockImplementation((idOrName: string) =>
|
|
idOrName === 'self-id' || idOrName === 'sencho',
|
|
);
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: {
|
|
ID: 'self-id',
|
|
Attributes: {
|
|
name: 'sencho',
|
|
'com.docker.compose.project': 'sencho',
|
|
},
|
|
},
|
|
});
|
|
await flushMicrotasks();
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'error',
|
|
'monitor_alert',
|
|
expect.stringContaining('Healthcheck failed'),
|
|
{ actor: 'system:docker-events' },
|
|
);
|
|
});
|
|
|
|
it('does not emit when global_crash is disabled', async () => {
|
|
mockGetGlobalSettings.mockReturnValue({ global_crash: '0' });
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'c6', Attributes: { exitCode: '1', name: 'web' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
|
|
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('clears crash dedup when container starts again', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
// First crash
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'c7', Attributes: { exitCode: '1', name: 'web' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// Start event clears dedup
|
|
stream.push({ Type: 'container', Action: 'start', Actor: { ID: 'c7' } });
|
|
|
|
// Second crash should fire again
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'c7', Attributes: { exitCode: '2', name: 'web' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
|
});
|
|
});
|
|
|
|
// ── Rate limiting ──────────────────────────────────────────────────────
|
|
|
|
describe('DockerEventService - rate limiting', () => {
|
|
it('batches overflow crashes into a single summary alert', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
// Push 22 die events (limit is 20).
|
|
for (let i = 0; i < 22; i++) {
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: `c-${i}`, Attributes: { exitCode: '1', name: `n-${i}` } },
|
|
});
|
|
}
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
|
|
const crashCalls = mockDispatchAlert.mock.calls.filter(c =>
|
|
typeof c[2] === 'string' && c[2].includes('Crash'));
|
|
expect(crashCalls).toHaveLength(20);
|
|
|
|
// After the rate window, a summary warning fires.
|
|
await vi.advanceTimersByTimeAsync(61_000);
|
|
const summaryCalls = mockDispatchAlert.mock.calls.filter(c =>
|
|
typeof c[2] === 'string' && c[2].includes('rate-limited'));
|
|
expect(summaryCalls).toHaveLength(1);
|
|
expect(summaryCalls[0][2]).toBe(
|
|
'2 additional container crash alerts were rate-limited in the last minute.',
|
|
);
|
|
});
|
|
});
|
|
|
|
// ── Health alert transition / dedup / rate sharing ─────────────────────
|
|
|
|
describe('DockerEventService - health alert gates', () => {
|
|
it('emits once for duplicate unhealthy while already unhealthy and keeps unhealthySince', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: {
|
|
ID: 'h1',
|
|
Attributes: { name: 'api', 'com.docker.compose.project': 'web' },
|
|
},
|
|
});
|
|
await flushMicrotasks();
|
|
const first = service.getContainerState('h1');
|
|
expect(first?.healthStatus).toBe('unhealthy');
|
|
expect(first?.unhealthySince).toBeTypeOf('number');
|
|
const since = first!.unhealthySince!;
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'h1', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
expect(service.getContainerState('h1')?.unhealthySince).toBe(since);
|
|
expect(service.getContainerState('h1')?.healthStatus).toBe('unhealthy');
|
|
});
|
|
|
|
it('suppresses a second flap while the first health dispatch is still pending', async () => {
|
|
let resolveFirst!: (value: { persisted: boolean }) => void;
|
|
mockDispatchAlert.mockImplementationOnce(
|
|
() => new Promise<{ persisted: boolean }>((resolve) => { resolveFirst = resolve; }),
|
|
);
|
|
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'h2', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: healthy',
|
|
Actor: { ID: 'h2', Attributes: { name: 'api' } },
|
|
});
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'h2', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
resolveFirst({ persisted: true });
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('retries a deferred flap when the in-flight dispatch fails to persist', async () => {
|
|
let resolveFirst!: (value: { persisted: boolean }) => void;
|
|
mockDispatchAlert
|
|
.mockImplementationOnce(
|
|
() => new Promise<{ persisted: boolean }>((resolve) => { resolveFirst = resolve; }),
|
|
)
|
|
.mockResolvedValue({ persisted: true });
|
|
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'h2b', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: healthy',
|
|
Actor: { ID: 'h2b', Attributes: { name: 'api' } },
|
|
});
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'h2b', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
resolveFirst({ persisted: false });
|
|
await flushMicrotasks();
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('does not advance health dedup when persisted is false, allowing a later transition', async () => {
|
|
mockDispatchAlert
|
|
.mockResolvedValueOnce({ persisted: false })
|
|
.mockResolvedValue({ persisted: true });
|
|
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'h3', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: healthy',
|
|
Actor: { ID: 'h3', Attributes: { name: 'api' } },
|
|
});
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'h3', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('keeps health dedup across the 10-minute prune window and re-emits after 60 minutes', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'h4', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// Recover so lastActivity ages without fresh unhealthy events; prune
|
|
// may drop containerState after 10m but health dedup must survive.
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: healthy',
|
|
Actor: { ID: 'h4', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
|
|
await vi.advanceTimersByTimeAsync(11 * 60_000);
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'h4', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
await vi.advanceTimersByTimeAsync(50 * 60_000);
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: healthy',
|
|
Actor: { ID: 'h4', Attributes: { name: 'api' } },
|
|
});
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'h4', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('does not let crash and health dedup suppress one another', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'h5', Attributes: { exitCode: '1', name: 'api' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'start',
|
|
Actor: { ID: 'h5', Attributes: { name: 'api' } },
|
|
});
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'h5', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
|
expect(mockDispatchAlert.mock.calls[1][2]).toContain('Healthcheck failed');
|
|
});
|
|
|
|
it('does not consume rate tokens for deduped duplicate health events', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'h6', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
|
|
for (let i = 0; i < 5; i++) {
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: healthy',
|
|
Actor: { ID: 'h6', Attributes: { name: 'api' } },
|
|
});
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'h6', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
}
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
for (let i = 0; i < 19; i++) {
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: `crash-${i}`, Attributes: { exitCode: '1', name: `n-${i}` } },
|
|
});
|
|
}
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
const crashCalls = mockDispatchAlert.mock.calls.filter(c =>
|
|
typeof c[2] === 'string' && c[2].includes('Crash'));
|
|
expect(crashCalls).toHaveLength(19);
|
|
});
|
|
|
|
it('emits an accurate health-only rate-limit roll-up', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
for (let i = 0; i < 22; i++) {
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: `hh-${i}`, Attributes: { name: `svc-${i}` } },
|
|
});
|
|
await flushMicrotasks();
|
|
}
|
|
|
|
const healthCalls = mockDispatchAlert.mock.calls.filter(c =>
|
|
typeof c[2] === 'string' && c[2].includes('Healthcheck failed'));
|
|
expect(healthCalls).toHaveLength(20);
|
|
|
|
await vi.advanceTimersByTimeAsync(61_000);
|
|
const summaryCalls = mockDispatchAlert.mock.calls.filter(c =>
|
|
typeof c[2] === 'string' && c[2].includes('rate-limited'));
|
|
expect(summaryCalls).toHaveLength(1);
|
|
expect(summaryCalls[0][2]).toBe(
|
|
'2 additional container health alerts were rate-limited in the last minute.',
|
|
);
|
|
});
|
|
|
|
it('emits an accurate mixed crash and health rate-limit roll-up', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
for (let i = 0; i < 10; i++) {
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: `mc-${i}`, Attributes: { exitCode: '1', name: `c-${i}` } },
|
|
});
|
|
}
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
|
|
for (let i = 0; i < 10; i++) {
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: `mh-${i}`, Attributes: { name: `h-${i}` } },
|
|
});
|
|
await flushMicrotasks();
|
|
}
|
|
|
|
for (let i = 10; i < 12; i++) {
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: `mc-${i}`, Attributes: { exitCode: '1', name: `c-${i}` } },
|
|
});
|
|
}
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
|
|
for (let i = 10; i < 13; i++) {
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: `mh-${i}`, Attributes: { name: `h-${i}` } },
|
|
});
|
|
await flushMicrotasks();
|
|
}
|
|
|
|
await vi.advanceTimersByTimeAsync(61_000);
|
|
const summaryCalls = mockDispatchAlert.mock.calls.filter(c =>
|
|
typeof c[2] === 'string' && c[2].includes('rate-limited'));
|
|
expect(summaryCalls).toHaveLength(1);
|
|
expect(summaryCalls[0][2]).toBe(
|
|
'5 additional container alerts were rate-limited in the last minute (2 crash, 3 health).',
|
|
);
|
|
});
|
|
|
|
it('keeps Auto-Heal health state when global_crash is off', async () => {
|
|
mockGetGlobalSettings.mockReturnValue({ global_crash: '0' });
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'h7', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
|
|
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
|
expect(service.getContainerState('h7')).toMatchObject({
|
|
healthStatus: 'unhealthy',
|
|
});
|
|
expect(service.getContainerState('h7')?.unhealthySince).toBeTypeOf('number');
|
|
});
|
|
|
|
it('does not refund a rate token into the next fixed window', async () => {
|
|
let resolvePersist!: (value: { persisted: boolean }) => void;
|
|
mockDispatchAlert.mockImplementation(
|
|
() =>
|
|
new Promise<{ persisted: boolean }>((resolve) => {
|
|
resolvePersist = resolve;
|
|
}),
|
|
);
|
|
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'cross-window', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// Cross the fixed window, then consume a token in the new window so
|
|
// refund would inflate W2's count if it ignored the issuing window.
|
|
await vi.advanceTimersByTimeAsync(60_000);
|
|
mockDispatchAlert.mockResolvedValue({ persisted: true });
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: {
|
|
ID: 'cw-roll',
|
|
Attributes: { exitCode: '1', name: 'roll' },
|
|
},
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
expect(
|
|
mockDispatchAlert.mock.calls.filter(
|
|
(c) => typeof c[2] === 'string' && c[2].includes('Crash Detected'),
|
|
),
|
|
).toHaveLength(1);
|
|
|
|
// Fail the original health persist: old code refunds into W2 (count 1→0).
|
|
resolvePersist({ persisted: false });
|
|
await flushMicrotasks();
|
|
|
|
// Fill the rest of W2 to the 20-alert cap (19 more crashes).
|
|
for (let i = 0; i < 19; i++) {
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: {
|
|
ID: `cw-${i}`,
|
|
Attributes: { exitCode: '1', name: `svc-${i}` },
|
|
},
|
|
});
|
|
}
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
|
|
// Cap must still hold: a cross-window refund would have allowed a 21st.
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: {
|
|
ID: 'cw-overflow',
|
|
Attributes: { exitCode: '1', name: 'overflow' },
|
|
},
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
|
|
const crashCalls = mockDispatchAlert.mock.calls.filter(
|
|
(c) => typeof c[2] === 'string' && c[2].includes('Crash Detected'),
|
|
);
|
|
expect(crashCalls).toHaveLength(20);
|
|
});
|
|
|
|
it('clears pending health retry when the container recovers before persist fails', async () => {
|
|
let resolvePersist!: (value: { persisted: boolean }) => void;
|
|
mockDispatchAlert.mockImplementation(
|
|
() =>
|
|
new Promise<{ persisted: boolean }>((resolve) => {
|
|
resolvePersist = resolve;
|
|
}),
|
|
);
|
|
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
// First unhealthy starts an in-flight dispatch.
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'recover-pending', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// Flap while in flight: recover then re-fail so a pending-retry marker is set.
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: healthy',
|
|
Actor: { ID: 'recover-pending', Attributes: { name: 'api' } },
|
|
});
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'recover-pending', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// Recover again before the first persist settles; marker must be dropped.
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: healthy',
|
|
Actor: { ID: 'recover-pending', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
|
|
resolvePersist({ persisted: false });
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// Later unrelated persist failure must not spuriously retry from a stale marker.
|
|
mockDispatchAlert.mockResolvedValueOnce({ persisted: false });
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'recover-pending', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('does not dispatch deferred health retry after shutdown', async () => {
|
|
let resolvePersist!: (value: { persisted: boolean }) => void;
|
|
mockDispatchAlert.mockImplementation(
|
|
() =>
|
|
new Promise<{ persisted: boolean }>((resolve) => {
|
|
resolvePersist = resolve;
|
|
}),
|
|
);
|
|
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'shutdown-pending', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
service.shutdown();
|
|
resolvePersist({ persisted: false });
|
|
await flushMicrotasks();
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('clears health alert bookkeeping on destroy', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'destroy-dedup', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'destroy',
|
|
Actor: { ID: 'destroy-dedup', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'destroy-dedup', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('does not retry a deferred health alert after destroy', async () => {
|
|
let resolvePersist!: (value: { persisted: boolean }) => void;
|
|
mockDispatchAlert.mockImplementation(
|
|
() =>
|
|
new Promise<{ persisted: boolean }>((resolve) => {
|
|
resolvePersist = resolve;
|
|
}),
|
|
);
|
|
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// Set a pending-retry marker while the first dispatch is in flight.
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: healthy',
|
|
Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } },
|
|
});
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'destroy',
|
|
Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } },
|
|
});
|
|
await flushMicrotasks();
|
|
|
|
resolvePersist({ persisted: false });
|
|
await flushMicrotasks();
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
});
|
|
});
|
|
|
|
// ── Malformed payloads ─────────────────────────────────────────────────
|
|
|
|
describe('DockerEventService - malformed payloads', () => {
|
|
it('tolerates a bad JSON line without tearing down the stream', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.pushRaw('not json\n');
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'ok', Attributes: { exitCode: '1', name: 'ok' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'error',
|
|
'monitor_alert',
|
|
expect.stringContaining('Container Crash Detected'),
|
|
expect.objectContaining({ containerName: 'ok' }),
|
|
);
|
|
});
|
|
});
|
|
|
|
// ── Reconciliation ─────────────────────────────────────────────────────
|
|
|
|
describe('DockerEventService - reconciliation', () => {
|
|
it('on first boot, records pre-existing exited containers as baseline and does not alert', async () => {
|
|
mockListContainers.mockResolvedValue([
|
|
{ Id: 'pre-1', State: 'exited' },
|
|
{ Id: 'pre-2', State: 'exited' },
|
|
{ Id: 'run-1', State: 'running' },
|
|
]);
|
|
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('emits mass-event summary on reconnect when >20% of containers newly exited', async () => {
|
|
// First connect: 10 running containers as baseline.
|
|
const running = Array.from({ length: 10 }, (_, i) => ({
|
|
Id: `c-${i}`,
|
|
State: 'running',
|
|
}));
|
|
mockListContainers.mockResolvedValueOnce(running);
|
|
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
// Simulate stream drop.
|
|
stream.error(new Error('connection reset'));
|
|
stream = makeStream();
|
|
mockGetEvents.mockImplementation(async () => stream);
|
|
|
|
// On reconnect: 5 of them now exited (>20%).
|
|
const mixed = running.map((c, i) => ({
|
|
Id: c.Id,
|
|
State: i < 5 ? 'exited' : 'running',
|
|
}));
|
|
mockListContainers.mockResolvedValueOnce(mixed);
|
|
|
|
// Drain reconnect backoff + reconciliation.
|
|
await vi.advanceTimersByTimeAsync(2_000);
|
|
// Flush microtasks spawned by the async reconnect + reconcile chain
|
|
// without running the recurring prune interval.
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
|
|
const massCall = mockDispatchAlert.mock.calls.find(c =>
|
|
typeof c[2] === 'string' && c[2].includes('daemon interruption'));
|
|
expect(massCall).toBeDefined();
|
|
});
|
|
|
|
it('classifies individual gap exits on reconnect when below mass threshold', async () => {
|
|
// Baseline: 10 containers running.
|
|
const baseline = Array.from({ length: 10 }, (_, i) => ({
|
|
Id: `c-${i}`,
|
|
State: 'running',
|
|
}));
|
|
mockListContainers.mockResolvedValueOnce(baseline);
|
|
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
// Drop + one new exit (10%, below 20% threshold).
|
|
stream.error(new Error('bad'));
|
|
stream = makeStream();
|
|
mockGetEvents.mockImplementation(async () => stream);
|
|
|
|
const postReconnect = baseline.map((c, i) => ({
|
|
Id: c.Id,
|
|
State: i === 0 ? 'exited' : 'running',
|
|
}));
|
|
mockListContainers.mockResolvedValueOnce(postReconnect);
|
|
|
|
mockInspect.mockResolvedValueOnce({
|
|
Name: '/crashed-app',
|
|
State: { ExitCode: 9, OOMKilled: false },
|
|
Config: { Labels: { 'com.docker.compose.project': 'my-stack' } },
|
|
});
|
|
|
|
await vi.advanceTimersByTimeAsync(2_000);
|
|
// Flush microtasks spawned by the async reconnect + reconcile chain
|
|
// without running the recurring prune interval.
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
|
|
const crashCall = mockDispatchAlert.mock.calls.find(c =>
|
|
typeof c[2] === 'string' && c[2].includes('crashed-app'));
|
|
expect(crashCall).toBeDefined();
|
|
expect(crashCall?.[3]).toMatchObject({ stackName: 'my-stack' });
|
|
});
|
|
});
|
|
|
|
// ── Reconnect lifecycle ────────────────────────────────────────────────
|
|
|
|
describe('DockerEventService - reconnect', () => {
|
|
it('emits one-time warning on first disconnect and info on reconnect', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.error(new Error('connection reset'));
|
|
|
|
// Reconnect succeeds immediately after backoff.
|
|
stream = makeStream();
|
|
mockGetEvents.mockImplementation(async () => stream);
|
|
|
|
await vi.advanceTimersByTimeAsync(2_000);
|
|
// Flush microtasks spawned by the async reconnect + reconcile chain
|
|
// without running the recurring prune interval.
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
|
|
const warn = mockDispatchAlert.mock.calls.find(c => c[0] === 'warning');
|
|
const info = mockDispatchAlert.mock.calls.find(c => c[0] === 'info');
|
|
expect(warn?.[2]).toBe('Lost connection to Docker daemon; monitoring paused.');
|
|
expect(info?.[2]).toBe('Reconnected to Docker daemon.');
|
|
});
|
|
|
|
it('shutdown cancels pending reconnect', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.error(new Error('broken'));
|
|
mockGetEvents.mockClear();
|
|
service.shutdown();
|
|
|
|
// Advance time well past the first backoff window; no new connect should run.
|
|
await vi.advanceTimersByTimeAsync(5_000);
|
|
expect(mockGetEvents).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
// ── Hardening: gap isolation / OOM fallback / concurrent dies ─────────
|
|
|
|
describe('DockerEventService - hardening', () => {
|
|
it('isolates failures inside classifyGap so one bad inspect does not abort the batch', async () => {
|
|
// 1 exit out of 10 (10%) stays below the 20% mass-event threshold, so
|
|
// the gap classifier inspects the container individually. That inspect
|
|
// fails; the service must isolate the failure and still classify a
|
|
// subsequent die as a crash.
|
|
const baseline = Array.from({ length: 10 }, (_, i) => ({
|
|
Id: `c-${i}`,
|
|
State: 'running',
|
|
}));
|
|
mockListContainers.mockResolvedValueOnce(baseline);
|
|
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.error(new Error('broken'));
|
|
stream = makeStream();
|
|
mockGetEvents.mockImplementation(async () => stream);
|
|
|
|
const postReconnect = baseline.map((c, i) => ({
|
|
Id: c.Id,
|
|
State: i < 1 ? 'exited' : 'running',
|
|
}));
|
|
mockListContainers.mockResolvedValueOnce(postReconnect);
|
|
mockInspect.mockRejectedValueOnce(new Error('gone'));
|
|
|
|
await vi.advanceTimersByTimeAsync(2_000);
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
|
|
mockDispatchAlert.mockClear();
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'post-recovery', Attributes: { exitCode: '1', name: 'app' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'error',
|
|
'monitor_alert',
|
|
expect.stringContaining('Container Crash Detected'),
|
|
expect.objectContaining({ containerName: 'app' }),
|
|
);
|
|
});
|
|
|
|
it('classifies exit code 137 as OOM when container inspect reports OOMKilled (no oom event)', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
// Inspect fallback: no prior `oom` event, but the container's
|
|
// State.OOMKilled is true.
|
|
mockInspect.mockResolvedValueOnce({
|
|
State: { OOMKilled: true, ExitCode: 137 },
|
|
});
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'cgroup-killed', Attributes: { exitCode: '137', name: 'hog' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
// Allow the awaited inspect in classifyDie to resolve.
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
|
|
const oomCall = mockDispatchAlert.mock.calls.find(c =>
|
|
typeof c[2] === 'string' && c[2].includes('OOM Kill'));
|
|
const crashCall = mockDispatchAlert.mock.calls.find(c =>
|
|
typeof c[2] === 'string' && c[2].includes('Crash Detected'));
|
|
expect(oomCall).toBeDefined();
|
|
expect(crashCall).toBeUndefined();
|
|
});
|
|
|
|
it('falls back to crash when exit 137 die inspect fails', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
mockInspect.mockRejectedValueOnce(new Error('no such container'));
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'gone', Attributes: { exitCode: '137', name: 'ephemeral' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(600);
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
|
|
// Inspect failed, so classification stays as the original 'crash'.
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'error',
|
|
'monitor_alert',
|
|
expect.stringContaining('Container Crash Detected'),
|
|
expect.objectContaining({ containerName: 'ephemeral' }),
|
|
);
|
|
});
|
|
|
|
it('collapses duplicate die events for the same container within the grace window', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
// Two dies for the same container within 500ms: the second cancels
|
|
// the first pending timer and reschedules. Exactly one crash alert
|
|
// must fire, using the later exit code.
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'dup', Attributes: { exitCode: '1', name: 'dup' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(100);
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'die',
|
|
Actor: { ID: 'dup', Attributes: { exitCode: '2', name: 'dup' } },
|
|
});
|
|
await vi.advanceTimersByTimeAsync(700);
|
|
|
|
const crashCalls = mockDispatchAlert.mock.calls.filter(c =>
|
|
typeof c[2] === 'string' && c[2].includes('Crash Detected'));
|
|
expect(crashCalls).toHaveLength(1);
|
|
expect(crashCalls[0][2]).toContain('Code: 2');
|
|
});
|
|
});
|
|
|
|
// ── State-invalidate broadcasts ────────────────────────────────────────
|
|
|
|
describe('DockerEventService - state-invalidate broadcasts', () => {
|
|
let invalidateSpy: ReturnType<typeof vi.spyOn>;
|
|
|
|
beforeEach(() => {
|
|
invalidateSpy = vi.spyOn(CacheService.getInstance(), 'invalidate');
|
|
});
|
|
|
|
afterEach(() => {
|
|
invalidateSpy.mockRestore();
|
|
});
|
|
|
|
it('broadcasts state-invalidate and drops the statuses cache on container start', async () => {
|
|
service = new DockerEventService(7, 'node-7');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'start',
|
|
Actor: { ID: 'aaa', Attributes: { 'com.docker.compose.project': 'web' } },
|
|
time: 1,
|
|
});
|
|
await vi.advanceTimersByTimeAsync(1);
|
|
|
|
expect(mockBroadcastEvent).toHaveBeenCalledWith(expect.objectContaining({
|
|
type: 'state-invalidate',
|
|
scope: 'stack',
|
|
nodeId: 7,
|
|
stackName: 'web',
|
|
containerId: 'aaa',
|
|
action: 'start',
|
|
}));
|
|
// The UI refetch this broadcast triggers must recompute, not hit a
|
|
// cache entry made stale by the event. Only the statuses key is
|
|
// touched; the full invalidateNodeCaches helper is not used here.
|
|
expect(invalidateSpy).toHaveBeenCalledTimes(1);
|
|
expect(invalidateSpy).toHaveBeenCalledWith('stack-statuses:7');
|
|
});
|
|
|
|
it('does not broadcast stack state-invalidate for Sencho self-container events', async () => {
|
|
mockIsOwnContainer.mockImplementation((idOrName: string) =>
|
|
idOrName === 'self-id' || idOrName === 'sencho',
|
|
);
|
|
service = new DockerEventService(7, 'node-7');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'start',
|
|
Actor: {
|
|
ID: 'self-id',
|
|
Attributes: {
|
|
name: 'sencho',
|
|
'com.docker.compose.project': 'sencho',
|
|
},
|
|
},
|
|
time: 1,
|
|
});
|
|
await vi.advanceTimersByTimeAsync(1);
|
|
|
|
expect(mockBroadcastEvent).not.toHaveBeenCalled();
|
|
expect(invalidateSpy).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('broadcasts state-invalidate on health_status:unhealthy', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'health_status: unhealthy',
|
|
Actor: { ID: 'bbb', Attributes: { 'com.docker.compose.project': 'api' } },
|
|
time: 1,
|
|
});
|
|
await vi.advanceTimersByTimeAsync(1);
|
|
|
|
const states = mockBroadcastEvent.mock.calls.filter(c =>
|
|
(c[0] as { type?: string }).type === 'state-invalidate');
|
|
expect(states.length).toBeGreaterThan(0);
|
|
expect(states[0][0]).toMatchObject({ action: 'health_status', stackName: 'api' });
|
|
expect(invalidateSpy).toHaveBeenCalledWith('stack-statuses:1');
|
|
});
|
|
|
|
it('does not broadcast state-invalidate on non-state actions like exec_create', async () => {
|
|
service = new DockerEventService(1, 'local');
|
|
await service.start();
|
|
|
|
stream.push({
|
|
Type: 'container',
|
|
Action: 'exec_create: /bin/sh',
|
|
Actor: { ID: 'ccc' },
|
|
time: 1,
|
|
});
|
|
await vi.advanceTimersByTimeAsync(1);
|
|
|
|
expect(mockBroadcastEvent).not.toHaveBeenCalled();
|
|
expect(invalidateSpy).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
// ── Diagnostics ────────────────────────────────────────────────────────
|
|
|
|
describe('DockerEventService - getStatus', () => {
|
|
it('reports connected after start', async () => {
|
|
service = new DockerEventService(42, 'my-node');
|
|
await service.start();
|
|
|
|
const status = service.getStatus();
|
|
expect(status.nodeId).toBe(42);
|
|
expect(status.nodeName).toBe('my-node');
|
|
expect(status.status).toBe('connected');
|
|
});
|
|
});
|