mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-20 23:32:19 +00:00
Anso
678c198faa
Fleet-aggregated reclaim alerts showed Node Local in the body while the badge already named the remote. Use a node-neutral message so identity comes from the hub badge; leave stored rows unchanged.
1683 lines
69 KiB
TypeScript
1683 lines
69 KiB
TypeScript
/**
|
|
* Unit tests for MonitorService — alert state machine, metric calculations,
|
|
* cleanup delegation, global settings evaluation, and concurrency guards.
|
|
*/
|
|
import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vitest';
|
|
import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsMock } from './helpers/arcstatsFsMock';
|
|
|
|
// ── Hoisted mocks ──────────────────────────────────────────────────────
|
|
|
|
const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContainerMetric,
|
|
mockCleanupOldMetrics, mockCleanupOldNotifications, mockCleanupOldAuditLogs,
|
|
mockUpdateStackAlertLastFired, mockGetSystemState, mockSetSystemState,
|
|
mockPruneScanHistoryPerImage, mockDeleteScansByImageRef,
|
|
mockGetRunningContainers, mockGetAllContainers, mockGetContainerStatsStream,
|
|
mockGetContainerRestartCount, mockGetDiskUsage, mockGetImages, mockGetStacks,
|
|
mockDispatchAlert,
|
|
mockCurrentLoad, mockMem, mockFsSize,
|
|
mockExecAsync,
|
|
mockFetchLatestSenchoVersion,
|
|
mockGetLatestVersion,
|
|
mockGetLatestVersionInfo,
|
|
mockGetSenchoVersion,
|
|
} = vi.hoisted(() => ({
|
|
mockGetGlobalSettings: vi.fn().mockReturnValue({}),
|
|
mockGetNodes: vi.fn().mockReturnValue([]),
|
|
mockGetStackAlerts: vi.fn().mockReturnValue([]),
|
|
mockAddContainerMetric: vi.fn(),
|
|
mockCleanupOldMetrics: vi.fn(),
|
|
mockCleanupOldNotifications: vi.fn(),
|
|
mockCleanupOldAuditLogs: vi.fn(),
|
|
mockUpdateStackAlertLastFired: vi.fn(),
|
|
mockGetSystemState: vi.fn().mockReturnValue(null),
|
|
mockSetSystemState: vi.fn(),
|
|
mockPruneScanHistoryPerImage: vi.fn().mockReturnValue(0),
|
|
mockDeleteScansByImageRef: vi.fn().mockReturnValue(0),
|
|
mockGetRunningContainers: vi.fn().mockResolvedValue([]),
|
|
mockGetAllContainers: vi.fn().mockResolvedValue([]),
|
|
mockGetContainerStatsStream: vi.fn().mockResolvedValue('{}'),
|
|
mockGetContainerRestartCount: vi.fn().mockResolvedValue(0),
|
|
mockGetDiskUsage: vi.fn().mockResolvedValue({
|
|
reclaimableImages: 0, reclaimableContainers: 0, reclaimableVolumes: 0, reclaimableBuildCache: 0,
|
|
reclaimableImageCount: 0, reclaimableContainerCount: 0, reclaimableVolumeCount: 0, reclaimableBuildCacheCount: 0,
|
|
}),
|
|
mockGetImages: vi.fn().mockResolvedValue([]),
|
|
mockGetStacks: vi.fn().mockResolvedValue([]),
|
|
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
|
|
mockCurrentLoad: vi.fn().mockResolvedValue({ currentLoad: 10 }),
|
|
mockMem: vi.fn().mockResolvedValue({ total: 16e9, used: 4e9, active: 4e9, available: 12e9, free: 12e9, buffcache: 0 }),
|
|
mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]),
|
|
mockExecAsync: vi.fn().mockResolvedValue({ stdout: '' }),
|
|
mockFetchLatestSenchoVersion: vi.fn().mockRejectedValue(new Error('not configured')),
|
|
mockGetLatestVersion: vi.fn().mockResolvedValue(null),
|
|
mockGetLatestVersionInfo: vi.fn().mockResolvedValue(null),
|
|
mockGetSenchoVersion: vi.fn().mockReturnValue(null),
|
|
}));
|
|
|
|
vi.mock('../services/DatabaseService', () => ({
|
|
DatabaseService: {
|
|
getInstance: () => ({
|
|
getGlobalSettings: mockGetGlobalSettings,
|
|
getNodes: mockGetNodes,
|
|
getStackAlerts: mockGetStackAlerts,
|
|
addContainerMetric: mockAddContainerMetric,
|
|
cleanupOldMetrics: mockCleanupOldMetrics,
|
|
cleanupOldNotifications: mockCleanupOldNotifications,
|
|
cleanupOldAuditLogs: mockCleanupOldAuditLogs,
|
|
updateStackAlertLastFired: mockUpdateStackAlertLastFired,
|
|
getSystemState: mockGetSystemState,
|
|
setSystemState: mockSetSystemState,
|
|
pruneScanHistoryPerImage: mockPruneScanHistoryPerImage,
|
|
deleteScansByImageRef: mockDeleteScansByImageRef,
|
|
}),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/DockerController', () => ({
|
|
default: {
|
|
getInstance: () => ({
|
|
getRunningContainers: mockGetRunningContainers,
|
|
getAllContainers: mockGetAllContainers,
|
|
getContainerStatsStream: mockGetContainerStatsStream,
|
|
getContainerRestartCount: mockGetContainerRestartCount,
|
|
getDiskUsage: mockGetDiskUsage,
|
|
getImages: mockGetImages,
|
|
}),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/FileSystemService', () => ({
|
|
FileSystemService: {
|
|
getInstance: () => ({
|
|
getStacks: mockGetStacks,
|
|
}),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../utils/version-check', () => ({
|
|
fetchLatestSenchoVersion: (...args: unknown[]) => mockFetchLatestSenchoVersion(...args),
|
|
getLatestVersion: (...args: unknown[]) => mockGetLatestVersion(...args),
|
|
getLatestVersionInfo: (...args: unknown[]) => mockGetLatestVersionInfo(...args),
|
|
}));
|
|
|
|
vi.mock('../services/CapabilityRegistry', async () => {
|
|
const semver = await import('semver');
|
|
return {
|
|
isValidVersion: (v: string | null | undefined): v is string =>
|
|
!!v && v !== 'unknown' && v !== '0.0.0-dev' && !!semver.default.valid(v),
|
|
getSenchoVersion: () => mockGetSenchoVersion(),
|
|
};
|
|
});
|
|
|
|
vi.mock('../services/NotificationService', () => ({
|
|
NotificationService: {
|
|
getInstance: () => ({
|
|
dispatchAlert: mockDispatchAlert,
|
|
}),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/NodeRegistry', () => ({
|
|
NodeRegistry: {
|
|
getInstance: () => ({
|
|
getDefaultNodeId: () => 1,
|
|
getNode: () => ({ id: 1, name: 'local-test', type: 'local' }),
|
|
}),
|
|
},
|
|
}));
|
|
|
|
vi.mock('systeminformation', () => ({
|
|
default: {
|
|
currentLoad: (...args: unknown[]) => mockCurrentLoad(...args),
|
|
mem: (...args: unknown[]) => mockMem(...args),
|
|
fsSize: (...args: unknown[]) => mockFsSize(...args),
|
|
},
|
|
}));
|
|
|
|
vi.mock('child_process', () => ({
|
|
exec: vi.fn(),
|
|
}));
|
|
|
|
vi.mock('util', () => ({
|
|
promisify: () => mockExecAsync,
|
|
}));
|
|
|
|
import { MonitorService, _resetHostAlertSuppressionStateForTests } from '../services/MonitorService';
|
|
|
|
// Host memory now reads ZFS ARC stats; intercept those reads so the suite does
|
|
// not depend on whether the machine running it is itself a ZFS host.
|
|
let arcFs: ArcstatsFsMock;
|
|
beforeAll(() => {
|
|
arcFs = installArcstatsFsMock();
|
|
});
|
|
|
|
beforeEach(() => {
|
|
vi.clearAllMocks();
|
|
arcFs.clear();
|
|
(MonitorService as any).instance = undefined;
|
|
_resetHostAlertSuppressionStateForTests();
|
|
mockGetSystemState.mockReturnValue(null);
|
|
});
|
|
|
|
// si.mem() returns active/available alongside used (used counts reclaimable page
|
|
// cache, active/available exclude it). The host-RAM percentage is derived from the
|
|
// cache-excluded figures, so mocks must supply a realistic shape. `realUsed` is the
|
|
// active working set; available/free are the remainder.
|
|
const memSample = (realUsed: number, total = 16e9) => ({
|
|
total, used: realUsed, active: realUsed,
|
|
available: total - realUsed, free: total - realUsed, buffcache: 0,
|
|
});
|
|
|
|
// ── Pure calculation helpers (accessed via private method reflection) ───
|
|
|
|
describe('MonitorService - calculateCpuPercent', () => {
|
|
function calcCpu(stats: any): number {
|
|
const svc = MonitorService.getInstance();
|
|
return (svc as any).calculateCpuPercent(stats);
|
|
}
|
|
|
|
it('returns correct percentage for normal stats', () => {
|
|
const stats = {
|
|
cpu_stats: { cpu_usage: { total_usage: 2000 }, system_cpu_usage: 10000, online_cpus: 4 },
|
|
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
|
};
|
|
// (1000 / 5000) * 4 * 100 = 80%
|
|
expect(calcCpu(stats)).toBeCloseTo(80, 1);
|
|
});
|
|
|
|
it('returns 0 when cpu_stats is missing', () => {
|
|
expect(calcCpu({})).toBe(0);
|
|
expect(calcCpu(null)).toBe(0);
|
|
expect(calcCpu({ cpu_stats: {} })).toBe(0);
|
|
});
|
|
|
|
it('returns 0 when systemDelta is zero', () => {
|
|
const stats = {
|
|
cpu_stats: { cpu_usage: { total_usage: 2000 }, system_cpu_usage: 5000, online_cpus: 1 },
|
|
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
|
};
|
|
expect(calcCpu(stats)).toBe(0);
|
|
});
|
|
|
|
it('accounts for online_cpus count', () => {
|
|
const stats = {
|
|
cpu_stats: { cpu_usage: { total_usage: 2000 }, system_cpu_usage: 10000, online_cpus: 8 },
|
|
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
|
};
|
|
// (1000/5000) * 8 * 100 = 160%
|
|
expect(calcCpu(stats)).toBeCloseTo(160, 1);
|
|
});
|
|
|
|
it('falls back to percpu_usage length when online_cpus missing', () => {
|
|
const stats = {
|
|
cpu_stats: { cpu_usage: { total_usage: 2000, percpu_usage: [0, 0] }, system_cpu_usage: 10000 },
|
|
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
|
};
|
|
// (1000/5000) * 2 * 100 = 40%
|
|
expect(calcCpu(stats)).toBeCloseTo(40, 1);
|
|
});
|
|
});
|
|
|
|
describe('MonitorService - calculateMemoryPercent', () => {
|
|
function calcMem(stats: any): number {
|
|
const svc = MonitorService.getInstance();
|
|
return (svc as any).calculateMemoryPercent(stats);
|
|
}
|
|
|
|
it('returns correct percentage subtracting cache', () => {
|
|
const stats = {
|
|
memory_stats: { usage: 500e6, limit: 1e9, stats: { cache: 100e6 } },
|
|
};
|
|
// (400e6 / 1e9) * 100 = 40%
|
|
expect(calcMem(stats)).toBeCloseTo(40, 1);
|
|
});
|
|
|
|
it('returns 0 when memory_stats is missing', () => {
|
|
expect(calcMem({})).toBe(0);
|
|
expect(calcMem({ memory_stats: {} })).toBe(0);
|
|
});
|
|
|
|
it('returns 0 when limit is zero', () => {
|
|
const stats = { memory_stats: { usage: 100, limit: 0 } };
|
|
expect(calcMem(stats)).toBe(0);
|
|
});
|
|
|
|
it('handles missing cache field', () => {
|
|
const stats = { memory_stats: { usage: 500e6, limit: 1e9 } };
|
|
// No cache → (500e6 / 1e9) * 100 = 50%
|
|
expect(calcMem(stats)).toBeCloseTo(50, 1);
|
|
});
|
|
});
|
|
|
|
describe('MonitorService - calculateNetwork', () => {
|
|
function calcNet(stats: any, dir: 'rx' | 'tx'): number {
|
|
const svc = MonitorService.getInstance();
|
|
return (svc as any).calculateNetwork(stats, dir);
|
|
}
|
|
|
|
it('sums rx_bytes across all interfaces', () => {
|
|
const stats = {
|
|
networks: {
|
|
eth0: { rx_bytes: 1024 * 1024, tx_bytes: 0 },
|
|
eth1: { rx_bytes: 2 * 1024 * 1024, tx_bytes: 0 },
|
|
},
|
|
};
|
|
expect(calcNet(stats, 'rx')).toBeCloseTo(3, 0); // 3 MB
|
|
});
|
|
|
|
it('sums tx_bytes across all interfaces', () => {
|
|
const stats = {
|
|
networks: {
|
|
eth0: { rx_bytes: 0, tx_bytes: 512 * 1024 },
|
|
},
|
|
};
|
|
expect(calcNet(stats, 'tx')).toBeCloseTo(0.5, 1); // 0.5 MB
|
|
});
|
|
|
|
it('returns 0 when no networks present', () => {
|
|
expect(calcNet({}, 'rx')).toBe(0);
|
|
expect(calcNet({ networks: null }, 'tx')).toBe(0);
|
|
});
|
|
});
|
|
|
|
describe('MonitorService - evaluateCondition', () => {
|
|
function evalCond(actual: number, operator: string, threshold: number): boolean {
|
|
const svc = MonitorService.getInstance();
|
|
return (svc as any).evaluateCondition(actual, operator, threshold);
|
|
}
|
|
|
|
it('handles > operator', () => {
|
|
expect(evalCond(81, '>', 80)).toBe(true);
|
|
expect(evalCond(80, '>', 80)).toBe(false);
|
|
});
|
|
|
|
it('handles < operator', () => {
|
|
expect(evalCond(79, '<', 80)).toBe(true);
|
|
expect(evalCond(80, '<', 80)).toBe(false);
|
|
});
|
|
|
|
it('handles >= operator at boundary', () => {
|
|
expect(evalCond(80, '>=', 80)).toBe(true);
|
|
expect(evalCond(79, '>=', 80)).toBe(false);
|
|
});
|
|
|
|
it('handles <= operator at boundary', () => {
|
|
expect(evalCond(80, '<=', 80)).toBe(true);
|
|
expect(evalCond(81, '<=', 80)).toBe(false);
|
|
});
|
|
|
|
it('handles == operator', () => {
|
|
expect(evalCond(80, '==', 80)).toBe(true);
|
|
expect(evalCond(81, '==', 80)).toBe(false);
|
|
});
|
|
|
|
it('returns false for unknown operator', () => {
|
|
expect(evalCond(80, '!=', 80)).toBe(false);
|
|
expect(evalCond(80, 'foo', 80)).toBe(false);
|
|
});
|
|
});
|
|
|
|
// ── Integration-level: evaluateGlobalSettings ──────────────────────────
|
|
|
|
describe('MonitorService - evaluateGlobalSettings', () => {
|
|
it('dispatches CPU warning when over threshold', async () => {
|
|
mockGetGlobalSettings.mockReturnValue({ host_cpu_limit: '50' });
|
|
mockCurrentLoad.mockResolvedValue({ currentLoad: 75 });
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' });
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));
|
|
});
|
|
|
|
it('does not dispatch when CPU below threshold', async () => {
|
|
mockCurrentLoad.mockResolvedValue({ currentLoad: 25 });
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' });
|
|
|
|
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));
|
|
});
|
|
|
|
it('dispatches RAM warning when over threshold', async () => {
|
|
mockMem.mockResolvedValue(memSample(15e9)); // ~94%
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
|
|
});
|
|
|
|
it('does not alert when most RAM is reclaimable page cache, not real usage', async () => {
|
|
// Busy host: 15.8G "used" including cache, but only 1.6G actively in use and
|
|
// 14.4G available. The naive used/total formula reads ~99% and would breach an
|
|
// 80% threshold; the active/total figure is 10% and must stay silent.
|
|
mockMem.mockResolvedValue({
|
|
total: 16e9, used: 15.8e9, active: 1.6e9, available: 14.4e9, free: 14.4e9, buffcache: 14.2e9,
|
|
});
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
|
|
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
|
|
});
|
|
|
|
it('alerts when the active working set is genuinely over threshold', async () => {
|
|
// Cache-heavy and genuinely busy: 14G active of 16G (87.5%) with 15.8G "used"
|
|
// including cache. active/total exceeds 80%, so the alert must still fire. This
|
|
// is the two-sided partner to the page-cache case above: keying off active must
|
|
// not suppress real breaches.
|
|
mockMem.mockResolvedValue({
|
|
total: 16e9, used: 15.8e9, active: 14e9, available: 2e9, free: 2e9, buffcache: 1.8e9,
|
|
});
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
|
|
});
|
|
|
|
it('does not alert when reclaimable ZFS ARC accounts for the memory pressure', async () => {
|
|
// Active working set reads 15G/16G (~94%, breaches 80%), but 5G of that is
|
|
// reclaimable ARC. Adding ARC back into available drops effective usage to
|
|
// ~62.5%, so no host-memory alert should fire.
|
|
mockMem.mockResolvedValue(memSample(15e9)); // available 1e9 -> 93.75%
|
|
arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(5e9, 0)); // reclaimable 5e9
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
|
|
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
|
|
});
|
|
|
|
it('dispatches disk warning when over threshold', async () => {
|
|
mockFsSize.mockResolvedValue([{ mount: '/', use: 92 }]);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateGlobalSettings({ host_disk_limit: '90' });
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Disk'));
|
|
});
|
|
|
|
it('skips host limits when threshold is 0 or NaN', async () => {
|
|
mockCurrentLoad.mockResolvedValue({ currentLoad: 99 });
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '0' });
|
|
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));
|
|
|
|
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: 'abc' });
|
|
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));
|
|
});
|
|
});
|
|
|
|
describe('MonitorService - host_alerts_enabled toggle', () => {
|
|
beforeEach(() => {
|
|
mockGetGlobalSettings.mockReturnValue({});
|
|
mockDispatchAlert.mockClear();
|
|
mockCurrentLoad.mockClear();
|
|
mockMem.mockClear();
|
|
mockFsSize.mockClear();
|
|
});
|
|
|
|
afterEach(() => {
|
|
// Restore default mock implementations so version-check state does not
|
|
// leak into sibling describe blocks (F-11 suppression, version check).
|
|
mockGetLatestVersion.mockResolvedValue(null);
|
|
mockGetLatestVersionInfo.mockResolvedValue(null);
|
|
mockGetSenchoVersion.mockReturnValue(null);
|
|
});
|
|
|
|
it('skips host metrics and dispatch when disabled', async () => {
|
|
mockCurrentLoad.mockResolvedValue({ currentLoad: 75 });
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '0', host_cpu_limit: '50' });
|
|
|
|
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
|
expect(mockCurrentLoad).not.toHaveBeenCalled();
|
|
expect(mockMem).not.toHaveBeenCalled();
|
|
expect(mockFsSize).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('does not call systeminformation when disabled', async () => {
|
|
mockMem.mockResolvedValue(memSample(15e9)); // ~94% - would breach
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '0', host_ram_limit: '80' });
|
|
|
|
expect(mockCurrentLoad).not.toHaveBeenCalled();
|
|
expect(mockMem).not.toHaveBeenCalled();
|
|
expect(mockFsSize).not.toHaveBeenCalled();
|
|
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('clears persisted suppression state when disabled', async () => {
|
|
const store: Record<string, string> = {};
|
|
mockGetSystemState.mockImplementation((key: string) => store[key] ?? null);
|
|
mockSetSystemState.mockImplementation((key: string, value: string) => { store[key] = value; });
|
|
|
|
// Simulate a prior breach that set the persisted suppression timestamp.
|
|
store['last_host_cpu_alert_ts'] = String(Date.now());
|
|
store['last_host_ram_alert_ts'] = String(Date.now());
|
|
store['last_host_disk_alert_ts'] = String(Date.now());
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '0' });
|
|
|
|
expect(store['last_host_cpu_alert_ts']).toBe('0');
|
|
expect(store['last_host_ram_alert_ts']).toBe('0');
|
|
expect(store['last_host_disk_alert_ts']).toBe('0');
|
|
});
|
|
|
|
it('re-enable fires fresh without suppressed suffix', async () => {
|
|
const store: Record<string, string> = {};
|
|
mockGetSystemState.mockImplementation((key: string) => store[key] ?? null);
|
|
mockSetSystemState.mockImplementation((key: string, value: string) => { store[key] = value; });
|
|
mockMem.mockResolvedValue(memSample(15e9)); // ~94% - breaches 80% limit
|
|
|
|
const svc = MonitorService.getInstance();
|
|
|
|
// Step 1: breach while enabled — fires alert and seeds suppression state.
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
|
|
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Suppressed'));
|
|
mockDispatchAlert.mockClear();
|
|
|
|
// Step 2: disable — clears suppression state and skips dispatch.
|
|
await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '0', host_ram_limit: '80' });
|
|
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
|
expect(store['last_host_ram_alert_ts']).toBe('0');
|
|
|
|
// Step 3: re-enable while still breaching — fires fresh, no "Suppressed" suffix.
|
|
await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '1', host_ram_limit: '80' });
|
|
const calls = mockDispatchAlert.mock.calls.filter(
|
|
(c: unknown[]) => c[0] === 'warning' && c[1] === 'monitor_alert',
|
|
);
|
|
expect(calls.length).toBe(1);
|
|
const msg = calls[0] as string[];
|
|
expect(msg[2]).toContain('Memory');
|
|
expect(msg[2]).not.toContain('Suppressed');
|
|
});
|
|
|
|
it('still runs version check when disabled', async () => {
|
|
mockGetSenchoVersion.mockReturnValue('0.45.0');
|
|
mockGetLatestVersionInfo.mockResolvedValue({ version: '0.46.0', publishPending: false });
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '0' });
|
|
|
|
expect(mockGetLatestVersionInfo).toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
// Crash + healthcheck detection now lives in DockerEventService (event-driven).
|
|
// Tests for those flows live in docker-event-service.test.ts.
|
|
|
|
// ── F-11: host-metric alert suppression ────────────────────────────────
|
|
|
|
describe('MonitorService - host alert suppression (F-11)', () => {
|
|
// Force RAM-over-threshold for every test in this block; CPU/disk are
|
|
// independently controlled per-test so a single mockMem set-up covers the
|
|
// common "I want a breach happening" case without test repetition.
|
|
beforeEach(() => {
|
|
mockMem.mockResolvedValue(memSample(15e9)); // ~94%
|
|
});
|
|
|
|
it('first breach dispatches immediately', async () => {
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
const [, , message] = mockDispatchAlert.mock.calls[0];
|
|
expect(message).toContain('Memory');
|
|
expect(message).not.toContain('Suppressed');
|
|
});
|
|
|
|
it('second breach within window does not dispatch', async () => {
|
|
const svc = MonitorService.getInstance();
|
|
const baseTime = 1_700_000_000_000;
|
|
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
|
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// 30 seconds later — well inside the 60-minute default window.
|
|
nowSpy.mockReturnValue(baseTime + 30_000);
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
nowSpy.mockRestore();
|
|
});
|
|
|
|
it('many breaches within window accumulate count without dispatching', async () => {
|
|
const svc = MonitorService.getInstance();
|
|
const baseTime = 1_700_000_000_000;
|
|
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
|
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
for (let i = 1; i < 10; i++) {
|
|
nowSpy.mockReturnValue(baseTime + i * 30_000);
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
}
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
nowSpy.mockRestore();
|
|
});
|
|
|
|
it('breach after window elapses dispatches follow-up with count summary and persists new timestamp', async () => {
|
|
const svc = MonitorService.getInstance();
|
|
const baseTime = 1_700_000_000_000;
|
|
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
|
|
|
// First dispatch.
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
expect(mockSetSystemState).toHaveBeenCalledWith('last_host_ram_alert_ts', String(baseTime));
|
|
|
|
// 5 cycles inside the window, each suppressed.
|
|
for (let i = 1; i <= 5; i++) {
|
|
nowSpy.mockReturnValue(baseTime + i * 30_000);
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
}
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// Jump past the suppression window — 61 minutes.
|
|
const followUpTime = baseTime + 61 * 60 * 1000;
|
|
nowSpy.mockReturnValue(followUpTime);
|
|
mockSetSystemState.mockClear();
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
|
const [, , followUpMessage] = mockDispatchAlert.mock.calls[1];
|
|
expect(followUpMessage).toMatch(/Suppressed 5 alerts in the last \d+m/);
|
|
expect(followUpMessage).toMatch(/first over threshold at \d{2}:\d{2} UTC/);
|
|
// Follow-up dispatch must persist the new timestamp so a subsequent
|
|
// restart-survivability seed picks up the most-recent fire, not the
|
|
// pre-window first fire (otherwise a restart 30min later would see a
|
|
// 90min-old persisted row and re-fire immediately).
|
|
expect(mockSetSystemState).toHaveBeenCalledWith('last_host_ram_alert_ts', String(followUpTime));
|
|
|
|
nowSpy.mockRestore();
|
|
});
|
|
|
|
it('dispatches at exactly the suppression window boundary', async () => {
|
|
const svc = MonitorService.getInstance();
|
|
const baseTime = 1_700_000_000_000;
|
|
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
|
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// Exactly 60 minutes later — at the boundary. The check is `<`, so the
|
|
// boundary tick should DISPATCH a follow-up rather than suppress.
|
|
nowSpy.mockReturnValue(baseTime + 60 * 60 * 1000);
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
|
|
|
nowSpy.mockRestore();
|
|
});
|
|
|
|
it('follow-up summary uses singular "alert" when exactly one cycle was suppressed', async () => {
|
|
const svc = MonitorService.getInstance();
|
|
const baseTime = 1_700_000_000_000;
|
|
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
|
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// One suppressed cycle.
|
|
nowSpy.mockReturnValue(baseTime + 30_000);
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
|
|
// Past window.
|
|
nowSpy.mockReturnValue(baseTime + 61 * 60 * 1000);
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
|
const [, , followUpMessage] = mockDispatchAlert.mock.calls[1];
|
|
expect(followUpMessage).toContain('Suppressed 1 alert in the last');
|
|
expect(followUpMessage).not.toContain('Suppressed 1 alerts'); // singular form, not plural
|
|
|
|
nowSpy.mockRestore();
|
|
});
|
|
|
|
it('disk-metric path dispatches and suppresses through the same mechanism', async () => {
|
|
const svc = MonitorService.getInstance();
|
|
mockFsSize.mockResolvedValue([{ mount: '/', use: 95 }]);
|
|
// Set RAM below threshold so it does not interfere with the disk-only assertions.
|
|
mockMem.mockResolvedValue(memSample(4e9));
|
|
const baseTime = 1_700_000_000_000;
|
|
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
|
|
|
await (svc as any).evaluateGlobalSettings({ host_disk_limit: '80' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
expect((mockDispatchAlert.mock.calls[0][2] as string)).toContain('Disk');
|
|
|
|
nowSpy.mockReturnValue(baseTime + 30_000);
|
|
await (svc as any).evaluateGlobalSettings({ host_disk_limit: '80' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1); // suppressed
|
|
|
|
nowSpy.mockRestore();
|
|
});
|
|
|
|
it('metric drop below threshold clears in-memory state and persisted timestamp', async () => {
|
|
// Recovery now reads system_state before deciding to write '0' (so it
|
|
// skips a redundant write when persisted is already null/0). Model the
|
|
// persistence chain so the recovery branch can observe the timestamp
|
|
// the first-fire path just wrote.
|
|
let persistedRamTs: string | null = null;
|
|
mockGetSystemState.mockImplementation((key: string) =>
|
|
key === 'last_host_ram_alert_ts' ? persistedRamTs : null,
|
|
);
|
|
mockSetSystemState.mockImplementation((key: string, value: string) => {
|
|
if (key === 'last_host_ram_alert_ts') persistedRamTs = value;
|
|
});
|
|
|
|
const svc = MonitorService.getInstance();
|
|
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
expect(persistedRamTs).not.toBeNull();
|
|
expect(persistedRamTs).not.toBe('0');
|
|
|
|
// Drop RAM back under threshold.
|
|
mockMem.mockResolvedValue(memSample(4e9)); // 25%
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
|
|
// Recovery branch resets the persisted timestamp to '0'.
|
|
expect(persistedRamTs).toBe('0');
|
|
});
|
|
|
|
it('re-breach after recovery fires fresh first alert (no Suppressed suffix)', async () => {
|
|
const svc = MonitorService.getInstance();
|
|
|
|
// Initial breach.
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// Recovery.
|
|
mockMem.mockResolvedValue(memSample(4e9));
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
|
|
// Re-breach.
|
|
mockMem.mockResolvedValue(memSample(15e9));
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
|
const [, , reBreachMessage] = mockDispatchAlert.mock.calls[1];
|
|
expect(reBreachMessage).not.toContain('Suppressed');
|
|
});
|
|
|
|
it('CPU and RAM suppression states are isolated per metric', async () => {
|
|
const svc = MonitorService.getInstance();
|
|
mockCurrentLoad.mockResolvedValue({ currentLoad: 95 });
|
|
|
|
// First cycle: both fire.
|
|
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
|
|
|
const messages1 = mockDispatchAlert.mock.calls.map(c => c[2] as string);
|
|
expect(messages1.some(m => m.includes('CPU'))).toBe(true);
|
|
expect(messages1.some(m => m.includes('Memory'))).toBe(true);
|
|
|
|
// Second cycle: both suppressed.
|
|
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
|
|
|
// CPU drops below threshold (recovers); RAM stays high.
|
|
mockCurrentLoad.mockResolvedValue({ currentLoad: 10 });
|
|
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });
|
|
|
|
// CPU re-breaches; RAM still in suppression window.
|
|
mockCurrentLoad.mockResolvedValue({ currentLoad: 95 });
|
|
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });
|
|
|
|
// CPU fires fresh; RAM stays silent.
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(3);
|
|
const newCpuMessage = mockDispatchAlert.mock.calls[2][2] as string;
|
|
expect(newCpuMessage).toContain('CPU');
|
|
expect(newCpuMessage).not.toContain('Suppressed');
|
|
});
|
|
|
|
it('respects custom host_alert_suppression_mins setting (5 minutes)', async () => {
|
|
const svc = MonitorService.getInstance();
|
|
const baseTime = 1_700_000_000_000;
|
|
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
|
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '5' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// 3 minutes — still within custom 5-minute window.
|
|
nowSpy.mockReturnValue(baseTime + 3 * 60 * 1000);
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '5' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// 6 minutes — past the custom window; follow-up should fire.
|
|
nowSpy.mockReturnValue(baseTime + 6 * 60 * 1000);
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '5' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
|
|
|
nowSpy.mockRestore();
|
|
});
|
|
|
|
it('post-restart recovery clears persisted timestamp so re-breach inside the window fires fresh', async () => {
|
|
// Codex audit scenario:
|
|
// T0: breach fires → system_state['last_host_ram_alert_ts'] = T0
|
|
// T1: process restart (in-memory state cleared, persisted T0 survives)
|
|
// T2: metric drops below threshold → clearHostMetricSuppression() runs
|
|
// T3: metric re-breaches within suppression window
|
|
// Expected: fresh first alert at T3 (no Suppressed suffix).
|
|
// Pre-fix bug: clearHostMetricSuppression early-returned on missing in-
|
|
// memory state, leaving persisted T0 alive; T3 hit the restart-survival
|
|
// path and was silently suppressed.
|
|
const baseTime = 1_700_000_000_000;
|
|
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime + 5 * 60 * 1000); // T1+T2 = T0+5min
|
|
|
|
// Simulate persisted state from a previous process's first fire.
|
|
let persistedRamTs: string | null = String(baseTime);
|
|
mockGetSystemState.mockImplementation((key: string) =>
|
|
key === 'last_host_ram_alert_ts' ? persistedRamTs : null,
|
|
);
|
|
mockSetSystemState.mockImplementation((key: string, value: string) => {
|
|
if (key === 'last_host_ram_alert_ts') persistedRamTs = value;
|
|
});
|
|
|
|
// T2: post-restart cycle finds metric BELOW threshold (recovered).
|
|
mockMem.mockResolvedValue(memSample(4e9));
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
// No dispatch on a non-breach cycle.
|
|
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
|
// Recovery branch MUST have reset persisted state so a re-breach is
|
|
// treated as fresh, not as a still-active cooldown.
|
|
expect(persistedRamTs).toBe('0');
|
|
|
|
// T3: re-breach 10 minutes later (well inside the original 60-min window).
|
|
nowSpy.mockReturnValue(baseTime + 15 * 60 * 1000);
|
|
mockMem.mockResolvedValue(memSample(15e9));
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
const [, , message] = mockDispatchAlert.mock.calls[0];
|
|
expect(message).not.toContain('Suppressed');
|
|
|
|
nowSpy.mockRestore();
|
|
});
|
|
|
|
it('caps host_alert_suppression_mins at 24h (1440) to defend against unvalidated single-key POST writes', async () => {
|
|
const svc = MonitorService.getInstance();
|
|
const baseTime = 1_700_000_000_000;
|
|
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
|
|
|
// The single-key POST /api/settings path stores allowlisted keys
|
|
// without zod re-validation. A 999999999-minute value (or any
|
|
// accidental garbage above 1440) must not let a metric stay
|
|
// suppressed for centuries; cap at 24h in the consumer.
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '999999999' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// 1441 minutes later — past the 24h cap.
|
|
nowSpy.mockReturnValue(baseTime + 1441 * 60 * 1000);
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '999999999' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
|
|
|
nowSpy.mockRestore();
|
|
});
|
|
|
|
it('post-restart with persisted timestamp inside window does not re-fire', async () => {
|
|
const baseTime = 1_700_000_000_000;
|
|
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime + 10 * 60 * 1000); // "now" = T+10min
|
|
|
|
// Simulate a previous process having persisted a fire 10 minutes ago.
|
|
mockGetSystemState.mockImplementation((key: string) =>
|
|
key === 'last_host_ram_alert_ts' ? String(baseTime) : null,
|
|
);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
|
|
|
// Post-restart cycle must NOT re-fire because the persisted cooldown
|
|
// is still active (10 min into the default 60 min window).
|
|
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
|
|
|
nowSpy.mockRestore();
|
|
});
|
|
|
|
it('zero or negative suppression_mins falls back to default', async () => {
|
|
const svc = MonitorService.getInstance();
|
|
const baseTime = 1_700_000_000_000;
|
|
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
|
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '0' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
// Default 60-minute window applies — a cycle 30 minutes later stays
|
|
// suppressed even though the setting said 0.
|
|
nowSpy.mockReturnValue(baseTime + 30 * 60 * 1000);
|
|
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '0' });
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
|
|
nowSpy.mockRestore();
|
|
});
|
|
});
|
|
|
|
// ── Alert breach state machine ─────────────────────────────────────────
|
|
|
|
describe('MonitorService - breach state machine', () => {
|
|
function setupAlertScenario(cpuPercent: number) {
|
|
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
|
mockGetRunningContainers.mockResolvedValue([{
|
|
Id: 'c1',
|
|
Labels: { 'com.docker.compose.project': 'my-stack' },
|
|
}]);
|
|
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
|
|
cpu_stats: { cpu_usage: { total_usage: 1000 + cpuPercent * 50 }, system_cpu_usage: 10000, online_cpus: 1 },
|
|
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
|
memory_stats: { usage: 100e6, limit: 1e9 },
|
|
}));
|
|
mockGetStackAlerts.mockReturnValue([{
|
|
id: 1,
|
|
stack_name: 'my-stack',
|
|
metric: 'cpu_percent',
|
|
operator: '>',
|
|
threshold: 80,
|
|
duration_mins: 0, // Fire immediately on breach
|
|
cooldown_mins: 60,
|
|
last_fired_at: 0,
|
|
}]);
|
|
mockGetGlobalSettings.mockReturnValue({});
|
|
}
|
|
|
|
it('fires alert when condition met and duration is 0', async () => {
|
|
setupAlertScenario(90); // Will produce CPU > 80%
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluate();
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: 'my-stack', actor: 'system:monitor' });
|
|
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
|
|
});
|
|
|
|
it('does not fire when condition not met', async () => {
|
|
setupAlertScenario(10); // Will produce CPU < 80%
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluate();
|
|
|
|
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'));
|
|
});
|
|
|
|
it('respects cooldown after firing', async () => {
|
|
setupAlertScenario(90);
|
|
// Simulate that alert was fired 30 minutes ago (within 60-min cooldown)
|
|
mockGetStackAlerts.mockReturnValue([{
|
|
id: 1,
|
|
stack_name: 'my-stack',
|
|
metric: 'cpu_percent',
|
|
operator: '>',
|
|
threshold: 80,
|
|
duration_mins: 0,
|
|
cooldown_mins: 60,
|
|
last_fired_at: Date.now() - 30 * 60 * 1000,
|
|
}]);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluate();
|
|
|
|
expect(mockUpdateStackAlertLastFired).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('resets breach state when condition clears', async () => {
|
|
const svc = MonitorService.getInstance();
|
|
|
|
// First: breach starts
|
|
setupAlertScenario(90);
|
|
mockGetStackAlerts.mockReturnValue([{
|
|
id: 42,
|
|
stack_name: 'my-stack',
|
|
metric: 'cpu_percent',
|
|
operator: '>',
|
|
threshold: 80,
|
|
duration_mins: 999, // Won't fire due to long duration
|
|
cooldown_mins: 0,
|
|
last_fired_at: 0,
|
|
}]);
|
|
await (svc as any).evaluate();
|
|
expect((svc as any).activeBreaches.has(42)).toBe(true);
|
|
|
|
// Second: condition clears
|
|
setupAlertScenario(10);
|
|
mockGetStackAlerts.mockReturnValue([{
|
|
id: 42,
|
|
stack_name: 'my-stack',
|
|
metric: 'cpu_percent',
|
|
operator: '>',
|
|
threshold: 80,
|
|
duration_mins: 999,
|
|
cooldown_mins: 0,
|
|
last_fired_at: 0,
|
|
}]);
|
|
await (svc as any).evaluate();
|
|
expect((svc as any).activeBreaches.has(42)).toBe(false);
|
|
});
|
|
});
|
|
|
|
// ── Cleanup triggers ───────────────────────────────────────────────────
|
|
|
|
describe('MonitorService - cleanup triggers', () => {
|
|
it('calls cleanup methods with configured retention', async () => {
|
|
mockGetNodes.mockReturnValue([]);
|
|
mockGetStackAlerts.mockReturnValue([]);
|
|
mockGetGlobalSettings.mockReturnValue({
|
|
metrics_retention_hours: '48',
|
|
log_retention_days: '7',
|
|
audit_retention_days: '30',
|
|
});
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluate();
|
|
|
|
expect(mockCleanupOldMetrics).toHaveBeenCalledWith(48);
|
|
expect(mockCleanupOldNotifications).toHaveBeenCalledWith(7);
|
|
expect(mockCleanupOldAuditLogs).toHaveBeenCalledWith(30);
|
|
});
|
|
|
|
it('uses defaults when settings are NaN', async () => {
|
|
mockGetNodes.mockReturnValue([]);
|
|
mockGetStackAlerts.mockReturnValue([]);
|
|
mockGetGlobalSettings.mockReturnValue({
|
|
metrics_retention_hours: 'bad',
|
|
log_retention_days: 'bad',
|
|
audit_retention_days: 'bad',
|
|
});
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluate();
|
|
|
|
expect(mockCleanupOldMetrics).toHaveBeenCalledWith(24);
|
|
expect(mockCleanupOldNotifications).toHaveBeenCalledWith(30);
|
|
expect(mockCleanupOldAuditLogs).toHaveBeenCalledWith(90);
|
|
});
|
|
});
|
|
|
|
// ── isProcessing guard ─────────────────────────────────────────────────
|
|
|
|
describe('MonitorService - isProcessing guard', () => {
|
|
it('skips evaluation if already processing', async () => {
|
|
mockGetGlobalSettings.mockReturnValue({});
|
|
mockGetNodes.mockReturnValue([]);
|
|
mockGetStackAlerts.mockReturnValue([]);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
(svc as any).isProcessing = true;
|
|
|
|
await (svc as any).evaluate();
|
|
|
|
// Should have been skipped — no DB calls
|
|
expect(mockGetGlobalSettings).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('resets isProcessing after evaluate completes (even on error)', async () => {
|
|
mockGetGlobalSettings.mockImplementationOnce(() => { throw new Error('boom'); });
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluate();
|
|
|
|
// isProcessing should be reset in finally block
|
|
expect((svc as any).isProcessing).toBe(false);
|
|
});
|
|
});
|
|
|
|
// ── restart_count metric ──────────────────────────────────────────────
|
|
|
|
describe('MonitorService - restart_count metric', () => {
|
|
function setupRestartScenario(restartCount: number, hasRestartRule: boolean) {
|
|
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
|
mockGetRunningContainers.mockResolvedValue([{
|
|
Id: 'c1',
|
|
Labels: { 'com.docker.compose.project': 'my-stack' },
|
|
}]);
|
|
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
|
|
cpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000, online_cpus: 1 },
|
|
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
|
memory_stats: { usage: 100e6, limit: 1e9 },
|
|
}));
|
|
mockGetContainerRestartCount.mockResolvedValue(restartCount);
|
|
const alerts = [];
|
|
if (hasRestartRule) {
|
|
alerts.push({
|
|
id: 100,
|
|
stack_name: 'my-stack',
|
|
metric: 'restart_count',
|
|
operator: '>',
|
|
threshold: 3,
|
|
duration_mins: 0,
|
|
cooldown_mins: 60,
|
|
last_fired_at: 0,
|
|
});
|
|
}
|
|
mockGetStackAlerts.mockReturnValue(alerts);
|
|
mockGetGlobalSettings.mockReturnValue({});
|
|
}
|
|
|
|
it('fetches restart count from Docker when a restart_count rule exists', async () => {
|
|
setupRestartScenario(5, true);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluate();
|
|
|
|
expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1');
|
|
// restart_count=5 > threshold=3, should fire
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Restart count'), { stackName: 'my-stack', actor: 'system:monitor' });
|
|
});
|
|
|
|
it('skips Docker inspect when no restart_count rules exist', async () => {
|
|
setupRestartScenario(5, false);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluate();
|
|
|
|
expect(mockGetContainerRestartCount).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('does not fire when restart count is below threshold', async () => {
|
|
setupRestartScenario(2, true);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluate();
|
|
|
|
expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1');
|
|
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Restart count'), expect.anything());
|
|
});
|
|
});
|
|
|
|
// ── Sencho version update check ───────────────────────────────────────
|
|
|
|
describe('MonitorService - Sencho version check', () => {
|
|
/** In-memory system_state so get/set round-trip within one evaluation. */
|
|
function wireStatefulSystemState(seed: Record<string, string> = {}) {
|
|
const store: Record<string, string> = { ...seed };
|
|
mockGetSystemState.mockImplementation((key: string) => store[key] ?? null);
|
|
mockSetSystemState.mockImplementation((key: string, value: string) => { store[key] = value; });
|
|
return store;
|
|
}
|
|
|
|
async function runEvaluate(): Promise<void> {
|
|
await (MonitorService.getInstance() as any).evaluate();
|
|
}
|
|
|
|
function updateAvailabilityCalls(): unknown[][] {
|
|
return mockDispatchAlert.mock.calls.filter(
|
|
(args: unknown[]) => args[1] === 'node_update_available',
|
|
);
|
|
}
|
|
|
|
beforeEach(() => {
|
|
mockGetGlobalSettings.mockReturnValue({});
|
|
mockGetNodes.mockReturnValue([]);
|
|
mockGetStackAlerts.mockReturnValue([]);
|
|
});
|
|
|
|
it('dispatches notification when newer version available', async () => {
|
|
mockGetSenchoVersion.mockReturnValue('0.45.0');
|
|
mockGetLatestVersionInfo.mockResolvedValue({ version: '0.46.0', publishPending: false });
|
|
mockGetSystemState.mockReturnValue(null);
|
|
|
|
await runEvaluate();
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('0.46.0'));
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('currently running 0.45.0'));
|
|
expect(mockSetSystemState).toHaveBeenCalledWith('last_sencho_update_notified_version', '0.46.0');
|
|
});
|
|
|
|
it('does not re-notify for the same version', async () => {
|
|
mockGetSenchoVersion.mockReturnValue('0.45.0');
|
|
mockGetLatestVersionInfo.mockResolvedValue({ version: '0.46.0', publishPending: false });
|
|
// Running < last notified: self-heal leaves the dedup key alone.
|
|
mockGetSystemState.mockReturnValue('0.46.0');
|
|
|
|
await runEvaluate();
|
|
|
|
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('0.46.0'));
|
|
});
|
|
|
|
it('handles version check failure gracefully and retries next cycle', async () => {
|
|
mockGetSenchoVersion.mockReturnValue('0.45.0');
|
|
mockGetLatestVersionInfo.mockResolvedValue(null);
|
|
mockGetSystemState.mockReturnValue(null);
|
|
|
|
await expect(runEvaluate()).resolves.toBeUndefined();
|
|
await expect(runEvaluate()).resolves.toBeUndefined();
|
|
|
|
expect(mockGetLatestVersionInfo).toHaveBeenCalledTimes(2);
|
|
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('available'));
|
|
});
|
|
|
|
it('skips version check when getSenchoVersion returns null', async () => {
|
|
mockGetSenchoVersion.mockReturnValue(null);
|
|
mockGetLatestVersionInfo.mockResolvedValue({ version: '0.46.0', publishPending: false });
|
|
mockGetSystemState.mockReturnValue(null);
|
|
|
|
await runEvaluate();
|
|
|
|
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('0.46.0'));
|
|
expect(mockSetSystemState).not.toHaveBeenCalledWith('last_sencho_update_notified_version', expect.anything());
|
|
expect(mockGetLatestVersionInfo).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('retries while a GitHub release is pending registry publish without notifying', async () => {
|
|
mockGetSenchoVersion.mockReturnValue('0.93.0');
|
|
mockGetLatestVersionInfo.mockResolvedValue({ version: '0.93.0', publishPending: true });
|
|
mockGetSystemState.mockReturnValue(null);
|
|
|
|
await runEvaluate();
|
|
await runEvaluate();
|
|
|
|
expect(mockGetLatestVersionInfo).toHaveBeenCalledTimes(2);
|
|
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('available'));
|
|
});
|
|
|
|
it('notifies on the next eval after an up-to-date success when a newer version appears', async () => {
|
|
const store = wireStatefulSystemState();
|
|
mockGetSenchoVersion.mockReturnValue('0.45.0');
|
|
mockGetLatestVersionInfo
|
|
.mockResolvedValueOnce({ version: '0.45.0', publishPending: false })
|
|
.mockResolvedValue({ version: '0.46.0', publishPending: false });
|
|
|
|
await runEvaluate();
|
|
expect(mockGetLatestVersionInfo).toHaveBeenCalledTimes(1);
|
|
expect(updateAvailabilityCalls()).toHaveLength(0);
|
|
expect(store.last_sencho_update_notified_version).toBeUndefined();
|
|
|
|
await runEvaluate();
|
|
expect(mockGetLatestVersionInfo).toHaveBeenCalledTimes(2);
|
|
const afterSecond = updateAvailabilityCalls();
|
|
expect(afterSecond).toHaveLength(1);
|
|
expect(afterSecond[0][2]).toContain('0.46.0');
|
|
expect(store.last_sencho_update_notified_version).toBe('0.46.0');
|
|
|
|
await runEvaluate();
|
|
expect(mockGetLatestVersionInfo).toHaveBeenCalledTimes(3);
|
|
expect(updateAvailabilityCalls()).toHaveLength(1);
|
|
});
|
|
|
|
it('notifies a newer release after a prior notify while still on the old version', async () => {
|
|
const store = wireStatefulSystemState({ last_sencho_update_notified_version: '0.46.0' });
|
|
mockGetSenchoVersion.mockReturnValue('0.45.0');
|
|
mockGetLatestVersionInfo.mockResolvedValue({ version: '0.47.0', publishPending: false });
|
|
|
|
await runEvaluate();
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('0.47.0'));
|
|
expect(store.last_sencho_update_notified_version).toBe('0.47.0');
|
|
});
|
|
|
|
it('self-heals dedup after user upgrades to the previously-notified version', async () => {
|
|
// Notified for 0.46.0 while on 0.45.0; now running 0.46.0 with 0.47.0 out.
|
|
const store = wireStatefulSystemState({ last_sencho_update_notified_version: '0.46.0' });
|
|
mockGetSenchoVersion.mockReturnValue('0.46.0');
|
|
mockGetLatestVersionInfo.mockResolvedValue({ version: '0.47.0', publishPending: false });
|
|
|
|
await runEvaluate();
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('0.47.0'));
|
|
expect(store.last_sencho_update_notified_version).toBe('0.47.0');
|
|
});
|
|
|
|
it('notifies once the registry publish completes after a pending GitHub release', async () => {
|
|
mockGetSenchoVersion.mockReturnValue('0.93.0');
|
|
mockGetSystemState.mockReturnValue(null);
|
|
|
|
mockGetLatestVersionInfo.mockResolvedValueOnce({ version: '0.93.0', publishPending: true });
|
|
await runEvaluate();
|
|
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('0.94.0'));
|
|
|
|
mockGetLatestVersionInfo.mockResolvedValueOnce({ version: '0.94.0', publishPending: false });
|
|
await runEvaluate();
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('0.94.0'));
|
|
});
|
|
});
|
|
|
|
// ── Per-container parallel fan-out ────────────────────────────────────
|
|
|
|
describe('MonitorService - parallel container processing', () => {
|
|
/** Build a stats payload that yields a positive CPU percent so the
|
|
* metric pipeline runs end-to-end (calculateCpuPercent + DB write). */
|
|
function statsPayload(): string {
|
|
return JSON.stringify({
|
|
cpu_stats: { cpu_usage: { total_usage: 2000 }, system_cpu_usage: 10000, online_cpus: 1 },
|
|
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
|
memory_stats: { usage: 100e6, limit: 1e9 },
|
|
});
|
|
}
|
|
|
|
beforeEach(() => {
|
|
mockGetGlobalSettings.mockReturnValue({});
|
|
mockGetStackAlerts.mockReturnValue([]);
|
|
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
|
});
|
|
|
|
it('fans out per-container stats fetches in parallel (wall time ~ max not sum)', async () => {
|
|
const containerCount = 10;
|
|
const perCallDelayMs = 200;
|
|
const containers = Array.from({ length: containerCount }, (_, i) => ({
|
|
Id: `container-${i}`,
|
|
Labels: { 'com.docker.compose.project': 'stack-x' },
|
|
}));
|
|
mockGetRunningContainers.mockResolvedValue(containers);
|
|
mockGetContainerStatsStream.mockImplementation(
|
|
() => new Promise((resolve) => setTimeout(() => resolve(statsPayload()), perCallDelayMs)),
|
|
);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
const start = Date.now();
|
|
await (svc as any).evaluate();
|
|
const elapsed = Date.now() - start;
|
|
|
|
// Serial would be containerCount * perCallDelayMs = 2000ms; parallel
|
|
// collapses to ~perCallDelayMs plus a little dispatch overhead. Allow
|
|
// generous headroom (3x the per-call delay) to avoid CI flake.
|
|
expect(elapsed).toBeLessThan(perCallDelayMs * 3);
|
|
// All containers were processed: one stats call and one metric write each.
|
|
expect(mockGetContainerStatsStream).toHaveBeenCalledTimes(containerCount);
|
|
expect(mockAddContainerMetric).toHaveBeenCalledTimes(containerCount);
|
|
});
|
|
|
|
it('isolates per-container failures: one rejection does not abort siblings', async () => {
|
|
const containers = Array.from({ length: 5 }, (_, i) => ({
|
|
Id: `container-${i}`,
|
|
Labels: { 'com.docker.compose.project': 'stack-x' },
|
|
}));
|
|
mockGetRunningContainers.mockResolvedValue(containers);
|
|
mockGetContainerStatsStream.mockImplementation(async (id: string) => {
|
|
if (id === 'container-2') {
|
|
const err = Object.assign(new Error('no such container'), { statusCode: 404 });
|
|
throw err;
|
|
}
|
|
return statsPayload();
|
|
});
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluate();
|
|
|
|
// 4 successful containers each wrote a metric; the 404 container is silently skipped.
|
|
expect(mockAddContainerMetric).toHaveBeenCalledTimes(4);
|
|
});
|
|
|
|
it('dispatches a stack alert exactly once per cycle even when multiple containers in the stack breach', async () => {
|
|
// 5 containers in the same stack, all breaching the same rule. Without
|
|
// the per-cycle dedup, parallel workers race past the cooldown check
|
|
// and each fire dispatchAlert before any DB write lands.
|
|
const containers = Array.from({ length: 5 }, (_, i) => ({
|
|
Id: `container-${i}`,
|
|
Labels: { 'com.docker.compose.project': 'shared-stack' },
|
|
}));
|
|
mockGetRunningContainers.mockResolvedValue(containers);
|
|
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
|
|
// Produces CPU = 90% to breach threshold of 80%.
|
|
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
|
|
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
|
memory_stats: { usage: 100e6, limit: 1e9 },
|
|
}));
|
|
mockGetStackAlerts.mockReturnValue([{
|
|
id: 7,
|
|
stack_name: 'shared-stack',
|
|
metric: 'cpu_percent',
|
|
operator: '>',
|
|
threshold: 80,
|
|
duration_mins: 0,
|
|
cooldown_mins: 60,
|
|
last_fired_at: 0,
|
|
}]);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluate();
|
|
|
|
const cpuDispatches = mockDispatchAlert.mock.calls.filter(
|
|
(args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'),
|
|
);
|
|
expect(cpuDispatches).toHaveLength(1);
|
|
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('caps simultaneous Docker calls at MAX_CONTAINER_CONCURRENCY', async () => {
|
|
// 25 containers with a long per-call delay; observe that no more than
|
|
// 10 (MAX_CONTAINER_CONCURRENCY) are in flight at the same time.
|
|
const containerCount = 25;
|
|
const perCallDelayMs = 100;
|
|
let inFlight = 0;
|
|
let peakInFlight = 0;
|
|
const containers = Array.from({ length: containerCount }, (_, i) => ({
|
|
Id: `container-${i}`,
|
|
Labels: { 'com.docker.compose.project': 'stack-x' },
|
|
}));
|
|
mockGetRunningContainers.mockResolvedValue(containers);
|
|
mockGetContainerStatsStream.mockImplementation(
|
|
() => new Promise((resolve) => {
|
|
inFlight += 1;
|
|
if (inFlight > peakInFlight) peakInFlight = inFlight;
|
|
setTimeout(() => {
|
|
inFlight -= 1;
|
|
resolve(statsPayload());
|
|
}, perCallDelayMs);
|
|
}),
|
|
);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluate();
|
|
|
|
expect(peakInFlight).toBeLessThanOrEqual(10);
|
|
// And we still saw real concurrency (not serialized), so peak should be
|
|
// close to the cap when we have many more items than workers.
|
|
expect(peakInFlight).toBeGreaterThan(1);
|
|
expect(mockAddContainerMetric).toHaveBeenCalledTimes(containerCount);
|
|
});
|
|
});
|
|
|
|
// ── Janitor cycle and circuit breaker (F-6) ────────────────────────────
|
|
|
|
describe('MonitorService - janitor cycle and circuit breaker', () => {
|
|
// Convenience: builds a never-settling promise used to simulate a hung df().
|
|
function hangForever(): Promise<never> {
|
|
return new Promise<never>(() => { /* never resolves */ });
|
|
}
|
|
|
|
// Reclaimable payload large enough to cross a 0.5 GB janitor threshold.
|
|
const RECLAIMABLE_3GB = {
|
|
reclaimableImages: 3 * 1024 * 1024 * 1024,
|
|
reclaimableContainers: 0,
|
|
reclaimableVolumes: 0,
|
|
reclaimableBuildCache: 0,
|
|
reclaimableImageCount: 5,
|
|
reclaimableContainerCount: 0,
|
|
reclaimableVolumeCount: 0,
|
|
reclaimableBuildCacheCount: 0,
|
|
};
|
|
|
|
it('evaluate() does NOT call getDiskUsage (decoupling guardrail)', async () => {
|
|
// F-6 regression guard: the 30s monitor cycle must never call df().
|
|
// If someone re-couples the janitor into evaluate(), this test fails.
|
|
mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
|
|
mockGetNodes.mockReturnValue([]);
|
|
mockGetStackAlerts.mockReturnValue([]);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluate();
|
|
|
|
expect(mockGetDiskUsage).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('evaluate() completes promptly even when getDiskUsage would hang (F-6 regression)', async () => {
|
|
// If df() were still on the 30s cycle, hangForever would compound the
|
|
// cycle beyond its 25s threshold. Decoupled, evaluate() must return
|
|
// within a small wall-clock budget regardless.
|
|
mockGetDiskUsage.mockReturnValue(hangForever());
|
|
mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
|
|
mockGetNodes.mockReturnValue([]);
|
|
mockGetStackAlerts.mockReturnValue([]);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
const t0 = Date.now();
|
|
await (svc as any).evaluate();
|
|
const elapsed = Date.now() - t0;
|
|
|
|
expect(elapsed).toBeLessThan(2000);
|
|
expect(mockGetDiskUsage).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('evaluateJanitor() honors isJanitorProcessing re-entrancy guard', async () => {
|
|
mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
|
|
mockGetDiskUsage.mockResolvedValue(RECLAIMABLE_3GB);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
(svc as any).isJanitorProcessing = true;
|
|
|
|
await (svc as any).evaluateJanitor();
|
|
|
|
// Second concurrent call must skip without touching settings or df.
|
|
expect(mockGetGlobalSettings).not.toHaveBeenCalled();
|
|
expect(mockGetDiskUsage).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('skips when docker_janitor_gb is unset, zero, or NaN', async () => {
|
|
const svc = MonitorService.getInstance();
|
|
|
|
mockGetGlobalSettings.mockReturnValue({});
|
|
await (svc as any).evaluateJanitor();
|
|
mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0' });
|
|
await (svc as any).evaluateJanitor();
|
|
mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: 'abc' });
|
|
await (svc as any).evaluateJanitor();
|
|
|
|
expect(mockGetDiskUsage).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('dispatches a node-neutral janitor alert', async () => {
|
|
mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
|
|
mockGetDiskUsage.mockResolvedValue(RECLAIMABLE_3GB);
|
|
mockGetSystemState.mockReturnValue('0'); // No prior alert; cooldown elapsed.
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateJanitor();
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'info',
|
|
'system',
|
|
'This node has accumulated 3.0 GB of unused Docker data. Open Resources to reclaim space, or set up a Prune Node Resources schedule.',
|
|
{ stackName: undefined, actor: 'system:monitor' },
|
|
);
|
|
});
|
|
|
|
it('does NOT alert when reclaimable is below MIN_RECLAIMABLE_GB even if threshold is aggressive', async () => {
|
|
mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.001' });
|
|
mockGetDiskUsage.mockResolvedValue({
|
|
...RECLAIMABLE_3GB,
|
|
reclaimableImages: 50 * 1024 * 1024, // 50 MB, below the 100 MB floor
|
|
});
|
|
mockGetSystemState.mockReturnValue('0');
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateJanitor();
|
|
|
|
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('opens the circuit breaker after JANITOR_BREAKER_THRESHOLD consecutive timeouts', async () => {
|
|
mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
|
|
mockGetDiskUsage.mockReturnValue(hangForever());
|
|
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateJanitor();
|
|
await (svc as any).evaluateJanitor();
|
|
expect(warnSpy).not.toHaveBeenCalled(); // Threshold not yet reached.
|
|
await (svc as any).evaluateJanitor(); // Third timeout trips the breaker.
|
|
|
|
const breakerLine = warnSpy.mock.calls.find(
|
|
(args) => typeof args[0] === 'string' && args[0].includes('circuit breaker opened'),
|
|
);
|
|
expect(breakerLine).toBeDefined();
|
|
expect((svc as any).janitorBreakerUntil).toBeGreaterThan(Date.now());
|
|
// Counter resets on open so the cooldown is what gates the next attempt.
|
|
expect((svc as any).janitorConsecutiveTimeouts).toBe(0);
|
|
|
|
warnSpy.mockRestore();
|
|
});
|
|
|
|
it('respects the breaker cooldown; does not call getDiskUsage while open', async () => {
|
|
mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
|
|
|
|
const svc = MonitorService.getInstance();
|
|
(svc as any).janitorBreakerUntil = Date.now() + 60 * 60 * 1000;
|
|
|
|
await (svc as any).evaluateJanitor();
|
|
|
|
expect(mockGetDiskUsage).not.toHaveBeenCalled();
|
|
expect(mockGetGlobalSettings).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('resets the timeout counter on a successful call after partial failures', async () => {
|
|
mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
|
|
mockGetDiskUsage
|
|
.mockReturnValueOnce(hangForever())
|
|
.mockReturnValueOnce(hangForever())
|
|
.mockResolvedValueOnce(RECLAIMABLE_3GB);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateJanitor();
|
|
await (svc as any).evaluateJanitor();
|
|
expect((svc as any).janitorConsecutiveTimeouts).toBe(2);
|
|
|
|
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
|
|
await (svc as any).evaluateJanitor();
|
|
|
|
expect((svc as any).janitorConsecutiveTimeouts).toBe(0);
|
|
const recoveryLine = logSpy.mock.calls.find(
|
|
(args) => typeof args[0] === 'string' && args[0].includes('recovered'),
|
|
);
|
|
expect(recoveryLine).toBeDefined();
|
|
logSpy.mockRestore();
|
|
});
|
|
|
|
it('logs recovery on the first successful call after a full breaker-open cooldown', async () => {
|
|
// After the breaker opens, the counter is zeroed; once the cooldown
|
|
// lapses, a successful call must still emit the recovered log so the
|
|
// operator observability story is symmetric with the partial-failure
|
|
// recovery path.
|
|
mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
|
|
|
|
const svc = MonitorService.getInstance();
|
|
(svc as any).janitorBreakerUntil = Date.now() - 1; // cooldown just elapsed
|
|
(svc as any).janitorConsecutiveTimeouts = 0; // zeroed on open
|
|
mockGetDiskUsage.mockResolvedValue(RECLAIMABLE_3GB);
|
|
|
|
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
|
|
await (svc as any).evaluateJanitor();
|
|
|
|
const recoveryLine = logSpy.mock.calls.find(
|
|
(args) => typeof args[0] === 'string' && args[0].includes('recovered'),
|
|
);
|
|
expect(recoveryLine).toBeDefined();
|
|
expect((svc as any).janitorBreakerUntil).toBe(0);
|
|
expect((svc as any).janitorConsecutiveTimeouts).toBe(0);
|
|
logSpy.mockRestore();
|
|
});
|
|
|
|
it('does NOT advance the breaker counter on non-timeout errors', async () => {
|
|
mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
|
|
mockGetDiskUsage.mockRejectedValue(Object.assign(new Error('daemon unreachable'), { statusCode: 500 }));
|
|
const errSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
|
|
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).evaluateJanitor();
|
|
await (svc as any).evaluateJanitor();
|
|
await (svc as any).evaluateJanitor();
|
|
await (svc as any).evaluateJanitor();
|
|
|
|
expect((svc as any).janitorConsecutiveTimeouts).toBe(0);
|
|
expect((svc as any).janitorBreakerUntil).toBe(0);
|
|
// The original support-grep line must still fire on every non-timeout error.
|
|
const janitorErrorLines = errSpy.mock.calls.filter(
|
|
(args) => typeof args[0] === 'string' && args[0].includes('Error checking docker janitor limits'),
|
|
);
|
|
expect(janitorErrorLines.length).toBe(4);
|
|
|
|
errSpy.mockRestore();
|
|
});
|
|
|
|
it('stop() clears both intervals AND both deferred first-tick timeouts', () => {
|
|
// Without canceling the first-tick setTimeouts, stop() would let
|
|
// evaluate() / evaluateJanitor() fire on a service that the caller
|
|
// believes is dormant. The 5s and 45s windows are wider than typical
|
|
// graceful-shutdown budgets, so this matters.
|
|
const svc = MonitorService.getInstance();
|
|
svc.start();
|
|
expect((svc as any).intervalId).not.toBeNull();
|
|
expect((svc as any).firstTickTimeoutId).not.toBeNull();
|
|
expect((svc as any).janitorIntervalId).not.toBeNull();
|
|
expect((svc as any).janitorFirstTickTimeoutId).not.toBeNull();
|
|
|
|
svc.stop();
|
|
|
|
expect((svc as any).intervalId).toBeNull();
|
|
expect((svc as any).firstTickTimeoutId).toBeNull();
|
|
expect((svc as any).janitorIntervalId).toBeNull();
|
|
expect((svc as any).janitorFirstTickTimeoutId).toBeNull();
|
|
});
|
|
});
|
|
|
|
describe('MonitorService - reconcileOrphanedScans', () => {
|
|
function makeDb(refs: string[]) {
|
|
return {
|
|
getDistinctScanImageRefs: vi.fn().mockReturnValue(refs),
|
|
deleteScansByImageRef: vi.fn().mockReturnValue(1),
|
|
};
|
|
}
|
|
|
|
async function run(db: ReturnType<typeof makeDb>, settings: Record<string, string>) {
|
|
const svc = MonitorService.getInstance();
|
|
await (svc as any).reconcileOrphanedScans(db, settings);
|
|
return db.deleteScansByImageRef.mock.calls.map((c) => c[1] as string).sort();
|
|
}
|
|
|
|
it('does nothing when prune_orphaned_scans is not "1"', async () => {
|
|
const db = makeDb(['nginx:1']);
|
|
await run(db, { prune_orphaned_scans: '0' });
|
|
expect(db.getDistinctScanImageRefs).not.toHaveBeenCalled();
|
|
expect(db.deleteScansByImageRef).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('purges only the image and stack scans whose artifact is gone, scoped to the local node', async () => {
|
|
mockGetImages.mockResolvedValue([
|
|
{ RepoTags: ['nginx:1', 'redis:7'] },
|
|
{ RepoTags: ['<none>:<none>'] },
|
|
{ RepoTags: undefined }, // dangling image with no tags
|
|
]);
|
|
mockGetStacks.mockResolvedValue(['web']);
|
|
const db = makeDb(['nginx:1', 'ghost:9', 'stack:web', 'stack:old']);
|
|
expect(await run(db, { prune_orphaned_scans: '1' })).toEqual(['ghost:9', 'stack:old']);
|
|
// Per-instance: reconciliation reads and deletes against the local node id only.
|
|
expect(db.getDistinctScanImageRefs).toHaveBeenCalledWith(1);
|
|
for (const call of db.deleteScansByImageRef.mock.calls) {
|
|
expect(call[0]).toBe(1);
|
|
}
|
|
});
|
|
|
|
it('keeps scans whose ref matches a live image after normalization (untagged, registry-qualified, digest-pinned)', async () => {
|
|
mockGetImages.mockResolvedValue([
|
|
{ RepoTags: ['alpine:latest', 'nginx:1.14'], RepoDigests: ['redis@sha256:abc'] },
|
|
]);
|
|
mockGetStacks.mockResolvedValue(['web']);
|
|
// Stored refs in non-canonical forms that all resolve to a present image:
|
|
// alpine -> alpine:latest, docker.io/library/nginx:1.14 -> nginx:1.14,
|
|
// docker.io/library/redis@sha256:abc -> redis@sha256:abc (digest).
|
|
const db = makeDb(['alpine', 'docker.io/library/nginx:1.14', 'docker.io/library/redis@sha256:abc', 'ghost:9']);
|
|
expect(await run(db, { prune_orphaned_scans: '1' })).toEqual(['ghost:9']);
|
|
});
|
|
|
|
it('skips stack reconciliation when getStacks returns empty (ambiguous), still purges image orphans', async () => {
|
|
mockGetImages.mockResolvedValue([{ RepoTags: ['nginx:1'] }]);
|
|
mockGetStacks.mockResolvedValue([]);
|
|
const db = makeDb(['stack:web', 'ghost:9']);
|
|
const deleted = await run(db, { prune_orphaned_scans: '1' });
|
|
expect(deleted).toContain('ghost:9');
|
|
expect(deleted).not.toContain('stack:web');
|
|
});
|
|
|
|
it('purges nothing when the Docker image list cannot be read (fail-safe)', async () => {
|
|
mockGetImages.mockRejectedValue(new Error('docker down'));
|
|
mockGetStacks.mockResolvedValue(['web']);
|
|
const db = makeDb(['ghost:9', 'stack:old']);
|
|
await run(db, { prune_orphaned_scans: '1' });
|
|
expect(db.deleteScansByImageRef).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('purges every image scan when there are zero live images (empty but successful read)', async () => {
|
|
mockGetImages.mockResolvedValue([]);
|
|
mockGetStacks.mockResolvedValue(['web']);
|
|
const db = makeDb(['nginx:1', 'redis:7', 'stack:web']);
|
|
expect(await run(db, { prune_orphaned_scans: '1' })).toEqual(['nginx:1', 'redis:7']);
|
|
});
|
|
});
|