mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-09 18:32:52 +00:00
fix(monitor): collapse repeated host-metric alerts into per-window summary (F-11) (#1175)
* fix(monitor): collapse repeated host-metric alerts into per-window summary (F-11) A host metric over threshold previously dispatched one notification every 5 minutes for the duration of the breach, producing 7+ identical messages in 35 minutes and spamming Discord/Slack routes. Replace the hardcoded 5-minute cooldown for CPU/RAM/disk with a per-metric suppression window (default 60 min, configurable via host_alert_suppression_mins). The first breach fires immediately; subsequent cycles within the window are silently counted; the next dispatch after the window elapses carries a summary suffix listing how many cycles were suppressed and when the breach first crossed threshold. Recovery clears the counter so re-breach fires fresh. The pattern mirrors PolicyEnforcement.notifyTrivyMissingOnce: module-scope Map, in-memory only, in-cycle dedup, with a test-reset helper. The existing system_state row keeps post-restart re-fires bounded. Janitor and per-stack alert rules are unchanged; they already have adequate cadence and per-rule cooldown respectively. * fix(ci): restore backend and frontend checks * fix(e2e): remove create button timing race * fix(e2e): harden create double-click test * fix(monitor): clear persisted F-11 timestamp on recovery + clamp suppression window Independent audit on the previous commit surfaced two issues. 1. clearHostMetricSuppression early-returned on missing in-memory state, leaving a stale system_state.last_host_*_alert_ts row alive after a process restart. Scenario: breach fires + persists timestamp, process restarts, metric recovers before another evaluate cycle re-seeds the in-memory Map, recovery cleanup early-returns. Next re-breach inside the original window hits the restart-survivability branch and is silently suppressed instead of firing fresh. Fix: read persisted state in clearHostMetricSuppression and reset to '0' independently of in-memory presence. The read-before-write also skips redundant writes when the row is already cleared. 2. host_alert_suppression_mins is validated by zod on the bulk PATCH path but the single-key POST /api/settings path accepts allowlisted keys without re-validation. A 999999999-minute value would silence host alerts for centuries. Add MAX_HOST_ALERT_SUPPRESSION_MIN = 1440 mirroring the zod max, and clamp via Math.min in evaluateGlobalSettings. Two new vitest cases (restart-then-recovery-then-rebreach; the 1440 clamp) confirmed failing before the fix, passing after. The existing "metric drop" case updated to use a mock-backed persistence pattern consistent with the new restart-scenario tests. 73/73 monitor-service tests green; full backend suite 2507/2510 (same pre-existing Windows EBUSY flake on filesystem-backup.test.ts as baseline).
This commit is contained in:
Anso
@@ -122,11 +122,13 @@ vi.mock('util', () => ({
|
||||
promisify: () => mockExecAsync,
|
||||
}));
|
||||
|
||||
import { MonitorService } from '../services/MonitorService';
|
||||
import { MonitorService, _resetHostAlertSuppressionStateForTests } from '../services/MonitorService';
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
(MonitorService as any).instance = undefined;
|
||||
_resetHostAlertSuppressionStateForTests();
|
||||
mockGetSystemState.mockReturnValue(null);
|
||||
});
|
||||
|
||||
// ── Pure calculation helpers (accessed via private method reflection) ───
|
||||
@@ -288,7 +290,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' });
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: undefined });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));
|
||||
});
|
||||
|
||||
it('does not dispatch when CPU below threshold', async () => {
|
||||
@@ -297,7 +299,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' });
|
||||
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: undefined });
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));
|
||||
});
|
||||
|
||||
it('dispatches RAM warning when over threshold', async () => {
|
||||
@@ -306,7 +308,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'), { stackName: undefined });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
|
||||
});
|
||||
|
||||
it('dispatches disk warning when over threshold', async () => {
|
||||
@@ -315,7 +317,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ host_disk_limit: '90' });
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Disk'), { stackName: undefined });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Disk'));
|
||||
});
|
||||
|
||||
it('skips host limits when threshold is 0 or NaN', async () => {
|
||||
@@ -323,16 +325,364 @@ describe('MonitorService - evaluateGlobalSettings', () => {
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '0' });
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: undefined });
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));
|
||||
|
||||
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: 'abc' });
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: undefined });
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));
|
||||
});
|
||||
});
|
||||
|
||||
// Crash + healthcheck detection now lives in DockerEventService (event-driven).
|
||||
// Tests for those flows live in docker-event-service.test.ts.
|
||||
|
||||
// ── F-11: host-metric alert suppression ────────────────────────────────
|
||||
|
||||
describe('MonitorService - host alert suppression (F-11)', () => {
|
||||
// Force RAM-over-threshold for every test in this block; CPU/disk are
|
||||
// independently controlled per-test so a single mockMem set-up covers the
|
||||
// common "I want a breach happening" case without test repetition.
|
||||
beforeEach(() => {
|
||||
mockMem.mockResolvedValue({ used: 15e9, total: 16e9 }); // ~94%
|
||||
});
|
||||
|
||||
it('first breach dispatches immediately', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
const [, , message] = mockDispatchAlert.mock.calls[0];
|
||||
expect(message).toContain('Memory');
|
||||
expect(message).not.toContain('Suppressed');
|
||||
});
|
||||
|
||||
it('second breach within window does not dispatch', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
const baseTime = 1_700_000_000_000;
|
||||
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
||||
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
|
||||
// 30 seconds later — well inside the 60-minute default window.
|
||||
nowSpy.mockReturnValue(baseTime + 30_000);
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
|
||||
nowSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('many breaches within window accumulate count without dispatching', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
const baseTime = 1_700_000_000_000;
|
||||
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
||||
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
for (let i = 1; i < 10; i++) {
|
||||
nowSpy.mockReturnValue(baseTime + i * 30_000);
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
}
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
nowSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('breach after window elapses dispatches follow-up with count summary and persists new timestamp', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
const baseTime = 1_700_000_000_000;
|
||||
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
||||
|
||||
// First dispatch.
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
expect(mockSetSystemState).toHaveBeenCalledWith('last_host_ram_alert_ts', String(baseTime));
|
||||
|
||||
// 5 cycles inside the window, each suppressed.
|
||||
for (let i = 1; i <= 5; i++) {
|
||||
nowSpy.mockReturnValue(baseTime + i * 30_000);
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
}
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Jump past the suppression window — 61 minutes.
|
||||
const followUpTime = baseTime + 61 * 60 * 1000;
|
||||
nowSpy.mockReturnValue(followUpTime);
|
||||
mockSetSystemState.mockClear();
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
||||
const [, , followUpMessage] = mockDispatchAlert.mock.calls[1];
|
||||
expect(followUpMessage).toMatch(/Suppressed 5 alerts in the last \d+m/);
|
||||
expect(followUpMessage).toMatch(/first over threshold at \d{2}:\d{2} UTC/);
|
||||
// Follow-up dispatch must persist the new timestamp so a subsequent
|
||||
// restart-survivability seed picks up the most-recent fire, not the
|
||||
// pre-window first fire (otherwise a restart 30min later would see a
|
||||
// 90min-old persisted row and re-fire immediately).
|
||||
expect(mockSetSystemState).toHaveBeenCalledWith('last_host_ram_alert_ts', String(followUpTime));
|
||||
|
||||
nowSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('dispatches at exactly the suppression window boundary', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
const baseTime = 1_700_000_000_000;
|
||||
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
||||
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Exactly 60 minutes later — at the boundary. The check is `<`, so the
|
||||
// boundary tick should DISPATCH a follow-up rather than suppress.
|
||||
nowSpy.mockReturnValue(baseTime + 60 * 60 * 1000);
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
||||
|
||||
nowSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('follow-up summary uses singular "alert" when exactly one cycle was suppressed', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
const baseTime = 1_700_000_000_000;
|
||||
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
||||
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
|
||||
// One suppressed cycle.
|
||||
nowSpy.mockReturnValue(baseTime + 30_000);
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
|
||||
// Past window.
|
||||
nowSpy.mockReturnValue(baseTime + 61 * 60 * 1000);
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
||||
const [, , followUpMessage] = mockDispatchAlert.mock.calls[1];
|
||||
expect(followUpMessage).toContain('Suppressed 1 alert in the last');
|
||||
expect(followUpMessage).not.toContain('Suppressed 1 alerts'); // singular form, not plural
|
||||
|
||||
nowSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('disk-metric path dispatches and suppresses through the same mechanism', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
mockFsSize.mockResolvedValue([{ mount: '/', use: 95 }]);
|
||||
// Set RAM below threshold so it does not interfere with the disk-only assertions.
|
||||
mockMem.mockResolvedValue({ used: 4e9, total: 16e9 });
|
||||
const baseTime = 1_700_000_000_000;
|
||||
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
||||
|
||||
await (svc as any).evaluateGlobalSettings({ host_disk_limit: '80' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
expect((mockDispatchAlert.mock.calls[0][2] as string)).toContain('Disk');
|
||||
|
||||
nowSpy.mockReturnValue(baseTime + 30_000);
|
||||
await (svc as any).evaluateGlobalSettings({ host_disk_limit: '80' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1); // suppressed
|
||||
|
||||
nowSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('metric drop below threshold clears in-memory state and persisted timestamp', async () => {
|
||||
// Recovery now reads system_state before deciding to write '0' (so it
|
||||
// skips a redundant write when persisted is already null/0). Model the
|
||||
// persistence chain so the recovery branch can observe the timestamp
|
||||
// the first-fire path just wrote.
|
||||
let persistedRamTs: string | null = null;
|
||||
mockGetSystemState.mockImplementation((key: string) =>
|
||||
key === 'last_host_ram_alert_ts' ? persistedRamTs : null,
|
||||
);
|
||||
mockSetSystemState.mockImplementation((key: string, value: string) => {
|
||||
if (key === 'last_host_ram_alert_ts') persistedRamTs = value;
|
||||
});
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
expect(persistedRamTs).not.toBeNull();
|
||||
expect(persistedRamTs).not.toBe('0');
|
||||
|
||||
// Drop RAM back under threshold.
|
||||
mockMem.mockResolvedValue({ used: 4e9, total: 16e9 }); // 25%
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
|
||||
// Recovery branch resets the persisted timestamp to '0'.
|
||||
expect(persistedRamTs).toBe('0');
|
||||
});
|
||||
|
||||
it('re-breach after recovery fires fresh first alert (no Suppressed suffix)', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
|
||||
// Initial breach.
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Recovery.
|
||||
mockMem.mockResolvedValue({ used: 4e9, total: 16e9 });
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
|
||||
// Re-breach.
|
||||
mockMem.mockResolvedValue({ used: 15e9, total: 16e9 });
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
||||
const [, , reBreachMessage] = mockDispatchAlert.mock.calls[1];
|
||||
expect(reBreachMessage).not.toContain('Suppressed');
|
||||
});
|
||||
|
||||
it('CPU and RAM suppression states are isolated per metric', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
mockCurrentLoad.mockResolvedValue({ currentLoad: 95 });
|
||||
|
||||
// First cycle: both fire.
|
||||
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
||||
|
||||
const messages1 = mockDispatchAlert.mock.calls.map(c => c[2] as string);
|
||||
expect(messages1.some(m => m.includes('CPU'))).toBe(true);
|
||||
expect(messages1.some(m => m.includes('Memory'))).toBe(true);
|
||||
|
||||
// Second cycle: both suppressed.
|
||||
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
||||
|
||||
// CPU drops below threshold (recovers); RAM stays high.
|
||||
mockCurrentLoad.mockResolvedValue({ currentLoad: 10 });
|
||||
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });
|
||||
|
||||
// CPU re-breaches; RAM still in suppression window.
|
||||
mockCurrentLoad.mockResolvedValue({ currentLoad: 95 });
|
||||
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });
|
||||
|
||||
// CPU fires fresh; RAM stays silent.
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(3);
|
||||
const newCpuMessage = mockDispatchAlert.mock.calls[2][2] as string;
|
||||
expect(newCpuMessage).toContain('CPU');
|
||||
expect(newCpuMessage).not.toContain('Suppressed');
|
||||
});
|
||||
|
||||
it('respects custom host_alert_suppression_mins setting (5 minutes)', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
const baseTime = 1_700_000_000_000;
|
||||
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
||||
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '5' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
|
||||
// 3 minutes — still within custom 5-minute window.
|
||||
nowSpy.mockReturnValue(baseTime + 3 * 60 * 1000);
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '5' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
|
||||
// 6 minutes — past the custom window; follow-up should fire.
|
||||
nowSpy.mockReturnValue(baseTime + 6 * 60 * 1000);
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '5' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
||||
|
||||
nowSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('post-restart recovery clears persisted timestamp so re-breach inside the window fires fresh', async () => {
|
||||
// Codex audit scenario:
|
||||
// T0: breach fires → system_state['last_host_ram_alert_ts'] = T0
|
||||
// T1: process restart (in-memory state cleared, persisted T0 survives)
|
||||
// T2: metric drops below threshold → clearHostMetricSuppression() runs
|
||||
// T3: metric re-breaches within suppression window
|
||||
// Expected: fresh first alert at T3 (no Suppressed suffix).
|
||||
// Pre-fix bug: clearHostMetricSuppression early-returned on missing in-
|
||||
// memory state, leaving persisted T0 alive; T3 hit the restart-survival
|
||||
// path and was silently suppressed.
|
||||
const baseTime = 1_700_000_000_000;
|
||||
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime + 5 * 60 * 1000); // T1+T2 = T0+5min
|
||||
|
||||
// Simulate persisted state from a previous process's first fire.
|
||||
let persistedRamTs: string | null = String(baseTime);
|
||||
mockGetSystemState.mockImplementation((key: string) =>
|
||||
key === 'last_host_ram_alert_ts' ? persistedRamTs : null,
|
||||
);
|
||||
mockSetSystemState.mockImplementation((key: string, value: string) => {
|
||||
if (key === 'last_host_ram_alert_ts') persistedRamTs = value;
|
||||
});
|
||||
|
||||
// T2: post-restart cycle finds metric BELOW threshold (recovered).
|
||||
mockMem.mockResolvedValue({ used: 4e9, total: 16e9 });
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
// No dispatch on a non-breach cycle.
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
||||
// Recovery branch MUST have reset persisted state so a re-breach is
|
||||
// treated as fresh, not as a still-active cooldown.
|
||||
expect(persistedRamTs).toBe('0');
|
||||
|
||||
// T3: re-breach 10 minutes later (well inside the original 60-min window).
|
||||
nowSpy.mockReturnValue(baseTime + 15 * 60 * 1000);
|
||||
mockMem.mockResolvedValue({ used: 15e9, total: 16e9 });
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
const [, , message] = mockDispatchAlert.mock.calls[0];
|
||||
expect(message).not.toContain('Suppressed');
|
||||
|
||||
nowSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('caps host_alert_suppression_mins at 24h (1440) to defend against unvalidated single-key POST writes', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
const baseTime = 1_700_000_000_000;
|
||||
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
||||
|
||||
// The single-key POST /api/settings path stores allowlisted keys
|
||||
// without zod re-validation. A 999999999-minute value (or any
|
||||
// accidental garbage above 1440) must not let a metric stay
|
||||
// suppressed for centuries; cap at 24h in the consumer.
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '999999999' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
|
||||
// 1441 minutes later — past the 24h cap.
|
||||
nowSpy.mockReturnValue(baseTime + 1441 * 60 * 1000);
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '999999999' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
||||
|
||||
nowSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('post-restart with persisted timestamp inside window does not re-fire', async () => {
|
||||
const baseTime = 1_700_000_000_000;
|
||||
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime + 10 * 60 * 1000); // "now" = T+10min
|
||||
|
||||
// Simulate a previous process having persisted a fire 10 minutes ago.
|
||||
mockGetSystemState.mockImplementation((key: string) =>
|
||||
key === 'last_host_ram_alert_ts' ? String(baseTime) : null,
|
||||
);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
|
||||
// Post-restart cycle must NOT re-fire because the persisted cooldown
|
||||
// is still active (10 min into the default 60 min window).
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
||||
|
||||
nowSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('zero or negative suppression_mins falls back to default', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
const baseTime = 1_700_000_000_000;
|
||||
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
|
||||
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '0' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Default 60-minute window applies — a cycle 30 minutes later stays
|
||||
// suppressed even though the setting said 0.
|
||||
nowSpy.mockReturnValue(baseTime + 30 * 60 * 1000);
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '0' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
|
||||
nowSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Alert breach state machine ─────────────────────────────────────────
|
||||
|
||||
describe('MonitorService - breach state machine', () => {
|
||||
|
||||
@@ -14,6 +14,7 @@ const ALLOWED_SETTING_KEYS = new Set([
|
||||
'host_cpu_limit',
|
||||
'host_ram_limit',
|
||||
'host_disk_limit',
|
||||
'host_alert_suppression_mins',
|
||||
'docker_janitor_gb',
|
||||
'global_crash',
|
||||
'developer_mode',
|
||||
@@ -28,6 +29,7 @@ const SettingsPatchSchema = z.object({
|
||||
host_cpu_limit: z.coerce.number().int().min(1).max(100).transform(String),
|
||||
host_ram_limit: z.coerce.number().int().min(1).max(100).transform(String),
|
||||
host_disk_limit: z.coerce.number().int().min(1).max(100).transform(String),
|
||||
host_alert_suppression_mins: z.coerce.number().int().min(1).max(1440).transform(String),
|
||||
docker_janitor_gb: z.coerce.number().min(0).transform(String),
|
||||
global_crash: z.enum(['0', '1']),
|
||||
developer_mode: z.enum(['0', '1']),
|
||||
|
||||
@@ -1229,6 +1229,7 @@ export class DatabaseService {
|
||||
stmt.run('host_cpu_limit', '90');
|
||||
stmt.run('host_ram_limit', '90');
|
||||
stmt.run('host_disk_limit', '90');
|
||||
stmt.run('host_alert_suppression_mins', '60');
|
||||
stmt.run('global_crash', '1');
|
||||
stmt.run('docker_janitor_gb', '5');
|
||||
stmt.run('developer_mode', '0');
|
||||
|
||||
@@ -58,6 +58,31 @@ const HOST_ALERT_KEYS = {
|
||||
janitor: 'last_janitor_alert_timestamp',
|
||||
} as const;
|
||||
|
||||
type HostMetricKey = 'cpu' | 'ram' | 'disk';
|
||||
|
||||
interface HostAlertSuppressionState {
|
||||
firstFiredAt: number;
|
||||
lastFiredAt: number;
|
||||
suppressedCount: number;
|
||||
}
|
||||
|
||||
// F-11 dedup for host-metric notifications (CPU, RAM, disk). The previous
|
||||
// 5-minute cooldown produced 7+ identical messages over a 35-minute window
|
||||
// when a host stayed over threshold, flooding Discord/Slack routes. State is
|
||||
// per-metric and module-scope so a singleton-replacement (only happens in
|
||||
// tests) does not lose tracking. Cleared on process restart; the persisted
|
||||
// system_state row keeps post-restart re-fires bounded.
|
||||
const hostAlertSuppressionState = new Map<HostMetricKey, HostAlertSuppressionState>();
|
||||
const DEFAULT_HOST_ALERT_SUPPRESSION_MIN = 60;
|
||||
// Mirror the zod schema upper bound in `routes/settings.ts` so the single-key
|
||||
// POST path (which lacks zod re-validation) cannot push the consumer past
|
||||
// the validated range.
|
||||
const MAX_HOST_ALERT_SUPPRESSION_MIN = 1440;
|
||||
|
||||
export function _resetHostAlertSuppressionStateForTests(): void {
|
||||
hostAlertSuppressionState.clear();
|
||||
}
|
||||
|
||||
const STATS_TIMEOUT_MS = 10_000;
|
||||
const FLOAT_EQ_EPSILON = 0.01;
|
||||
|
||||
@@ -226,7 +251,17 @@ export class MonitorService {
|
||||
}
|
||||
|
||||
private async evaluateGlobalSettings(settings: Record<string, string>) {
|
||||
const HOST_ALERT_COOLDOWN_MS = 5 * 60 * 1000; // 5 minutes between repeat alerts
|
||||
// F-11 suppression window. Default 60 min; configurable via global
|
||||
// settings. NaN / non-positive values fall back to the default so a
|
||||
// misconfigured row never disables the dedup entirely. Cap at 24h
|
||||
// to defend against the single-key POST /api/settings path (which
|
||||
// stores allowlisted keys without zod re-validation): an accidental
|
||||
// 999999999-minute value must not silence host alerts for centuries.
|
||||
const suppressionMinRaw = parseInt(settings['host_alert_suppression_mins'] || '', 10);
|
||||
const suppressionMin = isNaN(suppressionMinRaw) || suppressionMinRaw <= 0
|
||||
? DEFAULT_HOST_ALERT_SUPPRESSION_MIN
|
||||
: Math.min(suppressionMinRaw, MAX_HOST_ALERT_SUPPRESSION_MIN);
|
||||
const suppressionMs = suppressionMin * 60 * 1000;
|
||||
|
||||
// 1. Host Limits — fetch CPU, RAM, disk concurrently
|
||||
try {
|
||||
@@ -239,23 +274,29 @@ export class MonitorService {
|
||||
const cpuUsage = currentLoad.currentLoad;
|
||||
const cpuLimit = parseFloat(settings['host_cpu_limit']);
|
||||
if (!isNaN(cpuLimit) && cpuLimit > 0 && cpuUsage > cpuLimit) {
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.cpu, HOST_ALERT_COOLDOWN_MS, 'warning', 'monitor_alert',
|
||||
await this.dispatchHostMetricAlert('cpu', 'warning', suppressionMs,
|
||||
`Host CPU utilization is critically high: ${cpuUsage.toFixed(1)}% (Threshold: ${cpuLimit}%)`);
|
||||
} else {
|
||||
this.clearHostMetricSuppression('cpu');
|
||||
}
|
||||
|
||||
const ramUsage = (mem.used / mem.total) * 100;
|
||||
const ramLimit = parseFloat(settings['host_ram_limit']);
|
||||
if (!isNaN(ramLimit) && ramLimit > 0 && ramUsage > ramLimit) {
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.ram, HOST_ALERT_COOLDOWN_MS, 'warning', 'monitor_alert',
|
||||
await this.dispatchHostMetricAlert('ram', 'warning', suppressionMs,
|
||||
`Host Memory utilization is critically high: ${ramUsage.toFixed(1)}% (Threshold: ${ramLimit}%)`);
|
||||
} else {
|
||||
this.clearHostMetricSuppression('ram');
|
||||
}
|
||||
|
||||
const mainDisk = fsSize.find(fs => fs.mount === '/' || fs.mount === 'C:') || fsSize[0];
|
||||
if (mainDisk) {
|
||||
const diskLimit = parseFloat(settings['host_disk_limit']);
|
||||
if (!isNaN(diskLimit) && diskLimit > 0 && mainDisk.use > diskLimit) {
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.disk, HOST_ALERT_COOLDOWN_MS, 'warning', 'monitor_alert',
|
||||
await this.dispatchHostMetricAlert('disk', 'warning', suppressionMs,
|
||||
`Host Disk space utilization is critically high: ${mainDisk.use.toFixed(1)}% (Threshold: ${diskLimit}%)`);
|
||||
} else {
|
||||
this.clearHostMetricSuppression('disk');
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
@@ -687,6 +728,104 @@ export class MonitorService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* F-11 host-metric dispatch with per-key suppression window + count
|
||||
* summary. Replaces the old 5-minute hardcoded cooldown for CPU/RAM/disk.
|
||||
*
|
||||
* Behaviour:
|
||||
* - First breach (no in-memory state, no recent persisted timestamp) →
|
||||
* dispatch immediately and seed in-memory state.
|
||||
* - Restart on a still-breaching host (no in-memory state, but persisted
|
||||
* timestamp falls inside the window) → silently re-seed from the
|
||||
* persisted timestamp so post-restart cycles do not re-fire.
|
||||
* - Breach within an open suppression window → silently increment
|
||||
* suppressedCount; no dispatch, no broadcast.
|
||||
* - Breach after window elapses → fire one follow-up with the suffix
|
||||
* `Suppressed N alerts in the last Xm; first over threshold at HH:MM UTC.`
|
||||
*
|
||||
* Recovery is handled by the caller via clearHostMetricSuppression(): when
|
||||
* the metric drops below threshold, in-memory state and the persisted
|
||||
* timestamp are cleared so the next breach fires fresh.
|
||||
*/
|
||||
private async dispatchHostMetricAlert(
|
||||
key: HostMetricKey,
|
||||
severity: 'info' | 'warning' | 'error',
|
||||
suppressionMs: number,
|
||||
baseMessage: string,
|
||||
): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const stateKey = HOST_ALERT_KEYS[key];
|
||||
const now = Date.now();
|
||||
const state = hostAlertSuppressionState.get(key);
|
||||
|
||||
if (!state) {
|
||||
const persistedLast = parseInt(db.getSystemState(stateKey) || '0', 10);
|
||||
if (persistedLast > 0 && now - persistedLast < suppressionMs) {
|
||||
// Post-restart on a still-breaching host: respect the
|
||||
// persisted cooldown so we do not re-fire immediately. Seed
|
||||
// in-memory state from the persisted timestamp; count this
|
||||
// cycle as suppressed for an honest follow-up summary.
|
||||
hostAlertSuppressionState.set(key, {
|
||||
firstFiredAt: persistedLast,
|
||||
lastFiredAt: persistedLast,
|
||||
suppressedCount: 1,
|
||||
});
|
||||
return;
|
||||
}
|
||||
// First-ever fire (fresh process, or post-recovery re-breach).
|
||||
await NotificationService.getInstance().dispatchAlert(severity, 'monitor_alert', baseMessage);
|
||||
hostAlertSuppressionState.set(key, {
|
||||
firstFiredAt: now,
|
||||
lastFiredAt: now,
|
||||
suppressedCount: 0,
|
||||
});
|
||||
db.setSystemState(stateKey, now.toString());
|
||||
return;
|
||||
}
|
||||
|
||||
if (now - state.lastFiredAt < suppressionMs) {
|
||||
state.suppressedCount += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const windowMin = Math.max(1, Math.round((now - state.lastFiredAt) / 60000));
|
||||
const firstAt = new Date(state.firstFiredAt).toISOString().slice(11, 16);
|
||||
const plural = state.suppressedCount === 1 ? '' : 's';
|
||||
const suffix = ` Suppressed ${state.suppressedCount} alert${plural} in the last ${windowMin}m; first over threshold at ${firstAt} UTC.`;
|
||||
await NotificationService.getInstance().dispatchAlert(severity, 'monitor_alert', baseMessage + suffix);
|
||||
state.lastFiredAt = now;
|
||||
state.suppressedCount = 0;
|
||||
db.setSystemState(stateKey, now.toString());
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear in-memory + persisted suppression state for a host metric. Called
|
||||
* when the metric drops below threshold so the next breach fires fresh
|
||||
* (no "Suppressed N" suffix). Resetting the persisted row to '0' is
|
||||
* essential: a stale persisted timestamp inside the window would silently
|
||||
* absorb the next breach via the restart-survivability path.
|
||||
*
|
||||
* The persisted reset must run independently of in-memory state because
|
||||
* the two halves can drift after a process restart: the in-memory Map is
|
||||
* empty on a fresh process, but the system_state row from the previous
|
||||
* process still points at the old fire timestamp. If recovery happens
|
||||
* before another evaluate cycle re-seeds in-memory state, an early-return
|
||||
* on the in-memory check alone would leave the persisted row alive, and
|
||||
* the next re-breach within the original window would be silently
|
||||
* suppressed instead of firing fresh (Codex audit on PR #1175).
|
||||
*/
|
||||
private clearHostMetricSuppression(key: HostMetricKey): void {
|
||||
const stateKey = HOST_ALERT_KEYS[key];
|
||||
const db = DatabaseService.getInstance();
|
||||
const hadInMemory = hostAlertSuppressionState.delete(key);
|
||||
const persisted = db.getSystemState(stateKey);
|
||||
const needsPersistedReset = persisted !== null && persisted !== '0';
|
||||
if (!hadInMemory && !needsPersistedReset) return;
|
||||
if (needsPersistedReset) {
|
||||
db.setSystemState(stateKey, '0');
|
||||
}
|
||||
}
|
||||
|
||||
private calculateCpuPercent(stats: DockerContainerStats): number {
|
||||
let cpuPercent = 0.0;
|
||||
if (!stats?.cpu_stats?.cpu_usage || !stats?.precpu_stats?.cpu_usage) return 0.0;
|
||||
|
||||
Reference in New Issue
Block a user