fix(monitor): collapse repeated host-metric alerts into per-window summary (F-11) (#1175)

* fix(monitor): collapse repeated host-metric alerts into per-window summary (F-11) A host metric over threshold previously dispatched one notification every 5 minutes for the duration of the breach, producing 7+ identical messages in 35 minutes and spamming Discord/Slack routes. Replace the hardcoded 5-minute cooldown for CPU/RAM/disk with a per-metric suppression window (default 60 min, configurable via host_alert_suppression_mins). The first breach fires immediately; subsequent cycles within the window are silently counted; the next dispatch after the window elapses carries a summary suffix listing how many cycles were suppressed and when the breach first crossed threshold. Recovery clears the counter so re-breach fires fresh. The pattern mirrors PolicyEnforcement.notifyTrivyMissingOnce: module-scope Map, in-memory only, in-cycle dedup, with a test-reset helper. The existing system_state row keeps post-restart re-fires bounded. Janitor and per-stack alert rules are unchanged; they already have adequate cadence and per-rule cooldown respectively. * fix(ci): restore backend and frontend checks * fix(e2e): remove create button timing race * fix(e2e): harden create double-click test * fix(monitor): clear persisted F-11 timestamp on recovery + clamp suppression window Independent audit on the previous commit surfaced two issues. 1. clearHostMetricSuppression early-returned on missing in-memory state, leaving a stale system_state.last_host_*_alert_ts row alive after a process restart. Scenario: breach fires + persists timestamp, process restarts, metric recovers before another evaluate cycle re-seeds the in-memory Map, recovery cleanup early-returns. Next re-breach inside the original window hits the restart-survivability branch and is silently suppressed instead of firing fresh. Fix: read persisted state in clearHostMetricSuppression and reset to '0' independently of in-memory presence. The read-before-write also skips redundant writes when the row is already cleared. 2. host_alert_suppression_mins is validated by zod on the bulk PATCH path but the single-key POST /api/settings path accepts allowlisted keys without re-validation. A 999999999-minute value would silence host alerts for centuries. Add MAX_HOST_ALERT_SUPPRESSION_MIN = 1440 mirroring the zod max, and clamp via Math.min in evaluateGlobalSettings. Two new vitest cases (restart-then-recovery-then-rebreach; the 1440 clamp) confirmed failing before the fix, passing after. The existing "metric drop" case updated to use a mock-backed persistence pattern consistent with the new restart-scenario tests. 73/73 monitor-service tests green; full backend suite 2507/2510 (same pre-existing Windows EBUSY flake on filesystem-backup.test.ts as baseline).
2026-07-26 11:49:16 +00:00 · 2026-05-23 06:28:01 -04:00
parent e46f6980f8
commit fcff8e9047
8 changed files with 560 additions and 30 deletions
@@ -122,11 +122,13 @@ vi.mock('util', () => ({
  promisify: () => mockExecAsync,
 }));

-import { MonitorService } from '../services/MonitorService';
+import { MonitorService, _resetHostAlertSuppressionStateForTests } from '../services/MonitorService';

 beforeEach(() => {
  vi.clearAllMocks();
  (MonitorService as any).instance = undefined;
+  _resetHostAlertSuppressionStateForTests();
+  mockGetSystemState.mockReturnValue(null);
 });

 // ── Pure calculation helpers (accessed via private method reflection) ───
@@ -288,7 +290,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' });

-    expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: undefined });
+    expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));
  });

  it('does not dispatch when CPU below threshold', async () => {
@@ -297,7 +299,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' });

-    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: undefined });
+    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));
  });

  it('dispatches RAM warning when over threshold', async () => {
@@ -306,7 +308,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });

-    expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'), { stackName: undefined });
+    expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
  });

  it('dispatches disk warning when over threshold', async () => {
@@ -315,7 +317,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_disk_limit: '90' });

-    expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Disk'), { stackName: undefined });
+    expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Disk'));
  });

  it('skips host limits when threshold is 0 or NaN', async () => {
@@ -323,16 +325,364 @@ describe('MonitorService - evaluateGlobalSettings', () => {

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '0' });
-    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: undefined });
+    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));

    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: 'abc' });
-    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: undefined });
+    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));
  });
 });

 // Crash + healthcheck detection now lives in DockerEventService (event-driven).
 // Tests for those flows live in docker-event-service.test.ts.

+// ── F-11: host-metric alert suppression ────────────────────────────────
+
+describe('MonitorService - host alert suppression (F-11)', () => {
+  // Force RAM-over-threshold for every test in this block; CPU/disk are
+  // independently controlled per-test so a single mockMem set-up covers the
+  // common "I want a breach happening" case without test repetition.
+  beforeEach(() => {
+    mockMem.mockResolvedValue({ used: 15e9, total: 16e9 }); // ~94%
+  });
+
+  it('first breach dispatches immediately', async () => {
+    const svc = MonitorService.getInstance();
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+    const [, , message] = mockDispatchAlert.mock.calls[0];
+    expect(message).toContain('Memory');
+    expect(message).not.toContain('Suppressed');
+  });
+
+  it('second breach within window does not dispatch', async () => {
+    const svc = MonitorService.getInstance();
+    const baseTime = 1_700_000_000_000;
+    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
+
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+
+    // 30 seconds later — well inside the 60-minute default window.
+    nowSpy.mockReturnValue(baseTime + 30_000);
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+
+    nowSpy.mockRestore();
+  });
+
+  it('many breaches within window accumulate count without dispatching', async () => {
+    const svc = MonitorService.getInstance();
+    const baseTime = 1_700_000_000_000;
+    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
+
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+    for (let i = 1; i < 10; i++) {
+      nowSpy.mockReturnValue(baseTime + i * 30_000);
+      await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+    }
+
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+    nowSpy.mockRestore();
+  });
+
+  it('breach after window elapses dispatches follow-up with count summary and persists new timestamp', async () => {
+    const svc = MonitorService.getInstance();
+    const baseTime = 1_700_000_000_000;
+    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
+
+    // First dispatch.
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+    expect(mockSetSystemState).toHaveBeenCalledWith('last_host_ram_alert_ts', String(baseTime));
+
+    // 5 cycles inside the window, each suppressed.
+    for (let i = 1; i <= 5; i++) {
+      nowSpy.mockReturnValue(baseTime + i * 30_000);
+      await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+    }
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+
+    // Jump past the suppression window — 61 minutes.
+    const followUpTime = baseTime + 61 * 60 * 1000;
+    nowSpy.mockReturnValue(followUpTime);
+    mockSetSystemState.mockClear();
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
+    const [, , followUpMessage] = mockDispatchAlert.mock.calls[1];
+    expect(followUpMessage).toMatch(/Suppressed 5 alerts in the last \d+m/);
+    expect(followUpMessage).toMatch(/first over threshold at \d{2}:\d{2} UTC/);
+    // Follow-up dispatch must persist the new timestamp so a subsequent
+    // restart-survivability seed picks up the most-recent fire, not the
+    // pre-window first fire (otherwise a restart 30min later would see a
+    // 90min-old persisted row and re-fire immediately).
+    expect(mockSetSystemState).toHaveBeenCalledWith('last_host_ram_alert_ts', String(followUpTime));
+
+    nowSpy.mockRestore();
+  });
+
+  it('dispatches at exactly the suppression window boundary', async () => {
+    const svc = MonitorService.getInstance();
+    const baseTime = 1_700_000_000_000;
+    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
+
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+
+    // Exactly 60 minutes later — at the boundary. The check is `<`, so the
+    // boundary tick should DISPATCH a follow-up rather than suppress.
+    nowSpy.mockReturnValue(baseTime + 60 * 60 * 1000);
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
+
+    nowSpy.mockRestore();
+  });
+
+  it('follow-up summary uses singular "alert" when exactly one cycle was suppressed', async () => {
+    const svc = MonitorService.getInstance();
+    const baseTime = 1_700_000_000_000;
+    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
+
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+
+    // One suppressed cycle.
+    nowSpy.mockReturnValue(baseTime + 30_000);
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+
+    // Past window.
+    nowSpy.mockReturnValue(baseTime + 61 * 60 * 1000);
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
+    const [, , followUpMessage] = mockDispatchAlert.mock.calls[1];
+    expect(followUpMessage).toContain('Suppressed 1 alert in the last');
+    expect(followUpMessage).not.toContain('Suppressed 1 alerts'); // singular form, not plural
+
+    nowSpy.mockRestore();
+  });
+
+  it('disk-metric path dispatches and suppresses through the same mechanism', async () => {
+    const svc = MonitorService.getInstance();
+    mockFsSize.mockResolvedValue([{ mount: '/', use: 95 }]);
+    // Set RAM below threshold so it does not interfere with the disk-only assertions.
+    mockMem.mockResolvedValue({ used: 4e9, total: 16e9 });
+    const baseTime = 1_700_000_000_000;
+    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
+
+    await (svc as any).evaluateGlobalSettings({ host_disk_limit: '80' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+    expect((mockDispatchAlert.mock.calls[0][2] as string)).toContain('Disk');
+
+    nowSpy.mockReturnValue(baseTime + 30_000);
+    await (svc as any).evaluateGlobalSettings({ host_disk_limit: '80' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1); // suppressed
+
+    nowSpy.mockRestore();
+  });
+
+  it('metric drop below threshold clears in-memory state and persisted timestamp', async () => {
+    // Recovery now reads system_state before deciding to write '0' (so it
+    // skips a redundant write when persisted is already null/0). Model the
+    // persistence chain so the recovery branch can observe the timestamp
+    // the first-fire path just wrote.
+    let persistedRamTs: string | null = null;
+    mockGetSystemState.mockImplementation((key: string) =>
+      key === 'last_host_ram_alert_ts' ? persistedRamTs : null,
+    );
+    mockSetSystemState.mockImplementation((key: string, value: string) => {
+      if (key === 'last_host_ram_alert_ts') persistedRamTs = value;
+    });
+
+    const svc = MonitorService.getInstance();
+
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+    expect(persistedRamTs).not.toBeNull();
+    expect(persistedRamTs).not.toBe('0');
+
+    // Drop RAM back under threshold.
+    mockMem.mockResolvedValue({ used: 4e9, total: 16e9 }); // 25%
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+
+    // Recovery branch resets the persisted timestamp to '0'.
+    expect(persistedRamTs).toBe('0');
+  });
+
+  it('re-breach after recovery fires fresh first alert (no Suppressed suffix)', async () => {
+    const svc = MonitorService.getInstance();
+
+    // Initial breach.
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+
+    // Recovery.
+    mockMem.mockResolvedValue({ used: 4e9, total: 16e9 });
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+
+    // Re-breach.
+    mockMem.mockResolvedValue({ used: 15e9, total: 16e9 });
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
+    const [, , reBreachMessage] = mockDispatchAlert.mock.calls[1];
+    expect(reBreachMessage).not.toContain('Suppressed');
+  });
+
+  it('CPU and RAM suppression states are isolated per metric', async () => {
+    const svc = MonitorService.getInstance();
+    mockCurrentLoad.mockResolvedValue({ currentLoad: 95 });
+
+    // First cycle: both fire.
+    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
+
+    const messages1 = mockDispatchAlert.mock.calls.map(c => c[2] as string);
+    expect(messages1.some(m => m.includes('CPU'))).toBe(true);
+    expect(messages1.some(m => m.includes('Memory'))).toBe(true);
+
+    // Second cycle: both suppressed.
+    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
+
+    // CPU drops below threshold (recovers); RAM stays high.
+    mockCurrentLoad.mockResolvedValue({ currentLoad: 10 });
+    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });
+
+    // CPU re-breaches; RAM still in suppression window.
+    mockCurrentLoad.mockResolvedValue({ currentLoad: 95 });
+    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });
+
+    // CPU fires fresh; RAM stays silent.
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(3);
+    const newCpuMessage = mockDispatchAlert.mock.calls[2][2] as string;
+    expect(newCpuMessage).toContain('CPU');
+    expect(newCpuMessage).not.toContain('Suppressed');
+  });
+
+  it('respects custom host_alert_suppression_mins setting (5 minutes)', async () => {
+    const svc = MonitorService.getInstance();
+    const baseTime = 1_700_000_000_000;
+    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
+
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '5' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+
+    // 3 minutes — still within custom 5-minute window.
+    nowSpy.mockReturnValue(baseTime + 3 * 60 * 1000);
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '5' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+
+    // 6 minutes — past the custom window; follow-up should fire.
+    nowSpy.mockReturnValue(baseTime + 6 * 60 * 1000);
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '5' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
+
+    nowSpy.mockRestore();
+  });
+
+  it('post-restart recovery clears persisted timestamp so re-breach inside the window fires fresh', async () => {
+    // Codex audit scenario:
+    //   T0: breach fires → system_state['last_host_ram_alert_ts'] = T0
+    //   T1: process restart (in-memory state cleared, persisted T0 survives)
+    //   T2: metric drops below threshold → clearHostMetricSuppression() runs
+    //   T3: metric re-breaches within suppression window
+    // Expected: fresh first alert at T3 (no Suppressed suffix).
+    // Pre-fix bug: clearHostMetricSuppression early-returned on missing in-
+    // memory state, leaving persisted T0 alive; T3 hit the restart-survival
+    // path and was silently suppressed.
+    const baseTime = 1_700_000_000_000;
+    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime + 5 * 60 * 1000); // T1+T2 = T0+5min
+
+    // Simulate persisted state from a previous process's first fire.
+    let persistedRamTs: string | null = String(baseTime);
+    mockGetSystemState.mockImplementation((key: string) =>
+      key === 'last_host_ram_alert_ts' ? persistedRamTs : null,
+    );
+    mockSetSystemState.mockImplementation((key: string, value: string) => {
+      if (key === 'last_host_ram_alert_ts') persistedRamTs = value;
+    });
+
+    // T2: post-restart cycle finds metric BELOW threshold (recovered).
+    mockMem.mockResolvedValue({ used: 4e9, total: 16e9 });
+    const svc = MonitorService.getInstance();
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+    // No dispatch on a non-breach cycle.
+    expect(mockDispatchAlert).not.toHaveBeenCalled();
+    // Recovery branch MUST have reset persisted state so a re-breach is
+    // treated as fresh, not as a still-active cooldown.
+    expect(persistedRamTs).toBe('0');
+
+    // T3: re-breach 10 minutes later (well inside the original 60-min window).
+    nowSpy.mockReturnValue(baseTime + 15 * 60 * 1000);
+    mockMem.mockResolvedValue({ used: 15e9, total: 16e9 });
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+    const [, , message] = mockDispatchAlert.mock.calls[0];
+    expect(message).not.toContain('Suppressed');
+
+    nowSpy.mockRestore();
+  });
+
+  it('caps host_alert_suppression_mins at 24h (1440) to defend against unvalidated single-key POST writes', async () => {
+    const svc = MonitorService.getInstance();
+    const baseTime = 1_700_000_000_000;
+    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
+
+    // The single-key POST /api/settings path stores allowlisted keys
+    // without zod re-validation. A 999999999-minute value (or any
+    // accidental garbage above 1440) must not let a metric stay
+    // suppressed for centuries; cap at 24h in the consumer.
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '999999999' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+
+    // 1441 minutes later — past the 24h cap.
+    nowSpy.mockReturnValue(baseTime + 1441 * 60 * 1000);
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '999999999' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
+
+    nowSpy.mockRestore();
+  });
+
+  it('post-restart with persisted timestamp inside window does not re-fire', async () => {
+    const baseTime = 1_700_000_000_000;
+    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime + 10 * 60 * 1000); // "now" = T+10min
+
+    // Simulate a previous process having persisted a fire 10 minutes ago.
+    mockGetSystemState.mockImplementation((key: string) =>
+      key === 'last_host_ram_alert_ts' ? String(baseTime) : null,
+    );
+
+    const svc = MonitorService.getInstance();
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+
+    // Post-restart cycle must NOT re-fire because the persisted cooldown
+    // is still active (10 min into the default 60 min window).
+    expect(mockDispatchAlert).not.toHaveBeenCalled();
+
+    nowSpy.mockRestore();
+  });
+
+  it('zero or negative suppression_mins falls back to default', async () => {
+    const svc = MonitorService.getInstance();
+    const baseTime = 1_700_000_000_000;
+    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
+
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '0' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+
+    // Default 60-minute window applies — a cycle 30 minutes later stays
+    // suppressed even though the setting said 0.
+    nowSpy.mockReturnValue(baseTime + 30 * 60 * 1000);
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '0' });
+    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
+
+    nowSpy.mockRestore();
+  });
+});
+
 // ── Alert breach state machine ─────────────────────────────────────────

 describe('MonitorService - breach state machine', () => {
@@ -14,6 +14,7 @@ const ALLOWED_SETTING_KEYS = new Set([
  'host_cpu_limit',
  'host_ram_limit',
  'host_disk_limit',
+  'host_alert_suppression_mins',
  'docker_janitor_gb',
  'global_crash',
  'developer_mode',
@@ -28,6 +29,7 @@ const SettingsPatchSchema = z.object({
  host_cpu_limit: z.coerce.number().int().min(1).max(100).transform(String),
  host_ram_limit: z.coerce.number().int().min(1).max(100).transform(String),
  host_disk_limit: z.coerce.number().int().min(1).max(100).transform(String),
+  host_alert_suppression_mins: z.coerce.number().int().min(1).max(1440).transform(String),
  docker_janitor_gb: z.coerce.number().min(0).transform(String),
  global_crash: z.enum(['0', '1']),
  developer_mode: z.enum(['0', '1']),
@@ -1229,6 +1229,7 @@ export class DatabaseService {
        stmt.run('host_cpu_limit', '90');
        stmt.run('host_ram_limit', '90');
        stmt.run('host_disk_limit', '90');
+        stmt.run('host_alert_suppression_mins', '60');
        stmt.run('global_crash', '1');
        stmt.run('docker_janitor_gb', '5');
        stmt.run('developer_mode', '0');
@@ -58,6 +58,31 @@ const HOST_ALERT_KEYS = {
    janitor: 'last_janitor_alert_timestamp',
 } as const;

+type HostMetricKey = 'cpu' | 'ram' | 'disk';
+
+interface HostAlertSuppressionState {
+    firstFiredAt: number;
+    lastFiredAt: number;
+    suppressedCount: number;
+}
+
+// F-11 dedup for host-metric notifications (CPU, RAM, disk). The previous
+// 5-minute cooldown produced 7+ identical messages over a 35-minute window
+// when a host stayed over threshold, flooding Discord/Slack routes. State is
+// per-metric and module-scope so a singleton-replacement (only happens in
+// tests) does not lose tracking. Cleared on process restart; the persisted
+// system_state row keeps post-restart re-fires bounded.
+const hostAlertSuppressionState = new Map<HostMetricKey, HostAlertSuppressionState>();
+const DEFAULT_HOST_ALERT_SUPPRESSION_MIN = 60;
+// Mirror the zod schema upper bound in `routes/settings.ts` so the single-key
+// POST path (which lacks zod re-validation) cannot push the consumer past
+// the validated range.
+const MAX_HOST_ALERT_SUPPRESSION_MIN = 1440;
+
+export function _resetHostAlertSuppressionStateForTests(): void {
+    hostAlertSuppressionState.clear();
+}
+
 const STATS_TIMEOUT_MS = 10_000;
 const FLOAT_EQ_EPSILON = 0.01;

@@ -226,7 +251,17 @@ export class MonitorService {
    }

    private async evaluateGlobalSettings(settings: Record<string, string>) {
-        const HOST_ALERT_COOLDOWN_MS = 5 * 60 * 1000; // 5 minutes between repeat alerts
+        // F-11 suppression window. Default 60 min; configurable via global
+        // settings. NaN / non-positive values fall back to the default so a
+        // misconfigured row never disables the dedup entirely. Cap at 24h
+        // to defend against the single-key POST /api/settings path (which
+        // stores allowlisted keys without zod re-validation): an accidental
+        // 999999999-minute value must not silence host alerts for centuries.
+        const suppressionMinRaw = parseInt(settings['host_alert_suppression_mins'] || '', 10);
+        const suppressionMin = isNaN(suppressionMinRaw) || suppressionMinRaw <= 0
+            ? DEFAULT_HOST_ALERT_SUPPRESSION_MIN
+            : Math.min(suppressionMinRaw, MAX_HOST_ALERT_SUPPRESSION_MIN);
+        const suppressionMs = suppressionMin * 60 * 1000;

        // 1. Host Limits — fetch CPU, RAM, disk concurrently
        try {
@@ -239,23 +274,29 @@ export class MonitorService {
            const cpuUsage = currentLoad.currentLoad;
            const cpuLimit = parseFloat(settings['host_cpu_limit']);
            if (!isNaN(cpuLimit) && cpuLimit > 0 && cpuUsage > cpuLimit) {
-                await this.dispatchWithCooldown(HOST_ALERT_KEYS.cpu, HOST_ALERT_COOLDOWN_MS, 'warning', 'monitor_alert',
+                await this.dispatchHostMetricAlert('cpu', 'warning', suppressionMs,
                    `Host CPU utilization is critically high: ${cpuUsage.toFixed(1)}% (Threshold: ${cpuLimit}%)`);
+            } else {
+                this.clearHostMetricSuppression('cpu');
            }

            const ramUsage = (mem.used / mem.total) * 100;
            const ramLimit = parseFloat(settings['host_ram_limit']);
            if (!isNaN(ramLimit) && ramLimit > 0 && ramUsage > ramLimit) {
-                await this.dispatchWithCooldown(HOST_ALERT_KEYS.ram, HOST_ALERT_COOLDOWN_MS, 'warning', 'monitor_alert',
+                await this.dispatchHostMetricAlert('ram', 'warning', suppressionMs,
                    `Host Memory utilization is critically high: ${ramUsage.toFixed(1)}% (Threshold: ${ramLimit}%)`);
+            } else {
+                this.clearHostMetricSuppression('ram');
            }

            const mainDisk = fsSize.find(fs => fs.mount === '/' || fs.mount === 'C:') || fsSize[0];
            if (mainDisk) {
                const diskLimit = parseFloat(settings['host_disk_limit']);
                if (!isNaN(diskLimit) && diskLimit > 0 && mainDisk.use > diskLimit) {
-                    await this.dispatchWithCooldown(HOST_ALERT_KEYS.disk, HOST_ALERT_COOLDOWN_MS, 'warning', 'monitor_alert',
+                    await this.dispatchHostMetricAlert('disk', 'warning', suppressionMs,
                        `Host Disk space utilization is critically high: ${mainDisk.use.toFixed(1)}% (Threshold: ${diskLimit}%)`);
+                } else {
+                    this.clearHostMetricSuppression('disk');
                }
            }
        } catch (e) {
@@ -687,6 +728,104 @@ export class MonitorService {
        }
    }

+    /**
+     * F-11 host-metric dispatch with per-key suppression window + count
+     * summary. Replaces the old 5-minute hardcoded cooldown for CPU/RAM/disk.
+     *
+     * Behaviour:
+     *  - First breach (no in-memory state, no recent persisted timestamp) →
+     *    dispatch immediately and seed in-memory state.
+     *  - Restart on a still-breaching host (no in-memory state, but persisted
+     *    timestamp falls inside the window) → silently re-seed from the
+     *    persisted timestamp so post-restart cycles do not re-fire.
+     *  - Breach within an open suppression window → silently increment
+     *    suppressedCount; no dispatch, no broadcast.
+     *  - Breach after window elapses → fire one follow-up with the suffix
+     *    `Suppressed N alerts in the last Xm; first over threshold at HH:MM UTC.`
+     *
+     * Recovery is handled by the caller via clearHostMetricSuppression(): when
+     * the metric drops below threshold, in-memory state and the persisted
+     * timestamp are cleared so the next breach fires fresh.
+     */
+    private async dispatchHostMetricAlert(
+        key: HostMetricKey,
+        severity: 'info' | 'warning' | 'error',
+        suppressionMs: number,
+        baseMessage: string,
+    ): Promise<void> {
+        const db = DatabaseService.getInstance();
+        const stateKey = HOST_ALERT_KEYS[key];
+        const now = Date.now();
+        const state = hostAlertSuppressionState.get(key);
+
+        if (!state) {
+            const persistedLast = parseInt(db.getSystemState(stateKey) || '0', 10);
+            if (persistedLast > 0 && now - persistedLast < suppressionMs) {
+                // Post-restart on a still-breaching host: respect the
+                // persisted cooldown so we do not re-fire immediately. Seed
+                // in-memory state from the persisted timestamp; count this
+                // cycle as suppressed for an honest follow-up summary.
+                hostAlertSuppressionState.set(key, {
+                    firstFiredAt: persistedLast,
+                    lastFiredAt: persistedLast,
+                    suppressedCount: 1,
+                });
+                return;
+            }
+            // First-ever fire (fresh process, or post-recovery re-breach).
+            await NotificationService.getInstance().dispatchAlert(severity, 'monitor_alert', baseMessage);
+            hostAlertSuppressionState.set(key, {
+                firstFiredAt: now,
+                lastFiredAt: now,
+                suppressedCount: 0,
+            });
+            db.setSystemState(stateKey, now.toString());
+            return;
+        }
+
+        if (now - state.lastFiredAt < suppressionMs) {
+            state.suppressedCount += 1;
+            return;
+        }
+
+        const windowMin = Math.max(1, Math.round((now - state.lastFiredAt) / 60000));
+        const firstAt = new Date(state.firstFiredAt).toISOString().slice(11, 16);
+        const plural = state.suppressedCount === 1 ? '' : 's';
+        const suffix = ` Suppressed ${state.suppressedCount} alert${plural} in the last ${windowMin}m; first over threshold at ${firstAt} UTC.`;
+        await NotificationService.getInstance().dispatchAlert(severity, 'monitor_alert', baseMessage + suffix);
+        state.lastFiredAt = now;
+        state.suppressedCount = 0;
+        db.setSystemState(stateKey, now.toString());
+    }
+
+    /**
+     * Clear in-memory + persisted suppression state for a host metric. Called
+     * when the metric drops below threshold so the next breach fires fresh
+     * (no "Suppressed N" suffix). Resetting the persisted row to '0' is
+     * essential: a stale persisted timestamp inside the window would silently
+     * absorb the next breach via the restart-survivability path.
+     *
+     * The persisted reset must run independently of in-memory state because
+     * the two halves can drift after a process restart: the in-memory Map is
+     * empty on a fresh process, but the system_state row from the previous
+     * process still points at the old fire timestamp. If recovery happens
+     * before another evaluate cycle re-seeds in-memory state, an early-return
+     * on the in-memory check alone would leave the persisted row alive, and
+     * the next re-breach within the original window would be silently
+     * suppressed instead of firing fresh (Codex audit on PR #1175).
+     */
+    private clearHostMetricSuppression(key: HostMetricKey): void {
+        const stateKey = HOST_ALERT_KEYS[key];
+        const db = DatabaseService.getInstance();
+        const hadInMemory = hostAlertSuppressionState.delete(key);
+        const persisted = db.getSystemState(stateKey);
+        const needsPersistedReset = persisted !== null && persisted !== '0';
+        if (!hadInMemory && !needsPersistedReset) return;
+        if (needsPersistedReset) {
+            db.setSystemState(stateKey, '0');
+        }
+    }
+
    private calculateCpuPercent(stats: DockerContainerStats): number {
        let cpuPercent = 0.0;
        if (!stats?.cpu_stats?.cpu_usage || !stats?.precpu_stats?.cpu_usage) return 0.0;
@@ -282,7 +282,11 @@ Real-time on the Docker event stream:

 ### Host thresholds

-`warning`/`monitor_alert` for host CPU, RAM, and disk when the configured threshold is exceeded. 5-minute cooldown per signal. Example: `Host CPU utilization is critically high: 92% (Threshold: 90%)`. Evaluated on the 30-second monitor tick.
+`warning`/`monitor_alert` for host CPU, RAM, and disk when the configured threshold is exceeded. Evaluated on the 30-second monitor tick. Example: `Host CPU utilization is critically high: 92% (Threshold: 90%)`.
+
+Each metric (CPU, RAM, disk) carries its own suppression window. The first time a metric crosses its threshold, one notification fires immediately. While the metric stays over threshold within the window, additional cycles are silently counted but not dispatched. The next dispatch after the window elapses carries a summary suffix: `Suppressed 119 alerts in the last 60m; first over threshold at 14:23 UTC.` When the metric drops below threshold, the counter resets so the next breach fires fresh.
+
+The suppression window defaults to 60 minutes and is configured per node in **Settings → System → Host thresholds → Alert suppression**.

 ### Docker janitor

@@ -58,17 +58,35 @@ test.describe('Stack management', () => {
    await loginAs(page);
    await waitForStacksLoaded(page);

-    // Count POST /api/stacks calls and hold the response briefly so the
-    // disabled-button window is observable from the test. Awaiting
-    // route.continue() avoids the fire-and-forget pattern that can subtly
-    // stall a request on a busy CI runner.
    let postCount = 0;
+    let releaseCreate: () => void = () => undefined;
+    let markFirstPostSeen: () => void = () => undefined;
+    let releasedCreate = false;
+    const createMayContinue = new Promise<void>((resolve) => {
+      releaseCreate = () => {
+        if (releasedCreate) return;
+        releasedCreate = true;
+        resolve();
+      };
+    });
+    const firstPostSeen = new Promise<void>((resolve) => {
+      markFirstPostSeen = resolve;
+    });
+
+    // Count POST /api/stacks calls and hold the first response until after a
+    // second click has been attempted. Continuing the original route after an
+    // assertion failure can race Playwright cleanup, so fetch and fulfill it
+    // explicitly once the test is ready to let the request complete.
    await page.route('**/api/stacks', async (route) => {
      if (route.request().method() === 'POST') {
        postCount += 1;
-        await new Promise((resolve) => setTimeout(resolve, 400));
+        markFirstPostSeen();
+        await createMayContinue;
+        const response = await route.fetch();
+        await route.fulfill({ response });
+        return;
      }
-      await route.continue();
+      await route.fallback();
    });

    try {
@@ -78,22 +96,21 @@ test.describe('Stack management', () => {

      const createBtn = page.locator('[role="dialog"]').getByRole('button', { name: /^Create/ });
      await createBtn.click();
+      await firstPostSeen;

-      // The synchronous useRef guard plus React's disabled re-render must
-      // surface a disabled Create button while the POST is in flight. If this
-      // assertion ever times out, the busy-state path has regressed.
-      await expect(createBtn).toBeDisabled({ timeout: 1_500 });
-
-      // Best-effort second click against the now-disabled native button. A real
-      // user mash translates to a no-op at the browser level on a disabled
-      // submit button; the synchronous ref guard catches it even if Playwright
-      // managed to dispatch the click anyway.
+      // Best-effort second click while the first POST is still in flight. A
+      // real user mash translates to a no-op once the button disables; the
+      // synchronous ref guard catches it even if Playwright manages to dispatch
+      // the click before React commits the disabled state.
      await createBtn.click({ force: true, timeout: 500 }).catch(() => undefined);
+      await page.waitForTimeout(100);
+      expect(postCount).toBe(1);

+      releaseCreate();
      await expect(page.getByRole('dialog')).toBeHidden({ timeout: 8_000 });
      await expect(page.getByText(`Stack "${stackName}" created.`)).toBeVisible({ timeout: 5_000 });
-      expect(postCount).toBe(1);
    } finally {
+      releaseCreate();
      await page.unroute('**/api/stacks');
      // Cleanup
      await page.evaluate(async (name) => {
@@ -132,12 +132,13 @@ function SettingsSkeleton() {
    );
 }

-type SystemFields = Pick<PatchableSettings, 'host_cpu_limit' | 'host_ram_limit' | 'host_disk_limit' | 'docker_janitor_gb' | 'global_crash'>;
+type SystemFields = Pick<PatchableSettings, 'host_cpu_limit' | 'host_ram_limit' | 'host_disk_limit' | 'host_alert_suppression_mins' | 'docker_janitor_gb' | 'global_crash'>;

 const DEFAULT_SYSTEM: SystemFields = {
    host_cpu_limit: DEFAULT_SETTINGS.host_cpu_limit,
    host_ram_limit: DEFAULT_SETTINGS.host_ram_limit,
    host_disk_limit: DEFAULT_SETTINGS.host_disk_limit,
+    host_alert_suppression_mins: DEFAULT_SETTINGS.host_alert_suppression_mins,
    docker_janitor_gb: DEFAULT_SETTINGS.docker_janitor_gb,
    global_crash: DEFAULT_SETTINGS.global_crash,
 };
@@ -155,6 +156,7 @@ export function SystemSection({ onDirtyChange }: SystemSectionProps) {
        if (settings.host_cpu_limit !== baseline.host_cpu_limit) n++;
        if (settings.host_ram_limit !== baseline.host_ram_limit) n++;
        if (settings.host_disk_limit !== baseline.host_disk_limit) n++;
+        if (settings.host_alert_suppression_mins !== baseline.host_alert_suppression_mins) n++;
        if (settings.docker_janitor_gb !== baseline.docker_janitor_gb) n++;
        if (settings.global_crash !== baseline.global_crash) n++;
        return n;
@@ -188,6 +190,7 @@ export function SystemSection({ onDirtyChange }: SystemSectionProps) {
                    host_cpu_limit: nodeData.host_cpu_limit ?? DEFAULT_SETTINGS.host_cpu_limit,
                    host_ram_limit: nodeData.host_ram_limit ?? DEFAULT_SETTINGS.host_ram_limit,
                    host_disk_limit: nodeData.host_disk_limit ?? DEFAULT_SETTINGS.host_disk_limit,
+                    host_alert_suppression_mins: nodeData.host_alert_suppression_mins ?? DEFAULT_SETTINGS.host_alert_suppression_mins,
                    docker_janitor_gb: nodeData.docker_janitor_gb ?? DEFAULT_SETTINGS.docker_janitor_gb,
                    global_crash: (nodeData.global_crash as '0' | '1') ?? DEFAULT_SETTINGS.global_crash,
                };
@@ -235,7 +238,7 @@ export function SystemSection({ onDirtyChange }: SystemSectionProps) {
            <SettingsSection title="Host thresholds">
                <SettingsField
                    label="CPU limit"
-                    helper="Alerts fire when the 5-minute average exceeds this percentage."
+                    helper="Alerts fire when host CPU utilization exceeds this percentage."
                >
                    <NumberChip
                        value={settings.host_cpu_limit || '90'}
@@ -272,6 +275,18 @@ export function SystemSection({ onDirtyChange }: SystemSectionProps) {
                        warnOver={95}
                    />
                </SettingsField>
+                <SettingsField
+                    label="Alert suppression"
+                    helper="How long to wait before resending a host alert while the metric stays over threshold. The follow-up message includes a count of suppressed cycles."
+                >
+                    <NumberChip
+                        value={settings.host_alert_suppression_mins || '60'}
+                        onChange={(v) => onSettingChange('host_alert_suppression_mins', v)}
+                        suffix="min"
+                        min={1}
+                        max={1440}
+                    />
+                </SettingsField>
            </SettingsSection>

            <SettingsSection title="Docker hygiene">
@@ -2,6 +2,7 @@ export interface PatchableSettings {
    host_cpu_limit?: string;
    host_ram_limit?: string;
    host_disk_limit?: string;
+    host_alert_suppression_mins?: string;
    docker_janitor_gb?: string;
    global_crash?: '0' | '1';
    developer_mode?: '0' | '1';
@@ -15,6 +16,7 @@ export const DEFAULT_SETTINGS: PatchableSettings = {
    host_cpu_limit: '90',
    host_ram_limit: '90',
    host_disk_limit: '90',
+    host_alert_suppression_mins: '60',
    global_crash: '1',
    docker_janitor_gb: '5',
    developer_mode: '0',