sencho/backend/src/__tests__/monitor-service.test.ts

/**
 * Unit tests for MonitorService — alert state machine, metric calculations,
 * cleanup delegation, global settings evaluation, and concurrency guards.
 */
import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vitest';
import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsMock } from './helpers/arcstatsFsMock';

// ── Hoisted mocks ──────────────────────────────────────────────────────

const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContainerMetric,
  mockCleanupOldMetrics, mockCleanupOldNotifications, mockCleanupOldAuditLogs,
  mockUpdateStackAlertLastFired, mockGetStackAlertServiceCooldown,
  mockHasAnyStackAlertServiceCooldown, mockUpsertStackAlertServiceCooldown,
  mockGetSystemState, mockSetSystemState,
  mockPruneScanHistoryPerImage, mockDeleteScansByImageRef,
  mockGetRunningContainers, mockGetAllContainers, mockGetContainerStatsStream,
  mockGetContainerRestartCount, mockGetDiskUsage, mockGetImages, mockGetStacks,
  mockDispatchAlert,
  mockCurrentLoad, mockMem, mockFsSize,
  mockExecAsync,
  mockFetchLatestSenchoVersion,
  mockGetLatestVersion,
  mockGetLatestVersionInfo,
  mockGetSenchoVersion,
  mockGetPinInfo,
  mockGetIdentity,
  mockGetAuthForRegistry,
  mockDetectSelfDevBuildUpdate,
} = vi.hoisted(() => ({
  mockGetGlobalSettings: vi.fn().mockReturnValue({}),
  mockGetNodes: vi.fn().mockReturnValue([]),
  mockGetStackAlerts: vi.fn().mockReturnValue([]),
  mockAddContainerMetric: vi.fn(),
  mockCleanupOldMetrics: vi.fn(),
  mockCleanupOldNotifications: vi.fn(),
  mockCleanupOldAuditLogs: vi.fn(),
  mockUpdateStackAlertLastFired: vi.fn(),
  mockGetStackAlertServiceCooldown: vi.fn().mockReturnValue(null),
  mockHasAnyStackAlertServiceCooldown: vi.fn().mockReturnValue(false),
  mockUpsertStackAlertServiceCooldown: vi.fn(),
  mockGetSystemState: vi.fn().mockReturnValue(null),
  mockSetSystemState: vi.fn(),
  mockPruneScanHistoryPerImage: vi.fn().mockReturnValue(0),
  mockDeleteScansByImageRef: vi.fn().mockReturnValue(0),
  mockGetRunningContainers: vi.fn().mockResolvedValue([]),
  mockGetAllContainers: vi.fn().mockResolvedValue([]),
  mockGetContainerStatsStream: vi.fn().mockResolvedValue('{}'),
  mockGetContainerRestartCount: vi.fn().mockResolvedValue(0),
  mockGetDiskUsage: vi.fn().mockResolvedValue({
    reclaimableImages: 0, reclaimableContainers: 0, reclaimableVolumes: 0, reclaimableBuildCache: 0,
    reclaimableImageCount: 0, reclaimableContainerCount: 0, reclaimableVolumeCount: 0, reclaimableBuildCacheCount: 0,
  }),
  mockGetImages: vi.fn().mockResolvedValue([]),
  mockGetStacks: vi.fn().mockResolvedValue([]),
  mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
  mockCurrentLoad: vi.fn().mockResolvedValue({ currentLoad: 10 }),
  mockMem: vi.fn().mockResolvedValue({ total: 16e9, used: 4e9, active: 4e9, available: 12e9, free: 12e9, buffcache: 0 }),
  mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]),
  mockExecAsync: vi.fn().mockResolvedValue({ stdout: '' }),
  mockFetchLatestSenchoVersion: vi.fn().mockRejectedValue(new Error('not configured')),
  mockGetLatestVersion: vi.fn().mockResolvedValue(null),
  mockGetLatestVersionInfo: vi.fn().mockResolvedValue(null),
  mockGetSenchoVersion: vi.fn().mockReturnValue(null),
  // Default: no known compose pin, so checkSenchoVersion() falls through
  // unchanged and checkSenchoDevBuild() is a no-op, matching pre-existing
  // test expectations.
  mockGetPinInfo: vi.fn().mockResolvedValue(null),
  mockGetIdentity: vi.fn().mockReturnValue({
    containerId: null, containerName: null, composeProjectName: null,
    imageId: null, networkNames: [], volumeNames: [],
  }),
  mockGetAuthForRegistry: vi.fn().mockResolvedValue(null),
  mockDetectSelfDevBuildUpdate: vi.fn().mockResolvedValue({ kind: 'up_to_date' }),
}));

vi.mock('../services/DatabaseService', () => ({
  DatabaseService: {
    getInstance: () => ({
      getGlobalSettings: mockGetGlobalSettings,
      getNodes: mockGetNodes,
      getStackAlerts: mockGetStackAlerts,
      addContainerMetric: mockAddContainerMetric,
      cleanupOldMetrics: mockCleanupOldMetrics,
      cleanupOldNotifications: mockCleanupOldNotifications,
      cleanupOldAuditLogs: mockCleanupOldAuditLogs,
      updateStackAlertLastFired: mockUpdateStackAlertLastFired,
      getStackAlertServiceCooldown: mockGetStackAlertServiceCooldown,
      hasAnyStackAlertServiceCooldown: mockHasAnyStackAlertServiceCooldown,
      upsertStackAlertServiceCooldown: mockUpsertStackAlertServiceCooldown,
      getSystemState: mockGetSystemState,
      setSystemState: mockSetSystemState,
      pruneScanHistoryPerImage: mockPruneScanHistoryPerImage,
      deleteScansByImageRef: mockDeleteScansByImageRef,
    }),
  },
}));

vi.mock('../services/DockerController', () => ({
  default: {
    getInstance: () => ({
      getRunningContainers: mockGetRunningContainers,
      getAllContainers: mockGetAllContainers,
      getContainerStatsStream: mockGetContainerStatsStream,
      getContainerRestartCount: mockGetContainerRestartCount,
      getDiskUsage: mockGetDiskUsage,
      getImages: mockGetImages,
    }),
  },
}));

vi.mock('../services/FileSystemService', () => ({
  FileSystemService: {
    getInstance: () => ({
      getStacks: mockGetStacks,
    }),
  },
}));

vi.mock('../utils/version-check', () => ({
  fetchLatestSenchoVersion: (...args: unknown[]) => mockFetchLatestSenchoVersion(...args),
  getLatestVersion: (...args: unknown[]) => mockGetLatestVersion(...args),
  getLatestVersionInfo: (...args: unknown[]) => mockGetLatestVersionInfo(...args),
}));

vi.mock('../services/CapabilityRegistry', async () => {
  const semver = await import('semver');
  return {
    isValidVersion: (v: string | null | undefined): v is string =>
      !!v && v !== 'unknown' && v !== '0.0.0-dev' && !!semver.default.valid(v),
    getSenchoVersion: () => mockGetSenchoVersion(),
  };
});

vi.mock('../services/NotificationService', () => ({
  NotificationService: {
    getInstance: () => ({
      dispatchAlert: mockDispatchAlert,
    }),
  },
}));

vi.mock('../services/SelfUpdateService', () => ({
  default: {
    getInstance: () => ({
      getPinInfo: mockGetPinInfo,
    }),
  },
}));

vi.mock('../services/SelfIdentityService', () => ({
  default: {
    getInstance: () => ({
      getIdentity: mockGetIdentity,
    }),
  },
}));

vi.mock('../services/RegistryService', () => ({
  RegistryService: {
    getInstance: () => ({
      getAuthForRegistry: mockGetAuthForRegistry,
    }),
  },
}));

vi.mock('../services/selfDevBuildDetect', () => ({
  detectSelfDevBuildUpdate: (...args: unknown[]) => mockDetectSelfDevBuildUpdate(...args),
}));

vi.mock('../services/NodeRegistry', () => ({
  NodeRegistry: {
    getInstance: () => ({
      getDefaultNodeId: () => 1,
      getNode: () => ({ id: 1, name: 'local-test', type: 'local' }),
    }),
  },
}));

vi.mock('systeminformation', () => ({
  default: {
    currentLoad: (...args: unknown[]) => mockCurrentLoad(...args),
    mem: (...args: unknown[]) => mockMem(...args),
    fsSize: (...args: unknown[]) => mockFsSize(...args),
  },
}));

vi.mock('child_process', () => ({
  exec: vi.fn(),
}));

vi.mock('util', () => ({
  promisify: () => mockExecAsync,
}));

import { MonitorService, _resetHostAlertSuppressionStateForTests } from '../services/MonitorService';

// Host memory now reads ZFS ARC stats; intercept those reads so the suite does
// not depend on whether the machine running it is itself a ZFS host.
let arcFs: ArcstatsFsMock;
beforeAll(() => {
  arcFs = installArcstatsFsMock();
});

beforeEach(() => {
  vi.clearAllMocks();
  arcFs.clear();
  (MonitorService as any).instance = undefined;
  _resetHostAlertSuppressionStateForTests();
  mockGetSystemState.mockReturnValue(null);
  mockGetPinInfo.mockResolvedValue(null);
  mockGetIdentity.mockReturnValue({
    containerId: null, containerName: null, composeProjectName: null,
    imageId: null, networkNames: [], volumeNames: [],
  });
  mockGetAuthForRegistry.mockResolvedValue(null);
  mockDetectSelfDevBuildUpdate.mockResolvedValue({ kind: 'up_to_date' });
});

// si.mem() returns active/available alongside used (used counts reclaimable page
// cache, active/available exclude it). The host-RAM percentage is derived from the
// cache-excluded figures, so mocks must supply a realistic shape. `realUsed` is the
// active working set; available/free are the remainder.
const memSample = (realUsed: number, total = 16e9) => ({
  total, used: realUsed, active: realUsed,
  available: total - realUsed, free: total - realUsed, buffcache: 0,
});

// ── Pure calculation helpers (accessed via private method reflection) ───

describe('MonitorService - calculateCpuPercent', () => {
  function calcCpu(stats: any): number {
    const svc = MonitorService.getInstance();
    return (svc as any).calculateCpuPercent(stats);
  }

  it('returns correct percentage for normal stats', () => {
    const stats = {
      cpu_stats: { cpu_usage: { total_usage: 2000 }, system_cpu_usage: 10000, online_cpus: 4 },
      precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
    };
    // (1000 / 5000) * 4 * 100 = 80%
    expect(calcCpu(stats)).toBeCloseTo(80, 1);
  });

  it('returns 0 when cpu_stats is missing', () => {
    expect(calcCpu({})).toBe(0);
    expect(calcCpu(null)).toBe(0);
    expect(calcCpu({ cpu_stats: {} })).toBe(0);
  });

  it('returns 0 when systemDelta is zero', () => {
    const stats = {
      cpu_stats: { cpu_usage: { total_usage: 2000 }, system_cpu_usage: 5000, online_cpus: 1 },
      precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
    };
    expect(calcCpu(stats)).toBe(0);
  });

  it('accounts for online_cpus count', () => {
    const stats = {
      cpu_stats: { cpu_usage: { total_usage: 2000 }, system_cpu_usage: 10000, online_cpus: 8 },
      precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
    };
    // (1000/5000) * 8 * 100 = 160%
    expect(calcCpu(stats)).toBeCloseTo(160, 1);
  });

  it('falls back to percpu_usage length when online_cpus missing', () => {
    const stats = {
      cpu_stats: { cpu_usage: { total_usage: 2000, percpu_usage: [0, 0] }, system_cpu_usage: 10000 },
      precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
    };
    // (1000/5000) * 2 * 100 = 40%
    expect(calcCpu(stats)).toBeCloseTo(40, 1);
  });
});

describe('MonitorService - calculateMemoryPercent', () => {
  function calcMem(stats: any): number {
    const svc = MonitorService.getInstance();
    return (svc as any).calculateMemoryPercent(stats);
  }

  it('returns correct percentage subtracting cache', () => {
    const stats = {
      memory_stats: { usage: 500e6, limit: 1e9, stats: { cache: 100e6 } },
    };
    // (400e6 / 1e9) * 100 = 40%
    expect(calcMem(stats)).toBeCloseTo(40, 1);
  });

  it('returns 0 when memory_stats is missing', () => {
    expect(calcMem({})).toBe(0);
    expect(calcMem({ memory_stats: {} })).toBe(0);
  });

  it('returns 0 when limit is zero', () => {
    const stats = { memory_stats: { usage: 100, limit: 0 } };
    expect(calcMem(stats)).toBe(0);
  });

  it('handles missing cache field', () => {
    const stats = { memory_stats: { usage: 500e6, limit: 1e9 } };
    // No cache → (500e6 / 1e9) * 100 = 50%
    expect(calcMem(stats)).toBeCloseTo(50, 1);
  });
});

describe('MonitorService - calculateNetwork', () => {
  function calcNet(stats: any, dir: 'rx' | 'tx'): number {
    const svc = MonitorService.getInstance();
    return (svc as any).calculateNetwork(stats, dir);
  }

  it('sums rx_bytes across all interfaces', () => {
    const stats = {
      networks: {
        eth0: { rx_bytes: 1024 * 1024, tx_bytes: 0 },
        eth1: { rx_bytes: 2 * 1024 * 1024, tx_bytes: 0 },
      },
    };
    expect(calcNet(stats, 'rx')).toBeCloseTo(3, 0); // 3 MB
  });

  it('sums tx_bytes across all interfaces', () => {
    const stats = {
      networks: {
        eth0: { rx_bytes: 0, tx_bytes: 512 * 1024 },
      },
    };
    expect(calcNet(stats, 'tx')).toBeCloseTo(0.5, 1); // 0.5 MB
  });

  it('returns 0 when no networks present', () => {
    expect(calcNet({}, 'rx')).toBe(0);
    expect(calcNet({ networks: null }, 'tx')).toBe(0);
  });
});

describe('MonitorService - evaluateCondition', () => {
  function evalCond(actual: number, operator: string, threshold: number): boolean {
    const svc = MonitorService.getInstance();
    return (svc as any).evaluateCondition(actual, operator, threshold);
  }

  it('handles > operator', () => {
    expect(evalCond(81, '>', 80)).toBe(true);
    expect(evalCond(80, '>', 80)).toBe(false);
  });

  it('handles < operator', () => {
    expect(evalCond(79, '<', 80)).toBe(true);
    expect(evalCond(80, '<', 80)).toBe(false);
  });

  it('handles >= operator at boundary', () => {
    expect(evalCond(80, '>=', 80)).toBe(true);
    expect(evalCond(79, '>=', 80)).toBe(false);
  });

  it('handles <= operator at boundary', () => {
    expect(evalCond(80, '<=', 80)).toBe(true);
    expect(evalCond(81, '<=', 80)).toBe(false);
  });

  it('handles == operator', () => {
    expect(evalCond(80, '==', 80)).toBe(true);
    expect(evalCond(81, '==', 80)).toBe(false);
  });

  it('returns false for unknown operator', () => {
    expect(evalCond(80, '!=', 80)).toBe(false);
    expect(evalCond(80, 'foo', 80)).toBe(false);
  });
});

// ── Integration-level: evaluateGlobalSettings ──────────────────────────

describe('MonitorService - evaluateGlobalSettings', () => {
  it('dispatches CPU warning when over threshold', async () => {
    mockGetGlobalSettings.mockReturnValue({ host_cpu_limit: '50' });
    mockCurrentLoad.mockResolvedValue({ currentLoad: 75 });

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' });

    expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));
  });

  it('does not dispatch when CPU below threshold', async () => {
    mockCurrentLoad.mockResolvedValue({ currentLoad: 25 });

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' });

    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));
  });

  it('dispatches RAM warning when over threshold', async () => {
    mockMem.mockResolvedValue(memSample(15e9)); // ~94%

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });

    expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
  });

  it('does not alert when most RAM is reclaimable page cache, not real usage', async () => {
    // Busy host: 15.8G "used" including cache, but only 1.6G actively in use and
    // 14.4G available. The naive used/total formula reads ~99% and would breach an
    // 80% threshold; the active/total figure is 10% and must stay silent.
    mockMem.mockResolvedValue({
      total: 16e9, used: 15.8e9, active: 1.6e9, available: 14.4e9, free: 14.4e9, buffcache: 14.2e9,
    });

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });

    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
  });

  it('alerts when the active working set is genuinely over threshold', async () => {
    // Cache-heavy and genuinely busy: 14G active of 16G (87.5%) with 15.8G "used"
    // including cache. active/total exceeds 80%, so the alert must still fire. This
    // is the two-sided partner to the page-cache case above: keying off active must
    // not suppress real breaches.
    mockMem.mockResolvedValue({
      total: 16e9, used: 15.8e9, active: 14e9, available: 2e9, free: 2e9, buffcache: 1.8e9,
    });

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });

    expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
  });

  it('does not alert when reclaimable ZFS ARC accounts for the memory pressure', async () => {
    // Active working set reads 15G/16G (~94%, breaches 80%), but 5G of that is
    // reclaimable ARC. Adding ARC back into available drops effective usage to
    // ~62.5%, so no host-memory alert should fire.
    mockMem.mockResolvedValue(memSample(15e9)); // available 1e9 -> 93.75%
    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(5e9, 0)); // reclaimable 5e9

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });

    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
  });

  it('dispatches disk warning when over threshold', async () => {
    mockFsSize.mockResolvedValue([{ mount: '/', use: 92 }]);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_disk_limit: '90' });

    expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Disk'));
  });

  it('skips host limits when threshold is 0 or NaN', async () => {
    mockCurrentLoad.mockResolvedValue({ currentLoad: 99 });

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '0' });
    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));

    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: 'abc' });
    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'));
  });
});

describe('MonitorService - host_alerts_enabled toggle', () => {
  beforeEach(() => {
    mockGetGlobalSettings.mockReturnValue({});
    mockDispatchAlert.mockClear();
    mockCurrentLoad.mockClear();
    mockMem.mockClear();
    mockFsSize.mockClear();
  });

  afterEach(() => {
    // Restore default mock implementations so version-check state does not
    // leak into sibling describe blocks (F-11 suppression, version check).
    mockGetLatestVersion.mockResolvedValue(null);
    mockGetLatestVersionInfo.mockResolvedValue(null);
    mockGetSenchoVersion.mockReturnValue(null);
  });

  it('skips host metrics and dispatch when disabled', async () => {
    mockCurrentLoad.mockResolvedValue({ currentLoad: 75 });

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '0', host_cpu_limit: '50' });

    expect(mockDispatchAlert).not.toHaveBeenCalled();
    expect(mockCurrentLoad).not.toHaveBeenCalled();
    expect(mockMem).not.toHaveBeenCalled();
    expect(mockFsSize).not.toHaveBeenCalled();
  });

  it('does not call systeminformation when disabled', async () => {
    mockMem.mockResolvedValue(memSample(15e9)); // ~94% - would breach

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '0', host_ram_limit: '80' });

    expect(mockCurrentLoad).not.toHaveBeenCalled();
    expect(mockMem).not.toHaveBeenCalled();
    expect(mockFsSize).not.toHaveBeenCalled();
    expect(mockDispatchAlert).not.toHaveBeenCalled();
  });

  it('clears persisted suppression state when disabled', async () => {
    const store: Record<string, string> = {};
    mockGetSystemState.mockImplementation((key: string) => store[key] ?? null);
    mockSetSystemState.mockImplementation((key: string, value: string) => { store[key] = value; });

    // Simulate a prior breach that set the persisted suppression timestamp.
    store['last_host_cpu_alert_ts'] = String(Date.now());
    store['last_host_ram_alert_ts'] = String(Date.now());
    store['last_host_disk_alert_ts'] = String(Date.now());

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '0' });

    expect(store['last_host_cpu_alert_ts']).toBe('0');
    expect(store['last_host_ram_alert_ts']).toBe('0');
    expect(store['last_host_disk_alert_ts']).toBe('0');
  });

  it('re-enable fires fresh without suppressed suffix', async () => {
    const store: Record<string, string> = {};
    mockGetSystemState.mockImplementation((key: string) => store[key] ?? null);
    mockSetSystemState.mockImplementation((key: string, value: string) => { store[key] = value; });
    mockMem.mockResolvedValue(memSample(15e9)); // ~94% - breaches 80% limit

    const svc = MonitorService.getInstance();

    // Step 1: breach while enabled — fires alert and seeds suppression state.
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
    expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Suppressed'));
    mockDispatchAlert.mockClear();

    // Step 2: disable — clears suppression state and skips dispatch.
    await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '0', host_ram_limit: '80' });
    expect(mockDispatchAlert).not.toHaveBeenCalled();
    expect(store['last_host_ram_alert_ts']).toBe('0');

    // Step 3: re-enable while still breaching — fires fresh, no "Suppressed" suffix.
    await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '1', host_ram_limit: '80' });
    const calls = mockDispatchAlert.mock.calls.filter(
      (c: unknown[]) => c[0] === 'warning' && c[1] === 'monitor_alert',
    );
    expect(calls.length).toBe(1);
    const msg = calls[0] as string[];
    expect(msg[2]).toContain('Memory');
    expect(msg[2]).not.toContain('Suppressed');
  });

  it('still runs version check when disabled', async () => {
    mockGetSenchoVersion.mockReturnValue('0.45.0');
    mockGetLatestVersionInfo.mockResolvedValue({ version: '0.46.0', publishPending: false });

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '0' });

    expect(mockGetLatestVersionInfo).toHaveBeenCalled();
  });
});

// Crash + healthcheck detection now lives in DockerEventService (event-driven).
// Tests for those flows live in docker-event-service.test.ts.

// ── F-11: host-metric alert suppression ────────────────────────────────

describe('MonitorService - host alert suppression (F-11)', () => {
  // Force RAM-over-threshold for every test in this block; CPU/disk are
  // independently controlled per-test so a single mockMem set-up covers the
  // common "I want a breach happening" case without test repetition.
  beforeEach(() => {
    mockMem.mockResolvedValue(memSample(15e9)); // ~94%
  });

  it('first breach dispatches immediately', async () => {
    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });

    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
    const [, , message] = mockDispatchAlert.mock.calls[0];
    expect(message).toContain('Memory');
    expect(message).not.toContain('Suppressed');
  });

  it('second breach within window does not dispatch', async () => {
    const svc = MonitorService.getInstance();
    const baseTime = 1_700_000_000_000;
    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);

    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);

    // 30 seconds later — well inside the 60-minute default window.
    nowSpy.mockReturnValue(baseTime + 30_000);
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);

    nowSpy.mockRestore();
  });

  it('many breaches within window accumulate count without dispatching', async () => {
    const svc = MonitorService.getInstance();
    const baseTime = 1_700_000_000_000;
    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);

    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
    for (let i = 1; i < 10; i++) {
      nowSpy.mockReturnValue(baseTime + i * 30_000);
      await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
    }

    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
    nowSpy.mockRestore();
  });

  it('breach after window elapses dispatches follow-up with count summary and persists new timestamp', async () => {
    const svc = MonitorService.getInstance();
    const baseTime = 1_700_000_000_000;
    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);

    // First dispatch.
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
    expect(mockSetSystemState).toHaveBeenCalledWith('last_host_ram_alert_ts', String(baseTime));

    // 5 cycles inside the window, each suppressed.
    for (let i = 1; i <= 5; i++) {
      nowSpy.mockReturnValue(baseTime + i * 30_000);
      await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
    }
    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);

    // Jump past the suppression window — 61 minutes.
    const followUpTime = baseTime + 61 * 60 * 1000;
    nowSpy.mockReturnValue(followUpTime);
    mockSetSystemState.mockClear();
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });

    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
    const [, , followUpMessage] = mockDispatchAlert.mock.calls[1];
    expect(followUpMessage).toMatch(/Suppressed 5 alerts in the last \d+m/);
    expect(followUpMessage).toMatch(/first over threshold at \d{2}:\d{2} UTC/);
    // Follow-up dispatch must persist the new timestamp so a subsequent
    // restart-survivability seed picks up the most-recent fire, not the
    // pre-window first fire (otherwise a restart 30min later would see a
    // 90min-old persisted row and re-fire immediately).
    expect(mockSetSystemState).toHaveBeenCalledWith('last_host_ram_alert_ts', String(followUpTime));

    nowSpy.mockRestore();
  });

  it('dispatches at exactly the suppression window boundary', async () => {
    const svc = MonitorService.getInstance();
    const baseTime = 1_700_000_000_000;
    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);

    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);

    // Exactly 60 minutes later — at the boundary. The check is `<`, so the
    // boundary tick should DISPATCH a follow-up rather than suppress.
    nowSpy.mockReturnValue(baseTime + 60 * 60 * 1000);
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);

    nowSpy.mockRestore();
  });

  it('follow-up summary uses singular "alert" when exactly one cycle was suppressed', async () => {
    const svc = MonitorService.getInstance();
    const baseTime = 1_700_000_000_000;
    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);

    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);

    // One suppressed cycle.
    nowSpy.mockReturnValue(baseTime + 30_000);
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });

    // Past window.
    nowSpy.mockReturnValue(baseTime + 61 * 60 * 1000);
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });

    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
    const [, , followUpMessage] = mockDispatchAlert.mock.calls[1];
    expect(followUpMessage).toContain('Suppressed 1 alert in the last');
    expect(followUpMessage).not.toContain('Suppressed 1 alerts'); // singular form, not plural

    nowSpy.mockRestore();
  });

  it('disk-metric path dispatches and suppresses through the same mechanism', async () => {
    const svc = MonitorService.getInstance();
    mockFsSize.mockResolvedValue([{ mount: '/', use: 95 }]);
    // Set RAM below threshold so it does not interfere with the disk-only assertions.
    mockMem.mockResolvedValue(memSample(4e9));
    const baseTime = 1_700_000_000_000;
    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);

    await (svc as any).evaluateGlobalSettings({ host_disk_limit: '80' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
    expect((mockDispatchAlert.mock.calls[0][2] as string)).toContain('Disk');

    nowSpy.mockReturnValue(baseTime + 30_000);
    await (svc as any).evaluateGlobalSettings({ host_disk_limit: '80' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(1); // suppressed

    nowSpy.mockRestore();
  });

  it('metric drop below threshold clears in-memory state and persisted timestamp', async () => {
    // Recovery now reads system_state before deciding to write '0' (so it
    // skips a redundant write when persisted is already null/0). Model the
    // persistence chain so the recovery branch can observe the timestamp
    // the first-fire path just wrote.
    let persistedRamTs: string | null = null;
    mockGetSystemState.mockImplementation((key: string) =>
      key === 'last_host_ram_alert_ts' ? persistedRamTs : null,
    );
    mockSetSystemState.mockImplementation((key: string, value: string) => {
      if (key === 'last_host_ram_alert_ts') persistedRamTs = value;
    });

    const svc = MonitorService.getInstance();

    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
    expect(persistedRamTs).not.toBeNull();
    expect(persistedRamTs).not.toBe('0');

    // Drop RAM back under threshold.
    mockMem.mockResolvedValue(memSample(4e9)); // 25%
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });

    // Recovery branch resets the persisted timestamp to '0'.
    expect(persistedRamTs).toBe('0');
  });

  it('re-breach after recovery fires fresh first alert (no Suppressed suffix)', async () => {
    const svc = MonitorService.getInstance();

    // Initial breach.
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);

    // Recovery.
    mockMem.mockResolvedValue(memSample(4e9));
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });

    // Re-breach.
    mockMem.mockResolvedValue(memSample(15e9));
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });

    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
    const [, , reBreachMessage] = mockDispatchAlert.mock.calls[1];
    expect(reBreachMessage).not.toContain('Suppressed');
  });

  it('CPU and RAM suppression states are isolated per metric', async () => {
    const svc = MonitorService.getInstance();
    mockCurrentLoad.mockResolvedValue({ currentLoad: 95 });

    // First cycle: both fire.
    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);

    const messages1 = mockDispatchAlert.mock.calls.map(c => c[2] as string);
    expect(messages1.some(m => m.includes('CPU'))).toBe(true);
    expect(messages1.some(m => m.includes('Memory'))).toBe(true);

    // Second cycle: both suppressed.
    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);

    // CPU drops below threshold (recovers); RAM stays high.
    mockCurrentLoad.mockResolvedValue({ currentLoad: 10 });
    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });

    // CPU re-breaches; RAM still in suppression window.
    mockCurrentLoad.mockResolvedValue({ currentLoad: 95 });
    await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '80', host_ram_limit: '80' });

    // CPU fires fresh; RAM stays silent.
    expect(mockDispatchAlert).toHaveBeenCalledTimes(3);
    const newCpuMessage = mockDispatchAlert.mock.calls[2][2] as string;
    expect(newCpuMessage).toContain('CPU');
    expect(newCpuMessage).not.toContain('Suppressed');
  });

  it('respects custom host_alert_suppression_mins setting (5 minutes)', async () => {
    const svc = MonitorService.getInstance();
    const baseTime = 1_700_000_000_000;
    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);

    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '5' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);

    // 3 minutes — still within custom 5-minute window.
    nowSpy.mockReturnValue(baseTime + 3 * 60 * 1000);
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '5' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);

    // 6 minutes — past the custom window; follow-up should fire.
    nowSpy.mockReturnValue(baseTime + 6 * 60 * 1000);
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '5' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);

    nowSpy.mockRestore();
  });

  it('post-restart recovery clears persisted timestamp so re-breach inside the window fires fresh', async () => {
    // Codex audit scenario:
    //   T0: breach fires → system_state['last_host_ram_alert_ts'] = T0
    //   T1: process restart (in-memory state cleared, persisted T0 survives)
    //   T2: metric drops below threshold → clearHostMetricSuppression() runs
    //   T3: metric re-breaches within suppression window
    // Expected: fresh first alert at T3 (no Suppressed suffix).
    // Pre-fix bug: clearHostMetricSuppression early-returned on missing in-
    // memory state, leaving persisted T0 alive; T3 hit the restart-survival
    // path and was silently suppressed.
    const baseTime = 1_700_000_000_000;
    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime + 5 * 60 * 1000); // T1+T2 = T0+5min

    // Simulate persisted state from a previous process's first fire.
    let persistedRamTs: string | null = String(baseTime);
    mockGetSystemState.mockImplementation((key: string) =>
      key === 'last_host_ram_alert_ts' ? persistedRamTs : null,
    );
    mockSetSystemState.mockImplementation((key: string, value: string) => {
      if (key === 'last_host_ram_alert_ts') persistedRamTs = value;
    });

    // T2: post-restart cycle finds metric BELOW threshold (recovered).
    mockMem.mockResolvedValue(memSample(4e9));
    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
    // No dispatch on a non-breach cycle.
    expect(mockDispatchAlert).not.toHaveBeenCalled();
    // Recovery branch MUST have reset persisted state so a re-breach is
    // treated as fresh, not as a still-active cooldown.
    expect(persistedRamTs).toBe('0');

    // T3: re-breach 10 minutes later (well inside the original 60-min window).
    nowSpy.mockReturnValue(baseTime + 15 * 60 * 1000);
    mockMem.mockResolvedValue(memSample(15e9));
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });

    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
    const [, , message] = mockDispatchAlert.mock.calls[0];
    expect(message).not.toContain('Suppressed');

    nowSpy.mockRestore();
  });

  it('caps host_alert_suppression_mins at 24h (1440) to defend against unvalidated single-key POST writes', async () => {
    const svc = MonitorService.getInstance();
    const baseTime = 1_700_000_000_000;
    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);

    // The single-key POST /api/settings path stores allowlisted keys
    // without zod re-validation. A 999999999-minute value (or any
    // accidental garbage above 1440) must not let a metric stay
    // suppressed for centuries; cap at 24h in the consumer.
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '999999999' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);

    // 1441 minutes later — past the 24h cap.
    nowSpy.mockReturnValue(baseTime + 1441 * 60 * 1000);
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '999999999' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);

    nowSpy.mockRestore();
  });

  it('post-restart with persisted timestamp inside window does not re-fire', async () => {
    const baseTime = 1_700_000_000_000;
    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime + 10 * 60 * 1000); // "now" = T+10min

    // Simulate a previous process having persisted a fire 10 minutes ago.
    mockGetSystemState.mockImplementation((key: string) =>
      key === 'last_host_ram_alert_ts' ? String(baseTime) : null,
    );

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });

    // Post-restart cycle must NOT re-fire because the persisted cooldown
    // is still active (10 min into the default 60 min window).
    expect(mockDispatchAlert).not.toHaveBeenCalled();

    nowSpy.mockRestore();
  });

  it('zero or negative suppression_mins falls back to default', async () => {
    const svc = MonitorService.getInstance();
    const baseTime = 1_700_000_000_000;
    const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);

    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '0' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);

    // Default 60-minute window applies — a cycle 30 minutes later stays
    // suppressed even though the setting said 0.
    nowSpy.mockReturnValue(baseTime + 30 * 60 * 1000);
    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80', host_alert_suppression_mins: '0' });
    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);

    nowSpy.mockRestore();
  });
});

// ── Alert breach state machine ─────────────────────────────────────────

describe('MonitorService - breach state machine', () => {
  function setupAlertScenario(cpuPercent: number) {
    mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
    mockGetRunningContainers.mockResolvedValue([{
      Id: 'c1abcdefghijk',
      Names: ['/my-stack-api-1'],
      Labels: {
        'com.docker.compose.project': 'my-stack',
        'com.docker.compose.service': 'api',
      },
    }]);
    mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
      cpu_stats: { cpu_usage: { total_usage: 1000 + cpuPercent * 50 }, system_cpu_usage: 10000, online_cpus: 1 },
      precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
      memory_stats: { usage: 100e6, limit: 1e9 },
    }));
    mockGetStackAlerts.mockReturnValue([{
      id: 1,
      stack_name: 'my-stack',
      service_name: null,
      metric: 'cpu_percent',
      operator: '>',
      threshold: 80,
      duration_mins: 0, // Fire immediately on breach
      cooldown_mins: 60,
      last_fired_at: 0,
    }]);
    mockGetGlobalSettings.mockReturnValue({});
    mockGetStackAlertServiceCooldown.mockReturnValue(null);
    mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
  }

  it('fires alert when condition met and duration is 0', async () => {
    setupAlertScenario(90); // Will produce CPU > 80%

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockDispatchAlert).toHaveBeenCalledWith(
      'warning',
      'monitor_alert',
      'The **CPU usage** for **api** in **my-stack** (container **my-stack-api-1**) has exceeded your threshold of **80%** (Currently: 90%).',
      { stackName: 'my-stack', containerName: 'my-stack-api-1', actor: 'system:monitor' },
    );
    expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number));
    expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
  });

  it('does not fire when condition not met', async () => {
    setupAlertScenario(10); // Will produce CPU < 80%

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'));
  });

  it('respects cooldown after firing via persisted service cooldown', async () => {
    setupAlertScenario(90);
    mockGetStackAlertServiceCooldown.mockReturnValue(Date.now() - 30 * 60 * 1000);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled();
  });

  it('respects legacy last_fired_at floor when no child cooldown rows exist', async () => {
    setupAlertScenario(90);
    mockGetStackAlerts.mockReturnValue([{
      id: 1,
      stack_name: 'my-stack',
      service_name: null,
      metric: 'cpu_percent',
      operator: '>',
      threshold: 80,
      duration_mins: 0,
      cooldown_mins: 60,
      last_fired_at: Date.now() - 30 * 60 * 1000,
    }]);
    mockGetStackAlertServiceCooldown.mockReturnValue(null);
    mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled();
  });

  it('fires after legacy last_fired_at cooldown expires and writes a child cooldown row', async () => {
    setupAlertScenario(90);
    mockGetStackAlerts.mockReturnValue([{
      id: 1,
      stack_name: 'my-stack',
      service_name: null,
      metric: 'cpu_percent',
      operator: '>',
      threshold: 80,
      duration_mins: 0,
      cooldown_mins: 60,
      last_fired_at: Date.now() - 90 * 60 * 1000,
    }]);
    mockGetStackAlertServiceCooldown.mockReturnValue(null);
    mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
    expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number));
    expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
  });

  it('does not advance cooldown when notification history is not persisted', async () => {
    setupAlertScenario(90);
    mockDispatchAlert.mockResolvedValueOnce({ persisted: false });

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
    expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled();
    expect(mockUpdateStackAlertLastFired).not.toHaveBeenCalled();

    mockDispatchAlert.mockResolvedValueOnce({ persisted: true });
    await (svc as any).evaluate();

    expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
    expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number));
    expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
  });

  it('drops in-memory breach timers when a rule is deleted', async () => {
    const svc = MonitorService.getInstance();
    (svc as any).activeBreaches.set('55:gone-container', { breachStartedAt: Date.now() });
    mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
    mockGetRunningContainers.mockResolvedValue([]);
    mockGetStackAlerts.mockReturnValue([]);
    mockGetGlobalSettings.mockReturnValue({});

    await (svc as any).evaluate();

    expect((svc as any).activeBreaches.has('55:gone-container')).toBe(false);
  });

  it('names unlabeled containers as unknown service in the alert body', async () => {
    setupAlertScenario(90);
    mockGetRunningContainers.mockResolvedValue([{
      Id: 'orphan123456789',
      Names: ['/orphan'],
      Labels: {
        'com.docker.compose.project': 'my-stack',
      },
    }]);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockDispatchAlert).toHaveBeenCalledWith(
      'warning',
      'monitor_alert',
      expect.stringContaining('**unknown service**'),
      expect.objectContaining({ stackName: 'my-stack', containerName: 'orphan' }),
    );
    expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, '_unlabeled', expect.any(Number));
  });

  it('resets breach state when condition clears for that container only', async () => {
    const svc = MonitorService.getInstance();

    // First: breach starts
    setupAlertScenario(90);
    mockGetStackAlerts.mockReturnValue([{
      id: 42,
      stack_name: 'my-stack',
      service_name: null,
      metric: 'cpu_percent',
      operator: '>',
      threshold: 80,
      duration_mins: 999, // Won't fire due to long duration
      cooldown_mins: 0,
      last_fired_at: 0,
    }]);
    await (svc as any).evaluate();
    expect((svc as any).activeBreaches.has('42:c1abcdefghijk')).toBe(true);

    // Second: condition clears
    setupAlertScenario(10);
    mockGetStackAlerts.mockReturnValue([{
      id: 42,
      stack_name: 'my-stack',
      service_name: null,
      metric: 'cpu_percent',
      operator: '>',
      threshold: 80,
      duration_mins: 999,
      cooldown_mins: 0,
      last_fired_at: 0,
    }]);
    await (svc as any).evaluate();
    expect((svc as any).activeBreaches.has('42:c1abcdefghijk')).toBe(false);
  });

  it('falls back to short container id when Names is absent', async () => {
    setupAlertScenario(90);
    mockGetRunningContainers.mockResolvedValue([{
      Id: 'abcdef1234567890',
      Labels: {
        'com.docker.compose.project': 'my-stack',
        'com.docker.compose.service': 'api',
      },
    }]);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockDispatchAlert).toHaveBeenCalledWith(
      'warning',
      'monitor_alert',
      'The **CPU usage** for **api** in **my-stack** (container **abcdef123456**) has exceeded your threshold of **80%** (Currently: 90%).',
      { stackName: 'my-stack', containerName: 'abcdef123456', actor: 'system:monitor' },
    );
  });
});

// ── Cleanup triggers ───────────────────────────────────────────────────

describe('MonitorService - cleanup triggers', () => {
  it('calls cleanup methods with configured retention', async () => {
    mockGetNodes.mockReturnValue([]);
    mockGetStackAlerts.mockReturnValue([]);
    mockGetGlobalSettings.mockReturnValue({
      metrics_retention_hours: '48',
      log_retention_days: '7',
      audit_retention_days: '30',
    });

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockCleanupOldMetrics).toHaveBeenCalledWith(48);
    expect(mockCleanupOldNotifications).toHaveBeenCalledWith(7);
    expect(mockCleanupOldAuditLogs).toHaveBeenCalledWith(30);
  });

  it('uses defaults when settings are NaN', async () => {
    mockGetNodes.mockReturnValue([]);
    mockGetStackAlerts.mockReturnValue([]);
    mockGetGlobalSettings.mockReturnValue({
      metrics_retention_hours: 'bad',
      log_retention_days: 'bad',
      audit_retention_days: 'bad',
    });

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockCleanupOldMetrics).toHaveBeenCalledWith(24);
    expect(mockCleanupOldNotifications).toHaveBeenCalledWith(30);
    expect(mockCleanupOldAuditLogs).toHaveBeenCalledWith(90);
  });
});

// ── isProcessing guard ─────────────────────────────────────────────────

describe('MonitorService - isProcessing guard', () => {
  it('skips evaluation if already processing', async () => {
    mockGetGlobalSettings.mockReturnValue({});
    mockGetNodes.mockReturnValue([]);
    mockGetStackAlerts.mockReturnValue([]);

    const svc = MonitorService.getInstance();
    (svc as any).isProcessing = true;

    await (svc as any).evaluate();

    // Should have been skipped — no DB calls
    expect(mockGetGlobalSettings).not.toHaveBeenCalled();
  });

  it('resets isProcessing after evaluate completes (even on error)', async () => {
    mockGetGlobalSettings.mockImplementationOnce(() => { throw new Error('boom'); });

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    // isProcessing should be reset in finally block
    expect((svc as any).isProcessing).toBe(false);
  });
});

// ── restart_count metric ──────────────────────────────────────────────

describe('MonitorService - restart_count metric', () => {
  function setupRestartScenario(restartCount: number, hasRestartRule: boolean) {
    mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
    mockGetRunningContainers.mockResolvedValue([{
      Id: 'c1',
      Labels: { 'com.docker.compose.project': 'my-stack' },
    }]);
    mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
      cpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000, online_cpus: 1 },
      precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
      memory_stats: { usage: 100e6, limit: 1e9 },
    }));
    mockGetContainerRestartCount.mockResolvedValue(restartCount);
    const alerts = [];
    if (hasRestartRule) {
      alerts.push({
        id: 100,
        stack_name: 'my-stack',
        metric: 'restart_count',
        operator: '>',
        threshold: 3,
        duration_mins: 0,
        cooldown_mins: 60,
        last_fired_at: 0,
      });
    }
    mockGetStackAlerts.mockReturnValue(alerts);
    mockGetGlobalSettings.mockReturnValue({});
  }

  it('fetches restart count from Docker when a restart_count rule exists', async () => {
    setupRestartScenario(5, true);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1');
    // restart_count=5 > threshold=3, should fire
    expect(mockDispatchAlert).toHaveBeenCalledWith(
      'warning',
      'monitor_alert',
      expect.stringContaining('Restart count'),
      { stackName: 'my-stack', containerName: 'c1', actor: 'system:monitor' },
    );
  });

  it('skips Docker inspect when no restart_count rules exist', async () => {
    setupRestartScenario(5, false);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockGetContainerRestartCount).not.toHaveBeenCalled();
  });

  it('does not fire when restart count is below threshold', async () => {
    setupRestartScenario(2, true);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1');
    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Restart count'), expect.anything());
  });
});

// ── Sencho version update check ───────────────────────────────────────

describe('MonitorService - Sencho version check', () => {
  /** In-memory system_state so get/set round-trip within one evaluation. */
  function wireStatefulSystemState(seed: Record<string, string> = {}) {
    const store: Record<string, string> = { ...seed };
    mockGetSystemState.mockImplementation((key: string) => store[key] ?? null);
    mockSetSystemState.mockImplementation((key: string, value: string) => { store[key] = value; });
    return store;
  }

  async function runEvaluate(): Promise<void> {
    await (MonitorService.getInstance() as any).evaluate();
  }

  function updateAvailabilityCalls(): unknown[][] {
    return mockDispatchAlert.mock.calls.filter(
      (args: unknown[]) => args[1] === 'node_update_available',
    );
  }

  beforeEach(() => {
    mockGetGlobalSettings.mockReturnValue({});
    mockGetNodes.mockReturnValue([]);
    mockGetStackAlerts.mockReturnValue([]);
  });

  it('dispatches notification when newer version available', async () => {
    mockGetSenchoVersion.mockReturnValue('0.45.0');
    mockGetLatestVersionInfo.mockResolvedValue({ version: '0.46.0', publishPending: false });
    mockGetSystemState.mockReturnValue(null);

    await runEvaluate();

    expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('0.46.0'));
    expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('currently running 0.45.0'));
    expect(mockSetSystemState).toHaveBeenCalledWith('last_sencho_update_notified_version', '0.46.0');
  });

  it('does not re-notify for the same version', async () => {
    mockGetSenchoVersion.mockReturnValue('0.45.0');
    mockGetLatestVersionInfo.mockResolvedValue({ version: '0.46.0', publishPending: false });
    // Running < last notified: self-heal leaves the dedup key alone.
    mockGetSystemState.mockReturnValue('0.46.0');

    await runEvaluate();

    expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('0.46.0'));
  });

  it('handles version check failure gracefully and retries next cycle', async () => {
    mockGetSenchoVersion.mockReturnValue('0.45.0');
    mockGetLatestVersionInfo.mockResolvedValue(null);
    mockGetSystemState.mockReturnValue(null);

    await expect(runEvaluate()).resolves.toBeUndefined();
    await expect(runEvaluate()).resolves.toBeUndefined();

    expect(mockGetLatestVersionInfo).toHaveBeenCalledTimes(2);
    expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('available'));
  });

  it('skips version check when getSenchoVersion returns null', async () => {
    mockGetSenchoVersion.mockReturnValue(null);
    mockGetLatestVersionInfo.mockResolvedValue({ version: '0.46.0', publishPending: false });
    mockGetSystemState.mockReturnValue(null);

    await runEvaluate();

    expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('0.46.0'));
    expect(mockSetSystemState).not.toHaveBeenCalledWith('last_sencho_update_notified_version', expect.anything());
    expect(mockGetLatestVersionInfo).not.toHaveBeenCalled();
  });

  it('retries while a GitHub release is pending registry publish without notifying', async () => {
    mockGetSenchoVersion.mockReturnValue('0.93.0');
    mockGetLatestVersionInfo.mockResolvedValue({ version: '0.93.0', publishPending: true });
    mockGetSystemState.mockReturnValue(null);

    await runEvaluate();
    await runEvaluate();

    expect(mockGetLatestVersionInfo).toHaveBeenCalledTimes(2);
    expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('available'));
  });

  it('notifies on the next eval after an up-to-date success when a newer version appears', async () => {
    const store = wireStatefulSystemState();
    mockGetSenchoVersion.mockReturnValue('0.45.0');
    mockGetLatestVersionInfo
      .mockResolvedValueOnce({ version: '0.45.0', publishPending: false })
      .mockResolvedValue({ version: '0.46.0', publishPending: false });

    await runEvaluate();
    expect(mockGetLatestVersionInfo).toHaveBeenCalledTimes(1);
    expect(updateAvailabilityCalls()).toHaveLength(0);
    expect(store.last_sencho_update_notified_version).toBeUndefined();

    await runEvaluate();
    expect(mockGetLatestVersionInfo).toHaveBeenCalledTimes(2);
    const afterSecond = updateAvailabilityCalls();
    expect(afterSecond).toHaveLength(1);
    expect(afterSecond[0][2]).toContain('0.46.0');
    expect(store.last_sencho_update_notified_version).toBe('0.46.0');

    await runEvaluate();
    expect(mockGetLatestVersionInfo).toHaveBeenCalledTimes(3);
    expect(updateAvailabilityCalls()).toHaveLength(1);
  });

  it('notifies a newer release after a prior notify while still on the old version', async () => {
    const store = wireStatefulSystemState({ last_sencho_update_notified_version: '0.46.0' });
    mockGetSenchoVersion.mockReturnValue('0.45.0');
    mockGetLatestVersionInfo.mockResolvedValue({ version: '0.47.0', publishPending: false });

    await runEvaluate();

    expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('0.47.0'));
    expect(store.last_sencho_update_notified_version).toBe('0.47.0');
  });

  it('self-heals dedup after user upgrades to the previously-notified version', async () => {
    // Notified for 0.46.0 while on 0.45.0; now running 0.46.0 with 0.47.0 out.
    const store = wireStatefulSystemState({ last_sencho_update_notified_version: '0.46.0' });
    mockGetSenchoVersion.mockReturnValue('0.46.0');
    mockGetLatestVersionInfo.mockResolvedValue({ version: '0.47.0', publishPending: false });

    await runEvaluate();

    expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('0.47.0'));
    expect(store.last_sencho_update_notified_version).toBe('0.47.0');
  });

  it('notifies once the registry publish completes after a pending GitHub release', async () => {
    mockGetSenchoVersion.mockReturnValue('0.93.0');
    mockGetSystemState.mockReturnValue(null);

    mockGetLatestVersionInfo.mockResolvedValueOnce({ version: '0.93.0', publishPending: true });
    await runEvaluate();
    expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('0.94.0'));

    mockGetLatestVersionInfo.mockResolvedValueOnce({ version: '0.94.0', publishPending: false });
    await runEvaluate();
    expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('0.94.0'));
  });
});

describe('MonitorService - Sencho dev build check', () => {
  const DEV_PIN = { pinKind: 'floating' as const, composeImageRef: 'ghcr.io/studio-saelix/sencho-dev:dev', filePath: '/compose/docker-compose.yml' };
  const DEV_SHA_PIN = { pinKind: 'floating' as const, composeImageRef: 'ghcr.io/studio-saelix/sencho-dev:dev-abc1234', filePath: '/compose/docker-compose.yml' };
  const DEV_DIGEST_PIN = { pinKind: 'digest' as const, composeImageRef: `ghcr.io/studio-saelix/sencho-dev@sha256:${'a'.repeat(64)}`, filePath: '/compose/docker-compose.yml' };
  const STABLE_PIN = { pinKind: 'semver' as const, composeImageRef: 'ghcr.io/studio-saelix/sencho:0.97.1', filePath: '/compose/docker-compose.yml' };

  /** In-memory system_state so get/set round-trip within one evaluation. */
  function wireStatefulSystemState(seed: Record<string, string> = {}) {
    const store: Record<string, string> = { ...seed };
    mockGetSystemState.mockImplementation((key: string) => store[key] ?? null);
    mockSetSystemState.mockImplementation((key: string, value: string) => { store[key] = value; });
    return store;
  }

  async function runEvaluate(): Promise<void> {
    await (MonitorService.getInstance() as any).evaluate();
  }

  /**
   * checkSenchoDevBuild() is cadence-gated (5 or 30 minutes), so back-to-back
   * calls in a single test are otherwise short-circuited before the detector
   * ever runs. Tests that exercise a second check bypass the gate directly;
   * tests targeting the gate itself assert `lastDevBuildCheckGateMs` instead.
   */
  function bypassCadenceGate() {
    (MonitorService.getInstance() as any).lastDevBuildCheckAt = 0;
  }

  function devBuildCalls(): unknown[][] {
    return mockDispatchAlert.mock.calls.filter(
      (args: unknown[]) => args[1] === 'dev_build_update_available',
    );
  }

  beforeEach(() => {
    mockGetGlobalSettings.mockReturnValue({});
    mockGetNodes.mockReturnValue([]);
    mockGetStackAlerts.mockReturnValue([]);
    mockGetIdentity.mockReturnValue({
      containerId: null, containerName: null, composeProjectName: null,
      imageId: 'deadbeefcafe0000', networkNames: [], volumeNames: [],
    });
  });

  it('dispatches and persists dedup state on first detected update', async () => {
    const store = wireStatefulSystemState();
    mockGetPinInfo.mockResolvedValue(DEV_PIN);
    mockDetectSelfDevBuildUpdate.mockResolvedValue({ kind: 'update', digest: 'sha256:d1' });

    await runEvaluate();

    expect(devBuildCalls()).toHaveLength(1);
    expect(devBuildCalls()[0][2]).toContain('ghcr.io/studio-saelix/sencho-dev:dev');
    expect(store.sencho_dev_build_available_digest).toBe('sha256:d1');
    expect(store.last_sencho_dev_build_notified_digest).toBe('sha256:d1');
  });

  it('does not re-notify for the same digest (dedup)', async () => {
    const store = wireStatefulSystemState();
    mockGetPinInfo.mockResolvedValue(DEV_PIN);
    mockDetectSelfDevBuildUpdate.mockResolvedValue({ kind: 'update', digest: 'sha256:d1' });
    await runEvaluate();
    expect(devBuildCalls()).toHaveLength(1);

    bypassCadenceGate();
    await runEvaluate();

    expect(devBuildCalls()).toHaveLength(1);
    expect(store.sencho_dev_build_available_digest).toBe('sha256:d1');
    expect(store.last_sencho_dev_build_notified_digest).toBe('sha256:d1');
  });

  it('notifies again when a later digest appears', async () => {
    const store = wireStatefulSystemState();
    mockGetPinInfo.mockResolvedValue(DEV_PIN);
    mockDetectSelfDevBuildUpdate.mockResolvedValue({ kind: 'update', digest: 'sha256:d1' });
    await runEvaluate();

    bypassCadenceGate();
    mockDetectSelfDevBuildUpdate.mockResolvedValue({ kind: 'update', digest: 'sha256:d2' });
    await runEvaluate();

    expect(devBuildCalls()).toHaveLength(2);
    expect(store.sencho_dev_build_available_digest).toBe('sha256:d2');
    expect(store.last_sencho_dev_build_notified_digest).toBe('sha256:d2');
  });

  it('clears the availability key when up to date', async () => {
    const store = wireStatefulSystemState({ sencho_dev_build_available_digest: 'sha256:old' });
    mockGetPinInfo.mockResolvedValue(DEV_PIN);
    mockDetectSelfDevBuildUpdate.mockResolvedValue({ kind: 'up_to_date' });

    await runEvaluate();

    expect(store.sencho_dev_build_available_digest).toBe('');
    expect(devBuildCalls()).toHaveLength(0);
  });

  it('does not touch state when inconclusive, and uses the short cadence gate for the next check', async () => {
    const store = wireStatefulSystemState({ sencho_dev_build_available_digest: 'sha256:existing' });
    mockGetPinInfo.mockResolvedValue(DEV_PIN);
    mockDetectSelfDevBuildUpdate.mockResolvedValue({ kind: 'inconclusive', reason: 'registry unreachable' });

    await runEvaluate();

    expect(store.sencho_dev_build_available_digest).toBe('sha256:existing');
    expect(store.last_sencho_dev_build_notified_digest).toBeUndefined();
    expect(devBuildCalls()).toHaveLength(0);
    expect((MonitorService.getInstance() as any).lastDevBuildCheckGateMs).toBe(5 * 60 * 1000);
  });

  it('keeps the availability key set but leaves the notified key unchanged when the dispatch is not persisted, and retries next check', async () => {
    const store = wireStatefulSystemState();
    mockGetPinInfo.mockResolvedValue(DEV_PIN);
    mockDetectSelfDevBuildUpdate.mockResolvedValue({ kind: 'update', digest: 'sha256:d1' });
    mockDispatchAlert.mockResolvedValueOnce({ persisted: false });

    await runEvaluate();

    expect(store.sencho_dev_build_available_digest).toBe('sha256:d1');
    expect(store.last_sencho_dev_build_notified_digest).toBeUndefined();
    expect(devBuildCalls()).toHaveLength(1);
    expect((MonitorService.getInstance() as any).lastDevBuildCheckGateMs).toBe(5 * 60 * 1000);

    bypassCadenceGate();
    mockDispatchAlert.mockResolvedValueOnce({ persisted: true });
    await runEvaluate();

    expect(devBuildCalls()).toHaveLength(2);
    expect(store.last_sencho_dev_build_notified_digest).toBe('sha256:d1');
  });

  it('makes no registry call for an immutable dev-<sha> tag', async () => {
    mockGetPinInfo.mockResolvedValue(DEV_SHA_PIN);

    await runEvaluate();

    expect(mockDetectSelfDevBuildUpdate).not.toHaveBeenCalled();
    expect(mockSetSystemState).not.toHaveBeenCalled();
    expect(devBuildCalls()).toHaveLength(0);
  });

  it('makes no registry call for a digest-pinned dev-repo ref', async () => {
    mockGetPinInfo.mockResolvedValue(DEV_DIGEST_PIN);

    await runEvaluate();

    expect(mockDetectSelfDevBuildUpdate).not.toHaveBeenCalled();
    expect(mockSetSystemState).not.toHaveBeenCalled();
    expect(devBuildCalls()).toHaveLength(0);
  });

  it('is a no-op for a stable-pinned node, and leaves the existing stable version-update behavior unchanged', async () => {
    mockGetPinInfo.mockResolvedValue(STABLE_PIN);
    mockGetSenchoVersion.mockReturnValueOnce('0.45.0');
    mockGetLatestVersionInfo.mockResolvedValueOnce({ version: '0.46.0', publishPending: false });

    await runEvaluate();

    expect(mockDetectSelfDevBuildUpdate).not.toHaveBeenCalled();
    expect(devBuildCalls()).toHaveLength(0);
    expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'node_update_available', expect.stringContaining('0.46.0'));
  });

  it('suppresses the stable version-update notification for a dev-repo :dev pin even when a stable update is available', async () => {
    mockGetPinInfo.mockResolvedValue(DEV_PIN);
    mockGetSenchoVersion.mockReturnValueOnce('0.45.0');
    mockGetLatestVersionInfo.mockResolvedValueOnce({ version: '0.46.0', publishPending: false });
    mockDetectSelfDevBuildUpdate.mockResolvedValue({ kind: 'up_to_date' });

    await runEvaluate();

    expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'node_update_available', expect.anything());
  });

  it('suppresses the stable version-update notification for an immutable dev-<sha> pin', async () => {
    mockGetPinInfo.mockResolvedValue(DEV_SHA_PIN);
    mockGetSenchoVersion.mockReturnValueOnce('0.45.0');
    mockGetLatestVersionInfo.mockResolvedValueOnce({ version: '0.46.0', publishPending: false });

    await runEvaluate();

    expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'node_update_available', expect.anything());
  });

  it('suppresses the stable version-update notification for a digest-pinned dev-repo ref', async () => {
    mockGetPinInfo.mockResolvedValue(DEV_DIGEST_PIN);
    mockGetSenchoVersion.mockReturnValueOnce('0.45.0');
    mockGetLatestVersionInfo.mockResolvedValueOnce({ version: '0.46.0', publishPending: false });

    await runEvaluate();

    expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'node_update_available', expect.anything());
  });
});

// ── Per-container parallel fan-out ────────────────────────────────────

describe('MonitorService - parallel container processing', () => {
  /** Build a stats payload that yields a positive CPU percent so the
   *  metric pipeline runs end-to-end (calculateCpuPercent + DB write). */
  function statsPayload(): string {
    return JSON.stringify({
      cpu_stats: { cpu_usage: { total_usage: 2000 }, system_cpu_usage: 10000, online_cpus: 1 },
      precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
      memory_stats: { usage: 100e6, limit: 1e9 },
    });
  }

  beforeEach(() => {
    mockGetGlobalSettings.mockReturnValue({});
    mockGetStackAlerts.mockReturnValue([]);
    mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
    mockGetStackAlertServiceCooldown.mockReturnValue(null);
    mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
  });

  it('fans out per-container stats fetches in parallel (wall time ~ max not sum)', async () => {
    const containerCount = 10;
    const perCallDelayMs = 200;
    const containers = Array.from({ length: containerCount }, (_, i) => ({
      Id: `container-${i}`,
      Labels: { 'com.docker.compose.project': 'stack-x' },
    }));
    mockGetRunningContainers.mockResolvedValue(containers);
    mockGetContainerStatsStream.mockImplementation(
      () => new Promise((resolve) => setTimeout(() => resolve(statsPayload()), perCallDelayMs)),
    );

    const svc = MonitorService.getInstance();
    const start = Date.now();
    await (svc as any).evaluate();
    const elapsed = Date.now() - start;

    // Serial would be containerCount * perCallDelayMs = 2000ms; parallel
    // collapses to ~perCallDelayMs plus a little dispatch overhead. Allow
    // generous headroom (3x the per-call delay) to avoid CI flake.
    expect(elapsed).toBeLessThan(perCallDelayMs * 3);
    // All containers were processed: one stats call and one metric write each.
    expect(mockGetContainerStatsStream).toHaveBeenCalledTimes(containerCount);
    expect(mockAddContainerMetric).toHaveBeenCalledTimes(containerCount);
  });

  it('isolates per-container failures: one rejection does not abort siblings', async () => {
    const containers = Array.from({ length: 5 }, (_, i) => ({
      Id: `container-${i}`,
      Labels: { 'com.docker.compose.project': 'stack-x' },
    }));
    mockGetRunningContainers.mockResolvedValue(containers);
    mockGetContainerStatsStream.mockImplementation(async (id: string) => {
      if (id === 'container-2') {
        const err = Object.assign(new Error('no such container'), { statusCode: 404 });
        throw err;
      }
      return statsPayload();
    });

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    // 4 successful containers each wrote a metric; the 404 container is silently skipped.
    expect(mockAddContainerMetric).toHaveBeenCalledTimes(4);
  });

  it('dispatches a stack alert exactly once per cycle even when multiple containers in the stack breach', async () => {
    // 5 replicas of the same Compose service, all breaching. Dedup is per
    // rule+service, so only one fire should land this cycle.
    const containers = Array.from({ length: 5 }, (_, i) => ({
      Id: `container-${i}`,
      Names: [`/shared-stack-web-${i}`],
      Labels: {
        'com.docker.compose.project': 'shared-stack',
        'com.docker.compose.service': 'web',
      },
    }));
    mockGetRunningContainers.mockResolvedValue(containers);
    mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
      // Produces CPU = 90% to breach threshold of 80%.
      cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
      precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
      memory_stats: { usage: 100e6, limit: 1e9 },
    }));
    mockGetStackAlerts.mockReturnValue([{
      id: 7,
      stack_name: 'shared-stack',
      service_name: null,
      metric: 'cpu_percent',
      operator: '>',
      threshold: 80,
      duration_mins: 0,
      cooldown_mins: 60,
      last_fired_at: 0,
    }]);
    mockGetStackAlertServiceCooldown.mockReturnValue(null);
    mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    const cpuDispatches = mockDispatchAlert.mock.calls.filter(
      (args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'),
    );
    expect(cpuDispatches).toHaveLength(1);
    expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledTimes(1);
  });

  it('allows different services on an All-services rule to fire in the same cycle', async () => {
    mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
    mockGetRunningContainers.mockResolvedValue([
      {
        Id: 'api-1',
        Names: ['/stack-api-1'],
        Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
      },
      {
        Id: 'db-1',
        Names: ['/stack-db-1'],
        Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
      },
    ]);
    mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
      cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
      precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
      memory_stats: { usage: 100e6, limit: 1e9 },
    }));
    mockGetStackAlerts.mockReturnValue([{
      id: 8,
      stack_name: 'shared-stack',
      service_name: null,
      metric: 'cpu_percent',
      operator: '>',
      threshold: 80,
      duration_mins: 0,
      cooldown_mins: 60,
      last_fired_at: 0,
    }]);
    mockGetStackAlertServiceCooldown.mockReturnValue(null);
    mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
    mockGetGlobalSettings.mockReturnValue({});

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    const cpuDispatches = mockDispatchAlert.mock.calls.filter(
      (args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'),
    );
    expect(cpuDispatches).toHaveLength(2);
    expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(8, 'api', expect.any(Number));
    expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(8, 'database', expect.any(Number));
  });

  it('does not let a healthy sibling clear another container breach timer', async () => {
    const svc = MonitorService.getInstance();
    mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
    mockGetGlobalSettings.mockReturnValue({});
    mockGetStackAlerts.mockReturnValue([{
      id: 9,
      stack_name: 'shared-stack',
      service_name: null,
      metric: 'cpu_percent',
      operator: '>',
      threshold: 80,
      duration_mins: 999,
      cooldown_mins: 0,
      last_fired_at: 0,
    }]);

    const highCpu = JSON.stringify({
      cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
      precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
      memory_stats: { usage: 100e6, limit: 1e9 },
    });
    const lowCpu = JSON.stringify({
      cpu_stats: { cpu_usage: { total_usage: 1100 }, system_cpu_usage: 10000, online_cpus: 1 },
      precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
      memory_stats: { usage: 100e6, limit: 1e9 },
    });

    mockGetRunningContainers.mockResolvedValue([
      {
        Id: 'api-hot',
        Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
      },
      {
        Id: 'db-cool',
        Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
      },
    ]);
    mockGetContainerStatsStream.mockImplementation(async (id: string) => (id === 'api-hot' ? highCpu : lowCpu));

    await (svc as any).evaluate();

    expect((svc as any).activeBreaches.has('9:api-hot')).toBe(true);
    expect((svc as any).activeBreaches.has('9:db-cool')).toBe(false);
  });

  it('ignores sibling services for a service-scoped rule', async () => {
    mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
    mockGetGlobalSettings.mockReturnValue({});
    mockGetRunningContainers.mockResolvedValue([
      {
        Id: 'api-1',
        Names: ['/stack-api-1'],
        Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
      },
      {
        Id: 'db-1',
        Names: ['/stack-db-1'],
        Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
      },
    ]);
    mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
      cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
      precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
      memory_stats: { usage: 100e6, limit: 1e9 },
    }));
    mockGetStackAlerts.mockReturnValue([{
      id: 10,
      stack_name: 'shared-stack',
      service_name: 'api',
      metric: 'cpu_percent',
      operator: '>',
      threshold: 80,
      duration_mins: 0,
      cooldown_mins: 0,
      last_fired_at: 0,
    }]);
    mockGetStackAlertServiceCooldown.mockReturnValue(null);
    mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
    expect(mockDispatchAlert.mock.calls[0][2]).toContain('**api**');
    expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(10, 'api', expect.any(Number));
  });

  it('does not block service B after service A fires under new-schema cooldown', async () => {
    mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
    mockGetGlobalSettings.mockReturnValue({});
    mockGetRunningContainers.mockResolvedValue([
      {
        Id: 'db-1',
        Names: ['/stack-db-1'],
        Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
      },
    ]);
    mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
      cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
      precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
      memory_stats: { usage: 100e6, limit: 1e9 },
    }));
    mockGetStackAlerts.mockReturnValue([{
      id: 11,
      stack_name: 'shared-stack',
      service_name: null,
      metric: 'cpu_percent',
      operator: '>',
      threshold: 80,
      duration_mins: 0,
      cooldown_mins: 60,
      last_fired_at: Date.now() - 5 * 60 * 1000, // would block under old rule-wide semantics
    }]);
    // api already has a cooldown row; database does not, so database may still fire.
    mockGetStackAlertServiceCooldown.mockImplementation((_id: number, service: string) =>
      service === 'api' ? Date.now() - 5 * 60 * 1000 : null,
    );
    mockHasAnyStackAlertServiceCooldown.mockReturnValue(true);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(11, 'database', expect.any(Number));
  });

  it('honors persisted per-service cooldown after a fresh MonitorService instance', async () => {
    mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
    mockGetGlobalSettings.mockReturnValue({});
    mockGetRunningContainers.mockResolvedValue([
      {
        Id: 'api-1',
        Names: ['/stack-api-1'],
        Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
      },
      {
        Id: 'api-2',
        Names: ['/stack-api-2'],
        Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
      },
      {
        Id: 'db-1',
        Names: ['/stack-db-1'],
        Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
      },
    ]);
    mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
      cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
      precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
      memory_stats: { usage: 100e6, limit: 1e9 },
    }));
    mockGetStackAlerts.mockReturnValue([{
      id: 13,
      stack_name: 'shared-stack',
      service_name: null,
      metric: 'cpu_percent',
      operator: '>',
      threshold: 80,
      duration_mins: 0,
      cooldown_mins: 60,
      last_fired_at: 0,
    }]);
    mockGetStackAlertServiceCooldown.mockImplementation((_id: number, service: string) =>
      service === 'api' ? Date.now() - 5 * 60 * 1000 : null,
    );
    mockHasAnyStackAlertServiceCooldown.mockReturnValue(true);

    // beforeEach already cleared the singleton; this instance has empty maps.
    const svc = MonitorService.getInstance();
    expect((svc as any).activeBreaches.size).toBe(0);
    expect((svc as any).firedThisCycle.size).toBe(0);

    await (svc as any).evaluate();

    const cpuDispatches = mockDispatchAlert.mock.calls.filter(
      (args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'),
    );
    expect(cpuDispatches).toHaveLength(1);
    expect(cpuDispatches[0][2]).toContain('**database**');
    expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(13, 'database', expect.any(Number));
    expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalledWith(13, 'api', expect.any(Number));
  });

  it('preserves breach timers when container enumeration fails', async () => {
    const svc = MonitorService.getInstance();
    (svc as any).activeBreaches.set('12:still-running', { breachStartedAt: Date.now() });
    mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
    mockGetStackAlerts.mockReturnValue([{
      id: 12,
      stack_name: 'shared-stack',
      service_name: null,
      metric: 'cpu_percent',
      operator: '>',
      threshold: 80,
      duration_mins: 999,
      cooldown_mins: 0,
      last_fired_at: 0,
    }]);
    mockGetGlobalSettings.mockReturnValue({});
    mockGetRunningContainers.mockRejectedValue(new Error('docker down'));

    await (svc as any).evaluate();

    expect((svc as any).activeBreaches.has('12:still-running')).toBe(true);
  });

  it('removes breach timers for stopped containers after successful enumeration', async () => {
    const svc = MonitorService.getInstance();
    (svc as any).activeBreaches.set('13:gone', { breachStartedAt: Date.now() });
    mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
    mockGetStackAlerts.mockReturnValue([{
      id: 13,
      stack_name: 'shared-stack',
      service_name: null,
      metric: 'cpu_percent',
      operator: '>',
      threshold: 80,
      duration_mins: 999,
      cooldown_mins: 0,
      last_fired_at: 0,
    }]);
    mockGetGlobalSettings.mockReturnValue({});
    mockGetRunningContainers.mockResolvedValue([
      {
        Id: 'still-here',
        Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
      },
    ]);
    mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
      cpu_stats: { cpu_usage: { total_usage: 1100 }, system_cpu_usage: 10000, online_cpus: 1 },
      precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
      memory_stats: { usage: 100e6, limit: 1e9 },
    }));

    await (svc as any).evaluate();

    expect((svc as any).activeBreaches.has('13:gone')).toBe(false);
  });

  it('caps simultaneous Docker calls at MAX_CONTAINER_CONCURRENCY', async () => {
    // 25 containers with a long per-call delay; observe that no more than
    // 10 (MAX_CONTAINER_CONCURRENCY) are in flight at the same time.
    const containerCount = 25;
    const perCallDelayMs = 100;
    let inFlight = 0;
    let peakInFlight = 0;
    const containers = Array.from({ length: containerCount }, (_, i) => ({
      Id: `container-${i}`,
      Labels: { 'com.docker.compose.project': 'stack-x' },
    }));
    mockGetRunningContainers.mockResolvedValue(containers);
    mockGetContainerStatsStream.mockImplementation(
      () => new Promise((resolve) => {
        inFlight += 1;
        if (inFlight > peakInFlight) peakInFlight = inFlight;
        setTimeout(() => {
          inFlight -= 1;
          resolve(statsPayload());
        }, perCallDelayMs);
      }),
    );

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(peakInFlight).toBeLessThanOrEqual(10);
    // And we still saw real concurrency (not serialized), so peak should be
    // close to the cap when we have many more items than workers.
    expect(peakInFlight).toBeGreaterThan(1);
    expect(mockAddContainerMetric).toHaveBeenCalledTimes(containerCount);
  });
});

// ── Janitor cycle and circuit breaker (F-6) ────────────────────────────

describe('MonitorService - janitor cycle and circuit breaker', () => {
  // Convenience: builds a never-settling promise used to simulate a hung df().
  function hangForever(): Promise<never> {
    return new Promise<never>(() => { /* never resolves */ });
  }

  // Reclaimable payload large enough to cross a 0.5 GB janitor threshold.
  const RECLAIMABLE_3GB = {
    reclaimableImages: 3 * 1024 * 1024 * 1024,
    reclaimableContainers: 0,
    reclaimableVolumes: 0,
    reclaimableBuildCache: 0,
    reclaimableImageCount: 5,
    reclaimableContainerCount: 0,
    reclaimableVolumeCount: 0,
    reclaimableBuildCacheCount: 0,
  };

  it('evaluate() does NOT call getDiskUsage (decoupling guardrail)', async () => {
    // F-6 regression guard: the 30s monitor cycle must never call df().
    // If someone re-couples the janitor into evaluate(), this test fails.
    mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
    mockGetNodes.mockReturnValue([]);
    mockGetStackAlerts.mockReturnValue([]);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluate();

    expect(mockGetDiskUsage).not.toHaveBeenCalled();
  });

  it('evaluate() completes promptly even when getDiskUsage would hang (F-6 regression)', async () => {
    // If df() were still on the 30s cycle, hangForever would compound the
    // cycle beyond its 25s threshold. Decoupled, evaluate() must return
    // within a small wall-clock budget regardless.
    mockGetDiskUsage.mockReturnValue(hangForever());
    mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
    mockGetNodes.mockReturnValue([]);
    mockGetStackAlerts.mockReturnValue([]);

    const svc = MonitorService.getInstance();
    const t0 = Date.now();
    await (svc as any).evaluate();
    const elapsed = Date.now() - t0;

    expect(elapsed).toBeLessThan(2000);
    expect(mockGetDiskUsage).not.toHaveBeenCalled();
  });

  it('evaluateJanitor() honors isJanitorProcessing re-entrancy guard', async () => {
    mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
    mockGetDiskUsage.mockResolvedValue(RECLAIMABLE_3GB);

    const svc = MonitorService.getInstance();
    (svc as any).isJanitorProcessing = true;

    await (svc as any).evaluateJanitor();

    // Second concurrent call must skip without touching settings or df.
    expect(mockGetGlobalSettings).not.toHaveBeenCalled();
    expect(mockGetDiskUsage).not.toHaveBeenCalled();
  });

  it('skips when docker_janitor_gb is unset, zero, or NaN', async () => {
    const svc = MonitorService.getInstance();

    mockGetGlobalSettings.mockReturnValue({});
    await (svc as any).evaluateJanitor();
    mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0' });
    await (svc as any).evaluateJanitor();
    mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: 'abc' });
    await (svc as any).evaluateJanitor();

    expect(mockGetDiskUsage).not.toHaveBeenCalled();
  });

  it('dispatches a node-neutral janitor alert', async () => {
    mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
    mockGetDiskUsage.mockResolvedValue(RECLAIMABLE_3GB);
    mockGetSystemState.mockReturnValue('0'); // No prior alert; cooldown elapsed.

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateJanitor();

    expect(mockDispatchAlert).toHaveBeenCalledWith(
      'info',
      'system',
      'This node has accumulated 3.0 GB of unused Docker data. Open Resources to reclaim space, or set up a Prune Node Resources schedule.',
      { stackName: undefined, actor: 'system:monitor' },
    );
  });

  it('does NOT alert when reclaimable is below MIN_RECLAIMABLE_GB even if threshold is aggressive', async () => {
    mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.001' });
    mockGetDiskUsage.mockResolvedValue({
      ...RECLAIMABLE_3GB,
      reclaimableImages: 50 * 1024 * 1024, // 50 MB, below the 100 MB floor
    });
    mockGetSystemState.mockReturnValue('0');

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateJanitor();

    expect(mockDispatchAlert).not.toHaveBeenCalled();
  });

  it('opens the circuit breaker after JANITOR_BREAKER_THRESHOLD consecutive timeouts', async () => {
    mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
    mockGetDiskUsage.mockReturnValue(hangForever());
    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateJanitor();
    await (svc as any).evaluateJanitor();
    expect(warnSpy).not.toHaveBeenCalled(); // Threshold not yet reached.
    await (svc as any).evaluateJanitor(); // Third timeout trips the breaker.

    const breakerLine = warnSpy.mock.calls.find(
      (args) => typeof args[0] === 'string' && args[0].includes('circuit breaker opened'),
    );
    expect(breakerLine).toBeDefined();
    expect((svc as any).janitorBreakerUntil).toBeGreaterThan(Date.now());
    // Counter resets on open so the cooldown is what gates the next attempt.
    expect((svc as any).janitorConsecutiveTimeouts).toBe(0);

    warnSpy.mockRestore();
  });

  it('respects the breaker cooldown; does not call getDiskUsage while open', async () => {
    mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });

    const svc = MonitorService.getInstance();
    (svc as any).janitorBreakerUntil = Date.now() + 60 * 60 * 1000;

    await (svc as any).evaluateJanitor();

    expect(mockGetDiskUsage).not.toHaveBeenCalled();
    expect(mockGetGlobalSettings).not.toHaveBeenCalled();
  });

  it('resets the timeout counter on a successful call after partial failures', async () => {
    mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
    mockGetDiskUsage
      .mockReturnValueOnce(hangForever())
      .mockReturnValueOnce(hangForever())
      .mockResolvedValueOnce(RECLAIMABLE_3GB);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateJanitor();
    await (svc as any).evaluateJanitor();
    expect((svc as any).janitorConsecutiveTimeouts).toBe(2);

    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
    await (svc as any).evaluateJanitor();

    expect((svc as any).janitorConsecutiveTimeouts).toBe(0);
    const recoveryLine = logSpy.mock.calls.find(
      (args) => typeof args[0] === 'string' && args[0].includes('recovered'),
    );
    expect(recoveryLine).toBeDefined();
    logSpy.mockRestore();
  });

  it('logs recovery on the first successful call after a full breaker-open cooldown', async () => {
    // After the breaker opens, the counter is zeroed; once the cooldown
    // lapses, a successful call must still emit the recovered log so the
    // operator observability story is symmetric with the partial-failure
    // recovery path.
    mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });

    const svc = MonitorService.getInstance();
    (svc as any).janitorBreakerUntil = Date.now() - 1; // cooldown just elapsed
    (svc as any).janitorConsecutiveTimeouts = 0;       // zeroed on open
    mockGetDiskUsage.mockResolvedValue(RECLAIMABLE_3GB);

    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
    await (svc as any).evaluateJanitor();

    const recoveryLine = logSpy.mock.calls.find(
      (args) => typeof args[0] === 'string' && args[0].includes('recovered'),
    );
    expect(recoveryLine).toBeDefined();
    expect((svc as any).janitorBreakerUntil).toBe(0);
    expect((svc as any).janitorConsecutiveTimeouts).toBe(0);
    logSpy.mockRestore();
  });

  it('does NOT advance the breaker counter on non-timeout errors', async () => {
    mockGetGlobalSettings.mockReturnValue({ docker_janitor_gb: '0.5' });
    mockGetDiskUsage.mockRejectedValue(Object.assign(new Error('daemon unreachable'), { statusCode: 500 }));
    const errSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);

    const svc = MonitorService.getInstance();
    await (svc as any).evaluateJanitor();
    await (svc as any).evaluateJanitor();
    await (svc as any).evaluateJanitor();
    await (svc as any).evaluateJanitor();

    expect((svc as any).janitorConsecutiveTimeouts).toBe(0);
    expect((svc as any).janitorBreakerUntil).toBe(0);
    // The original support-grep line must still fire on every non-timeout error.
    const janitorErrorLines = errSpy.mock.calls.filter(
      (args) => typeof args[0] === 'string' && args[0].includes('Error checking docker janitor limits'),
    );
    expect(janitorErrorLines.length).toBe(4);

    errSpy.mockRestore();
  });

  it('stop() clears both intervals AND both deferred first-tick timeouts', () => {
    // Without canceling the first-tick setTimeouts, stop() would let
    // evaluate() / evaluateJanitor() fire on a service that the caller
    // believes is dormant. The 5s and 45s windows are wider than typical
    // graceful-shutdown budgets, so this matters.
    const svc = MonitorService.getInstance();
    svc.start();
    expect((svc as any).intervalId).not.toBeNull();
    expect((svc as any).firstTickTimeoutId).not.toBeNull();
    expect((svc as any).janitorIntervalId).not.toBeNull();
    expect((svc as any).janitorFirstTickTimeoutId).not.toBeNull();

    svc.stop();

    expect((svc as any).intervalId).toBeNull();
    expect((svc as any).firstTickTimeoutId).toBeNull();
    expect((svc as any).janitorIntervalId).toBeNull();
    expect((svc as any).janitorFirstTickTimeoutId).toBeNull();
  });
});

describe('MonitorService - reconcileOrphanedScans', () => {
  function makeDb(refs: string[]) {
    return {
      getDistinctScanImageRefs: vi.fn().mockReturnValue(refs),
      deleteScansByImageRef: vi.fn().mockReturnValue(1),
    };
  }

  async function run(db: ReturnType<typeof makeDb>, settings: Record<string, string>) {
    const svc = MonitorService.getInstance();
    await (svc as any).reconcileOrphanedScans(db, settings);
    return db.deleteScansByImageRef.mock.calls.map((c) => c[1] as string).sort();
  }

  it('does nothing when prune_orphaned_scans is not "1"', async () => {
    const db = makeDb(['nginx:1']);
    await run(db, { prune_orphaned_scans: '0' });
    expect(db.getDistinctScanImageRefs).not.toHaveBeenCalled();
    expect(db.deleteScansByImageRef).not.toHaveBeenCalled();
  });

  it('purges only the image and stack scans whose artifact is gone, scoped to the local node', async () => {
    mockGetImages.mockResolvedValue([
      { RepoTags: ['nginx:1', 'redis:7'] },
      { RepoTags: ['<none>:<none>'] },
      { RepoTags: undefined }, // dangling image with no tags
    ]);
    mockGetStacks.mockResolvedValue(['web']);
    const db = makeDb(['nginx:1', 'ghost:9', 'stack:web', 'stack:old']);
    expect(await run(db, { prune_orphaned_scans: '1' })).toEqual(['ghost:9', 'stack:old']);
    // Per-instance: reconciliation reads and deletes against the local node id only.
    expect(db.getDistinctScanImageRefs).toHaveBeenCalledWith(1);
    for (const call of db.deleteScansByImageRef.mock.calls) {
      expect(call[0]).toBe(1);
    }
  });

  it('keeps scans whose ref matches a live image after normalization (untagged, registry-qualified, digest-pinned)', async () => {
    mockGetImages.mockResolvedValue([
      { RepoTags: ['alpine:latest', 'nginx:1.14'], RepoDigests: ['redis@sha256:abc'] },
    ]);
    mockGetStacks.mockResolvedValue(['web']);
    // Stored refs in non-canonical forms that all resolve to a present image:
    //   alpine -> alpine:latest, docker.io/library/nginx:1.14 -> nginx:1.14,
    //   docker.io/library/redis@sha256:abc -> redis@sha256:abc (digest).
    const db = makeDb(['alpine', 'docker.io/library/nginx:1.14', 'docker.io/library/redis@sha256:abc', 'ghost:9']);
    expect(await run(db, { prune_orphaned_scans: '1' })).toEqual(['ghost:9']);
  });

  it('skips stack reconciliation when getStacks returns empty (ambiguous), still purges image orphans', async () => {
    mockGetImages.mockResolvedValue([{ RepoTags: ['nginx:1'] }]);
    mockGetStacks.mockResolvedValue([]);
    const db = makeDb(['stack:web', 'ghost:9']);
    const deleted = await run(db, { prune_orphaned_scans: '1' });
    expect(deleted).toContain('ghost:9');
    expect(deleted).not.toContain('stack:web');
  });

  it('purges nothing when the Docker image list cannot be read (fail-safe)', async () => {
    mockGetImages.mockRejectedValue(new Error('docker down'));
    mockGetStacks.mockResolvedValue(['web']);
    const db = makeDb(['ghost:9', 'stack:old']);
    await run(db, { prune_orphaned_scans: '1' });
    expect(db.deleteScansByImageRef).not.toHaveBeenCalled();
  });

  it('purges every image scan when there are zero live images (empty but successful read)', async () => {
    mockGetImages.mockResolvedValue([]);
    mockGetStacks.mockResolvedValue(['web']);
    const db = makeDb(['nginx:1', 'redis:7', 'stack:web']);
    expect(await run(db, { prune_orphaned_scans: '1' })).toEqual(['nginx:1', 'redis:7']);
  });
});