diff --git a/backend/src/__tests__/cache-endpoints.test.ts b/backend/src/__tests__/cache-endpoints.test.ts index d498135a..2021f7ad 100644 --- a/backend/src/__tests__/cache-endpoints.test.ts +++ b/backend/src/__tests__/cache-endpoints.test.ts @@ -111,7 +111,10 @@ beforeEach(() => { mockGetBulkStackStatuses.mockResolvedValue({}); mockGetStacks.mockResolvedValue([]); mockCurrentLoad.mockResolvedValue({ currentLoad: 42.5, cpus: [{}, {}] }); - mockMem.mockResolvedValue({ total: 1000, used: 500, free: 500 }); + // active/available exclude reclaimable cache; used/free include it. The route + // reports the cache-excluded figures, so the mock supplies the full shape: + // active(400) + available(600) = total; used(500) = active + 100 cache. + mockMem.mockResolvedValue({ total: 1000, used: 500, active: 400, free: 500, available: 600, buffcache: 100 }); mockFsSize.mockResolvedValue([{ fs: '/dev/sda1', mount: '/', size: 1000, used: 500, available: 500, use: 50 }]); }); @@ -169,6 +172,19 @@ describe('GET /api/system/stats caching', () => { // CPU/mem/disk sample is cached; network is fresh per request. expect(mockCurrentLoad).toHaveBeenCalledTimes(1); }); + + it('reports memory from the active working set, excluding reclaimable cache', async () => { + const res = await request(app).get('/api/system/stats').set('Cookie', authCookie); + expect(res.status).toBe(200); + // Figures come from mem.active / mem.available (cache-excluded), not the + // cache-inclusive mem.used / mem.free, so a busy host does not read ~100%. + expect(res.body.memory).toMatchObject({ + total: 1000, + used: 400, // mem.active, not mem.used (500) + free: 600, // mem.available, not mem.free (500) + usagePercent: '40.0', // 400 / 1000, not 500 / 1000 + }); + }); }); // ── /api/stacks/statuses ─────────────────────────────────────────────── diff --git a/backend/src/__tests__/monitor-service.test.ts b/backend/src/__tests__/monitor-service.test.ts index de54aa64..120f84c8 100644 --- a/backend/src/__tests__/monitor-service.test.ts +++ b/backend/src/__tests__/monitor-service.test.ts @@ -38,7 +38,7 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine }), mockDispatchAlert: vi.fn().mockResolvedValue(undefined), mockCurrentLoad: vi.fn().mockResolvedValue({ currentLoad: 10 }), - mockMem: vi.fn().mockResolvedValue({ used: 4e9, total: 16e9 }), + mockMem: vi.fn().mockResolvedValue({ total: 16e9, used: 4e9, active: 4e9, available: 12e9, free: 12e9, buffcache: 0 }), mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]), mockExecAsync: vi.fn().mockResolvedValue({ stdout: '' }), mockFetchLatestSenchoVersion: vi.fn().mockRejectedValue(new Error('not configured')), @@ -131,6 +131,15 @@ beforeEach(() => { mockGetSystemState.mockReturnValue(null); }); +// si.mem() returns active/available alongside used (used counts reclaimable page +// cache, active/available exclude it). The host-RAM percentage is derived from the +// cache-excluded figures, so mocks must supply a realistic shape. `realUsed` is the +// active working set; available/free are the remainder. +const memSample = (realUsed: number, total = 16e9) => ({ + total, used: realUsed, active: realUsed, + available: total - realUsed, free: total - realUsed, buffcache: 0, +}); + // ── Pure calculation helpers (accessed via private method reflection) ─── describe('MonitorService - calculateCpuPercent', () => { @@ -303,7 +312,36 @@ describe('MonitorService - evaluateGlobalSettings', () => { }); it('dispatches RAM warning when over threshold', async () => { - mockMem.mockResolvedValue({ used: 15e9, total: 16e9 }); // ~94% + mockMem.mockResolvedValue(memSample(15e9)); // ~94% + + const svc = MonitorService.getInstance(); + await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' }); + + expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory')); + }); + + it('does not alert when most RAM is reclaimable page cache, not real usage', async () => { + // Busy host: 15.8G "used" including cache, but only 1.6G actively in use and + // 14.4G available. The naive used/total formula reads ~99% and would breach an + // 80% threshold; the active/total figure is 10% and must stay silent. + mockMem.mockResolvedValue({ + total: 16e9, used: 15.8e9, active: 1.6e9, available: 14.4e9, free: 14.4e9, buffcache: 14.2e9, + }); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' }); + + expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory')); + }); + + it('alerts when the active working set is genuinely over threshold', async () => { + // Cache-heavy and genuinely busy: 14G active of 16G (87.5%) with 15.8G "used" + // including cache. active/total exceeds 80%, so the alert must still fire. This + // is the two-sided partner to the page-cache case above: keying off active must + // not suppress real breaches. + mockMem.mockResolvedValue({ + total: 16e9, used: 15.8e9, active: 14e9, available: 2e9, free: 2e9, buffcache: 1.8e9, + }); const svc = MonitorService.getInstance(); await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' }); @@ -342,7 +380,7 @@ describe('MonitorService - host alert suppression (F-11)', () => { // independently controlled per-test so a single mockMem set-up covers the // common "I want a breach happening" case without test repetition. beforeEach(() => { - mockMem.mockResolvedValue({ used: 15e9, total: 16e9 }); // ~94% + mockMem.mockResolvedValue(memSample(15e9)); // ~94% }); it('first breach dispatches immediately', async () => { @@ -467,7 +505,7 @@ describe('MonitorService - host alert suppression (F-11)', () => { const svc = MonitorService.getInstance(); mockFsSize.mockResolvedValue([{ mount: '/', use: 95 }]); // Set RAM below threshold so it does not interfere with the disk-only assertions. - mockMem.mockResolvedValue({ used: 4e9, total: 16e9 }); + mockMem.mockResolvedValue(memSample(4e9)); const baseTime = 1_700_000_000_000; const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime); @@ -503,7 +541,7 @@ describe('MonitorService - host alert suppression (F-11)', () => { expect(persistedRamTs).not.toBe('0'); // Drop RAM back under threshold. - mockMem.mockResolvedValue({ used: 4e9, total: 16e9 }); // 25% + mockMem.mockResolvedValue(memSample(4e9)); // 25% await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' }); // Recovery branch resets the persisted timestamp to '0'. @@ -518,11 +556,11 @@ describe('MonitorService - host alert suppression (F-11)', () => { expect(mockDispatchAlert).toHaveBeenCalledTimes(1); // Recovery. - mockMem.mockResolvedValue({ used: 4e9, total: 16e9 }); + mockMem.mockResolvedValue(memSample(4e9)); await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' }); // Re-breach. - mockMem.mockResolvedValue({ used: 15e9, total: 16e9 }); + mockMem.mockResolvedValue(memSample(15e9)); await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' }); expect(mockDispatchAlert).toHaveBeenCalledTimes(2); @@ -605,7 +643,7 @@ describe('MonitorService - host alert suppression (F-11)', () => { }); // T2: post-restart cycle finds metric BELOW threshold (recovered). - mockMem.mockResolvedValue({ used: 4e9, total: 16e9 }); + mockMem.mockResolvedValue(memSample(4e9)); const svc = MonitorService.getInstance(); await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' }); // No dispatch on a non-breach cycle. @@ -616,7 +654,7 @@ describe('MonitorService - host alert suppression (F-11)', () => { // T3: re-breach 10 minutes later (well inside the original 60-min window). nowSpy.mockReturnValue(baseTime + 15 * 60 * 1000); - mockMem.mockResolvedValue({ used: 15e9, total: 16e9 }); + mockMem.mockResolvedValue(memSample(15e9)); await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' }); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); diff --git a/backend/src/routes/fleet.ts b/backend/src/routes/fleet.ts index ef5d2be8..b7494ed4 100644 --- a/backend/src/routes/fleet.ts +++ b/backend/src/routes/fleet.ts @@ -233,9 +233,12 @@ async function fetchLocalNodeOverview(node: Node): Promise { cpu: { usage: currentLoad.currentLoad.toFixed(1), cores: currentLoad.cpus.length }, memory: { total: mem.total, - used: mem.used, - free: mem.free, - usagePercent: ((mem.used / mem.total) * 100).toFixed(1), + // Percentage is keyed off mem.active (the real working set), not mem.used: + // on Linux/BSD/macOS mem.used counts reclaimable page cache and reads ~99% + // on a busy host. mem.available is total - active, so used + free = total. + used: mem.active, + free: mem.available, + usagePercent: ((mem.active / mem.total) * 100).toFixed(1), }, disk: mainDisk ? { total: mainDisk.size, diff --git a/backend/src/routes/metrics.ts b/backend/src/routes/metrics.ts index 16e74897..e6077c5b 100644 --- a/backend/src/routes/metrics.ts +++ b/backend/src/routes/metrics.ts @@ -311,9 +311,12 @@ metricsRouter.get('/system/stats', authMiddleware, async (req: Request, res: Res }, memory: { total: mem.total, - used: mem.used, - free: mem.free, - usagePercent: ((mem.used / mem.total) * 100).toFixed(1), + // Percentage is keyed off mem.active (the real working set), not mem.used: + // on Linux/BSD/macOS mem.used counts reclaimable page cache and reads ~99% + // on a busy host. mem.available is total - active, so used + free = total. + used: mem.active, + free: mem.available, + usagePercent: ((mem.active / mem.total) * 100).toFixed(1), }, disk: mainDisk ? { fs: mainDisk.fs, diff --git a/backend/src/services/MonitorService.ts b/backend/src/services/MonitorService.ts index f02a4a4d..5afd7db0 100644 --- a/backend/src/services/MonitorService.ts +++ b/backend/src/services/MonitorService.ts @@ -299,7 +299,10 @@ export class MonitorService { this.clearHostMetricSuppression('cpu'); } - const ramUsage = (mem.used / mem.total) * 100; + // Key off mem.active (the real working set), not mem.used: on Linux/BSD/macOS + // mem.used counts reclaimable page cache and reads ~99% on a busy host, which + // would fire spurious host-memory alerts. + const ramUsage = (mem.active / mem.total) * 100; const ramLimit = parseFloat(settings['host_ram_limit']); if (!isNaN(ramLimit) && ramLimit > 0 && ramUsage > ramLimit) { await this.dispatchHostMetricAlert('ram', 'warning', suppressionMs,