fix(metrics): host memory usage excludes reclaimable page cache (#1297)

* fix(metrics): host memory usage excludes reclaimable page cache

Host RAM was computed as mem.used / mem.total via systeminformation, but mem.used counts
reclaimable buffers/cache as used, so a busy Linux host read ~99%. Switch the dashboard stats,
the fleet node self-report, and the host-RAM alert threshold to mem.active / mem.total
(cache-excluded), and report used: mem.active and free: mem.available so the byte readout stays
consistent with the percentage. Add regression tests for a cache-heavy host (no false alert) and
a genuinely busy host (alert still fires).

* test(metrics): assert system stats memory excludes reclaimable cache

The cached /api/system/stats test mocked si.mem() without active/available, so after
the route switched to the cache-excluded fields it produced a NaN percentage that the
shape-only assertions did not catch. Fill the mock with a realistic shape and assert the
route reports the active working set, not the cache-inclusive used/free.

backend/src/__tests__/cache-endpoints.test.ts

@@ -111,7 +111,10 @@ beforeEach(() => {
   mockGetBulkStackStatuses.mockResolvedValue({});
   mockGetStacks.mockResolvedValue([]);
   mockCurrentLoad.mockResolvedValue({ currentLoad: 42.5, cpus: [{}, {}] });
-  mockMem.mockResolvedValue({ total: 1000, used: 500, free: 500 });
+  // active/available exclude reclaimable cache; used/free include it. The route
+  // reports the cache-excluded figures, so the mock supplies the full shape:
+  // active(400) + available(600) = total; used(500) = active + 100 cache.
+  mockMem.mockResolvedValue({ total: 1000, used: 500, active: 400, free: 500, available: 600, buffcache: 100 });
   mockFsSize.mockResolvedValue([{ fs: '/dev/sda1', mount: '/', size: 1000, used: 500, available: 500, use: 50 }]);
 });
 
@@ -169,6 +172,19 @@ describe('GET /api/system/stats caching', () => {
     // CPU/mem/disk sample is cached; network is fresh per request.
     expect(mockCurrentLoad).toHaveBeenCalledTimes(1);
   });
+
+  it('reports memory from the active working set, excluding reclaimable cache', async () => {
+    const res = await request(app).get('/api/system/stats').set('Cookie', authCookie);
+    expect(res.status).toBe(200);
+    // Figures come from mem.active / mem.available (cache-excluded), not the
+    // cache-inclusive mem.used / mem.free, so a busy host does not read ~100%.
+    expect(res.body.memory).toMatchObject({
+      total: 1000,
+      used: 400,            // mem.active, not mem.used (500)
+      free: 600,            // mem.available, not mem.free (500)
+      usagePercent: '40.0', // 400 / 1000, not 500 / 1000
+    });
+  });
 });
 
 // ── /api/stacks/statuses ───────────────────────────────────────────────

backend/src/__tests__/monitor-service.test.ts

@@ -38,7 +38,7 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
   }),
   mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
   mockCurrentLoad: vi.fn().mockResolvedValue({ currentLoad: 10 }),
-  mockMem: vi.fn().mockResolvedValue({ used: 4e9, total: 16e9 }),
+  mockMem: vi.fn().mockResolvedValue({ total: 16e9, used: 4e9, active: 4e9, available: 12e9, free: 12e9, buffcache: 0 }),
   mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]),
   mockExecAsync: vi.fn().mockResolvedValue({ stdout: '' }),
   mockFetchLatestSenchoVersion: vi.fn().mockRejectedValue(new Error('not configured')),
@@ -131,6 +131,15 @@ beforeEach(() => {
   mockGetSystemState.mockReturnValue(null);
 });
 
+// si.mem() returns active/available alongside used (used counts reclaimable page
+// cache, active/available exclude it). The host-RAM percentage is derived from the
+// cache-excluded figures, so mocks must supply a realistic shape. `realUsed` is the
+// active working set; available/free are the remainder.
+const memSample = (realUsed: number, total = 16e9) => ({
+  total, used: realUsed, active: realUsed,
+  available: total - realUsed, free: total - realUsed, buffcache: 0,
+});
+
 // ── Pure calculation helpers (accessed via private method reflection) ───
 
 describe('MonitorService - calculateCpuPercent', () => {
@@ -303,7 +312,36 @@ describe('MonitorService - evaluateGlobalSettings', () => {
   });
 
   it('dispatches RAM warning when over threshold', async () => {
-    mockMem.mockResolvedValue({ used: 15e9, total: 16e9 }); // ~94%
+    mockMem.mockResolvedValue(memSample(15e9)); // ~94%
+
+    const svc = MonitorService.getInstance();
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+
+    expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
+  });
+
+  it('does not alert when most RAM is reclaimable page cache, not real usage', async () => {
+    // Busy host: 15.8G "used" including cache, but only 1.6G actively in use and
+    // 14.4G available. The naive used/total formula reads ~99% and would breach an
+    // 80% threshold; the active/total figure is 10% and must stay silent.
+    mockMem.mockResolvedValue({
+      total: 16e9, used: 15.8e9, active: 1.6e9, available: 14.4e9, free: 14.4e9, buffcache: 14.2e9,
+    });
+
+    const svc = MonitorService.getInstance();
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+
+    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
+  });
+
+  it('alerts when the active working set is genuinely over threshold', async () => {
+    // Cache-heavy and genuinely busy: 14G active of 16G (87.5%) with 15.8G "used"
+    // including cache. active/total exceeds 80%, so the alert must still fire. This
+    // is the two-sided partner to the page-cache case above: keying off active must
+    // not suppress real breaches.
+    mockMem.mockResolvedValue({
+      total: 16e9, used: 15.8e9, active: 14e9, available: 2e9, free: 2e9, buffcache: 1.8e9,
+    });
 
     const svc = MonitorService.getInstance();
     await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
@@ -342,7 +380,7 @@ describe('MonitorService - host alert suppression (F-11)', () => {
   // independently controlled per-test so a single mockMem set-up covers the
   // common "I want a breach happening" case without test repetition.
   beforeEach(() => {
-    mockMem.mockResolvedValue({ used: 15e9, total: 16e9 }); // ~94%
+    mockMem.mockResolvedValue(memSample(15e9)); // ~94%
   });
 
   it('first breach dispatches immediately', async () => {
@@ -467,7 +505,7 @@ describe('MonitorService - host alert suppression (F-11)', () => {
     const svc = MonitorService.getInstance();
     mockFsSize.mockResolvedValue([{ mount: '/', use: 95 }]);
     // Set RAM below threshold so it does not interfere with the disk-only assertions.
-    mockMem.mockResolvedValue({ used: 4e9, total: 16e9 });
+    mockMem.mockResolvedValue(memSample(4e9));
     const baseTime = 1_700_000_000_000;
     const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseTime);
 
@@ -503,7 +541,7 @@ describe('MonitorService - host alert suppression (F-11)', () => {
     expect(persistedRamTs).not.toBe('0');
 
     // Drop RAM back under threshold.
-    mockMem.mockResolvedValue({ used: 4e9, total: 16e9 }); // 25%
+    mockMem.mockResolvedValue(memSample(4e9)); // 25%
     await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
 
     // Recovery branch resets the persisted timestamp to '0'.
@@ -518,11 +556,11 @@ describe('MonitorService - host alert suppression (F-11)', () => {
     expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
 
     // Recovery.
-    mockMem.mockResolvedValue({ used: 4e9, total: 16e9 });
+    mockMem.mockResolvedValue(memSample(4e9));
     await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
 
     // Re-breach.
-    mockMem.mockResolvedValue({ used: 15e9, total: 16e9 });
+    mockMem.mockResolvedValue(memSample(15e9));
     await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
 
     expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
@@ -605,7 +643,7 @@ describe('MonitorService - host alert suppression (F-11)', () => {
     });
 
     // T2: post-restart cycle finds metric BELOW threshold (recovered).
-    mockMem.mockResolvedValue({ used: 4e9, total: 16e9 });
+    mockMem.mockResolvedValue(memSample(4e9));
     const svc = MonitorService.getInstance();
     await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
     // No dispatch on a non-breach cycle.
@@ -616,7 +654,7 @@ describe('MonitorService - host alert suppression (F-11)', () => {
 
     // T3: re-breach 10 minutes later (well inside the original 60-min window).
     nowSpy.mockReturnValue(baseTime + 15 * 60 * 1000);
-    mockMem.mockResolvedValue({ used: 15e9, total: 16e9 });
+    mockMem.mockResolvedValue(memSample(15e9));
     await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
 
     expect(mockDispatchAlert).toHaveBeenCalledTimes(1);

backend/src/routes/fleet.ts

@@ -233,9 +233,12 @@ async function fetchLocalNodeOverview(node: Node): Promise<FleetNodeOverview> {
         cpu: { usage: currentLoad.currentLoad.toFixed(1), cores: currentLoad.cpus.length },
         memory: {
           total: mem.total,
-          used: mem.used,
-          free: mem.free,
-          usagePercent: ((mem.used / mem.total) * 100).toFixed(1),
+          // Percentage is keyed off mem.active (the real working set), not mem.used:
+          // on Linux/BSD/macOS mem.used counts reclaimable page cache and reads ~99%
+          // on a busy host. mem.available is total - active, so used + free = total.
+          used: mem.active,
+          free: mem.available,
+          usagePercent: ((mem.active / mem.total) * 100).toFixed(1),
         },
         disk: mainDisk ? {
           total: mainDisk.size,

backend/src/routes/metrics.ts

@@ -311,9 +311,12 @@ metricsRouter.get('/system/stats', authMiddleware, async (req: Request, res: Res
           },
           memory: {
             total: mem.total,
-            used: mem.used,
-            free: mem.free,
-            usagePercent: ((mem.used / mem.total) * 100).toFixed(1),
+            // Percentage is keyed off mem.active (the real working set), not mem.used:
+            // on Linux/BSD/macOS mem.used counts reclaimable page cache and reads ~99%
+            // on a busy host. mem.available is total - active, so used + free = total.
+            used: mem.active,
+            free: mem.available,
+            usagePercent: ((mem.active / mem.total) * 100).toFixed(1),
           },
           disk: mainDisk ? {
             fs: mainDisk.fs,

backend/src/services/MonitorService.ts

@@ -299,7 +299,10 @@ export class MonitorService {
                 this.clearHostMetricSuppression('cpu');
             }
 
-            const ramUsage = (mem.used / mem.total) * 100;
+            // Key off mem.active (the real working set), not mem.used: on Linux/BSD/macOS
+            // mem.used counts reclaimable page cache and reads ~99% on a busy host, which
+            // would fire spurious host-memory alerts.
+            const ramUsage = (mem.active / mem.total) * 100;
             const ramLimit = parseFloat(settings['host_ram_limit']);
             if (!isNaN(ramLimit) && ramLimit > 0 && ramUsage > ramLimit) {
                 await this.dispatchHostMetricAlert('ram', 'warning', suppressionMs,