fix: make host memory usage ZFS ARC-aware (#1547)

backend/src/__tests__/cache-endpoints.test.ts

@@ -17,6 +17,7 @@
 import { describe, it, expect, beforeAll, afterAll, vi, beforeEach } from 'vitest';
 import request from 'supertest';
 import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_PASSWORD } from './helpers/setupTestDb';
+import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsMock } from './helpers/arcstatsFsMock';
 import { GitSourceService } from '../services/GitSourceService';
 import type { PublicGitSource } from '../services/GitSourceService';
 
@@ -80,9 +81,13 @@ let tmpDir: string;
 let app: import('express').Express;
 let authCookie: string;
 let CacheService: typeof import('../services/CacheService').CacheService;
+let arcFs: ArcstatsFsMock;
 
 beforeAll(async () => {
   tmpDir = await setupTestDb();
+  // Host memory reads ZFS ARC stats; intercept those reads so results do not
+  // depend on whether the CI host is itself ZFS. Default: no ARC present.
+  arcFs = installArcstatsFsMock();
   ({ app } = await import('../index'));
   ({ CacheService } = await import('../services/CacheService'));
 
@@ -98,6 +103,7 @@ afterAll(() => {
 
 beforeEach(() => {
   CacheService.getInstance().flush();
+  arcFs.clear();
 
   mockGetAllContainers.mockReset();
   mockGetBulkStackStatuses.mockReset();
@@ -178,8 +184,9 @@ describe('GET /api/system/stats caching', () => {
   it('reports memory from the active working set, excluding reclaimable cache', async () => {
     const res = await request(app).get('/api/system/stats').set('Cookie', authCookie);
     expect(res.status).toBe(200);
-    // Figures come from mem.active / mem.available (cache-excluded), not the
-    // cache-inclusive mem.used / mem.free, so a busy host does not read ~100%.
+    // With no ARC present, effective used is total - available (which equals
+    // mem.active), not the cache-inclusive mem.used / mem.free, so a busy host
+    // does not read ~100%.
     expect(res.body.memory).toMatchObject({
       total: 1000,
       used: 400,            // mem.active, not mem.used (500)
@@ -187,6 +194,36 @@ describe('GET /api/system/stats caching', () => {
       usagePercent: '40.0', // 400 / 1000, not 500 / 1000
     });
   });
+
+  it('adds reclaimable ZFS ARC back into available memory', async () => {
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100)); // reclaimable 200
+    const res = await request(app).get('/api/system/stats').set('Cookie', authCookie);
+    expect(res.status).toBe(200);
+    // available 600 + 200 reclaimable ARC = 800 effective free; used drops to 200.
+    expect(res.body.memory).toMatchObject({
+      total: 1000,
+      used: 200,
+      free: 800,
+      usagePercent: '20.0',
+    });
+  });
+});
+
+// ── /api/fleet/overview (local node) ───────────────────────────────────
+
+describe('GET /api/fleet/overview local-node memory', () => {
+  it('reports ARC-adjusted memory for the local node', async () => {
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100)); // reclaimable 200
+    const res = await request(app).get('/api/fleet/overview').set('Cookie', authCookie);
+    expect(res.status).toBe(200);
+    const local = res.body.find((n: { type: string }) => n.type === 'local');
+    expect(local?.systemStats?.memory).toMatchObject({
+      total: 1000,
+      used: 200,
+      free: 800,
+      usagePercent: '20.0',
+    });
+  });
 });
 
 // ── /api/stacks/statuses ───────────────────────────────────────────────

backend/src/__tests__/helpers/arcstatsFsMock.ts

@@ -0,0 +1,98 @@
+import { promises as fs } from 'fs';
+import { vi } from 'vitest';
+import { ARCSTATS_FIXED_PATHS } from '../../helpers/hostMemory';
+
+/**
+ * Path-aware partial mock of `fs.promises` for ZFS arcstats reads.
+ *
+ * `helpers/hostMemory.ts` reads `/proc/spl/kstat/zfs/arcstats` (and optional
+ * variants) to compute reclaimable ARC. Tests may run on a ZFS host, so a real
+ * read would make results host-dependent. This installs a spy that intercepts
+ * ONLY registered/ARC-candidate paths and delegates every other
+ * `readFile`/`stat` to the real filesystem, so `setupTestDb` and
+ * `DatabaseService` keep working. Default behavior: ARC candidates reject with
+ * ENOENT (no ARC), so consumers fall back to the plain `active/total` reading.
+ */
+
+// Sourced from the helper so the mock cannot silently drift from the paths the
+// production code actually reads.
+export const ARC_CANDIDATE_PATHS = ARCSTATS_FIXED_PATHS;
+
+/** Second fixed candidate; the default path fixtures are served from. */
+export const DEFAULT_ARC_PATH = ARC_CANDIDATE_PATHS[1];
+
+type StatDescriptor = { isFile: boolean; size: number };
+
+export interface ArcstatsFsMock {
+  /** Serve `content` when `path` is read. */
+  setRead(path: string, content: string): void;
+  /** Reject a read of `path` with `err` (e.g. an EACCES/EIO error). */
+  setReadError(path: string, err: NodeJS.ErrnoException): void;
+  /** Control `stat(path)` result (for override-path guard tests). */
+  setStat(path: string, descriptor: StatDescriptor | NodeJS.ErrnoException): void;
+  /** Forget all registered paths (back to default no-ARC). */
+  clear(): void;
+}
+
+function enoent(path: string): NodeJS.ErrnoException {
+  return Object.assign(new Error(`ENOENT: no such file, open '${path}'`), { code: 'ENOENT' });
+}
+
+/**
+ * Install the spy. Call once per test file (e.g. in `beforeAll`); use the
+ * returned setters per test and `clear()` in `beforeEach`.
+ */
+export function installArcstatsFsMock(): ArcstatsFsMock {
+  const realReadFile = fs.readFile.bind(fs);
+  const realStat = fs.stat.bind(fs);
+  const reads = new Map<string, string | NodeJS.ErrnoException>();
+  const stats = new Map<string, StatDescriptor | NodeJS.ErrnoException>();
+  const isArcCandidate = (p: string): boolean => ARC_CANDIDATE_PATHS.includes(p);
+
+  vi.spyOn(fs, 'readFile').mockImplementation((async (p: unknown, ...rest: unknown[]) => {
+    const key = String(p);
+    if (reads.has(key)) {
+      const v = reads.get(key)!;
+      if (v instanceof Error) throw v;
+      return v;
+    }
+    if (isArcCandidate(key)) throw enoent(key);
+    return (realReadFile as (...a: unknown[]) => unknown)(p, ...rest);
+  }) as unknown as typeof fs.readFile);
+
+  vi.spyOn(fs, 'stat').mockImplementation((async (p: unknown, ...rest: unknown[]) => {
+    const key = String(p);
+    if (stats.has(key)) {
+      const v = stats.get(key)!;
+      if (v instanceof Error) throw v;
+      return { isFile: () => v.isFile, size: v.size };
+    }
+    // A registered read with no explicit stat implies a small regular file.
+    if (reads.has(key)) {
+      const v = reads.get(key);
+      const size = typeof v === 'string' ? Buffer.byteLength(v) : 0;
+      return { isFile: () => true, size };
+    }
+    if (isArcCandidate(key)) throw enoent(key);
+    return (realStat as (...a: unknown[]) => unknown)(p, ...rest);
+  }) as unknown as typeof fs.stat);
+
+  return {
+    setRead: (path, content) => reads.set(path, content),
+    setReadError: (path, err) => reads.set(path, err),
+    setStat: (path, descriptor) => stats.set(path, descriptor),
+    clear: () => { reads.clear(); stats.clear(); },
+  };
+}
+
+/** Build a minimal arcstats kstat body with the given `size` and `c_min` rows. */
+export function arcstatsBody(sizeRow: string | number, cMinRow: string | number): string {
+  return [
+    'name                            type data',
+    `hits                            4    123456`,
+    `c_min                           4    ${cMinRow}`,
+    `size                            4    ${sizeRow}`,
+    `c_max                           4    9999999999`,
+    '',
+  ].join('\n');
+}

backend/src/__tests__/host-memory.test.ts

@@ -0,0 +1,204 @@
+/**
+ * Unit tests for the ZFS ARC-aware host-memory helper.
+ *
+ * `adjustForArc` is exercised directly; `readReclaimableArc` and
+ * `parseArcstats` stay module-internal and are exercised through
+ * `getHostMemory` with a path-aware fs mock (see helpers/arcstatsFsMock.ts).
+ */
+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';
+import {
+  installArcstatsFsMock,
+  arcstatsBody,
+  DEFAULT_ARC_PATH,
+  ARC_CANDIDATE_PATHS,
+  type ArcstatsFsMock,
+} from './helpers/arcstatsFsMock';
+
+const mockMem = vi.fn();
+
+vi.mock('systeminformation', () => ({
+  default: { mem: (...args: unknown[]) => mockMem(...args) },
+}));
+
+import { getHostMemory, adjustForArc } from '../helpers/hostMemory';
+
+// mem.active === total - available on Linux, so used/free below mirror the
+// real systeminformation shape the helper consumes.
+const memSample = (total: number, available: number) => ({
+  total,
+  available,
+  active: total - available,
+  used: total - available,
+  free: available,
+  buffcache: 0,
+});
+
+let arcFs: ArcstatsFsMock;
+
+beforeAll(() => {
+  arcFs = installArcstatsFsMock();
+});
+
+beforeEach(() => {
+  arcFs.clear();
+  mockMem.mockReset();
+  delete process.env.SENCHO_ZFS_ARCSTATS_PATH;
+});
+
+describe('adjustForArc', () => {
+  it('reproduces active/total when reclaimable ARC is 0', () => {
+    const result = adjustForArc(memSample(1000, 600), 0);
+    expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
+  });
+
+  it('adds reclaimable ARC back into available, lowering usage', () => {
+    const result = adjustForArc(memSample(1000, 600), 200);
+    expect(result).toEqual({ total: 1000, used: 200, free: 800, usagePercent: 20 });
+  });
+
+  it('clamps effective available to total when ARC exceeds the gap', () => {
+    const result = adjustForArc(memSample(1000, 600), 5000);
+    expect(result).toEqual({ total: 1000, used: 0, free: 1000, usagePercent: 0 });
+  });
+
+  it('guards against a zero total', () => {
+    const result = adjustForArc(memSample(0, 0), 0);
+    expect(result.usagePercent).toBe(0);
+  });
+});
+
+describe('getHostMemory ARC discovery', () => {
+  it('falls back to active/total when no ARC stats are present', async () => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    const result = await getHostMemory();
+    expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
+  });
+
+  it('subtracts reclaimable ARC (size - c_min) from used', async () => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100)); // reclaimable 200
+    const result = await getHostMemory();
+    expect(result).toEqual({ total: 1000, used: 200, free: 800, usagePercent: 20 });
+  });
+
+  it('prefers the operator override path over the fixed candidates', async () => {
+    process.env.SENCHO_ZFS_ARCSTATS_PATH = '/custom/arcstats';
+    mockMem.mockResolvedValue(memSample(2000, 600));
+    arcFs.setRead('/custom/arcstats', arcstatsBody(500, 100)); // reclaimable 400
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100));   // fixed would be 200
+    const result = await getHostMemory();
+    expect(result.used).toBe(1000); // 2000 - (600 + 400 override); fixed would give 1200
+    expect(result.free).toBe(1000);
+  });
+
+  it('reads the host-mounted candidate and prefers it over /proc', async () => {
+    // ARC_CANDIDATE_PATHS[0] is /host/proc/..., the path docker-compose mounts
+    // into the container, so this covers the real deployment path and precedence.
+    mockMem.mockResolvedValue(memSample(2000, 600));
+    arcFs.setRead(ARC_CANDIDATE_PATHS[0], arcstatsBody(500, 100)); // /host/proc: reclaimable 400
+    arcFs.setRead(ARC_CANDIDATE_PATHS[1], arcstatsBody(300, 100)); // /proc: would be 200
+    const result = await getHostMemory();
+    expect(result.used).toBe(1000); // 2000 - (600 + 400); /proc winning would give 1200
+  });
+
+  it('falls through to a fixed candidate when the override is unreadable', async () => {
+    process.env.SENCHO_ZFS_ARCSTATS_PATH = '/custom/arcstats';
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setReadError('/custom/arcstats', Object.assign(new Error('nope'), { code: 'ENOENT' }));
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100)); // reclaimable 200
+    const result = await getHostMemory();
+    expect(result.used).toBe(200);
+  });
+
+  it('resolves immediately to 0 reclaimable when size < c_min (ARC at floor)', async () => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(50, 100)); // size < c_min
+    const result = await getHostMemory();
+    expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
+  });
+
+  it.each([
+    ['non-numeric size', arcstatsBody('abc', 100)],
+    ['negative size', arcstatsBody(-5, 100)],
+    ['non-numeric c_min', arcstatsBody(300, 'xyz')],
+    ['negative c_min', arcstatsBody(300, -5)],
+    ['missing c_min', 'size                            4    300\n'],
+    ['missing size', 'c_min                           4    100\n'],
+    ['empty file', '   \n'],
+  ])('treats a %s record as unusable and yields no ARC', async (_label, body) => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setRead(DEFAULT_ARC_PATH, body);
+    const result = await getHostMemory();
+    expect(result.used).toBe(400); // fell through to active/total
+  });
+
+  it.each([
+    ['EACCES', 'EACCES'],
+    ['EIO', 'EIO'],
+    ['EMFILE', 'EMFILE'],
+  ])('fails open (ARC 0) on a %s read error', async (_label, code) => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setReadError(DEFAULT_ARC_PATH, Object.assign(new Error(code), { code }));
+    const result = await getHostMemory();
+    expect(result.used).toBe(400);
+  });
+
+  it('logs an unexpected read error (once per code) but stays silent on an expected one', async () => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+
+    // Expected fs error: silent fall-through.
+    arcFs.setReadError(ARC_CANDIDATE_PATHS[0], Object.assign(new Error('denied'), { code: 'EACCES' }));
+    arcFs.setReadError(ARC_CANDIDATE_PATHS[1], Object.assign(new Error('denied'), { code: 'EACCES' }));
+    await getHostMemory();
+    expect(warn).not.toHaveBeenCalled();
+
+    // Unexpected fs error: logged, but only once per error code across calls.
+    // Uses a code no other test triggers, since the once-per-code memo is
+    // process-global.
+    arcFs.setReadError(ARC_CANDIDATE_PATHS[0], Object.assign(new Error('stale'), { code: 'ESTALE' }));
+    arcFs.setReadError(ARC_CANDIDATE_PATHS[1], Object.assign(new Error('stale'), { code: 'ESTALE' }));
+    await getHostMemory();
+    await getHostMemory();
+    const unexpectedLogs = warn.mock.calls.filter(([msg]) => String(msg).includes('ESTALE'));
+    expect(unexpectedLogs).toHaveLength(1);
+    warn.mockRestore();
+  });
+
+  it('skips an override path that is not a regular file', async () => {
+    process.env.SENCHO_ZFS_ARCSTATS_PATH = '/custom/arcstats';
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setStat('/custom/arcstats', { isFile: false, size: 10 });
+    arcFs.setRead('/custom/arcstats', arcstatsBody(500, 100));
+    const result = await getHostMemory();
+    expect(result.used).toBe(400); // override skipped, no fixed ARC present
+  });
+
+  it('skips an override path that exceeds the size bound', async () => {
+    process.env.SENCHO_ZFS_ARCSTATS_PATH = '/custom/arcstats';
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setStat('/custom/arcstats', { isFile: true, size: 2 * 1024 * 1024 });
+    arcFs.setRead('/custom/arcstats', arcstatsBody(500, 100));
+    const result = await getHostMemory();
+    expect(result.used).toBe(400);
+  });
+
+  it('logs the selected path once and never the file contents', async () => {
+    process.env.SENCHO_ZFS_ARCSTATS_PATH = '/log-once/arcstats';
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setRead('/log-once/arcstats', arcstatsBody(300, 100));
+    const debug = vi.spyOn(console, 'debug').mockImplementation(() => {});
+    await getHostMemory();
+    await getHostMemory();
+    const pathLogs = debug.mock.calls.filter(([msg]) => String(msg).includes('/log-once/arcstats'));
+    expect(pathLogs).toHaveLength(1);
+    // The log names the path, never the kstat contents (size / c_min values).
+    expect(String(pathLogs[0][0])).not.toContain('300');
+    expect(String(pathLogs[0][0])).not.toContain('100');
+    debug.mockRestore();
+  });
+});
+
+afterEach(() => {
+  delete process.env.SENCHO_ZFS_ARCSTATS_PATH;
+});

backend/src/__tests__/monitor-service.test.ts

@@ -2,7 +2,8 @@
  * Unit tests for MonitorService — alert state machine, metric calculations,
  * cleanup delegation, global settings evaluation, and concurrency guards.
  */
-import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vitest';
+import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsMock } from './helpers/arcstatsFsMock';
 
 // ── Hoisted mocks ──────────────────────────────────────────────────────
 
@@ -138,8 +139,16 @@ vi.mock('util', () => ({
 
 import { MonitorService, _resetHostAlertSuppressionStateForTests } from '../services/MonitorService';
 
+// Host memory now reads ZFS ARC stats; intercept those reads so the suite does
+// not depend on whether the machine running it is itself a ZFS host.
+let arcFs: ArcstatsFsMock;
+beforeAll(() => {
+  arcFs = installArcstatsFsMock();
+});
+
 beforeEach(() => {
   vi.clearAllMocks();
+  arcFs.clear();
   (MonitorService as any).instance = undefined;
   _resetHostAlertSuppressionStateForTests();
   mockGetSystemState.mockReturnValue(null);
@@ -363,6 +372,19 @@ describe('MonitorService - evaluateGlobalSettings', () => {
     expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
   });
 
+  it('does not alert when reclaimable ZFS ARC accounts for the memory pressure', async () => {
+    // Active working set reads 15G/16G (~94%, breaches 80%), but 5G of that is
+    // reclaimable ARC. Adding ARC back into available drops effective usage to
+    // ~62.5%, so no host-memory alert should fire.
+    mockMem.mockResolvedValue(memSample(15e9)); // available 1e9 -> 93.75%
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(5e9, 0)); // reclaimable 5e9
+
+    const svc = MonitorService.getInstance();
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+
+    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
+  });
+
   it('dispatches disk warning when over threshold', async () => {
     mockFsSize.mockResolvedValue([{ mount: '/', use: 92 }]);