fix: make host memory usage ZFS ARC-aware (#1547)

backend/src/__tests__/cache-endpoints.test.ts

@@ -17,6 +17,7 @@
 import { describe, it, expect, beforeAll, afterAll, vi, beforeEach } from 'vitest';
 import request from 'supertest';
 import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_PASSWORD } from './helpers/setupTestDb';
+import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsMock } from './helpers/arcstatsFsMock';
 import { GitSourceService } from '../services/GitSourceService';
 import type { PublicGitSource } from '../services/GitSourceService';
 
@@ -80,9 +81,13 @@ let tmpDir: string;
 let app: import('express').Express;
 let authCookie: string;
 let CacheService: typeof import('../services/CacheService').CacheService;
+let arcFs: ArcstatsFsMock;
 
 beforeAll(async () => {
   tmpDir = await setupTestDb();
+  // Host memory reads ZFS ARC stats; intercept those reads so results do not
+  // depend on whether the CI host is itself ZFS. Default: no ARC present.
+  arcFs = installArcstatsFsMock();
   ({ app } = await import('../index'));
   ({ CacheService } = await import('../services/CacheService'));
 
@@ -98,6 +103,7 @@ afterAll(() => {
 
 beforeEach(() => {
   CacheService.getInstance().flush();
+  arcFs.clear();
 
   mockGetAllContainers.mockReset();
   mockGetBulkStackStatuses.mockReset();
@@ -178,8 +184,9 @@ describe('GET /api/system/stats caching', () => {
   it('reports memory from the active working set, excluding reclaimable cache', async () => {
     const res = await request(app).get('/api/system/stats').set('Cookie', authCookie);
     expect(res.status).toBe(200);
-    // Figures come from mem.active / mem.available (cache-excluded), not the
-    // cache-inclusive mem.used / mem.free, so a busy host does not read ~100%.
+    // With no ARC present, effective used is total - available (which equals
+    // mem.active), not the cache-inclusive mem.used / mem.free, so a busy host
+    // does not read ~100%.
     expect(res.body.memory).toMatchObject({
       total: 1000,
       used: 400,            // mem.active, not mem.used (500)
@@ -187,6 +194,36 @@ describe('GET /api/system/stats caching', () => {
       usagePercent: '40.0', // 400 / 1000, not 500 / 1000
     });
   });
+
+  it('adds reclaimable ZFS ARC back into available memory', async () => {
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100)); // reclaimable 200
+    const res = await request(app).get('/api/system/stats').set('Cookie', authCookie);
+    expect(res.status).toBe(200);
+    // available 600 + 200 reclaimable ARC = 800 effective free; used drops to 200.
+    expect(res.body.memory).toMatchObject({
+      total: 1000,
+      used: 200,
+      free: 800,
+      usagePercent: '20.0',
+    });
+  });
+});
+
+// ── /api/fleet/overview (local node) ───────────────────────────────────
+
+describe('GET /api/fleet/overview local-node memory', () => {
+  it('reports ARC-adjusted memory for the local node', async () => {
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100)); // reclaimable 200
+    const res = await request(app).get('/api/fleet/overview').set('Cookie', authCookie);
+    expect(res.status).toBe(200);
+    const local = res.body.find((n: { type: string }) => n.type === 'local');
+    expect(local?.systemStats?.memory).toMatchObject({
+      total: 1000,
+      used: 200,
+      free: 800,
+      usagePercent: '20.0',
+    });
+  });
 });
 
 // ── /api/stacks/statuses ───────────────────────────────────────────────

backend/src/__tests__/helpers/arcstatsFsMock.ts

@@ -0,0 +1,98 @@
+import { promises as fs } from 'fs';
+import { vi } from 'vitest';
+import { ARCSTATS_FIXED_PATHS } from '../../helpers/hostMemory';
+
+/**
+ * Path-aware partial mock of `fs.promises` for ZFS arcstats reads.
+ *
+ * `helpers/hostMemory.ts` reads `/proc/spl/kstat/zfs/arcstats` (and optional
+ * variants) to compute reclaimable ARC. Tests may run on a ZFS host, so a real
+ * read would make results host-dependent. This installs a spy that intercepts
+ * ONLY registered/ARC-candidate paths and delegates every other
+ * `readFile`/`stat` to the real filesystem, so `setupTestDb` and
+ * `DatabaseService` keep working. Default behavior: ARC candidates reject with
+ * ENOENT (no ARC), so consumers fall back to the plain `active/total` reading.
+ */
+
+// Sourced from the helper so the mock cannot silently drift from the paths the
+// production code actually reads.
+export const ARC_CANDIDATE_PATHS = ARCSTATS_FIXED_PATHS;
+
+/** Second fixed candidate; the default path fixtures are served from. */
+export const DEFAULT_ARC_PATH = ARC_CANDIDATE_PATHS[1];
+
+type StatDescriptor = { isFile: boolean; size: number };
+
+export interface ArcstatsFsMock {
+  /** Serve `content` when `path` is read. */
+  setRead(path: string, content: string): void;
+  /** Reject a read of `path` with `err` (e.g. an EACCES/EIO error). */
+  setReadError(path: string, err: NodeJS.ErrnoException): void;
+  /** Control `stat(path)` result (for override-path guard tests). */
+  setStat(path: string, descriptor: StatDescriptor | NodeJS.ErrnoException): void;
+  /** Forget all registered paths (back to default no-ARC). */
+  clear(): void;
+}
+
+function enoent(path: string): NodeJS.ErrnoException {
+  return Object.assign(new Error(`ENOENT: no such file, open '${path}'`), { code: 'ENOENT' });
+}
+
+/**
+ * Install the spy. Call once per test file (e.g. in `beforeAll`); use the
+ * returned setters per test and `clear()` in `beforeEach`.
+ */
+export function installArcstatsFsMock(): ArcstatsFsMock {
+  const realReadFile = fs.readFile.bind(fs);
+  const realStat = fs.stat.bind(fs);
+  const reads = new Map<string, string | NodeJS.ErrnoException>();
+  const stats = new Map<string, StatDescriptor | NodeJS.ErrnoException>();
+  const isArcCandidate = (p: string): boolean => ARC_CANDIDATE_PATHS.includes(p);
+
+  vi.spyOn(fs, 'readFile').mockImplementation((async (p: unknown, ...rest: unknown[]) => {
+    const key = String(p);
+    if (reads.has(key)) {
+      const v = reads.get(key)!;
+      if (v instanceof Error) throw v;
+      return v;
+    }
+    if (isArcCandidate(key)) throw enoent(key);
+    return (realReadFile as (...a: unknown[]) => unknown)(p, ...rest);
+  }) as unknown as typeof fs.readFile);
+
+  vi.spyOn(fs, 'stat').mockImplementation((async (p: unknown, ...rest: unknown[]) => {
+    const key = String(p);
+    if (stats.has(key)) {
+      const v = stats.get(key)!;
+      if (v instanceof Error) throw v;
+      return { isFile: () => v.isFile, size: v.size };
+    }
+    // A registered read with no explicit stat implies a small regular file.
+    if (reads.has(key)) {
+      const v = reads.get(key);
+      const size = typeof v === 'string' ? Buffer.byteLength(v) : 0;
+      return { isFile: () => true, size };
+    }
+    if (isArcCandidate(key)) throw enoent(key);
+    return (realStat as (...a: unknown[]) => unknown)(p, ...rest);
+  }) as unknown as typeof fs.stat);
+
+  return {
+    setRead: (path, content) => reads.set(path, content),
+    setReadError: (path, err) => reads.set(path, err),
+    setStat: (path, descriptor) => stats.set(path, descriptor),
+    clear: () => { reads.clear(); stats.clear(); },
+  };
+}
+
+/** Build a minimal arcstats kstat body with the given `size` and `c_min` rows. */
+export function arcstatsBody(sizeRow: string | number, cMinRow: string | number): string {
+  return [
+    'name                            type data',
+    `hits                            4    123456`,
+    `c_min                           4    ${cMinRow}`,
+    `size                            4    ${sizeRow}`,
+    `c_max                           4    9999999999`,
+    '',
+  ].join('\n');
+}

backend/src/__tests__/host-memory.test.ts

@@ -0,0 +1,204 @@
+/**
+ * Unit tests for the ZFS ARC-aware host-memory helper.
+ *
+ * `adjustForArc` is exercised directly; `readReclaimableArc` and
+ * `parseArcstats` stay module-internal and are exercised through
+ * `getHostMemory` with a path-aware fs mock (see helpers/arcstatsFsMock.ts).
+ */
+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';
+import {
+  installArcstatsFsMock,
+  arcstatsBody,
+  DEFAULT_ARC_PATH,
+  ARC_CANDIDATE_PATHS,
+  type ArcstatsFsMock,
+} from './helpers/arcstatsFsMock';
+
+const mockMem = vi.fn();
+
+vi.mock('systeminformation', () => ({
+  default: { mem: (...args: unknown[]) => mockMem(...args) },
+}));
+
+import { getHostMemory, adjustForArc } from '../helpers/hostMemory';
+
+// mem.active === total - available on Linux, so used/free below mirror the
+// real systeminformation shape the helper consumes.
+const memSample = (total: number, available: number) => ({
+  total,
+  available,
+  active: total - available,
+  used: total - available,
+  free: available,
+  buffcache: 0,
+});
+
+let arcFs: ArcstatsFsMock;
+
+beforeAll(() => {
+  arcFs = installArcstatsFsMock();
+});
+
+beforeEach(() => {
+  arcFs.clear();
+  mockMem.mockReset();
+  delete process.env.SENCHO_ZFS_ARCSTATS_PATH;
+});
+
+describe('adjustForArc', () => {
+  it('reproduces active/total when reclaimable ARC is 0', () => {
+    const result = adjustForArc(memSample(1000, 600), 0);
+    expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
+  });
+
+  it('adds reclaimable ARC back into available, lowering usage', () => {
+    const result = adjustForArc(memSample(1000, 600), 200);
+    expect(result).toEqual({ total: 1000, used: 200, free: 800, usagePercent: 20 });
+  });
+
+  it('clamps effective available to total when ARC exceeds the gap', () => {
+    const result = adjustForArc(memSample(1000, 600), 5000);
+    expect(result).toEqual({ total: 1000, used: 0, free: 1000, usagePercent: 0 });
+  });
+
+  it('guards against a zero total', () => {
+    const result = adjustForArc(memSample(0, 0), 0);
+    expect(result.usagePercent).toBe(0);
+  });
+});
+
+describe('getHostMemory ARC discovery', () => {
+  it('falls back to active/total when no ARC stats are present', async () => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    const result = await getHostMemory();
+    expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
+  });
+
+  it('subtracts reclaimable ARC (size - c_min) from used', async () => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100)); // reclaimable 200
+    const result = await getHostMemory();
+    expect(result).toEqual({ total: 1000, used: 200, free: 800, usagePercent: 20 });
+  });
+
+  it('prefers the operator override path over the fixed candidates', async () => {
+    process.env.SENCHO_ZFS_ARCSTATS_PATH = '/custom/arcstats';
+    mockMem.mockResolvedValue(memSample(2000, 600));
+    arcFs.setRead('/custom/arcstats', arcstatsBody(500, 100)); // reclaimable 400
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100));   // fixed would be 200
+    const result = await getHostMemory();
+    expect(result.used).toBe(1000); // 2000 - (600 + 400 override); fixed would give 1200
+    expect(result.free).toBe(1000);
+  });
+
+  it('reads the host-mounted candidate and prefers it over /proc', async () => {
+    // ARC_CANDIDATE_PATHS[0] is /host/proc/..., the path docker-compose mounts
+    // into the container, so this covers the real deployment path and precedence.
+    mockMem.mockResolvedValue(memSample(2000, 600));
+    arcFs.setRead(ARC_CANDIDATE_PATHS[0], arcstatsBody(500, 100)); // /host/proc: reclaimable 400
+    arcFs.setRead(ARC_CANDIDATE_PATHS[1], arcstatsBody(300, 100)); // /proc: would be 200
+    const result = await getHostMemory();
+    expect(result.used).toBe(1000); // 2000 - (600 + 400); /proc winning would give 1200
+  });
+
+  it('falls through to a fixed candidate when the override is unreadable', async () => {
+    process.env.SENCHO_ZFS_ARCSTATS_PATH = '/custom/arcstats';
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setReadError('/custom/arcstats', Object.assign(new Error('nope'), { code: 'ENOENT' }));
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100)); // reclaimable 200
+    const result = await getHostMemory();
+    expect(result.used).toBe(200);
+  });
+
+  it('resolves immediately to 0 reclaimable when size < c_min (ARC at floor)', async () => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(50, 100)); // size < c_min
+    const result = await getHostMemory();
+    expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
+  });
+
+  it.each([
+    ['non-numeric size', arcstatsBody('abc', 100)],
+    ['negative size', arcstatsBody(-5, 100)],
+    ['non-numeric c_min', arcstatsBody(300, 'xyz')],
+    ['negative c_min', arcstatsBody(300, -5)],
+    ['missing c_min', 'size                            4    300\n'],
+    ['missing size', 'c_min                           4    100\n'],
+    ['empty file', '   \n'],
+  ])('treats a %s record as unusable and yields no ARC', async (_label, body) => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setRead(DEFAULT_ARC_PATH, body);
+    const result = await getHostMemory();
+    expect(result.used).toBe(400); // fell through to active/total
+  });
+
+  it.each([
+    ['EACCES', 'EACCES'],
+    ['EIO', 'EIO'],
+    ['EMFILE', 'EMFILE'],
+  ])('fails open (ARC 0) on a %s read error', async (_label, code) => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setReadError(DEFAULT_ARC_PATH, Object.assign(new Error(code), { code }));
+    const result = await getHostMemory();
+    expect(result.used).toBe(400);
+  });
+
+  it('logs an unexpected read error (once per code) but stays silent on an expected one', async () => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+
+    // Expected fs error: silent fall-through.
+    arcFs.setReadError(ARC_CANDIDATE_PATHS[0], Object.assign(new Error('denied'), { code: 'EACCES' }));
+    arcFs.setReadError(ARC_CANDIDATE_PATHS[1], Object.assign(new Error('denied'), { code: 'EACCES' }));
+    await getHostMemory();
+    expect(warn).not.toHaveBeenCalled();
+
+    // Unexpected fs error: logged, but only once per error code across calls.
+    // Uses a code no other test triggers, since the once-per-code memo is
+    // process-global.
+    arcFs.setReadError(ARC_CANDIDATE_PATHS[0], Object.assign(new Error('stale'), { code: 'ESTALE' }));
+    arcFs.setReadError(ARC_CANDIDATE_PATHS[1], Object.assign(new Error('stale'), { code: 'ESTALE' }));
+    await getHostMemory();
+    await getHostMemory();
+    const unexpectedLogs = warn.mock.calls.filter(([msg]) => String(msg).includes('ESTALE'));
+    expect(unexpectedLogs).toHaveLength(1);
+    warn.mockRestore();
+  });
+
+  it('skips an override path that is not a regular file', async () => {
+    process.env.SENCHO_ZFS_ARCSTATS_PATH = '/custom/arcstats';
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setStat('/custom/arcstats', { isFile: false, size: 10 });
+    arcFs.setRead('/custom/arcstats', arcstatsBody(500, 100));
+    const result = await getHostMemory();
+    expect(result.used).toBe(400); // override skipped, no fixed ARC present
+  });
+
+  it('skips an override path that exceeds the size bound', async () => {
+    process.env.SENCHO_ZFS_ARCSTATS_PATH = '/custom/arcstats';
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setStat('/custom/arcstats', { isFile: true, size: 2 * 1024 * 1024 });
+    arcFs.setRead('/custom/arcstats', arcstatsBody(500, 100));
+    const result = await getHostMemory();
+    expect(result.used).toBe(400);
+  });
+
+  it('logs the selected path once and never the file contents', async () => {
+    process.env.SENCHO_ZFS_ARCSTATS_PATH = '/log-once/arcstats';
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setRead('/log-once/arcstats', arcstatsBody(300, 100));
+    const debug = vi.spyOn(console, 'debug').mockImplementation(() => {});
+    await getHostMemory();
+    await getHostMemory();
+    const pathLogs = debug.mock.calls.filter(([msg]) => String(msg).includes('/log-once/arcstats'));
+    expect(pathLogs).toHaveLength(1);
+    // The log names the path, never the kstat contents (size / c_min values).
+    expect(String(pathLogs[0][0])).not.toContain('300');
+    expect(String(pathLogs[0][0])).not.toContain('100');
+    debug.mockRestore();
+  });
+});
+
+afterEach(() => {
+  delete process.env.SENCHO_ZFS_ARCSTATS_PATH;
+});

backend/src/__tests__/monitor-service.test.ts

@@ -2,7 +2,8 @@
  * Unit tests for MonitorService — alert state machine, metric calculations,
  * cleanup delegation, global settings evaluation, and concurrency guards.
  */
-import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vitest';
+import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsMock } from './helpers/arcstatsFsMock';
 
 // ── Hoisted mocks ──────────────────────────────────────────────────────
 
@@ -138,8 +139,16 @@ vi.mock('util', () => ({
 
 import { MonitorService, _resetHostAlertSuppressionStateForTests } from '../services/MonitorService';
 
+// Host memory now reads ZFS ARC stats; intercept those reads so the suite does
+// not depend on whether the machine running it is itself a ZFS host.
+let arcFs: ArcstatsFsMock;
+beforeAll(() => {
+  arcFs = installArcstatsFsMock();
+});
+
 beforeEach(() => {
   vi.clearAllMocks();
+  arcFs.clear();
   (MonitorService as any).instance = undefined;
   _resetHostAlertSuppressionStateForTests();
   mockGetSystemState.mockReturnValue(null);
@@ -363,6 +372,19 @@ describe('MonitorService - evaluateGlobalSettings', () => {
     expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
   });
 
+  it('does not alert when reclaimable ZFS ARC accounts for the memory pressure', async () => {
+    // Active working set reads 15G/16G (~94%, breaches 80%), but 5G of that is
+    // reclaimable ARC. Adding ARC back into available drops effective usage to
+    // ~62.5%, so no host-memory alert should fire.
+    mockMem.mockResolvedValue(memSample(15e9)); // available 1e9 -> 93.75%
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(5e9, 0)); // reclaimable 5e9
+
+    const svc = MonitorService.getInstance();
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+
+    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
+  });
+
   it('dispatches disk warning when over threshold', async () => {
     mockFsSize.mockResolvedValue([{ mount: '/', use: 92 }]);
 

backend/src/helpers/hostMemory.ts

@@ -0,0 +1,141 @@
+import si from 'systeminformation';
+import { promises as fs } from 'fs';
+
+/**
+ * Shared host-memory computation, ZFS ARC aware.
+ *
+ * `systeminformation.mem()` derives `active` as `total - available` on
+ * Linux/BSD/macOS, so keying usage off `active` already dodges page-cache
+ * inflation. It does NOT account for the OpenZFS ARC: the kernel's
+ * MemAvailable treats ARC as unavailable even though ARC shrinks under
+ * memory pressure, so on ZFS hosts a large ARC reads as hard-used memory
+ * and produces false host-memory alerts.
+ *
+ * When ARC kstats are readable we add the reclaimable portion
+ * (`max(size - c_min, 0)`) back into available memory. On non-ZFS hosts, or
+ * when the kstat file is not readable inside the container, ARC is treated as
+ * zero and the result is identical to the previous `active / total` behavior.
+ */
+
+/** Effective host memory after adding reclaimable ZFS ARC back into available. */
+export interface HostMemory {
+  total: number;
+  /** Effective used bytes (ARC-adjusted). */
+  used: number;
+  /** Effective available bytes (ARC-adjusted). */
+  free: number;
+  /** Effective used as a percentage of total (0 when total is 0). */
+  usagePercent: number;
+}
+
+type MemData = Awaited<ReturnType<typeof si.mem>>;
+
+/**
+ * Candidate arcstats paths in priority order. The operator override is only
+ * present when SENCHO_ZFS_ARCSTATS_PATH is set; the two fixed paths are the
+ * host-mounted and the standard container-visible kstat locations.
+ */
+export const ARCSTATS_FIXED_PATHS = [
+  '/host/proc/spl/kstat/zfs/arcstats',
+  '/proc/spl/kstat/zfs/arcstats',
+];
+
+/** Bound reads of the operator-supplied override path; arcstats is a few KB. */
+const MAX_ARCSTATS_BYTES = 1024 * 1024;
+
+// Memoized so a 30s monitor tick / dashboard poll does not log on every cycle.
+const loggedSelectedPaths = new Set<string>();
+const loggedErrorCodes = new Set<string>();
+
+function overridePath(): string | undefined {
+  const raw = process.env.SENCHO_ZFS_ARCSTATS_PATH?.trim();
+  return raw ? raw : undefined;
+}
+
+function isExpectedFsError(err: unknown): boolean {
+  const code = (err as NodeJS.ErrnoException)?.code;
+  return (
+    code === 'ENOENT' ||
+    code === 'EACCES' ||
+    code === 'EPERM' ||
+    code === 'EISDIR' ||
+    code === 'ENOTDIR' ||
+    code === 'ELOOP'
+  );
+}
+
+function logUnexpected(context: string, err: unknown): void {
+  const code = (err as NodeJS.ErrnoException)?.code ?? 'UNKNOWN';
+  if (loggedErrorCodes.has(code)) return;
+  loggedErrorCodes.add(code);
+  console.warn(`[HostMemory] Unexpected error reading ARC stats (${context}, ${code}); treating ARC as reclaimable=0`);
+}
+
+/** Parse the kstat table for the `size` and `c_min` rows (`<name> <type> <value>`). */
+function parseArcstats(raw: string): { size?: number; cMin?: number } {
+  let size: number | undefined;
+  let cMin: number | undefined;
+  for (const line of raw.split('\n')) {
+    const parts = line.trim().split(/\s+/);
+    if (parts.length < 3) continue;
+    if (parts[0] === 'size') size = Number(parts[2]);
+    else if (parts[0] === 'c_min') cMin = Number(parts[2]);
+  }
+  return { size, cMin };
+}
+
+/**
+ * Reclaimable ARC in bytes, or 0 when ARC stats are unavailable/unusable.
+ * Never throws: any error resolves to 0 so ARC awareness can only lower a
+ * false-positive reading, never break host-memory reporting.
+ */
+async function readReclaimableArc(): Promise<number> {
+  const override = overridePath();
+  const candidates = override ? [override, ...ARCSTATS_FIXED_PATHS] : ARCSTATS_FIXED_PATHS;
+  for (const candidatePath of candidates) {
+    try {
+      // The override path is operator-supplied: verify it is a regular file
+      // of bounded size before reading (guards against a named pipe or an
+      // accidentally huge target). The fixed kstat paths are trusted.
+      if (candidatePath === override) {
+        const info = await fs.stat(candidatePath);
+        if (!info.isFile() || info.size > MAX_ARCSTATS_BYTES) continue;
+      }
+      const raw = await fs.readFile(candidatePath, 'utf8');
+      const { size, cMin } = parseArcstats(raw);
+      if (size === undefined || cMin === undefined) continue;
+      if (!Number.isFinite(size) || !Number.isFinite(cMin) || size < 0 || cMin < 0) continue;
+      // A valid record resolves the lookup, even when reclaimable is 0
+      // (size < c_min means ARC is at its floor).
+      if (!loggedSelectedPaths.has(candidatePath)) {
+        loggedSelectedPaths.add(candidatePath);
+        console.debug(`[HostMemory] Using ZFS ARC stats from ${candidatePath}`);
+      }
+      return Math.max(size - cMin, 0);
+    } catch (err) {
+      // Fail open: a missing or unreadable kstat is the normal non-ZFS case
+      // (expected fs errors); an unexpected error is logged once but still
+      // falls through so ARC awareness can only lower a false positive.
+      if (isExpectedFsError(err)) continue;
+      logUnexpected(candidatePath, err);
+    }
+  }
+  return 0;
+}
+
+/**
+ * Pure ARC adjustment. With `arcReclaimable === 0` this reproduces the prior
+ * `active / total` percentage exactly (since `active === total - available`).
+ */
+export function adjustForArc(mem: Pick<MemData, 'total' | 'available'>, arcReclaimable: number): HostMemory {
+  const effectiveAvailable = Math.min(mem.total, mem.available + Math.max(arcReclaimable, 0));
+  const effectiveUsed = Math.max(mem.total - effectiveAvailable, 0);
+  const usagePercent = mem.total > 0 ? (effectiveUsed / mem.total) * 100 : 0;
+  return { total: mem.total, used: effectiveUsed, free: effectiveAvailable, usagePercent };
+}
+
+/** Fetch host memory and reclaimable ARC concurrently, return the adjusted view. */
+export async function getHostMemory(): Promise<HostMemory> {
+  const [mem, arcReclaimable] = await Promise.all([si.mem(), readReclaimableArc()]);
+  return adjustForArc(mem, arcReclaimable);
+}

backend/src/routes/fleet.ts

@@ -10,6 +10,7 @@ import { FleetUpdateTrackerService, type UpdateTracker, type TerminalStatus, UPD
 import { NodeRegistry } from '../services/NodeRegistry';
 import { computeNodeNetworkingSummary, type NodeNetworkingSummary } from '../services/network/networkingSummary';
 import DockerController from '../services/DockerController';
+import { getHostMemory } from '../helpers/hostMemory';
 import { FileSystemService } from '../services/FileSystemService';
 import { ComposeService } from '../services/ComposeService';
 import { StackOpLockService } from '../services/StackOpLockService';
@@ -222,11 +223,11 @@ async function getCompareTarget(gatewayVersion: string | null) {
 async function fetchLocalNodeOverview(node: Node): Promise<FleetNodeOverview> {
   try {
     const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(node.id));
-    const [allContainers, stacks, currentLoad, mem, fsSize] = await Promise.all([
+    const [allContainers, stacks, currentLoad, hostMem, fsSize] = await Promise.all([
       DockerController.getInstance(node.id).getAllContainers(),
       FileSystemService.getInstance(node.id).getStacks(),
       si.currentLoad(),
-      si.mem(),
+      getHostMemory(),
       si.fsSize(),
     ]);
 
@@ -255,13 +256,12 @@ async function fetchLocalNodeOverview(node: Node): Promise<FleetNodeOverview> {
       systemStats: {
         cpu: { usage: currentLoad.currentLoad.toFixed(1), cores: currentLoad.cpus.length },
         memory: {
-          total: mem.total,
-          // Percentage is keyed off mem.active (the real working set), not mem.used:
-          // on Linux/BSD/macOS mem.used counts reclaimable page cache and reads ~99%
-          // on a busy host. mem.available is total - active, so used + free = total.
-          used: mem.active,
-          free: mem.available,
-          usagePercent: ((mem.active / mem.total) * 100).toFixed(1),
+          total: hostMem.total,
+          // ZFS ARC aware: reclaimable ARC is added back into available so a
+          // large ARC cache is not reported as hard-used. See helpers/hostMemory.ts.
+          used: hostMem.used,
+          free: hostMem.free,
+          usagePercent: hostMem.usagePercent.toFixed(1),
         },
         disk: mainDisk ? {
           total: mainDisk.size,

backend/src/routes/metrics.ts

@@ -9,6 +9,7 @@ import { PilotTunnelManager } from '../services/PilotTunnelManager';
 import { authMiddleware } from '../middleware/auth';
 import { requireAdmin } from '../middleware/tierGates';
 import { STATS_CACHE_TTL_MS, SYSTEM_STATS_CACHE_TTL_MS } from '../helpers/constants';
+import { getHostMemory } from '../helpers/hostMemory';
 import { isDebugEnabled } from '../utils/debug';
 import { getErrorMessage } from '../utils/errors';
 import { isManagedByComposeDir } from '../utils/managed-containers';
@@ -296,9 +297,9 @@ metricsRouter.get('/system/stats', authMiddleware, async (req: Request, res: Res
       async () => {
         // Remote-node requests are intercepted and proxied upstream before
         // reaching here; this fetcher only runs for local nodes.
-        const [currentLoad, mem, fsSize] = await Promise.all([
+        const [currentLoad, hostMem, fsSize] = await Promise.all([
           si.currentLoad(),
-          si.mem(),
+          getHostMemory(),
           si.fsSize(),
         ]);
 
@@ -310,13 +311,12 @@ metricsRouter.get('/system/stats', authMiddleware, async (req: Request, res: Res
             cores: currentLoad.cpus.length,
           },
           memory: {
-            total: mem.total,
-            // Percentage is keyed off mem.active (the real working set), not mem.used:
-            // on Linux/BSD/macOS mem.used counts reclaimable page cache and reads ~99%
-            // on a busy host. mem.available is total - active, so used + free = total.
-            used: mem.active,
-            free: mem.available,
-            usagePercent: ((mem.active / mem.total) * 100).toFixed(1),
+            total: hostMem.total,
+            // ZFS ARC aware: reclaimable ARC is added back into available so a
+            // large ARC cache is not reported as hard-used. See helpers/hostMemory.ts.
+            used: hostMem.used,
+            free: hostMem.free,
+            usagePercent: hostMem.usagePercent.toFixed(1),
           },
           disk: mainDisk ? {
             fs: mainDisk.fs,

backend/src/services/MonitorService.ts

@@ -9,6 +9,7 @@ import { NotificationService } from './NotificationService';
 import { FleetUpdateTrackerService } from './FleetUpdateTrackerService';
 import { isValidVersion, getSenchoVersion } from './CapabilityRegistry';
 import { getLatestVersionInfo } from '../utils/version-check';
+import { getHostMemory } from '../helpers/hostMemory';
 import { isDebugEnabled } from '../utils/debug';
 import { withTimeout, TimeoutError } from '../utils/withTimeout';
 
@@ -287,9 +288,9 @@ export class MonitorService {
         // 1. Host Limits — fetch CPU, RAM, disk concurrently
         if (settings['host_alerts_enabled'] !== '0') {
             try {
-                const [currentLoad, mem, fsSize] = await Promise.all([
+                const [currentLoad, hostMem, fsSize] = await Promise.all([
                     withTimeout(si.currentLoad(), STATS_TIMEOUT_MS, 'host CPU stats'),
-                    withTimeout(si.mem(), STATS_TIMEOUT_MS, 'host RAM stats'),
+                    withTimeout(getHostMemory(), STATS_TIMEOUT_MS, 'host RAM stats'),
                     withTimeout(si.fsSize(), STATS_TIMEOUT_MS, 'host disk stats'),
                 ]);
 
@@ -302,10 +303,9 @@ export class MonitorService {
                     this.clearHostMetricSuppression('cpu');
                 }
 
-                // Key off mem.active (the real working set), not mem.used: on Linux/BSD/macOS
-                // mem.used counts reclaimable page cache and reads ~99% on a busy host, which
-                // would fire spurious host-memory alerts.
-                const ramUsage = (mem.active / mem.total) * 100;
+                // ZFS ARC aware: reclaimable ARC is added back into available so a large ARC
+                // cache does not fire spurious host-memory alerts. See helpers/hostMemory.ts.
+                const ramUsage = hostMem.usagePercent;
                 const ramLimit = parseFloat(settings['host_ram_limit']);
                 if (!isNaN(ramLimit) && ramLimit > 0 && ramUsage > ramLimit) {
                     await this.dispatchHostMetricAlert('ram', 'warning', suppressionMs,