fix: make host memory usage ZFS ARC-aware (#1547)

.env.example

@@ -148,3 +148,13 @@ GITSOURCE_MAX_CLONE_BYTES=104857600
 # working pull can be briefly silent while a large layer extracts; raise it on
 # slow links or heavy local builds. Default 600000 (10 minutes).
 # SENCHO_COMPOSE_STALL_TIMEOUT_MS=600000
+
+# Container-side path to the OpenZFS ARC stats file, for ZFS-aware host memory.
+# On ZFS hosts (TrueNAS SCALE, Proxmox, ZFS on Ubuntu/Debian) the ARC cache is
+# reclaimable but the kernel reports it as used, which can trigger false
+# host-memory alerts. When ARC stats are readable, Sencho adds reclaimable ARC
+# back into available memory. Sencho checks this path first, then
+# /host/proc/spl/kstat/zfs/arcstats, then /proc/spl/kstat/zfs/arcstats. Set this
+# only if your ARC stats live at a non-standard path inside the container. If no
+# ARC stats are readable, host memory reporting is unchanged.
+# SENCHO_ZFS_ARCSTATS_PATH=

backend/src/__tests__/cache-endpoints.test.ts

@@ -17,6 +17,7 @@
 import { describe, it, expect, beforeAll, afterAll, vi, beforeEach } from 'vitest';
 import request from 'supertest';
 import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_PASSWORD } from './helpers/setupTestDb';
+import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsMock } from './helpers/arcstatsFsMock';
 import { GitSourceService } from '../services/GitSourceService';
 import type { PublicGitSource } from '../services/GitSourceService';
 
@@ -80,9 +81,13 @@ let tmpDir: string;
 let app: import('express').Express;
 let authCookie: string;
 let CacheService: typeof import('../services/CacheService').CacheService;
+let arcFs: ArcstatsFsMock;
 
 beforeAll(async () => {
   tmpDir = await setupTestDb();
+  // Host memory reads ZFS ARC stats; intercept those reads so results do not
+  // depend on whether the CI host is itself ZFS. Default: no ARC present.
+  arcFs = installArcstatsFsMock();
   ({ app } = await import('../index'));
   ({ CacheService } = await import('../services/CacheService'));
 
@@ -98,6 +103,7 @@ afterAll(() => {
 
 beforeEach(() => {
   CacheService.getInstance().flush();
+  arcFs.clear();
 
   mockGetAllContainers.mockReset();
   mockGetBulkStackStatuses.mockReset();
@@ -178,8 +184,9 @@ describe('GET /api/system/stats caching', () => {
   it('reports memory from the active working set, excluding reclaimable cache', async () => {
     const res = await request(app).get('/api/system/stats').set('Cookie', authCookie);
     expect(res.status).toBe(200);
-    // Figures come from mem.active / mem.available (cache-excluded), not the
+    // With no ARC present, effective used is total - available (which equals
-    // cache-inclusive mem.used / mem.free, so a busy host does not read ~100%.
+    // mem.active), not the cache-inclusive mem.used / mem.free, so a busy host
+    // does not read ~100%.
     expect(res.body.memory).toMatchObject({
       total: 1000,
       used: 400,            // mem.active, not mem.used (500)
@@ -187,6 +194,36 @@ describe('GET /api/system/stats caching', () => {
       usagePercent: '40.0', // 400 / 1000, not 500 / 1000
     });
   });
+
+  it('adds reclaimable ZFS ARC back into available memory', async () => {
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100)); // reclaimable 200
+    const res = await request(app).get('/api/system/stats').set('Cookie', authCookie);
+    expect(res.status).toBe(200);
+    // available 600 + 200 reclaimable ARC = 800 effective free; used drops to 200.
+    expect(res.body.memory).toMatchObject({
+      total: 1000,
+      used: 200,
+      free: 800,
+      usagePercent: '20.0',
+    });
+  });
+});
+
+// ── /api/fleet/overview (local node) ───────────────────────────────────
+
+describe('GET /api/fleet/overview local-node memory', () => {
+  it('reports ARC-adjusted memory for the local node', async () => {
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100)); // reclaimable 200
+    const res = await request(app).get('/api/fleet/overview').set('Cookie', authCookie);
+    expect(res.status).toBe(200);
+    const local = res.body.find((n: { type: string }) => n.type === 'local');
+    expect(local?.systemStats?.memory).toMatchObject({
+      total: 1000,
+      used: 200,
+      free: 800,
+      usagePercent: '20.0',
+    });
+  });
 });
 
 // ── /api/stacks/statuses ───────────────────────────────────────────────

backend/src/__tests__/helpers/arcstatsFsMock.ts

@@ -0,0 +1,98 @@
+import { promises as fs } from 'fs';
+import { vi } from 'vitest';
+import { ARCSTATS_FIXED_PATHS } from '../../helpers/hostMemory';
+
+/**
+ * Path-aware partial mock of `fs.promises` for ZFS arcstats reads.
+ *
+ * `helpers/hostMemory.ts` reads `/proc/spl/kstat/zfs/arcstats` (and optional
+ * variants) to compute reclaimable ARC. Tests may run on a ZFS host, so a real
+ * read would make results host-dependent. This installs a spy that intercepts
+ * ONLY registered/ARC-candidate paths and delegates every other
+ * `readFile`/`stat` to the real filesystem, so `setupTestDb` and
+ * `DatabaseService` keep working. Default behavior: ARC candidates reject with
+ * ENOENT (no ARC), so consumers fall back to the plain `active/total` reading.
+ */
+
+// Sourced from the helper so the mock cannot silently drift from the paths the
+// production code actually reads.
+export const ARC_CANDIDATE_PATHS = ARCSTATS_FIXED_PATHS;
+
+/** Second fixed candidate; the default path fixtures are served from. */
+export const DEFAULT_ARC_PATH = ARC_CANDIDATE_PATHS[1];
+
+type StatDescriptor = { isFile: boolean; size: number };
+
+export interface ArcstatsFsMock {
+  /** Serve `content` when `path` is read. */
+  setRead(path: string, content: string): void;
+  /** Reject a read of `path` with `err` (e.g. an EACCES/EIO error). */
+  setReadError(path: string, err: NodeJS.ErrnoException): void;
+  /** Control `stat(path)` result (for override-path guard tests). */
+  setStat(path: string, descriptor: StatDescriptor | NodeJS.ErrnoException): void;
+  /** Forget all registered paths (back to default no-ARC). */
+  clear(): void;
+}
+
+function enoent(path: string): NodeJS.ErrnoException {
+  return Object.assign(new Error(`ENOENT: no such file, open '${path}'`), { code: 'ENOENT' });
+}
+
+/**
+ * Install the spy. Call once per test file (e.g. in `beforeAll`); use the
+ * returned setters per test and `clear()` in `beforeEach`.
+ */
+export function installArcstatsFsMock(): ArcstatsFsMock {
+  const realReadFile = fs.readFile.bind(fs);
+  const realStat = fs.stat.bind(fs);
+  const reads = new Map<string, string | NodeJS.ErrnoException>();
+  const stats = new Map<string, StatDescriptor | NodeJS.ErrnoException>();
+  const isArcCandidate = (p: string): boolean => ARC_CANDIDATE_PATHS.includes(p);
+
+  vi.spyOn(fs, 'readFile').mockImplementation((async (p: unknown, ...rest: unknown[]) => {
+    const key = String(p);
+    if (reads.has(key)) {
+      const v = reads.get(key)!;
+      if (v instanceof Error) throw v;
+      return v;
+    }
+    if (isArcCandidate(key)) throw enoent(key);
+    return (realReadFile as (...a: unknown[]) => unknown)(p, ...rest);
+  }) as unknown as typeof fs.readFile);
+
+  vi.spyOn(fs, 'stat').mockImplementation((async (p: unknown, ...rest: unknown[]) => {
+    const key = String(p);
+    if (stats.has(key)) {
+      const v = stats.get(key)!;
+      if (v instanceof Error) throw v;
+      return { isFile: () => v.isFile, size: v.size };
+    }
+    // A registered read with no explicit stat implies a small regular file.
+    if (reads.has(key)) {
+      const v = reads.get(key);
+      const size = typeof v === 'string' ? Buffer.byteLength(v) : 0;
+      return { isFile: () => true, size };
+    }
+    if (isArcCandidate(key)) throw enoent(key);
+    return (realStat as (...a: unknown[]) => unknown)(p, ...rest);
+  }) as unknown as typeof fs.stat);
+
+  return {
+    setRead: (path, content) => reads.set(path, content),
+    setReadError: (path, err) => reads.set(path, err),
+    setStat: (path, descriptor) => stats.set(path, descriptor),
+    clear: () => { reads.clear(); stats.clear(); },
+  };
+}
+
+/** Build a minimal arcstats kstat body with the given `size` and `c_min` rows. */
+export function arcstatsBody(sizeRow: string | number, cMinRow: string | number): string {
+  return [
+    'name                            type data',
+    `hits                            4    123456`,
+    `c_min                           4    ${cMinRow}`,
+    `size                            4    ${sizeRow}`,
+    `c_max                           4    9999999999`,
+    '',
+  ].join('\n');
+}

backend/src/__tests__/host-memory.test.ts

@@ -0,0 +1,204 @@
+/**
+ * Unit tests for the ZFS ARC-aware host-memory helper.
+ *
+ * `adjustForArc` is exercised directly; `readReclaimableArc` and
+ * `parseArcstats` stay module-internal and are exercised through
+ * `getHostMemory` with a path-aware fs mock (see helpers/arcstatsFsMock.ts).
+ */
+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';
+import {
+  installArcstatsFsMock,
+  arcstatsBody,
+  DEFAULT_ARC_PATH,
+  ARC_CANDIDATE_PATHS,
+  type ArcstatsFsMock,
+} from './helpers/arcstatsFsMock';
+
+const mockMem = vi.fn();
+
+vi.mock('systeminformation', () => ({
+  default: { mem: (...args: unknown[]) => mockMem(...args) },
+}));
+
+import { getHostMemory, adjustForArc } from '../helpers/hostMemory';
+
+// mem.active === total - available on Linux, so used/free below mirror the
+// real systeminformation shape the helper consumes.
+const memSample = (total: number, available: number) => ({
+  total,
+  available,
+  active: total - available,
+  used: total - available,
+  free: available,
+  buffcache: 0,
+});
+
+let arcFs: ArcstatsFsMock;
+
+beforeAll(() => {
+  arcFs = installArcstatsFsMock();
+});
+
+beforeEach(() => {
+  arcFs.clear();
+  mockMem.mockReset();
+  delete process.env.SENCHO_ZFS_ARCSTATS_PATH;
+});
+
+describe('adjustForArc', () => {
+  it('reproduces active/total when reclaimable ARC is 0', () => {
+    const result = adjustForArc(memSample(1000, 600), 0);
+    expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
+  });
+
+  it('adds reclaimable ARC back into available, lowering usage', () => {
+    const result = adjustForArc(memSample(1000, 600), 200);
+    expect(result).toEqual({ total: 1000, used: 200, free: 800, usagePercent: 20 });
+  });
+
+  it('clamps effective available to total when ARC exceeds the gap', () => {
+    const result = adjustForArc(memSample(1000, 600), 5000);
+    expect(result).toEqual({ total: 1000, used: 0, free: 1000, usagePercent: 0 });
+  });
+
+  it('guards against a zero total', () => {
+    const result = adjustForArc(memSample(0, 0), 0);
+    expect(result.usagePercent).toBe(0);
+  });
+});
+
+describe('getHostMemory ARC discovery', () => {
+  it('falls back to active/total when no ARC stats are present', async () => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    const result = await getHostMemory();
+    expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
+  });
+
+  it('subtracts reclaimable ARC (size - c_min) from used', async () => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100)); // reclaimable 200
+    const result = await getHostMemory();
+    expect(result).toEqual({ total: 1000, used: 200, free: 800, usagePercent: 20 });
+  });
+
+  it('prefers the operator override path over the fixed candidates', async () => {
+    process.env.SENCHO_ZFS_ARCSTATS_PATH = '/custom/arcstats';
+    mockMem.mockResolvedValue(memSample(2000, 600));
+    arcFs.setRead('/custom/arcstats', arcstatsBody(500, 100)); // reclaimable 400
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100));   // fixed would be 200
+    const result = await getHostMemory();
+    expect(result.used).toBe(1000); // 2000 - (600 + 400 override); fixed would give 1200
+    expect(result.free).toBe(1000);
+  });
+
+  it('reads the host-mounted candidate and prefers it over /proc', async () => {
+    // ARC_CANDIDATE_PATHS[0] is /host/proc/..., the path docker-compose mounts
+    // into the container, so this covers the real deployment path and precedence.
+    mockMem.mockResolvedValue(memSample(2000, 600));
+    arcFs.setRead(ARC_CANDIDATE_PATHS[0], arcstatsBody(500, 100)); // /host/proc: reclaimable 400
+    arcFs.setRead(ARC_CANDIDATE_PATHS[1], arcstatsBody(300, 100)); // /proc: would be 200
+    const result = await getHostMemory();
+    expect(result.used).toBe(1000); // 2000 - (600 + 400); /proc winning would give 1200
+  });
+
+  it('falls through to a fixed candidate when the override is unreadable', async () => {
+    process.env.SENCHO_ZFS_ARCSTATS_PATH = '/custom/arcstats';
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setReadError('/custom/arcstats', Object.assign(new Error('nope'), { code: 'ENOENT' }));
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100)); // reclaimable 200
+    const result = await getHostMemory();
+    expect(result.used).toBe(200);
+  });
+
+  it('resolves immediately to 0 reclaimable when size < c_min (ARC at floor)', async () => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(50, 100)); // size < c_min
+    const result = await getHostMemory();
+    expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
+  });
+
+  it.each([
+    ['non-numeric size', arcstatsBody('abc', 100)],
+    ['negative size', arcstatsBody(-5, 100)],
+    ['non-numeric c_min', arcstatsBody(300, 'xyz')],
+    ['negative c_min', arcstatsBody(300, -5)],
+    ['missing c_min', 'size                            4    300\n'],
+    ['missing size', 'c_min                           4    100\n'],
+    ['empty file', '   \n'],
+  ])('treats a %s record as unusable and yields no ARC', async (_label, body) => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setRead(DEFAULT_ARC_PATH, body);
+    const result = await getHostMemory();
+    expect(result.used).toBe(400); // fell through to active/total
+  });
+
+  it.each([
+    ['EACCES', 'EACCES'],
+    ['EIO', 'EIO'],
+    ['EMFILE', 'EMFILE'],
+  ])('fails open (ARC 0) on a %s read error', async (_label, code) => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setReadError(DEFAULT_ARC_PATH, Object.assign(new Error(code), { code }));
+    const result = await getHostMemory();
+    expect(result.used).toBe(400);
+  });
+
+  it('logs an unexpected read error (once per code) but stays silent on an expected one', async () => {
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+
+    // Expected fs error: silent fall-through.
+    arcFs.setReadError(ARC_CANDIDATE_PATHS[0], Object.assign(new Error('denied'), { code: 'EACCES' }));
+    arcFs.setReadError(ARC_CANDIDATE_PATHS[1], Object.assign(new Error('denied'), { code: 'EACCES' }));
+    await getHostMemory();
+    expect(warn).not.toHaveBeenCalled();
+
+    // Unexpected fs error: logged, but only once per error code across calls.
+    // Uses a code no other test triggers, since the once-per-code memo is
+    // process-global.
+    arcFs.setReadError(ARC_CANDIDATE_PATHS[0], Object.assign(new Error('stale'), { code: 'ESTALE' }));
+    arcFs.setReadError(ARC_CANDIDATE_PATHS[1], Object.assign(new Error('stale'), { code: 'ESTALE' }));
+    await getHostMemory();
+    await getHostMemory();
+    const unexpectedLogs = warn.mock.calls.filter(([msg]) => String(msg).includes('ESTALE'));
+    expect(unexpectedLogs).toHaveLength(1);
+    warn.mockRestore();
+  });
+
+  it('skips an override path that is not a regular file', async () => {
+    process.env.SENCHO_ZFS_ARCSTATS_PATH = '/custom/arcstats';
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setStat('/custom/arcstats', { isFile: false, size: 10 });
+    arcFs.setRead('/custom/arcstats', arcstatsBody(500, 100));
+    const result = await getHostMemory();
+    expect(result.used).toBe(400); // override skipped, no fixed ARC present
+  });
+
+  it('skips an override path that exceeds the size bound', async () => {
+    process.env.SENCHO_ZFS_ARCSTATS_PATH = '/custom/arcstats';
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setStat('/custom/arcstats', { isFile: true, size: 2 * 1024 * 1024 });
+    arcFs.setRead('/custom/arcstats', arcstatsBody(500, 100));
+    const result = await getHostMemory();
+    expect(result.used).toBe(400);
+  });
+
+  it('logs the selected path once and never the file contents', async () => {
+    process.env.SENCHO_ZFS_ARCSTATS_PATH = '/log-once/arcstats';
+    mockMem.mockResolvedValue(memSample(1000, 600));
+    arcFs.setRead('/log-once/arcstats', arcstatsBody(300, 100));
+    const debug = vi.spyOn(console, 'debug').mockImplementation(() => {});
+    await getHostMemory();
+    await getHostMemory();
+    const pathLogs = debug.mock.calls.filter(([msg]) => String(msg).includes('/log-once/arcstats'));
+    expect(pathLogs).toHaveLength(1);
+    // The log names the path, never the kstat contents (size / c_min values).
+    expect(String(pathLogs[0][0])).not.toContain('300');
+    expect(String(pathLogs[0][0])).not.toContain('100');
+    debug.mockRestore();
+  });
+});
+
+afterEach(() => {
+  delete process.env.SENCHO_ZFS_ARCSTATS_PATH;
+});

backend/src/__tests__/monitor-service.test.ts

@@ -2,7 +2,8 @@
  * Unit tests for MonitorService — alert state machine, metric calculations,
  * cleanup delegation, global settings evaluation, and concurrency guards.
  */
-import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vitest';
+import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsMock } from './helpers/arcstatsFsMock';
 
 // ── Hoisted mocks ──────────────────────────────────────────────────────
 
@@ -138,8 +139,16 @@ vi.mock('util', () => ({
 
 import { MonitorService, _resetHostAlertSuppressionStateForTests } from '../services/MonitorService';
 
+// Host memory now reads ZFS ARC stats; intercept those reads so the suite does
+// not depend on whether the machine running it is itself a ZFS host.
+let arcFs: ArcstatsFsMock;
+beforeAll(() => {
+  arcFs = installArcstatsFsMock();
+});
+
 beforeEach(() => {
   vi.clearAllMocks();
+  arcFs.clear();
   (MonitorService as any).instance = undefined;
   _resetHostAlertSuppressionStateForTests();
   mockGetSystemState.mockReturnValue(null);
@@ -363,6 +372,19 @@ describe('MonitorService - evaluateGlobalSettings', () => {
     expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
   });
 
+  it('does not alert when reclaimable ZFS ARC accounts for the memory pressure', async () => {
+    // Active working set reads 15G/16G (~94%, breaches 80%), but 5G of that is
+    // reclaimable ARC. Adding ARC back into available drops effective usage to
+    // ~62.5%, so no host-memory alert should fire.
+    mockMem.mockResolvedValue(memSample(15e9)); // available 1e9 -> 93.75%
+    arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(5e9, 0)); // reclaimable 5e9
+
+    const svc = MonitorService.getInstance();
+    await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
+
+    expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
+  });
+
   it('dispatches disk warning when over threshold', async () => {
     mockFsSize.mockResolvedValue([{ mount: '/', use: 92 }]);
 

backend/src/helpers/hostMemory.ts

@@ -0,0 +1,141 @@
+import si from 'systeminformation';
+import { promises as fs } from 'fs';
+
+/**
+ * Shared host-memory computation, ZFS ARC aware.
+ *
+ * `systeminformation.mem()` derives `active` as `total - available` on
+ * Linux/BSD/macOS, so keying usage off `active` already dodges page-cache
+ * inflation. It does NOT account for the OpenZFS ARC: the kernel's
+ * MemAvailable treats ARC as unavailable even though ARC shrinks under
+ * memory pressure, so on ZFS hosts a large ARC reads as hard-used memory
+ * and produces false host-memory alerts.
+ *
+ * When ARC kstats are readable we add the reclaimable portion
+ * (`max(size - c_min, 0)`) back into available memory. On non-ZFS hosts, or
+ * when the kstat file is not readable inside the container, ARC is treated as
+ * zero and the result is identical to the previous `active / total` behavior.
+ */
+
+/** Effective host memory after adding reclaimable ZFS ARC back into available. */
+export interface HostMemory {
+  total: number;
+  /** Effective used bytes (ARC-adjusted). */
+  used: number;
+  /** Effective available bytes (ARC-adjusted). */
+  free: number;
+  /** Effective used as a percentage of total (0 when total is 0). */
+  usagePercent: number;
+}
+
+type MemData = Awaited<ReturnType<typeof si.mem>>;
+
+/**
+ * Candidate arcstats paths in priority order. The operator override is only
+ * present when SENCHO_ZFS_ARCSTATS_PATH is set; the two fixed paths are the
+ * host-mounted and the standard container-visible kstat locations.
+ */
+export const ARCSTATS_FIXED_PATHS = [
+  '/host/proc/spl/kstat/zfs/arcstats',
+  '/proc/spl/kstat/zfs/arcstats',
+];
+
+/** Bound reads of the operator-supplied override path; arcstats is a few KB. */
+const MAX_ARCSTATS_BYTES = 1024 * 1024;
+
+// Memoized so a 30s monitor tick / dashboard poll does not log on every cycle.
+const loggedSelectedPaths = new Set<string>();
+const loggedErrorCodes = new Set<string>();
+
+function overridePath(): string | undefined {
+  const raw = process.env.SENCHO_ZFS_ARCSTATS_PATH?.trim();
+  return raw ? raw : undefined;
+}
+
+function isExpectedFsError(err: unknown): boolean {
+  const code = (err as NodeJS.ErrnoException)?.code;
+  return (
+    code === 'ENOENT' ||
+    code === 'EACCES' ||
+    code === 'EPERM' ||
+    code === 'EISDIR' ||
+    code === 'ENOTDIR' ||
+    code === 'ELOOP'
+  );
+}
+
+function logUnexpected(context: string, err: unknown): void {
+  const code = (err as NodeJS.ErrnoException)?.code ?? 'UNKNOWN';
+  if (loggedErrorCodes.has(code)) return;
+  loggedErrorCodes.add(code);
+  console.warn(`[HostMemory] Unexpected error reading ARC stats (${context}, ${code}); treating ARC as reclaimable=0`);
+}
+
+/** Parse the kstat table for the `size` and `c_min` rows (`<name> <type> <value>`). */
+function parseArcstats(raw: string): { size?: number; cMin?: number } {
+  let size: number | undefined;
+  let cMin: number | undefined;
+  for (const line of raw.split('\n')) {
+    const parts = line.trim().split(/\s+/);
+    if (parts.length < 3) continue;
+    if (parts[0] === 'size') size = Number(parts[2]);
+    else if (parts[0] === 'c_min') cMin = Number(parts[2]);
+  }
+  return { size, cMin };
+}
+
+/**
+ * Reclaimable ARC in bytes, or 0 when ARC stats are unavailable/unusable.
+ * Never throws: any error resolves to 0 so ARC awareness can only lower a
+ * false-positive reading, never break host-memory reporting.
+ */
+async function readReclaimableArc(): Promise<number> {
+  const override = overridePath();
+  const candidates = override ? [override, ...ARCSTATS_FIXED_PATHS] : ARCSTATS_FIXED_PATHS;
+  for (const candidatePath of candidates) {
+    try {
+      // The override path is operator-supplied: verify it is a regular file
+      // of bounded size before reading (guards against a named pipe or an
+      // accidentally huge target). The fixed kstat paths are trusted.
+      if (candidatePath === override) {
+        const info = await fs.stat(candidatePath);
+        if (!info.isFile() || info.size > MAX_ARCSTATS_BYTES) continue;
+      }
+      const raw = await fs.readFile(candidatePath, 'utf8');
+      const { size, cMin } = parseArcstats(raw);
+      if (size === undefined || cMin === undefined) continue;
+      if (!Number.isFinite(size) || !Number.isFinite(cMin) || size < 0 || cMin < 0) continue;
+      // A valid record resolves the lookup, even when reclaimable is 0
+      // (size < c_min means ARC is at its floor).
+      if (!loggedSelectedPaths.has(candidatePath)) {
+        loggedSelectedPaths.add(candidatePath);
+        console.debug(`[HostMemory] Using ZFS ARC stats from ${candidatePath}`);
+      }
+      return Math.max(size - cMin, 0);
+    } catch (err) {
+      // Fail open: a missing or unreadable kstat is the normal non-ZFS case
+      // (expected fs errors); an unexpected error is logged once but still
+      // falls through so ARC awareness can only lower a false positive.
+      if (isExpectedFsError(err)) continue;
+      logUnexpected(candidatePath, err);
+    }
+  }
+  return 0;
+}
+
+/**
+ * Pure ARC adjustment. With `arcReclaimable === 0` this reproduces the prior
+ * `active / total` percentage exactly (since `active === total - available`).
+ */
+export function adjustForArc(mem: Pick<MemData, 'total' | 'available'>, arcReclaimable: number): HostMemory {
+  const effectiveAvailable = Math.min(mem.total, mem.available + Math.max(arcReclaimable, 0));
+  const effectiveUsed = Math.max(mem.total - effectiveAvailable, 0);
+  const usagePercent = mem.total > 0 ? (effectiveUsed / mem.total) * 100 : 0;
+  return { total: mem.total, used: effectiveUsed, free: effectiveAvailable, usagePercent };
+}
+
+/** Fetch host memory and reclaimable ARC concurrently, return the adjusted view. */
+export async function getHostMemory(): Promise<HostMemory> {
+  const [mem, arcReclaimable] = await Promise.all([si.mem(), readReclaimableArc()]);
+  return adjustForArc(mem, arcReclaimable);
+}

backend/src/routes/fleet.ts

@@ -10,6 +10,7 @@ import { FleetUpdateTrackerService, type UpdateTracker, type TerminalStatus, UPD
 import { NodeRegistry } from '../services/NodeRegistry';
 import { computeNodeNetworkingSummary, type NodeNetworkingSummary } from '../services/network/networkingSummary';
 import DockerController from '../services/DockerController';
+import { getHostMemory } from '../helpers/hostMemory';
 import { FileSystemService } from '../services/FileSystemService';
 import { ComposeService } from '../services/ComposeService';
 import { StackOpLockService } from '../services/StackOpLockService';
@@ -222,11 +223,11 @@ async function getCompareTarget(gatewayVersion: string | null) {
 async function fetchLocalNodeOverview(node: Node): Promise<FleetNodeOverview> {
   try {
     const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(node.id));
-    const [allContainers, stacks, currentLoad, mem, fsSize] = await Promise.all([
+    const [allContainers, stacks, currentLoad, hostMem, fsSize] = await Promise.all([
       DockerController.getInstance(node.id).getAllContainers(),
       FileSystemService.getInstance(node.id).getStacks(),
       si.currentLoad(),
-      si.mem(),
+      getHostMemory(),
       si.fsSize(),
     ]);
 
@@ -255,13 +256,12 @@ async function fetchLocalNodeOverview(node: Node): Promise<FleetNodeOverview> {
       systemStats: {
         cpu: { usage: currentLoad.currentLoad.toFixed(1), cores: currentLoad.cpus.length },
         memory: {
-          total: mem.total,
+          total: hostMem.total,
-          // Percentage is keyed off mem.active (the real working set), not mem.used:
+          // ZFS ARC aware: reclaimable ARC is added back into available so a
-          // on Linux/BSD/macOS mem.used counts reclaimable page cache and reads ~99%
+          // large ARC cache is not reported as hard-used. See helpers/hostMemory.ts.
-          // on a busy host. mem.available is total - active, so used + free = total.
+          used: hostMem.used,
-          used: mem.active,
+          free: hostMem.free,
-          free: mem.available,
+          usagePercent: hostMem.usagePercent.toFixed(1),
-          usagePercent: ((mem.active / mem.total) * 100).toFixed(1),
         },
         disk: mainDisk ? {
           total: mainDisk.size,

backend/src/routes/metrics.ts

@@ -9,6 +9,7 @@ import { PilotTunnelManager } from '../services/PilotTunnelManager';
 import { authMiddleware } from '../middleware/auth';
 import { requireAdmin } from '../middleware/tierGates';
 import { STATS_CACHE_TTL_MS, SYSTEM_STATS_CACHE_TTL_MS } from '../helpers/constants';
+import { getHostMemory } from '../helpers/hostMemory';
 import { isDebugEnabled } from '../utils/debug';
 import { getErrorMessage } from '../utils/errors';
 import { isManagedByComposeDir } from '../utils/managed-containers';
@@ -296,9 +297,9 @@ metricsRouter.get('/system/stats', authMiddleware, async (req: Request, res: Res
       async () => {
         // Remote-node requests are intercepted and proxied upstream before
         // reaching here; this fetcher only runs for local nodes.
-        const [currentLoad, mem, fsSize] = await Promise.all([
+        const [currentLoad, hostMem, fsSize] = await Promise.all([
           si.currentLoad(),
-          si.mem(),
+          getHostMemory(),
           si.fsSize(),
         ]);
 
@@ -310,13 +311,12 @@ metricsRouter.get('/system/stats', authMiddleware, async (req: Request, res: Res
             cores: currentLoad.cpus.length,
           },
           memory: {
-            total: mem.total,
+            total: hostMem.total,
-            // Percentage is keyed off mem.active (the real working set), not mem.used:
+            // ZFS ARC aware: reclaimable ARC is added back into available so a
-            // on Linux/BSD/macOS mem.used counts reclaimable page cache and reads ~99%
+            // large ARC cache is not reported as hard-used. See helpers/hostMemory.ts.
-            // on a busy host. mem.available is total - active, so used + free = total.
+            used: hostMem.used,
-            used: mem.active,
+            free: hostMem.free,
-            free: mem.available,
+            usagePercent: hostMem.usagePercent.toFixed(1),
-            usagePercent: ((mem.active / mem.total) * 100).toFixed(1),
           },
           disk: mainDisk ? {
             fs: mainDisk.fs,

backend/src/services/MonitorService.ts

@@ -9,6 +9,7 @@ import { NotificationService } from './NotificationService';
 import { FleetUpdateTrackerService } from './FleetUpdateTrackerService';
 import { isValidVersion, getSenchoVersion } from './CapabilityRegistry';
 import { getLatestVersionInfo } from '../utils/version-check';
+import { getHostMemory } from '../helpers/hostMemory';
 import { isDebugEnabled } from '../utils/debug';
 import { withTimeout, TimeoutError } from '../utils/withTimeout';
 
@@ -287,9 +288,9 @@ export class MonitorService {
         // 1. Host Limits — fetch CPU, RAM, disk concurrently
         if (settings['host_alerts_enabled'] !== '0') {
             try {
-                const [currentLoad, mem, fsSize] = await Promise.all([
+                const [currentLoad, hostMem, fsSize] = await Promise.all([
                     withTimeout(si.currentLoad(), STATS_TIMEOUT_MS, 'host CPU stats'),
-                    withTimeout(si.mem(), STATS_TIMEOUT_MS, 'host RAM stats'),
+                    withTimeout(getHostMemory(), STATS_TIMEOUT_MS, 'host RAM stats'),
                     withTimeout(si.fsSize(), STATS_TIMEOUT_MS, 'host disk stats'),
                 ]);
 
@@ -302,10 +303,9 @@ export class MonitorService {
                     this.clearHostMetricSuppression('cpu');
                 }
 
-                // Key off mem.active (the real working set), not mem.used: on Linux/BSD/macOS
+                // ZFS ARC aware: reclaimable ARC is added back into available so a large ARC
-                // mem.used counts reclaimable page cache and reads ~99% on a busy host, which
+                // cache does not fire spurious host-memory alerts. See helpers/hostMemory.ts.
-                // would fire spurious host-memory alerts.
+                const ramUsage = hostMem.usagePercent;
-                const ramUsage = (mem.active / mem.total) * 100;
                 const ramLimit = parseFloat(settings['host_ram_limit']);
                 if (!isNaN(ramLimit) && ramLimit > 0 && ramUsage > ramLimit) {
                     await this.dispatchHostMetricAlert('ram', 'warning', suppressionMs,

docker-compose.yml

@@ -29,15 +29,27 @@ services:
       # (Optional but Recommended) Media/Data Drives
       # Mount your media drives here so Docker Compose inside Sencho can validate paths during deployment.
       - /path/to/your/media/drives:/path/to/your/media/drives
-      
+
+      # (Optional, ZFS hosts only) OpenZFS ARC stats for ZFS-aware host memory.
+      # ARC cache is reclaimable but the kernel reports it as used, which can
+      # trigger false host-memory alerts. Uncomment to let Sencho treat ARC as
+      # available memory. Usually already visible in the container; only needed
+      # if your runtime does not expose /proc/spl/kstat/zfs/arcstats.
+      # - /proc/spl/kstat/zfs/arcstats:/host/proc/spl/kstat/zfs/arcstats:ro
+
     environment:
       # ENVIRONMENT VARIABLES FOR INSIDE THE CONTAINER
-      
+
       # This points to the Container Path (right side) of your 1:1 mount above
       - COMPOSE_DIR=/path/to/your/docker/folder/compose
-      
+
       # This points to the Container Path (right side) of your database mount above. Leave this as /app/data.
       - DATA_DIR=/app/data
+
+      # (Optional, ZFS hosts only) Container-side path to the OpenZFS ARC stats
+      # file, if it is not at a standard location. Leave empty to auto-detect
+      # /host/proc/spl/kstat/zfs/arcstats then /proc/spl/kstat/zfs/arcstats.
+      - SENCHO_ZFS_ARCSTATS_PATH=${SENCHO_ZFS_ARCSTATS_PATH:-}
     
     # ⚠️ GLOBAL ENVIRONMENT VARIABLES ⚠️
     # If your compose files rely on host-level shell variables (like $PUID, $TZ)

docs/features/dashboard.mdx

@@ -47,6 +47,10 @@ The gauge bars (and the corresponding numeric values) pick up amber at 80% and r
 
 While the dashboard is loading the CPU tile reads `--` and the caption shows `collecting metrics…`; bars and sparklines render once the first sample arrives.
 
+<Note>
+  **ZFS hosts:** the memory tile and host RAM alerts are ZFS ARC-aware. Reclaimable ARC cache is treated as available memory rather than used, so a large ARC does not inflate the gauge or trigger false low-memory alerts. See [ZFS ARC-aware host memory](/getting-started/configuration#zfs-arc-aware-host-memory) for how to expose ARC stats to a Docker install.
+</Note>
+
 ## Stack health
 
 A mono table of every stack discovered in the active node's `COMPOSE_DIR`, sorted so the stacks demanding attention sit at the top.

docs/getting-started/configuration.mdx

@@ -43,9 +43,25 @@ These tune optional subsystems. Most deployments never set them; the defaults ar
 | `GITSOURCE_MAX_CLONE_BYTES` | `104857600` | Maximum bytes a single [Git Source](/features/git-sources) clone may download before it is aborted (100 MB). A shallow Compose clone is tiny; raise it only if you track Compose files in a legitimately large repository. |
 | `SENCHO_PUBLIC_URL` | *(request host)* | Set on the primary instance. Its externally reachable `http(s)://` URL, no trailing slash, baked into pilot enrollment so remote agents dial the public hostname rather than the address the admin used at setup. |
 | `SENCHO_COMPOSE_STALL_TIMEOUT_MS` | `600000` | Idle-output backstop for deploy and update Compose steps (pull and recreate). If a step produces no output for this long while still running, Sencho stops it so a hung image pull surfaces a clear failure and the in-app recovery actions instead of spinning. Raise it on slow links or for heavy local image builds. |
+| `SENCHO_ZFS_ARCSTATS_PATH` | *(auto)* | Path **inside the container** to the OpenZFS ARC kstat file, for [ZFS ARC-aware host memory](#zfs-arc-aware-host-memory). Sencho checks this path first, then `/host/proc/spl/kstat/zfs/arcstats`, then `/proc/spl/kstat/zfs/arcstats`. Set it only when your ARC stats live at a non-standard path. |
 
 Running a remote host as a pilot agent uses four more variables (`SENCHO_MODE`, `SENCHO_PRIMARY_URL`, `SENCHO_ENROLL_TOKEN`, and `SENCHO_PILOT_CA_FILE`), set only on the remote agent container. Sencho bakes them into the enrollment Compose file it generates, so you rarely write them by hand. See [Pilot Agent](/features/pilot-agent) for the full enrollment walkthrough.
 
+## ZFS ARC-aware host memory
+
+On OpenZFS hosts (TrueNAS SCALE, Proxmox, ZFS on Ubuntu or Debian) the ZFS ARC cache can hold a large share of RAM. ARC is reclaimable on demand, but the Linux kernel reports it as unavailable, so a naive reading counts ARC as used memory and can raise false host-memory alerts.
+
+Sencho reads the ARC kstat when it is available and adds the reclaimable portion back into available memory, so the dashboard memory gauge and host RAM alerts reflect real memory pressure. When no ARC stats are readable the behavior is unchanged.
+
+The ARC kstat is usually visible inside the container at `/proc/spl/kstat/zfs/arcstats` with no extra configuration. If your runtime does not expose it, mount it read-only:
+
+```yaml
+volumes:
+  - /proc/spl/kstat/zfs/arcstats:/host/proc/spl/kstat/zfs/arcstats:ro
+```
+
+Sencho checks `SENCHO_ZFS_ARCSTATS_PATH`, then `/host/proc/spl/kstat/zfs/arcstats`, then `/proc/spl/kstat/zfs/arcstats`. Set `SENCHO_ZFS_ARCSTATS_PATH` only if your ARC stats live somewhere else inside the container.
+
 ## Listen port
 
 Sencho always listens on `1852` inside the container. The port is fixed and is not read from an environment variable. To expose Sencho on a different host port, remap with Docker's `-p` flag (or the `ports:` key in your compose file):