From 71be1634a5b6214d39ee9159eadf4151a85500c8 Mon Sep 17 00:00:00 2001 From: Anso Date: Mon, 3 Aug 2026 13:44:20 -0400 Subject: [PATCH] feat: surface ZFS ARC reclaimable as dashboard context line (#1760) * feat: surface ZFS ARC reclaimable as dashboard context line Add arcReclaimable to the HostMemory interface and MemoryWire shape so the reclaimable ARC amount computed by readReclaimableArc() is exposed through /api/system/stats and /api/fleet/overview. Show it as a context line on the dashboard memory tile, matching the balloon pattern. ARC continues to feed the gauge percentage as before; this is a display-only addition for operator visibility. * docs: clarify ARC line requires nonzero reclaimable, not just readable stats --- backend/src/__tests__/host-memory.test.ts | 58 +++++++++++++++++-- backend/src/helpers/hostMemory.ts | 7 ++- docs/features/dashboard.mdx | 2 +- docs/getting-started/configuration.mdx | 2 +- frontend/src/components/FleetView/types.ts | 1 + .../components/dashboard/ResourceGauges.tsx | 5 ++ frontend/src/components/dashboard/types.ts | 1 + 7 files changed, 69 insertions(+), 7 deletions(-) diff --git a/backend/src/__tests__/host-memory.test.ts b/backend/src/__tests__/host-memory.test.ts index f021b800..81ee5b7d 100644 --- a/backend/src/__tests__/host-memory.test.ts +++ b/backend/src/__tests__/host-memory.test.ts @@ -23,7 +23,7 @@ vi.mock('systeminformation', () => ({ default: { mem: (...args: unknown[]) => mockMem(...args) }, })); -import { getHostMemory, adjustForArc, adjustForBalloon } from '../helpers/hostMemory'; +import { getHostMemory, adjustForArc, adjustForBalloon, memoryToWire } from '../helpers/hostMemory'; // mem.active === total - available on Linux, so used/free below mirror the // real systeminformation shape the helper consumes. @@ -53,16 +53,17 @@ describe('adjustForArc', () => { it('reproduces active/total when reclaimable ARC is 0', () => { const result = adjustForArc(memSample(1000, 600), 0); expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 }); + expect('arcReclaimable' in result).toBe(false); }); it('adds reclaimable ARC back into available, lowering usage', () => { const result = adjustForArc(memSample(1000, 600), 200); - expect(result).toEqual({ total: 1000, used: 200, free: 800, usagePercent: 20 }); + expect(result).toEqual({ total: 1000, used: 200, free: 800, usagePercent: 20, arcReclaimable: 200 }); }); it('clamps effective available to total when ARC exceeds the gap', () => { const result = adjustForArc(memSample(1000, 600), 5000); - expect(result).toEqual({ total: 1000, used: 0, free: 1000, usagePercent: 0 }); + expect(result).toEqual({ total: 1000, used: 0, free: 1000, usagePercent: 0, arcReclaimable: 5000 }); }); it('guards against a zero total', () => { @@ -82,7 +83,7 @@ describe('getHostMemory ARC discovery', () => { mockMem.mockResolvedValue(memSample(1000, 600)); arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100)); // reclaimable 200 const result = await getHostMemory(); - expect(result).toEqual({ total: 1000, used: 200, free: 800, usagePercent: 20 }); + expect(result).toEqual({ total: 1000, used: 200, free: 800, usagePercent: 20, arcReclaimable: 200 }); }); it('prefers the operator override path over the fixed candidates', async () => { @@ -295,6 +296,7 @@ describe('getHostMemory balloon discovery', () => { const result = await getHostMemory(); // ARC-adjusted: used = 16000 - (2000 + 4000) = 10000 expect(result.used).toBe(10000); + expect(result.arcReclaimable).toBe(4000); // Balloon-adjusted: effectiveUsed = 10000 - 2GiB expect(result.effectiveUsed).toBeDefined(); expect(result.effectiveUsed!).toBeLessThan(result.used); @@ -388,6 +390,54 @@ describe('getHostMemory balloon discovery', () => { }); }); +describe('memoryToWire', () => { + it('includes arcReclaimable when present on the HostMemory object', () => { + const wire = memoryToWire({ total: 1000, used: 400, free: 600, usagePercent: 40, arcReclaimable: 300 }); + expect(wire.arcReclaimable).toBe(300); + expect(wire.total).toBe(1000); + }); + + it('omits arcReclaimable when absent from the HostMemory object', () => { + const wire = memoryToWire({ total: 1000, used: 400, free: 600, usagePercent: 40 }); + expect('arcReclaimable' in wire).toBe(false); + }); + + it('includes both arcReclaimable and balloon fields when both are present', () => { + const hostMem = adjustForBalloon( + { total: 16000, used: 10000, free: 6000, usagePercent: 62.5, arcReclaimable: 4000 }, + 2_147_483_648, // 2 GiB + ); + const wire = memoryToWire(hostMem); + expect(wire.arcReclaimable).toBe(4000); + expect(wire.ballooned).toBe(2_147_483_648); + expect(wire.effectiveUsed).toBeDefined(); + expect(wire.effectiveUsagePercent).toBeDefined(); + }); +}); + +describe('getHostMemory ARC surfacing end-to-end', () => { + it('surfaces arcReclaimable through memoryToWire with mocked arcstats', async () => { + mockMem.mockResolvedValue(memSample(1000, 600)); + arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(300, 100)); // reclaimable 200 + const result = await getHostMemory(); + expect(result.arcReclaimable).toBe(200); + const wire = memoryToWire(result); + expect(wire.arcReclaimable).toBe(200); + }); + + it('surfaces both arcReclaimable and balloon fields through memoryToWire', async () => { + mockMem.mockResolvedValue(memSample(16000, 2000)); + arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(5000, 1000)); // reclaimable 4000 + arcFs.setRead(DEFAULT_MEMINFO_PATH, meminfoBody(2_097_152)); // 2 GiB balloon + const result = await getHostMemory(); + expect(result.arcReclaimable).toBe(4000); + const wire = memoryToWire(result); + expect(wire.arcReclaimable).toBe(4000); + expect(wire.ballooned).toBeGreaterThan(0); + expect(wire.effectiveUsed).toBeDefined(); + }); +}); + afterEach(() => { delete process.env.SENCHO_ZFS_ARCSTATS_PATH; delete process.env.SENCHO_PROC_MEMINFO_PATH; diff --git a/backend/src/helpers/hostMemory.ts b/backend/src/helpers/hostMemory.ts index e271d0b6..89b6174a 100644 --- a/backend/src/helpers/hostMemory.ts +++ b/backend/src/helpers/hostMemory.ts @@ -34,6 +34,8 @@ export interface HostMemory { free: number; /** Effective used as a percentage of total (0 when total is 0). */ usagePercent: number; + /** Reclaimable ARC bytes (from ZFS kstat). Present only when > 0. */ + arcReclaimable?: number; /** Balloon-reclaimed bytes (from /proc/meminfo). Present only when > 0. */ ballooned?: number; /** Total memory (same as `total`; provided for symmetric UI code). */ @@ -215,7 +217,8 @@ export function adjustForArc(mem: Pick, arcRecla const effectiveAvailable = Math.min(mem.total, mem.available + Math.max(arcReclaimable, 0)); const effectiveUsed = Math.max(mem.total - effectiveAvailable, 0); const usagePercent = mem.total > 0 ? (effectiveUsed / mem.total) * 100 : 0; - return { total: mem.total, used: effectiveUsed, free: effectiveAvailable, usagePercent }; + if (arcReclaimable <= 0) return { total: mem.total, used: effectiveUsed, free: effectiveAvailable, usagePercent }; + return { total: mem.total, used: effectiveUsed, free: effectiveAvailable, usagePercent, arcReclaimable }; } /** @@ -247,6 +250,7 @@ export interface MemoryWire { used: number; free: number; usagePercent: string; + arcReclaimable?: number; ballooned?: number; effectiveTotal?: number; effectiveUsed?: number; @@ -266,6 +270,7 @@ export function memoryToWire(hostMem: HostMemory): MemoryWire { used: hostMem.used, free: hostMem.free, usagePercent: hostMem.usagePercent.toFixed(1), + ...(hostMem.arcReclaimable !== undefined ? { arcReclaimable: hostMem.arcReclaimable } : {}), ...(hostMem.ballooned !== undefined ? { ballooned: hostMem.ballooned, diff --git a/docs/features/dashboard.mdx b/docs/features/dashboard.mdx index f54908c4..5f687215 100644 --- a/docs/features/dashboard.mdx +++ b/docs/features/dashboard.mdx @@ -48,7 +48,7 @@ The gauge bars (and the corresponding numeric values) pick up amber at 80% and r While the dashboard is loading the CPU tile reads `--` and the caption shows `collecting metrics…`; bars and sparklines render once the first sample arrives. - **ZFS hosts:** the memory tile and host RAM alerts are ZFS ARC-aware. Reclaimable ARC cache is treated as available memory rather than used, so a large ARC does not inflate the gauge or trigger false low-memory alerts. See [ZFS ARC-aware host memory](/getting-started/configuration#zfs-arc-aware-host-memory) for how to expose ARC stats to a Docker install. + **ZFS hosts:** the memory tile and host RAM alerts are ZFS ARC-aware. Reclaimable ARC cache is treated as available memory rather than used, so a large ARC does not inflate the gauge or trigger false low-memory alerts. The memory tile shows the reclaimable amount as a context line when ARC stats are readable and the reclaimable amount is nonzero. See [ZFS ARC-aware host memory](/getting-started/configuration#zfs-arc-aware-host-memory) for how to expose ARC stats to a Docker install. **Virtual machines:** the memory tile shows hypervisor-ballooned memory (TrueNAS/KVM, Proxmox) as informational context. Unlike ARC, ballooned pages are host-reclaimed and the guest cannot get them back on demand, so the gauge, health verdict, and alerts continue to use the standard working-set percentage. See [VM memory ballooning](/getting-started/configuration#vm-memory-ballooning) for details. diff --git a/docs/getting-started/configuration.mdx b/docs/getting-started/configuration.mdx index 82e68cb1..451ba29f 100644 --- a/docs/getting-started/configuration.mdx +++ b/docs/getting-started/configuration.mdx @@ -61,7 +61,7 @@ Running a remote host as a pilot agent uses four more variables (`SENCHO_MODE`, On OpenZFS hosts (TrueNAS SCALE, Proxmox, ZFS on Ubuntu or Debian) the ZFS ARC cache can hold a large share of RAM. ARC is reclaimable on demand, but the Linux kernel reports it as unavailable, so a naive reading counts ARC as used memory and can raise false host-memory alerts. -Sencho reads the ARC kstat when it is available and adds the reclaimable portion back into available memory, so the dashboard memory gauge and host RAM alerts reflect real memory pressure. When no ARC stats are readable the behavior is unchanged. +Sencho reads the ARC kstat when it is available and adds the reclaimable portion back into available memory, so the dashboard memory gauge and host RAM alerts reflect real memory pressure. The reclaimable amount is also shown on the dashboard memory tile as a context line. When no ARC stats are readable the behavior is unchanged. The ARC kstat is usually visible inside the container at `/proc/spl/kstat/zfs/arcstats` with no extra configuration. If your runtime does not expose it, mount it read-only: diff --git a/frontend/src/components/FleetView/types.ts b/frontend/src/components/FleetView/types.ts index 1c242256..6b1ba012 100644 --- a/frontend/src/components/FleetView/types.ts +++ b/frontend/src/components/FleetView/types.ts @@ -15,6 +15,7 @@ export interface FleetNodeSystemStats { used: number; free: number; usagePercent: string; + arcReclaimable?: number; ballooned?: number; effectiveTotal?: number; effectiveUsed?: number; diff --git a/frontend/src/components/dashboard/ResourceGauges.tsx b/frontend/src/components/dashboard/ResourceGauges.tsx index 291f7e47..b969bc00 100644 --- a/frontend/src/components/dashboard/ResourceGauges.tsx +++ b/frontend/src/components/dashboard/ResourceGauges.tsx @@ -116,6 +116,11 @@ export function ResourceGauges({ systemStats, cpuHistory, netHistory, historyEnd {ramEffectivePercent !== null ? ` (effective ${parseFloat(ramEffectivePercent).toFixed(0)}%)` : ''} ) : null} + {systemStats?.memory.arcReclaimable && systemStats.memory.arcReclaimable > 0 ? ( +
+ ZFS ARC reclaimable: {formatBytes(systemStats.memory.arcReclaimable)} +
+ ) : null} {systemStats ? : null} diff --git a/frontend/src/components/dashboard/types.ts b/frontend/src/components/dashboard/types.ts index 8aae9bfd..d2485b7d 100644 --- a/frontend/src/components/dashboard/types.ts +++ b/frontend/src/components/dashboard/types.ts @@ -16,6 +16,7 @@ export interface SystemStats { used: number; free: number; usagePercent: string; + arcReclaimable?: number; ballooned?: number; effectiveTotal?: number; effectiveUsed?: number;