mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-04 16:07:55 +00:00
feat: account for VM memory ballooning in host memory reporting (#1750)
* feat: account for VM memory ballooning in host memory reporting Extend hostMemory.ts with a readBalloonedMemory() function that parses the Balloon: field from /proc/meminfo, following the same fail-open pattern as the ZFS ARC integration. When a nonzero balloon is detected, effective memory fields (effectiveUsed, effectiveFree, effectiveUsagePercent) are computed and exposed through /api/system/stats and /api/fleet/overview. All consumers that derive meaning from host memory now prefer effective values when present: the dashboard gauge, Fleet card RAM bar, mobile views, health verdict, health status bar stat tile, and host RAM alerts. Backward compatible: missing /proc/meminfo or absent Balloon: line preserves exact current behavior. Old remote nodes without the new fields continue rendering normally. * refactor: extract shared helpers for balloon memory wiring Extract readCandidateFile() and logSelectedPath() in hostMemory.ts to deduplicate ARC and balloon file-read logic. Add memoryToWire() to centralize the optional-field spread used by /api/system/stats and /api/fleet/overview. Add getNodeMemUsed()/getNodeMemTotal() helpers in nodeUtils.ts for frontend byte-text consumers. * fix: make desktop fleet masthead aggregate balloon-aware The desktop fleet overview's memory aggregate in useFleetOverview.ts still summed raw memory.used, while the mobile fleet aggregate and per-node cards already used effective values. Update to use getNodeMemUsed/getNodeMemTotal helpers. * fix: revert balloon adjustment from alerting and health decisions Ballooned memory is host-reclaimed (unlike ZFS ARC, which the guest can reclaim on demand). The guest cannot get ballooned pages back until the hypervisor deflates them, so treating ballooned memory as available for alerting or health can mask real memory pressure. Keep balloon parsing, wire fields, and the dashboard context line as informational-only. The memory gauge, health verdict, and host RAM alerts now use the standard ARC-adjusted working-set percentage regardless of balloon. Updated configuration.mdx and dashboard.mdx to document that balloon data is informational and does not influence alerting.
This commit is contained in:
Anso
@@ -157,3 +157,8 @@ GITSOURCE_MAX_CLONE_BYTES=104857600
|
||||
# only if your ARC stats live at a non-standard path inside the container. If no
|
||||
# ARC stats are readable, host memory reporting is unchanged.
|
||||
# SENCHO_ZFS_ARCSTATS_PATH=
|
||||
|
||||
# Path inside the container to /proc/meminfo, for VM memory ballooning awareness.
|
||||
# Sencho checks this path first, then /host/proc/meminfo, then /proc/meminfo.
|
||||
# Set it only when your meminfo lives at a non-standard path inside the container.
|
||||
# SENCHO_PROC_MEMINFO_PATH=
|
||||
|
||||
@@ -1,26 +1,33 @@
|
||||
import { promises as fs } from 'fs';
|
||||
import { vi } from 'vitest';
|
||||
import { ARCSTATS_FIXED_PATHS } from '../../helpers/hostMemory';
|
||||
import { ARCSTATS_FIXED_PATHS, MEMINFO_FIXED_PATHS } from '../../helpers/hostMemory';
|
||||
|
||||
/**
|
||||
* Path-aware partial mock of `fs.promises` for ZFS arcstats reads.
|
||||
* Path-aware partial mock of `fs.promises` for ZFS arcstats and /proc/meminfo
|
||||
* reads.
|
||||
*
|
||||
* `helpers/hostMemory.ts` reads `/proc/spl/kstat/zfs/arcstats` (and optional
|
||||
* variants) to compute reclaimable ARC. Tests may run on a ZFS host, so a real
|
||||
* read would make results host-dependent. This installs a spy that intercepts
|
||||
* ONLY registered/ARC-candidate paths and delegates every other
|
||||
* `readFile`/`stat` to the real filesystem, so `setupTestDb` and
|
||||
* `DatabaseService` keep working. Default behavior: ARC candidates reject with
|
||||
* ENOENT (no ARC), so consumers fall back to the plain `active/total` reading.
|
||||
* `helpers/hostMemory.ts` reads `/proc/spl/kstat/zfs/arcstats` (for ARC) and
|
||||
* `/proc/meminfo` (for VM ballooning). Tests may run on a ZFS host or a
|
||||
* ballooned VM, so real reads would make results host-dependent. This installs
|
||||
* a spy that intercepts ONLY registered/candidate paths and delegates every
|
||||
* other `readFile`/`stat` to the real filesystem, so `setupTestDb` and
|
||||
* `DatabaseService` keep working. Default behavior: candidates reject with
|
||||
* ENOENT, so consumers fall back to the plain `active/total` reading.
|
||||
*/
|
||||
|
||||
// Sourced from the helper so the mock cannot silently drift from the paths the
|
||||
// production code actually reads.
|
||||
export const ARC_CANDIDATE_PATHS = ARCSTATS_FIXED_PATHS;
|
||||
|
||||
/** Second fixed candidate; the default path fixtures are served from. */
|
||||
/** Second fixed ARC candidate; the default path ARC fixtures are served from. */
|
||||
export const DEFAULT_ARC_PATH = ARC_CANDIDATE_PATHS[1];
|
||||
|
||||
/** Meminfo candidate paths (same source-of-truth import pattern as ARC). */
|
||||
export const MEMINFO_CANDIDATE_PATHS = MEMINFO_FIXED_PATHS;
|
||||
|
||||
/** Default meminfo path for test fixtures. */
|
||||
export const DEFAULT_MEMINFO_PATH = MEMINFO_CANDIDATE_PATHS[1];
|
||||
|
||||
type StatDescriptor = { isFile: boolean; size: number };
|
||||
|
||||
export interface ArcstatsFsMock {
|
||||
@@ -47,7 +54,8 @@ export function installArcstatsFsMock(): ArcstatsFsMock {
|
||||
const realStat = fs.stat.bind(fs);
|
||||
const reads = new Map<string, string | NodeJS.ErrnoException>();
|
||||
const stats = new Map<string, StatDescriptor | NodeJS.ErrnoException>();
|
||||
const isArcCandidate = (p: string): boolean => ARC_CANDIDATE_PATHS.includes(p);
|
||||
const isCandidatePath = (p: string): boolean =>
|
||||
ARC_CANDIDATE_PATHS.includes(p) || MEMINFO_CANDIDATE_PATHS.includes(p);
|
||||
|
||||
vi.spyOn(fs, 'readFile').mockImplementation((async (p: unknown, ...rest: unknown[]) => {
|
||||
const key = String(p);
|
||||
@@ -56,7 +64,7 @@ export function installArcstatsFsMock(): ArcstatsFsMock {
|
||||
if (v instanceof Error) throw v;
|
||||
return v;
|
||||
}
|
||||
if (isArcCandidate(key)) throw enoent(key);
|
||||
if (isCandidatePath(key)) throw enoent(key);
|
||||
return (realReadFile as (...a: unknown[]) => unknown)(p, ...rest);
|
||||
}) as unknown as typeof fs.readFile);
|
||||
|
||||
@@ -73,7 +81,7 @@ export function installArcstatsFsMock(): ArcstatsFsMock {
|
||||
const size = typeof v === 'string' ? Buffer.byteLength(v) : 0;
|
||||
return { isFile: () => true, size };
|
||||
}
|
||||
if (isArcCandidate(key)) throw enoent(key);
|
||||
if (isCandidatePath(key)) throw enoent(key);
|
||||
return (realStat as (...a: unknown[]) => unknown)(p, ...rest);
|
||||
}) as unknown as typeof fs.stat);
|
||||
|
||||
@@ -96,3 +104,29 @@ export function arcstatsBody(sizeRow: string | number, cMinRow: string | number)
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a realistic /proc/meminfo snippet with the given Balloon value in kB.
|
||||
* Pass undefined / a negative value to omit the Balloon line entirely.
|
||||
*/
|
||||
export function meminfoBody(balloonKb?: number): string {
|
||||
const balloonLine = balloonKb !== undefined && balloonKb >= 0
|
||||
? `Balloon: ${balloonKb} kB\n`
|
||||
: '';
|
||||
return [
|
||||
'MemTotal: 16433188 kB',
|
||||
'MemFree: 620452 kB',
|
||||
'MemAvailable: 3489624 kB',
|
||||
'Buffers: 158668 kB',
|
||||
'Cached: 3335960 kB',
|
||||
'SwapCached: 0 kB',
|
||||
'Active: 5280444 kB',
|
||||
'Inactive: 7478672 kB',
|
||||
balloonLine,
|
||||
'SwapTotal: 8388604 kB',
|
||||
'SwapFree: 8388604 kB',
|
||||
'Dirty: 124 kB',
|
||||
'Writeback: 0 kB',
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
@@ -9,8 +9,11 @@ import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vite
|
||||
import {
|
||||
installArcstatsFsMock,
|
||||
arcstatsBody,
|
||||
meminfoBody,
|
||||
DEFAULT_ARC_PATH,
|
||||
DEFAULT_MEMINFO_PATH,
|
||||
ARC_CANDIDATE_PATHS,
|
||||
MEMINFO_CANDIDATE_PATHS,
|
||||
type ArcstatsFsMock,
|
||||
} from './helpers/arcstatsFsMock';
|
||||
|
||||
@@ -20,7 +23,7 @@ vi.mock('systeminformation', () => ({
|
||||
default: { mem: (...args: unknown[]) => mockMem(...args) },
|
||||
}));
|
||||
|
||||
import { getHostMemory, adjustForArc } from '../helpers/hostMemory';
|
||||
import { getHostMemory, adjustForArc, adjustForBalloon } from '../helpers/hostMemory';
|
||||
|
||||
// mem.active === total - available on Linux, so used/free below mirror the
|
||||
// real systeminformation shape the helper consumes.
|
||||
@@ -43,6 +46,7 @@ beforeEach(() => {
|
||||
arcFs.clear();
|
||||
mockMem.mockReset();
|
||||
delete process.env.SENCHO_ZFS_ARCSTATS_PATH;
|
||||
delete process.env.SENCHO_PROC_MEMINFO_PATH;
|
||||
});
|
||||
|
||||
describe('adjustForArc', () => {
|
||||
@@ -199,6 +203,192 @@ describe('getHostMemory ARC discovery', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('adjustForBalloon', () => {
|
||||
const arcAdjusted = (total: number, used: number, free: number, usagePercent: number): ReturnType<typeof adjustForArc> =>
|
||||
({ total, used, free, usagePercent });
|
||||
|
||||
it('returns the input unchanged when ballooned is 0', () => {
|
||||
const input = arcAdjusted(1000, 400, 600, 40);
|
||||
const result = adjustForBalloon(input, 0);
|
||||
expect(result).toBe(input); // identity for zero
|
||||
});
|
||||
|
||||
it('returns the input unchanged when ballooned is negative', () => {
|
||||
const input = arcAdjusted(1000, 400, 600, 40);
|
||||
const result = adjustForBalloon(input, -5);
|
||||
expect(result).toBe(input);
|
||||
});
|
||||
|
||||
it('subtracts ballooned from used, adds to free, sets optional fields', () => {
|
||||
const input = arcAdjusted(1000, 400, 600, 40);
|
||||
const result = adjustForBalloon(input, 200);
|
||||
expect(result.ballooned).toBe(200);
|
||||
expect(result.effectiveTotal).toBe(1000);
|
||||
expect(result.effectiveUsed).toBe(200); // 400 - 200
|
||||
expect(result.effectiveFree).toBe(800); // 600 + 200
|
||||
expect(result.effectiveUsagePercent).toBe(20); // 200 / 1000 * 100
|
||||
expect(result.balloonSource).toBe('linux_proc_meminfo');
|
||||
// Base fields unchanged.
|
||||
expect(result.total).toBe(1000);
|
||||
expect(result.used).toBe(400);
|
||||
expect(result.free).toBe(600);
|
||||
});
|
||||
|
||||
it('clamps effectiveUsed at 0 when balloon exceeds used', () => {
|
||||
const input = arcAdjusted(1000, 100, 900, 10);
|
||||
const result = adjustForBalloon(input, 5000);
|
||||
expect(result.effectiveUsed).toBe(0);
|
||||
expect(result.effectiveFree).toBe(1000);
|
||||
expect(result.effectiveUsagePercent).toBe(0);
|
||||
});
|
||||
|
||||
it('handles zero total gracefully', () => {
|
||||
const input = arcAdjusted(0, 0, 0, 0);
|
||||
const result = adjustForBalloon(input, 100);
|
||||
expect(result.effectiveUsagePercent).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getHostMemory balloon discovery', () => {
|
||||
it('returns the base ARC-adjusted shape when no meminfo is present', async () => {
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
const result = await getHostMemory();
|
||||
expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
|
||||
});
|
||||
|
||||
it('returns the base ARC-adjusted shape when Balloon is missing from meminfo', async () => {
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
arcFs.setRead(DEFAULT_MEMINFO_PATH, meminfoBody()); // no Balloon line
|
||||
const result = await getHostMemory();
|
||||
expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
|
||||
});
|
||||
|
||||
it('returns the base ARC-adjusted shape when Balloon is 0 kB', async () => {
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
arcFs.setRead(DEFAULT_MEMINFO_PATH, meminfoBody(0));
|
||||
const result = await getHostMemory();
|
||||
expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
|
||||
});
|
||||
|
||||
it('subtracts ballooned memory and sets optional fields', async () => {
|
||||
mockMem.mockResolvedValue(memSample(16000, 4000)); // 16 GB total, 4 GB available → 75% used
|
||||
// 4 GiB balloon = 4194304 kB
|
||||
arcFs.setRead(DEFAULT_MEMINFO_PATH, meminfoBody(4_194_304));
|
||||
const result = await getHostMemory();
|
||||
// ARC=0, available=4000: used=12000
|
||||
// ballooned=4194304*1024 = 4_294_967_296 bytes
|
||||
// effectiveUsed = 12000 - ballooned ≈ 7705 MB
|
||||
expect(result.used).toBe(12000);
|
||||
expect(typeof result.ballooned).toBe('number');
|
||||
expect(result.ballooned!).toBeGreaterThan(0);
|
||||
expect(result.effectiveUsed).toBeDefined();
|
||||
expect(result.effectiveFree).toBeDefined();
|
||||
expect(result.effectiveUsagePercent).toBeDefined();
|
||||
expect(result.effectiveUsed!).toBeLessThan(result.used);
|
||||
expect(result.balloonSource).toBe('linux_proc_meminfo');
|
||||
});
|
||||
|
||||
it('combines ARC reclaim and balloon adjustment', async () => {
|
||||
mockMem.mockResolvedValue(memSample(16000, 2000)); // 12.5% available
|
||||
arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(5000, 1000)); // reclaimable ARC = 4000
|
||||
arcFs.setRead(DEFAULT_MEMINFO_PATH, meminfoBody(2_097_152)); // 2 GiB balloon
|
||||
const result = await getHostMemory();
|
||||
// ARC-adjusted: used = 16000 - (2000 + 4000) = 10000
|
||||
expect(result.used).toBe(10000);
|
||||
// Balloon-adjusted: effectiveUsed = 10000 - 2GiB
|
||||
expect(result.effectiveUsed).toBeDefined();
|
||||
expect(result.effectiveUsed!).toBeLessThan(result.used);
|
||||
});
|
||||
|
||||
it('prefers the meminfo override path', async () => {
|
||||
process.env.SENCHO_PROC_MEMINFO_PATH = '/custom/meminfo';
|
||||
mockMem.mockResolvedValue(memSample(16000, 4000));
|
||||
arcFs.setRead('/custom/meminfo', meminfoBody(4_194_304)); // 4 GiB balloon
|
||||
arcFs.setRead(DEFAULT_MEMINFO_PATH, meminfoBody(0)); // fixed path says no balloon
|
||||
const result = await getHostMemory();
|
||||
expect(result.ballooned).toBeGreaterThan(0); // override won
|
||||
});
|
||||
|
||||
it('falls through to a fixed meminfo path when the override is unreadable', async () => {
|
||||
process.env.SENCHO_PROC_MEMINFO_PATH = '/custom/meminfo';
|
||||
mockMem.mockResolvedValue(memSample(16000, 4000));
|
||||
arcFs.setReadError('/custom/meminfo', Object.assign(new Error('nope'), { code: 'ENOENT' }));
|
||||
arcFs.setRead(DEFAULT_MEMINFO_PATH, meminfoBody(2_097_152)); // 2 GiB
|
||||
const result = await getHostMemory();
|
||||
expect(result.ballooned).toBeGreaterThan(0); // fell through to fixed
|
||||
});
|
||||
|
||||
it('reads the host-mounted meminfo candidate and prefers it over /proc', async () => {
|
||||
mockMem.mockResolvedValue(memSample(16000, 4000));
|
||||
arcFs.setRead(MEMINFO_CANDIDATE_PATHS[0], meminfoBody(4_194_304)); // /host/proc: 4 GiB
|
||||
arcFs.setRead(MEMINFO_CANDIDATE_PATHS[1], meminfoBody(1_048_576)); // /proc: 1 GiB
|
||||
const result = await getHostMemory();
|
||||
// First candidate wins: 4 GiB balloon.
|
||||
expect(result.ballooned).toBeGreaterThan(0);
|
||||
expect(result.effectiveUsed).toBeDefined();
|
||||
});
|
||||
|
||||
it('skips an override path that is not a regular file', async () => {
|
||||
process.env.SENCHO_PROC_MEMINFO_PATH = '/custom/meminfo';
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
arcFs.setStat('/custom/meminfo', { isFile: false, size: 10 });
|
||||
arcFs.setRead('/custom/meminfo', meminfoBody(100));
|
||||
const result = await getHostMemory();
|
||||
expect(result.used).toBe(400); // override skipped, no meminfo on fixed
|
||||
});
|
||||
|
||||
it('skips an override path that exceeds the size bound', async () => {
|
||||
process.env.SENCHO_PROC_MEMINFO_PATH = '/custom/meminfo';
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
arcFs.setStat('/custom/meminfo', { isFile: true, size: 2 * 1024 * 1024 });
|
||||
arcFs.setRead('/custom/meminfo', meminfoBody(100));
|
||||
const result = await getHostMemory();
|
||||
expect(result.used).toBe(400);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['non-numeric value', 'Balloon: abc kB\n'],
|
||||
['negative value', 'Balloon: -100 kB\n'],
|
||||
['no kB suffix', 'Balloon: 100\n'],
|
||||
['wrong suffix', 'Balloon: 100 MB\n'],
|
||||
['extra token', 'Balloon: 100 kB extra\n'],
|
||||
])('treats a %s Balloon line as unusable and yields no balloon', async (_label, body) => {
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
arcFs.setRead(DEFAULT_MEMINFO_PATH, body);
|
||||
const result = await getHostMemory();
|
||||
expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
|
||||
});
|
||||
|
||||
it('fails open (balloon 0) on a read error', async () => {
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
arcFs.setReadError(DEFAULT_MEMINFO_PATH, Object.assign(new Error('nope'), { code: 'EACCES' }));
|
||||
const result = await getHostMemory();
|
||||
expect(result.used).toBe(400);
|
||||
});
|
||||
|
||||
it('logs an unexpected meminfo read error once per code', async () => {
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
|
||||
// Expected fs error: silent fall-through.
|
||||
arcFs.setReadError(MEMINFO_CANDIDATE_PATHS[0], Object.assign(new Error('denied'), { code: 'EACCES' }));
|
||||
arcFs.setReadError(MEMINFO_CANDIDATE_PATHS[1], Object.assign(new Error('denied'), { code: 'EACCES' }));
|
||||
await getHostMemory();
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
|
||||
// Unexpected fs error: logged once. Use EBADF to avoid collision with
|
||||
// the ARC test suite's own EIO trigger (loggedErrorCodes is shared).
|
||||
arcFs.setReadError(MEMINFO_CANDIDATE_PATHS[0], Object.assign(new Error('badf'), { code: 'EBADF' }));
|
||||
arcFs.setReadError(MEMINFO_CANDIDATE_PATHS[1], Object.assign(new Error('badf'), { code: 'EBADF' }));
|
||||
await getHostMemory();
|
||||
await getHostMemory();
|
||||
const unexpectedLogs = warn.mock.calls.filter(([msg]) => String(msg).includes('EBADF'));
|
||||
expect(unexpectedLogs).toHaveLength(1);
|
||||
warn.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.SENCHO_ZFS_ARCSTATS_PATH;
|
||||
delete process.env.SENCHO_PROC_MEMINFO_PATH;
|
||||
});
|
||||
|
||||
@@ -2,22 +2,30 @@ import si from 'systeminformation';
|
||||
import { promises as fs } from 'fs';
|
||||
|
||||
/**
|
||||
* Shared host-memory computation, ZFS ARC aware.
|
||||
* Shared host-memory computation, ZFS ARC and VM-balloon aware.
|
||||
*
|
||||
* `systeminformation.mem()` derives `active` as `total - available` on
|
||||
* Linux/BSD/macOS, so keying usage off `active` already dodges page-cache
|
||||
* inflation. It does NOT account for the OpenZFS ARC: the kernel's
|
||||
* MemAvailable treats ARC as unavailable even though ARC shrinks under
|
||||
* memory pressure, so on ZFS hosts a large ARC reads as hard-used memory
|
||||
* and produces false host-memory alerts.
|
||||
* inflation. It does NOT account for two sources of reclaimable memory:
|
||||
*
|
||||
* 1. **OpenZFS ARC**: the kernel's MemAvailable treats ARC as unavailable
|
||||
* even though ARC shrinks under memory pressure.
|
||||
* 2. **VM memory ballooning**: hypervisors (TrueNAS/KVM, Proxmox) reclaim
|
||||
* guest memory through a balloon driver. The reclaimed amount appears in
|
||||
* `/proc/meminfo` as `Balloon: N kB` but is invisible to
|
||||
* `systeminformation.mem()`, so a ballooned VM can read as memory-critical
|
||||
* when the guest is actually healthy.
|
||||
*
|
||||
* When ARC kstats are readable we add the reclaimable portion
|
||||
* (`max(size - c_min, 0)`) back into available memory. On non-ZFS hosts, or
|
||||
* when the kstat file is not readable inside the container, ARC is treated as
|
||||
* zero and the result is identical to the previous `active / total` behavior.
|
||||
* (`max(size - c_min, 0)`) back into available memory. When `/proc/meminfo`
|
||||
* reports a nonzero `Balloon:` value we subtract the ballooned amount from
|
||||
* used and recompute an effective usage percentage. On non-ZFS / non-VM
|
||||
* hosts, or when the files are not readable inside the container, both
|
||||
* adjustments resolve to zero and the result is identical to the previous
|
||||
* `active / total` behavior.
|
||||
*/
|
||||
|
||||
/** Effective host memory after adding reclaimable ZFS ARC back into available. */
|
||||
/** Effective host memory after ARC and balloon adjustments. */
|
||||
export interface HostMemory {
|
||||
total: number;
|
||||
/** Effective used bytes (ARC-adjusted). */
|
||||
@@ -26,6 +34,18 @@ export interface HostMemory {
|
||||
free: number;
|
||||
/** Effective used as a percentage of total (0 when total is 0). */
|
||||
usagePercent: number;
|
||||
/** Balloon-reclaimed bytes (from /proc/meminfo). Present only when > 0. */
|
||||
ballooned?: number;
|
||||
/** Total memory (same as `total`; provided for symmetric UI code). */
|
||||
effectiveTotal?: number;
|
||||
/** Used bytes after subtracting both ARC reclaim and balloon. */
|
||||
effectiveUsed?: number;
|
||||
/** Free bytes after adding both ARC reclaim and balloon. */
|
||||
effectiveFree?: number;
|
||||
/** Effective used as a percentage (balloon-adjusted). */
|
||||
effectiveUsagePercent?: number;
|
||||
/** Source identifier for the balloon reading. */
|
||||
balloonSource?: 'linux_proc_meminfo';
|
||||
}
|
||||
|
||||
type MemData = Awaited<ReturnType<typeof si.mem>>;
|
||||
@@ -40,18 +60,33 @@ export const ARCSTATS_FIXED_PATHS = [
|
||||
'/proc/spl/kstat/zfs/arcstats',
|
||||
];
|
||||
|
||||
/** Bound reads of the operator-supplied override path; arcstats is a few KB. */
|
||||
const MAX_ARCSTATS_BYTES = 1024 * 1024;
|
||||
/** Bound reads of the operator-supplied override path; both files are a few KB. */
|
||||
const MAX_CANDIDATE_BYTES = 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Candidate /proc/meminfo paths in priority order. The operator override is
|
||||
* only present when SENCHO_PROC_MEMINFO_PATH is set; the two fixed paths are
|
||||
* the host-mounted and the standard container-visible locations.
|
||||
*/
|
||||
export const MEMINFO_FIXED_PATHS = [
|
||||
'/host/proc/meminfo',
|
||||
'/proc/meminfo',
|
||||
];
|
||||
|
||||
// Memoized so a 30s monitor tick / dashboard poll does not log on every cycle.
|
||||
const loggedSelectedPaths = new Set<string>();
|
||||
const loggedErrorCodes = new Set<string>();
|
||||
|
||||
function overridePath(): string | undefined {
|
||||
function arcOverridePath(): string | undefined {
|
||||
const raw = process.env.SENCHO_ZFS_ARCSTATS_PATH?.trim();
|
||||
return raw ? raw : undefined;
|
||||
}
|
||||
|
||||
function meminfoOverridePath(): string | undefined {
|
||||
const raw = process.env.SENCHO_PROC_MEMINFO_PATH?.trim();
|
||||
return raw ? raw : undefined;
|
||||
}
|
||||
|
||||
function isExpectedFsError(err: unknown): boolean {
|
||||
const code = (err as NodeJS.ErrnoException)?.code;
|
||||
return (
|
||||
@@ -64,11 +99,39 @@ function isExpectedFsError(err: unknown): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
function logUnexpected(context: string, err: unknown): void {
|
||||
const code = (err as NodeJS.ErrnoException)?.code ?? 'UNKNOWN';
|
||||
if (loggedErrorCodes.has(code)) return;
|
||||
loggedErrorCodes.add(code);
|
||||
console.warn(`[HostMemory] Unexpected error reading ARC stats (${context}, ${code}); treating ARC as reclaimable=0`);
|
||||
function logSelectedPath(path: string, label: string): void {
|
||||
if (loggedSelectedPaths.has(path)) return;
|
||||
loggedSelectedPaths.add(path);
|
||||
console.debug(`[HostMemory] Using ${label} from ${path}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a single candidate file, applying the operator-override guard (regular
|
||||
* file, bounded size) and the fail-open error contract. Returns the raw text,
|
||||
* or undefined when the path is unusable so the caller falls through to the
|
||||
* next candidate. Unexpected errors are logged once per code.
|
||||
*/
|
||||
async function readCandidateFile(path: string, isOverride: boolean): Promise<string | undefined> {
|
||||
try {
|
||||
// The override path is operator-supplied: verify it is a regular file of
|
||||
// bounded size before reading (guards against a named pipe or an
|
||||
// accidentally huge target). The fixed paths are trusted.
|
||||
if (isOverride) {
|
||||
const info = await fs.stat(path);
|
||||
if (!info.isFile() || info.size > MAX_CANDIDATE_BYTES) return undefined;
|
||||
}
|
||||
return await fs.readFile(path, 'utf8');
|
||||
} catch (err) {
|
||||
// Fail open: a missing or unreadable file is the normal non-ZFS/non-VM
|
||||
// case (expected fs errors); an unexpected error is logged once but still
|
||||
// falls through so the adjustment can only lower a false positive.
|
||||
if (isExpectedFsError(err)) return undefined;
|
||||
const code = (err as NodeJS.ErrnoException)?.code ?? 'UNKNOWN';
|
||||
if (loggedErrorCodes.has(code)) return undefined;
|
||||
loggedErrorCodes.add(code);
|
||||
console.warn(`[HostMemory] Unexpected error reading ${path} (${code}); treating as 0`);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Parse the kstat table for the `size` and `c_min` rows (`<name> <type> <value>`). */
|
||||
@@ -84,41 +147,62 @@ function parseArcstats(raw: string): { size?: number; cMin?: number } {
|
||||
return { size, cMin };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a /proc/meminfo body for the `Balloon:` line. Returns bytes, or
|
||||
* undefined when the field is absent/malformed. Only the standard `<N> kB`
|
||||
* format is recognized.
|
||||
*/
|
||||
function parseMeminfoBalloon(raw: string): number | undefined {
|
||||
for (const line of raw.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed.startsWith('Balloon:')) continue;
|
||||
const parts = trimmed.split(/\s+/);
|
||||
// Expect "Balloon: <N> kB" (3 tokens).
|
||||
if (parts.length !== 3 || parts[2] !== 'kB') return undefined;
|
||||
const value = Number(parts[1]);
|
||||
if (!Number.isFinite(value) || value < 0) return undefined;
|
||||
return value * 1024;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ballooned memory in bytes, or 0 when the meminfo field is absent/unusable.
|
||||
* Never throws: any error resolves to 0 so balloon awareness can only lower a
|
||||
* false-positive reading, never break host-memory reporting.
|
||||
*/
|
||||
async function readBalloonedMemory(): Promise<number> {
|
||||
const override = meminfoOverridePath();
|
||||
const candidates = override ? [override, ...MEMINFO_FIXED_PATHS] : MEMINFO_FIXED_PATHS;
|
||||
for (const candidatePath of candidates) {
|
||||
const raw = await readCandidateFile(candidatePath, candidatePath === override);
|
||||
if (raw === undefined) continue;
|
||||
const ballooned = parseMeminfoBalloon(raw);
|
||||
if (ballooned === undefined) continue;
|
||||
logSelectedPath(candidatePath, 'meminfo');
|
||||
return ballooned;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reclaimable ARC in bytes, or 0 when ARC stats are unavailable/unusable.
|
||||
* Never throws: any error resolves to 0 so ARC awareness can only lower a
|
||||
* false-positive reading, never break host-memory reporting.
|
||||
*/
|
||||
async function readReclaimableArc(): Promise<number> {
|
||||
const override = overridePath();
|
||||
const override = arcOverridePath();
|
||||
const candidates = override ? [override, ...ARCSTATS_FIXED_PATHS] : ARCSTATS_FIXED_PATHS;
|
||||
for (const candidatePath of candidates) {
|
||||
try {
|
||||
// The override path is operator-supplied: verify it is a regular file
|
||||
// of bounded size before reading (guards against a named pipe or an
|
||||
// accidentally huge target). The fixed kstat paths are trusted.
|
||||
if (candidatePath === override) {
|
||||
const info = await fs.stat(candidatePath);
|
||||
if (!info.isFile() || info.size > MAX_ARCSTATS_BYTES) continue;
|
||||
}
|
||||
const raw = await fs.readFile(candidatePath, 'utf8');
|
||||
const { size, cMin } = parseArcstats(raw);
|
||||
if (size === undefined || cMin === undefined) continue;
|
||||
if (!Number.isFinite(size) || !Number.isFinite(cMin) || size < 0 || cMin < 0) continue;
|
||||
// A valid record resolves the lookup, even when reclaimable is 0
|
||||
// (size < c_min means ARC is at its floor).
|
||||
if (!loggedSelectedPaths.has(candidatePath)) {
|
||||
loggedSelectedPaths.add(candidatePath);
|
||||
console.debug(`[HostMemory] Using ZFS ARC stats from ${candidatePath}`);
|
||||
}
|
||||
return Math.max(size - cMin, 0);
|
||||
} catch (err) {
|
||||
// Fail open: a missing or unreadable kstat is the normal non-ZFS case
|
||||
// (expected fs errors); an unexpected error is logged once but still
|
||||
// falls through so ARC awareness can only lower a false positive.
|
||||
if (isExpectedFsError(err)) continue;
|
||||
logUnexpected(candidatePath, err);
|
||||
}
|
||||
const raw = await readCandidateFile(candidatePath, candidatePath === override);
|
||||
if (raw === undefined) continue;
|
||||
const { size, cMin } = parseArcstats(raw);
|
||||
if (size === undefined || cMin === undefined) continue;
|
||||
if (!Number.isFinite(size) || !Number.isFinite(cMin) || size < 0 || cMin < 0) continue;
|
||||
// A valid record resolves the lookup, even when reclaimable is 0
|
||||
// (size < c_min means ARC is at its floor).
|
||||
logSelectedPath(candidatePath, 'ZFS ARC stats');
|
||||
return Math.max(size - cMin, 0);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -134,8 +218,73 @@ export function adjustForArc(mem: Pick<MemData, 'total' | 'available'>, arcRecla
|
||||
return { total: mem.total, used: effectiveUsed, free: effectiveAvailable, usagePercent };
|
||||
}
|
||||
|
||||
/** Fetch host memory and reclaimable ARC concurrently, return the adjusted view. */
|
||||
export async function getHostMemory(): Promise<HostMemory> {
|
||||
const [mem, arcReclaimable] = await Promise.all([si.mem(), readReclaimableArc()]);
|
||||
return adjustForArc(mem, arcReclaimable);
|
||||
/**
|
||||
* Balloon adjustment layer. Applies on top of the ARC-adjusted result.
|
||||
* When `ballooned <= 0` the input is returned unchanged, preserving exact
|
||||
* `.toEqual()` backward compatibility for every existing test assertion.
|
||||
*/
|
||||
export function adjustForBalloon(hostMem: HostMemory, ballooned: number): HostMemory {
|
||||
if (ballooned <= 0) return hostMem;
|
||||
const effectiveUsed = Math.max(hostMem.used - ballooned, 0);
|
||||
const effectiveFree = Math.min(hostMem.free + ballooned, hostMem.total);
|
||||
const effectiveUsagePercent = hostMem.total > 0
|
||||
? (effectiveUsed / hostMem.total) * 100
|
||||
: 0;
|
||||
return {
|
||||
...hostMem,
|
||||
ballooned,
|
||||
effectiveTotal: hostMem.total,
|
||||
effectiveUsed,
|
||||
effectiveFree,
|
||||
effectiveUsagePercent,
|
||||
balloonSource: 'linux_proc_meminfo' as const,
|
||||
};
|
||||
}
|
||||
|
||||
/** Wire shape of host memory as served by /api/system/stats and fleet overviews. */
|
||||
export interface MemoryWire {
|
||||
total: number;
|
||||
used: number;
|
||||
free: number;
|
||||
usagePercent: string;
|
||||
ballooned?: number;
|
||||
effectiveTotal?: number;
|
||||
effectiveUsed?: number;
|
||||
effectiveFree?: number;
|
||||
effectiveUsagePercent?: string;
|
||||
balloonSource?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map adjusted host memory to the wire shape. Balloon fields are only
|
||||
* included when a balloon reading was present, so the non-VM shape stays
|
||||
* identical to the pre-balloon wire format.
|
||||
*/
|
||||
export function memoryToWire(hostMem: HostMemory): MemoryWire {
|
||||
return {
|
||||
total: hostMem.total,
|
||||
used: hostMem.used,
|
||||
free: hostMem.free,
|
||||
usagePercent: hostMem.usagePercent.toFixed(1),
|
||||
...(hostMem.ballooned !== undefined
|
||||
? {
|
||||
ballooned: hostMem.ballooned,
|
||||
effectiveTotal: hostMem.effectiveTotal,
|
||||
effectiveUsed: hostMem.effectiveUsed,
|
||||
effectiveFree: hostMem.effectiveFree,
|
||||
effectiveUsagePercent: hostMem.effectiveUsagePercent?.toFixed(1),
|
||||
balloonSource: hostMem.balloonSource,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Fetch host memory, reclaimable ARC, and ballooned memory concurrently. */
|
||||
export async function getHostMemory(): Promise<HostMemory> {
|
||||
const [mem, arcReclaimable, ballooned] = await Promise.all([
|
||||
si.mem(),
|
||||
readReclaimableArc(),
|
||||
readBalloonedMemory(),
|
||||
]);
|
||||
return adjustForBalloon(adjustForArc(mem, arcReclaimable), ballooned);
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ import { FleetUpdateTrackerService, type UpdateTracker, type TerminalStatus, UPD
|
||||
import { NodeRegistry } from '../services/NodeRegistry';
|
||||
import { computeNodeNetworkingSummary, type NodeNetworkingSummary } from '../services/network/networkingSummary';
|
||||
import DockerController from '../services/DockerController';
|
||||
import { getHostMemory } from '../helpers/hostMemory';
|
||||
import { getHostMemory, memoryToWire, type MemoryWire } from '../helpers/hostMemory';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
import { ComposeService } from '../services/ComposeService';
|
||||
import { StackOpLockService } from '../services/StackOpLockService';
|
||||
@@ -232,7 +232,7 @@ interface FleetNodeOverview {
|
||||
} | null;
|
||||
systemStats: {
|
||||
cpu: { usage: string; cores: number };
|
||||
memory: { total: number; used: number; free: number; usagePercent: string };
|
||||
memory: MemoryWire;
|
||||
disk: { total: number; used: number; free: number; usagePercent: string } | null;
|
||||
} | null;
|
||||
stacks: string[] | null;
|
||||
@@ -301,14 +301,9 @@ async function fetchLocalNodeOverview(node: Node): Promise<FleetNodeOverview> {
|
||||
stats: { active, managed, unmanaged, exited, total },
|
||||
systemStats: {
|
||||
cpu: { usage: currentLoad.currentLoad.toFixed(1), cores: currentLoad.cpus.length },
|
||||
memory: {
|
||||
total: hostMem.total,
|
||||
// ZFS ARC aware: reclaimable ARC is added back into available so a
|
||||
// large ARC cache is not reported as hard-used. See helpers/hostMemory.ts.
|
||||
used: hostMem.used,
|
||||
free: hostMem.free,
|
||||
usagePercent: hostMem.usagePercent.toFixed(1),
|
||||
},
|
||||
// ARC/balloon aware: reclaimable ARC is added back into available,
|
||||
// and ballooned memory is subtracted from used. See helpers/hostMemory.ts.
|
||||
memory: memoryToWire(hostMem),
|
||||
disk: mainDisk ? {
|
||||
total: mainDisk.size,
|
||||
used: mainDisk.used,
|
||||
@@ -389,7 +384,7 @@ async function fetchRemoteNodeOverview(node: Node, db: DatabaseService): Promise
|
||||
|
||||
interface RemoteSystemStats {
|
||||
cpu: { usage: string; cores: number };
|
||||
memory: { total: number; used: number; free: number; usagePercent: string };
|
||||
memory: MemoryWire;
|
||||
disk?: { total: number; used: number; free: number; usagePercent: string } | null;
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import { PilotTunnelManager } from '../services/PilotTunnelManager';
|
||||
import { authMiddleware } from '../middleware/auth';
|
||||
import { requireAdmin } from '../middleware/tierGates';
|
||||
import { STATS_CACHE_TTL_MS, SYSTEM_STATS_CACHE_TTL_MS } from '../helpers/constants';
|
||||
import { getHostMemory } from '../helpers/hostMemory';
|
||||
import { getHostMemory, memoryToWire } from '../helpers/hostMemory';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { isManagedByComposeDir } from '../utils/managed-containers';
|
||||
@@ -310,14 +310,9 @@ metricsRouter.get('/system/stats', authMiddleware, async (req: Request, res: Res
|
||||
usage: currentLoad.currentLoad.toFixed(1),
|
||||
cores: currentLoad.cpus.length,
|
||||
},
|
||||
memory: {
|
||||
total: hostMem.total,
|
||||
// ZFS ARC aware: reclaimable ARC is added back into available so a
|
||||
// large ARC cache is not reported as hard-used. See helpers/hostMemory.ts.
|
||||
used: hostMem.used,
|
||||
free: hostMem.free,
|
||||
usagePercent: hostMem.usagePercent.toFixed(1),
|
||||
},
|
||||
// ARC/balloon aware: reclaimable ARC is added back into available,
|
||||
// and ballooned memory is subtracted from used. See helpers/hostMemory.ts.
|
||||
memory: memoryToWire(hostMem),
|
||||
disk: mainDisk ? {
|
||||
fs: mainDisk.fs,
|
||||
mount: mainDisk.mount,
|
||||
|
||||
@@ -330,8 +330,12 @@ export class MonitorService {
|
||||
this.clearHostMetricSuppression('cpu');
|
||||
}
|
||||
|
||||
// ZFS ARC aware: reclaimable ARC is added back into available so a large ARC
|
||||
// cache does not fire spurious host-memory alerts. See helpers/hostMemory.ts.
|
||||
// ZFS ARC aware: reclaimable ARC is added back into available so a
|
||||
// large ARC cache does not fire spurious host-memory alerts.
|
||||
// Ballooned memory is deliberately NOT subtracted here: unlike
|
||||
// ARC, ballooned pages are reclaimed by the hypervisor and the
|
||||
// guest cannot get them back on demand. A ballooned VM with real
|
||||
// memory pressure must still alert. See helpers/hostMemory.ts.
|
||||
const ramUsage = hostMem.usagePercent;
|
||||
const ramLimit = parseFloat(settings['host_ram_limit']);
|
||||
if (!isNaN(ramLimit) && ramLimit > 0 && ramUsage > ramLimit) {
|
||||
|
||||
@@ -36,6 +36,8 @@ services:
|
||||
# available memory. Usually already visible in the container; only needed
|
||||
# if your runtime does not expose /proc/spl/kstat/zfs/arcstats.
|
||||
# - /proc/spl/kstat/zfs/arcstats:/host/proc/spl/kstat/zfs/arcstats:ro
|
||||
# VM ballooning: mount /proc/meminfo if your runtime does not expose it.
|
||||
# - /proc/meminfo:/host/proc/meminfo:ro
|
||||
|
||||
environment:
|
||||
# ENVIRONMENT VARIABLES FOR INSIDE THE CONTAINER
|
||||
@@ -50,6 +52,9 @@ services:
|
||||
# file, if it is not at a standard location. Leave empty to auto-detect
|
||||
# /host/proc/spl/kstat/zfs/arcstats then /proc/spl/kstat/zfs/arcstats.
|
||||
- SENCHO_ZFS_ARCSTATS_PATH=${SENCHO_ZFS_ARCSTATS_PATH:-}
|
||||
# Optional: mount /proc/meminfo for VM memory ballooning awareness
|
||||
# - /proc/meminfo:/host/proc/meminfo:ro
|
||||
- SENCHO_PROC_MEMINFO_PATH=${SENCHO_PROC_MEMINFO_PATH:-}
|
||||
|
||||
# ⚠️ GLOBAL ENVIRONMENT VARIABLES ⚠️
|
||||
# If your compose files rely on host-level shell variables (like $PUID, $TZ)
|
||||
|
||||
@@ -49,6 +49,8 @@ While the dashboard is loading the CPU tile reads `--` and the caption shows `co
|
||||
|
||||
<Note>
|
||||
**ZFS hosts:** the memory tile and host RAM alerts are ZFS ARC-aware. Reclaimable ARC cache is treated as available memory rather than used, so a large ARC does not inflate the gauge or trigger false low-memory alerts. See [ZFS ARC-aware host memory](/getting-started/configuration#zfs-arc-aware-host-memory) for how to expose ARC stats to a Docker install.
|
||||
|
||||
**Virtual machines:** the memory tile shows hypervisor-ballooned memory (TrueNAS/KVM, Proxmox) as informational context. Unlike ARC, ballooned pages are host-reclaimed and the guest cannot get them back on demand, so the gauge, health verdict, and alerts continue to use the standard working-set percentage. See [VM memory ballooning](/getting-started/configuration#vm-memory-ballooning) for details.
|
||||
</Note>
|
||||
|
||||
## Stack health
|
||||
|
||||
@@ -53,6 +53,7 @@ These tune optional subsystems. Most deployments never set them; the defaults ar
|
||||
| `SENCHO_COMPOSE_COMMAND_TIMEOUT_MS` | `1800000` | Hard timeout for a single Compose command (pull, up, down) during deploy and update, in milliseconds (30 minutes). Sencho kills the command and reports failure if it runs longer than this, regardless of whether it is still producing output. Raise it only for very large images or slow storage. |
|
||||
| `SENCHO_COMPOSE_STALL_TIMEOUT_MS` | `600000` | Idle-output backstop for deploy and update Compose steps (pull and recreate), separate from the hard timeout above. If a step produces no output for this long while still running, Sencho stops it so a hung image pull surfaces a clear failure and the in-app recovery actions instead of spinning. Raise it on slow links or for heavy local image builds. |
|
||||
| `SENCHO_ZFS_ARCSTATS_PATH` | *(auto)* | Path **inside the container** to the OpenZFS ARC kstat file, for [ZFS ARC-aware host memory](#zfs-arc-aware-host-memory). Sencho checks this path first, then `/host/proc/spl/kstat/zfs/arcstats`, then `/proc/spl/kstat/zfs/arcstats`. Set it only when your ARC stats live at a non-standard path. |
|
||||
| `SENCHO_PROC_MEMINFO_PATH` | *(auto)* | Path **inside the container** to `/proc/meminfo`, for [VM memory ballooning](#vm-memory-ballooning). Sencho checks this path first, then `/host/proc/meminfo`, then `/proc/meminfo`. Set it only when you need a custom meminfo path. |
|
||||
|
||||
Running a remote host as a pilot agent uses four more variables (`SENCHO_MODE`, `SENCHO_PRIMARY_URL`, `SENCHO_ENROLL_TOKEN`, and `SENCHO_PILOT_CA_FILE`), set only on the remote agent container. Sencho bakes them into the enrollment Compose file it generates, so you rarely write them by hand. See [Pilot Agent](/features/pilot-agent) for the full enrollment walkthrough.
|
||||
|
||||
@@ -71,6 +72,21 @@ volumes:
|
||||
|
||||
Sencho checks `SENCHO_ZFS_ARCSTATS_PATH`, then `/host/proc/spl/kstat/zfs/arcstats`, then `/proc/spl/kstat/zfs/arcstats`. Set `SENCHO_ZFS_ARCSTATS_PATH` only if your ARC stats live somewhere else inside the container.
|
||||
|
||||
## VM memory ballooning
|
||||
|
||||
On Linux virtual machines with memory ballooning enabled (TrueNAS/KVM, Proxmox, VMware), the hypervisor can reclaim guest memory through a balloon driver. The reclaimed amount is tracked in `/proc/meminfo` on the `Balloon:` line but standard memory counters do not account for it, so a ballooned VM can appear memory-critical when the guest workload is actually healthy.
|
||||
|
||||
Sencho reads the `Balloon:` field from `/proc/meminfo` when it is available and shows the ballooned amount on the dashboard memory tile alongside an effective-usage percentage. The memory gauge, health verdict, and host RAM alerts continue to use the standard working-set percentage: unlike ZFS ARC, ballooned memory is reclaimed by the hypervisor and the guest cannot get it back on demand, so balloon data is informational context rather than a factor in alerting or health decisions. When `/proc/meminfo` is unreadable or the `Balloon:` field is absent, the behavior is unchanged.
|
||||
|
||||
`/proc/meminfo` is usually visible inside the container at `/proc/meminfo` with no extra configuration. If your runtime does not expose it, mount it read-only:
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- /proc/meminfo:/host/proc/meminfo:ro
|
||||
```
|
||||
|
||||
Sencho checks `SENCHO_PROC_MEMINFO_PATH`, then `/host/proc/meminfo`, then `/proc/meminfo`. Set `SENCHO_PROC_MEMINFO_PATH` only if your meminfo lives somewhere else inside the container.
|
||||
|
||||
## Listen port
|
||||
|
||||
Sencho always listens on `1852` inside the container. The port is fixed and is not read from an environment variable. To expose Sencho on a different host port, remap with Docker's `-p` flag (or the `ports:` key in your compose file):
|
||||
|
||||
@@ -29,7 +29,7 @@ import { PinnedUpdateBadge } from './PinnedUpdateBadge';
|
||||
import { StackSection } from './NodeCardStackList';
|
||||
import type { Label as StackLabel } from '../label-types';
|
||||
import type { FleetNode, NodeUpdateStatus } from './types';
|
||||
import { getNodeCpu, getNodeMem, getNodeDisk, isCritical } from './nodeUtils';
|
||||
import { getNodeCpu, getNodeMem, getNodeMemUsed, getNodeMemTotal, getNodeDisk, isCritical } from './nodeUtils';
|
||||
|
||||
// --- Types ---
|
||||
|
||||
@@ -97,6 +97,8 @@ export function NodeCard({ node, onNavigate, onOpenNetworking, networkingSignal,
|
||||
const formattedLatest = formatVersion(updateStatus?.latestVersion);
|
||||
const cpuPercent = getNodeCpu(node);
|
||||
const memPercent = getNodeMem(node);
|
||||
const memUsed = getNodeMemUsed(node);
|
||||
const memTotal = getNodeMemTotal(node);
|
||||
const diskPercent = getNodeDisk(node);
|
||||
|
||||
const openCordonModal = () => {
|
||||
@@ -308,7 +310,7 @@ export function NodeCard({ node, onNavigate, onOpenNetworking, networkingSignal,
|
||||
<span className="flex items-center gap-1 text-muted-foreground">
|
||||
<MemoryStick className="w-3 h-3" /> RAM
|
||||
</span>
|
||||
<span className="font-medium">{formatBytes(node.systemStats.memory.used, 1)} / {formatBytes(node.systemStats.memory.total, 1)}</span>
|
||||
<span className="font-medium">{formatBytes(memUsed, 1)} / {formatBytes(memTotal, 1)}</span>
|
||||
</div>
|
||||
<UsageBar percent={memPercent} color={memPercent > 80 ? 'bg-destructive/80' : memPercent > 60 ? 'bg-warning' : 'bg-brand/60'} />
|
||||
</div>
|
||||
|
||||
@@ -2,7 +2,7 @@ import { useState, useCallback, useMemo, useRef } from 'react';
|
||||
import { apiFetch } from '@/lib/api';
|
||||
import { useFleetLabels, labelPaletteKey } from './useFleetLabels';
|
||||
import { useNodeLabels } from './useNodeLabels';
|
||||
import { isCritical, getNodeCpu, getNodeMem, getNodeDisk } from '../nodeUtils';
|
||||
import { isCritical, getNodeCpu, getNodeMem, getNodeMemUsed, getNodeMemTotal, getNodeDisk } from '../nodeUtils';
|
||||
import type { FleetNode, ViewMode, FleetPreferences, NodeUpdateStatus } from '../types';
|
||||
|
||||
interface MastheadStats {
|
||||
@@ -96,8 +96,8 @@ export function useFleetOverview({ prefs, updatePrefs, updateStatuses }: UseFlee
|
||||
const worstCpu = worstCpuNode
|
||||
? { name: worstCpuNode.name, percent: getNodeCpu(worstCpuNode) }
|
||||
: null;
|
||||
const totalMemUsed = onlineNodes.reduce((sum, n) => sum + (n.systemStats?.memory.used ?? 0), 0);
|
||||
const totalMemTotal = onlineNodes.reduce((sum, n) => sum + (n.systemStats?.memory.total ?? 0), 0);
|
||||
const totalMemUsed = onlineNodes.reduce((sum, n) => sum + getNodeMemUsed(n), 0);
|
||||
const totalMemTotal = onlineNodes.reduce((sum, n) => sum + getNodeMemTotal(n), 0);
|
||||
return {
|
||||
nodeCount: nodes.length,
|
||||
onlineCount,
|
||||
|
||||
@@ -5,7 +5,19 @@ export function getNodeCpu(node: FleetNode): number {
|
||||
}
|
||||
|
||||
export function getNodeMem(node: FleetNode): number {
|
||||
return node.systemStats ? parseFloat(node.systemStats.memory.usagePercent) : 0;
|
||||
if (!node.systemStats) return 0;
|
||||
const eff = node.systemStats.memory.effectiveUsagePercent;
|
||||
return parseFloat(eff ?? node.systemStats.memory.usagePercent);
|
||||
}
|
||||
|
||||
export function getNodeMemUsed(node: FleetNode): number {
|
||||
const mem = node.systemStats?.memory;
|
||||
return mem?.effectiveUsed ?? mem?.used ?? 0;
|
||||
}
|
||||
|
||||
export function getNodeMemTotal(node: FleetNode): number {
|
||||
const mem = node.systemStats?.memory;
|
||||
return mem?.effectiveTotal ?? mem?.total ?? 0;
|
||||
}
|
||||
|
||||
export function getNodeDisk(node: FleetNode): number {
|
||||
|
||||
@@ -10,7 +10,18 @@ export interface FleetNodeStats {
|
||||
|
||||
export interface FleetNodeSystemStats {
|
||||
cpu: { usage: string; cores: number };
|
||||
memory: { total: number; used: number; free: number; usagePercent: string };
|
||||
memory: {
|
||||
total: number;
|
||||
used: number;
|
||||
free: number;
|
||||
usagePercent: string;
|
||||
ballooned?: number;
|
||||
effectiveTotal?: number;
|
||||
effectiveUsed?: number;
|
||||
effectiveFree?: number;
|
||||
effectiveUsagePercent?: string;
|
||||
balloonSource?: string;
|
||||
};
|
||||
disk: { total: number; used: number; free: number; usagePercent: string } | null;
|
||||
}
|
||||
|
||||
|
||||
@@ -80,7 +80,7 @@ export function HealthStatusBar({
|
||||
const unreadAlerts = countVisibleUnread(notifications);
|
||||
const running = `${stats.active}/${stats.total}`;
|
||||
const cpuLabel = systemStats ? `${parseFloat(systemStats.cpu.usage).toFixed(0)}%` : '--';
|
||||
const memLabel = systemStats ? formatGib(systemStats.memory.used) : '--';
|
||||
const memLabel = systemStats ? formatGib(systemStats.memory.effectiveUsed ?? systemStats.memory.used) : '--';
|
||||
const lastSyncLabel = lastSyncAt ? `last sync ${formatAgo(now - lastSyncAt)}` : 'connecting…';
|
||||
const metaLine = `${activeNodeName} · ${nodeCount} ${nodeCount === 1 ? 'node' : 'nodes'} · ${lastSyncLabel}`;
|
||||
const reasonsLine = reasons.join(' · ');
|
||||
|
||||
@@ -49,7 +49,13 @@ function GaugeBar({ value, warn = 80, crit = 90 }: { value: number; warn?: numbe
|
||||
|
||||
export function ResourceGauges({ systemStats, cpuHistory, netHistory, historyEndAt }: ResourceGaugesProps) {
|
||||
const cpuVal = parseFloat(systemStats?.cpu.usage || '0');
|
||||
// Primary gauge driven by raw ARC-adjusted percent only. Ballooned memory
|
||||
// is host-reclaimed (not guest-reclaimable like ARC), so it must not mask
|
||||
// real memory pressure in the gauge color or percent hero.
|
||||
const ramVal = parseFloat(systemStats?.memory.usagePercent || '0');
|
||||
const ramEffectivePercent = systemStats?.memory.effectiveUsagePercent ?? null;
|
||||
const ramUsed = systemStats?.memory.effectiveUsed ?? systemStats?.memory.used ?? 0;
|
||||
const ramTotal = systemStats?.memory.effectiveTotal ?? systemStats?.memory.total ?? 0;
|
||||
const diskVal = parseFloat(systemStats?.disk?.usagePercent || '0');
|
||||
|
||||
const cpuPeak = cpuHistory.length > 0 ? Math.max(...cpuHistory) : 0;
|
||||
@@ -102,8 +108,14 @@ export function ResourceGauges({ systemStats, cpuHistory, netHistory, historyEnd
|
||||
{systemStats ? `${ramVal.toFixed(0)}%` : '--'}
|
||||
</div>
|
||||
<div className="mt-1.5 font-mono text-[11px] text-stat-subtitle">
|
||||
{systemStats ? `${formatBytes(systemStats.memory.used)} / ${formatBytes(systemStats.memory.total)}` : '\u00A0'}
|
||||
{systemStats ? `${formatBytes(ramUsed)} / ${formatBytes(ramTotal)}` : '\u00A0'}
|
||||
</div>
|
||||
{systemStats?.memory.ballooned && systemStats.memory.ballooned > 0 ? (
|
||||
<div className="mt-1 font-mono text-[10px] text-stat-subtitle/70">
|
||||
Ballooned to host: {formatBytes(systemStats.memory.ballooned)}
|
||||
{ramEffectivePercent !== null ? ` (effective ${parseFloat(ramEffectivePercent).toFixed(0)}%)` : ''}
|
||||
</div>
|
||||
) : null}
|
||||
{systemStats ? <GaugeBar value={ramVal} /> : null}
|
||||
</div>
|
||||
|
||||
|
||||
@@ -10,7 +10,10 @@ export interface HealthResult {
|
||||
// drift apart.
|
||||
export function deriveHealth(stats: Stats, systemStats: SystemStats | null, notifications: NotificationItem[]): HealthResult {
|
||||
const cpu = parseFloat(systemStats?.cpu.usage || '0');
|
||||
const ram = parseFloat(systemStats?.memory.usagePercent || '0');
|
||||
// Ballooned memory is NOT subtracted for health: unlike ARC, ballooned
|
||||
// pages are host-reclaimed and the guest cannot get them back on demand.
|
||||
// A ballooned VM with real memory pressure must still show degraded/critical.
|
||||
const ram = parseFloat(systemStats?.memory.usagePercent ?? '0');
|
||||
const disk = parseFloat(systemStats?.disk?.usagePercent || '0');
|
||||
const unreadErrors = notifications.filter(n => !n.is_read && n.level === 'error').length;
|
||||
|
||||
|
||||
@@ -16,6 +16,12 @@ export interface SystemStats {
|
||||
used: number;
|
||||
free: number;
|
||||
usagePercent: string;
|
||||
ballooned?: number;
|
||||
effectiveTotal?: number;
|
||||
effectiveUsed?: number;
|
||||
effectiveFree?: number;
|
||||
effectiveUsagePercent?: string;
|
||||
balloonSource?: string;
|
||||
};
|
||||
disk: {
|
||||
fs: string;
|
||||
|
||||
@@ -95,7 +95,7 @@ export function MobileDashboard({ notifications, headerActions, onNavigateToStac
|
||||
);
|
||||
|
||||
const cpuVal = parseFloat(data.systemStats?.cpu.usage || '0');
|
||||
const ramVal = parseFloat(data.systemStats?.memory.usagePercent || '0');
|
||||
const ramVal = parseFloat(data.systemStats?.memory.effectiveUsagePercent ?? data.systemStats?.memory.usagePercent ?? '0');
|
||||
const diskVal = parseFloat(data.systemStats?.disk?.usagePercent || '0');
|
||||
const netPerSec = (data.systemStats?.network?.rxSec ?? 0) + (data.systemStats?.network?.txSec ?? 0);
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import { cordonNode, uncordonNode } from '@/lib/nodesApi';
|
||||
import { toast } from '@/components/ui/toast-store';
|
||||
import { ConfirmModal } from '@/components/ui/modal';
|
||||
import { formatBytes } from '@/lib/utils';
|
||||
import { getNodeCpu, getNodeMem, getNodeDisk, isCritical } from '@/components/FleetView/nodeUtils';
|
||||
import { getNodeCpu, getNodeMem, getNodeMemUsed, getNodeMemTotal, getNodeDisk, isCritical } from '@/components/FleetView/nodeUtils';
|
||||
import type { FleetNode } from '@/components/FleetView/types';
|
||||
import { Bar, BackChip, Kicker, Masthead, MBtn, SectionHead, StateDot, StatePill } from './mobile-ui';
|
||||
import type { Tone as UiTone } from './mobile-ui';
|
||||
@@ -221,7 +221,7 @@ function NodeDetail({
|
||||
<ResourceRow
|
||||
label="mem"
|
||||
pct={getNodeMem(node)}
|
||||
detail={`${formatBytes(node.systemStats.memory.used, 1)} / ${formatBytes(node.systemStats.memory.total, 1)}`}
|
||||
detail={`${formatBytes(getNodeMemUsed(node), 1)} / ${formatBytes(getNodeMemTotal(node), 1)}`}
|
||||
/>
|
||||
{node.systemStats.disk ? (
|
||||
<ResourceRow
|
||||
@@ -309,8 +309,8 @@ export function MobileFleet({ headerActions, onInspectNode, onInspectStack }: Mo
|
||||
const totalStacks = nodes.reduce((sum, n) => sum + (n.stacks?.length ?? 0), 0);
|
||||
const running = nodes.reduce((sum, n) => sum + (n.stats?.active ?? 0), 0);
|
||||
const avgCpu = onlineNodes.length > 0 ? onlineNodes.reduce((s, n) => s + getNodeCpu(n), 0) / onlineNodes.length : 0;
|
||||
const memUsed = onlineNodes.reduce((s, n) => s + (n.systemStats?.memory.used ?? 0), 0);
|
||||
const memTotal = onlineNodes.reduce((s, n) => s + (n.systemStats?.memory.total ?? 0), 0);
|
||||
const memUsed = onlineNodes.reduce((s, n) => s + getNodeMemUsed(n), 0);
|
||||
const memTotal = onlineNodes.reduce((s, n) => s + getNodeMemTotal(n), 0);
|
||||
const memPct = memTotal > 0 ? (memUsed / memTotal) * 100 : 0;
|
||||
const syncLabel = lastSyncAt ? `last sync ${formatAgo(now - lastSyncAt)}` : 'connecting…';
|
||||
|
||||
|
||||
Reference in New Issue
Block a user