mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-22 08:06:42 +00:00
feat: account for VM memory ballooning in host memory reporting (#1750)
* feat: account for VM memory ballooning in host memory reporting Extend hostMemory.ts with a readBalloonedMemory() function that parses the Balloon: field from /proc/meminfo, following the same fail-open pattern as the ZFS ARC integration. When a nonzero balloon is detected, effective memory fields (effectiveUsed, effectiveFree, effectiveUsagePercent) are computed and exposed through /api/system/stats and /api/fleet/overview. All consumers that derive meaning from host memory now prefer effective values when present: the dashboard gauge, Fleet card RAM bar, mobile views, health verdict, health status bar stat tile, and host RAM alerts. Backward compatible: missing /proc/meminfo or absent Balloon: line preserves exact current behavior. Old remote nodes without the new fields continue rendering normally. * refactor: extract shared helpers for balloon memory wiring Extract readCandidateFile() and logSelectedPath() in hostMemory.ts to deduplicate ARC and balloon file-read logic. Add memoryToWire() to centralize the optional-field spread used by /api/system/stats and /api/fleet/overview. Add getNodeMemUsed()/getNodeMemTotal() helpers in nodeUtils.ts for frontend byte-text consumers. * fix: make desktop fleet masthead aggregate balloon-aware The desktop fleet overview's memory aggregate in useFleetOverview.ts still summed raw memory.used, while the mobile fleet aggregate and per-node cards already used effective values. Update to use getNodeMemUsed/getNodeMemTotal helpers. * fix: revert balloon adjustment from alerting and health decisions Ballooned memory is host-reclaimed (unlike ZFS ARC, which the guest can reclaim on demand). The guest cannot get ballooned pages back until the hypervisor deflates them, so treating ballooned memory as available for alerting or health can mask real memory pressure. Keep balloon parsing, wire fields, and the dashboard context line as informational-only. The memory gauge, health verdict, and host RAM alerts now use the standard ARC-adjusted working-set percentage regardless of balloon. Updated configuration.mdx and dashboard.mdx to document that balloon data is informational and does not influence alerting.
This commit is contained in:
Anso
@@ -1,26 +1,33 @@
|
||||
import { promises as fs } from 'fs';
|
||||
import { vi } from 'vitest';
|
||||
import { ARCSTATS_FIXED_PATHS } from '../../helpers/hostMemory';
|
||||
import { ARCSTATS_FIXED_PATHS, MEMINFO_FIXED_PATHS } from '../../helpers/hostMemory';
|
||||
|
||||
/**
|
||||
* Path-aware partial mock of `fs.promises` for ZFS arcstats reads.
|
||||
* Path-aware partial mock of `fs.promises` for ZFS arcstats and /proc/meminfo
|
||||
* reads.
|
||||
*
|
||||
* `helpers/hostMemory.ts` reads `/proc/spl/kstat/zfs/arcstats` (and optional
|
||||
* variants) to compute reclaimable ARC. Tests may run on a ZFS host, so a real
|
||||
* read would make results host-dependent. This installs a spy that intercepts
|
||||
* ONLY registered/ARC-candidate paths and delegates every other
|
||||
* `readFile`/`stat` to the real filesystem, so `setupTestDb` and
|
||||
* `DatabaseService` keep working. Default behavior: ARC candidates reject with
|
||||
* ENOENT (no ARC), so consumers fall back to the plain `active/total` reading.
|
||||
* `helpers/hostMemory.ts` reads `/proc/spl/kstat/zfs/arcstats` (for ARC) and
|
||||
* `/proc/meminfo` (for VM ballooning). Tests may run on a ZFS host or a
|
||||
* ballooned VM, so real reads would make results host-dependent. This installs
|
||||
* a spy that intercepts ONLY registered/candidate paths and delegates every
|
||||
* other `readFile`/`stat` to the real filesystem, so `setupTestDb` and
|
||||
* `DatabaseService` keep working. Default behavior: candidates reject with
|
||||
* ENOENT, so consumers fall back to the plain `active/total` reading.
|
||||
*/
|
||||
|
||||
// Sourced from the helper so the mock cannot silently drift from the paths the
|
||||
// production code actually reads.
|
||||
export const ARC_CANDIDATE_PATHS = ARCSTATS_FIXED_PATHS;
|
||||
|
||||
/** Second fixed candidate; the default path fixtures are served from. */
|
||||
/** Second fixed ARC candidate; the default path ARC fixtures are served from. */
|
||||
export const DEFAULT_ARC_PATH = ARC_CANDIDATE_PATHS[1];
|
||||
|
||||
/** Meminfo candidate paths (same source-of-truth import pattern as ARC). */
|
||||
export const MEMINFO_CANDIDATE_PATHS = MEMINFO_FIXED_PATHS;
|
||||
|
||||
/** Default meminfo path for test fixtures. */
|
||||
export const DEFAULT_MEMINFO_PATH = MEMINFO_CANDIDATE_PATHS[1];
|
||||
|
||||
type StatDescriptor = { isFile: boolean; size: number };
|
||||
|
||||
export interface ArcstatsFsMock {
|
||||
@@ -47,7 +54,8 @@ export function installArcstatsFsMock(): ArcstatsFsMock {
|
||||
const realStat = fs.stat.bind(fs);
|
||||
const reads = new Map<string, string | NodeJS.ErrnoException>();
|
||||
const stats = new Map<string, StatDescriptor | NodeJS.ErrnoException>();
|
||||
const isArcCandidate = (p: string): boolean => ARC_CANDIDATE_PATHS.includes(p);
|
||||
const isCandidatePath = (p: string): boolean =>
|
||||
ARC_CANDIDATE_PATHS.includes(p) || MEMINFO_CANDIDATE_PATHS.includes(p);
|
||||
|
||||
vi.spyOn(fs, 'readFile').mockImplementation((async (p: unknown, ...rest: unknown[]) => {
|
||||
const key = String(p);
|
||||
@@ -56,7 +64,7 @@ export function installArcstatsFsMock(): ArcstatsFsMock {
|
||||
if (v instanceof Error) throw v;
|
||||
return v;
|
||||
}
|
||||
if (isArcCandidate(key)) throw enoent(key);
|
||||
if (isCandidatePath(key)) throw enoent(key);
|
||||
return (realReadFile as (...a: unknown[]) => unknown)(p, ...rest);
|
||||
}) as unknown as typeof fs.readFile);
|
||||
|
||||
@@ -73,7 +81,7 @@ export function installArcstatsFsMock(): ArcstatsFsMock {
|
||||
const size = typeof v === 'string' ? Buffer.byteLength(v) : 0;
|
||||
return { isFile: () => true, size };
|
||||
}
|
||||
if (isArcCandidate(key)) throw enoent(key);
|
||||
if (isCandidatePath(key)) throw enoent(key);
|
||||
return (realStat as (...a: unknown[]) => unknown)(p, ...rest);
|
||||
}) as unknown as typeof fs.stat);
|
||||
|
||||
@@ -96,3 +104,29 @@ export function arcstatsBody(sizeRow: string | number, cMinRow: string | number)
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a realistic /proc/meminfo snippet with the given Balloon value in kB.
|
||||
* Pass undefined / a negative value to omit the Balloon line entirely.
|
||||
*/
|
||||
export function meminfoBody(balloonKb?: number): string {
|
||||
const balloonLine = balloonKb !== undefined && balloonKb >= 0
|
||||
? `Balloon: ${balloonKb} kB\n`
|
||||
: '';
|
||||
return [
|
||||
'MemTotal: 16433188 kB',
|
||||
'MemFree: 620452 kB',
|
||||
'MemAvailable: 3489624 kB',
|
||||
'Buffers: 158668 kB',
|
||||
'Cached: 3335960 kB',
|
||||
'SwapCached: 0 kB',
|
||||
'Active: 5280444 kB',
|
||||
'Inactive: 7478672 kB',
|
||||
balloonLine,
|
||||
'SwapTotal: 8388604 kB',
|
||||
'SwapFree: 8388604 kB',
|
||||
'Dirty: 124 kB',
|
||||
'Writeback: 0 kB',
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
@@ -9,8 +9,11 @@ import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vite
|
||||
import {
|
||||
installArcstatsFsMock,
|
||||
arcstatsBody,
|
||||
meminfoBody,
|
||||
DEFAULT_ARC_PATH,
|
||||
DEFAULT_MEMINFO_PATH,
|
||||
ARC_CANDIDATE_PATHS,
|
||||
MEMINFO_CANDIDATE_PATHS,
|
||||
type ArcstatsFsMock,
|
||||
} from './helpers/arcstatsFsMock';
|
||||
|
||||
@@ -20,7 +23,7 @@ vi.mock('systeminformation', () => ({
|
||||
default: { mem: (...args: unknown[]) => mockMem(...args) },
|
||||
}));
|
||||
|
||||
import { getHostMemory, adjustForArc } from '../helpers/hostMemory';
|
||||
import { getHostMemory, adjustForArc, adjustForBalloon } from '../helpers/hostMemory';
|
||||
|
||||
// mem.active === total - available on Linux, so used/free below mirror the
|
||||
// real systeminformation shape the helper consumes.
|
||||
@@ -43,6 +46,7 @@ beforeEach(() => {
|
||||
arcFs.clear();
|
||||
mockMem.mockReset();
|
||||
delete process.env.SENCHO_ZFS_ARCSTATS_PATH;
|
||||
delete process.env.SENCHO_PROC_MEMINFO_PATH;
|
||||
});
|
||||
|
||||
describe('adjustForArc', () => {
|
||||
@@ -199,6 +203,192 @@ describe('getHostMemory ARC discovery', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('adjustForBalloon', () => {
|
||||
const arcAdjusted = (total: number, used: number, free: number, usagePercent: number): ReturnType<typeof adjustForArc> =>
|
||||
({ total, used, free, usagePercent });
|
||||
|
||||
it('returns the input unchanged when ballooned is 0', () => {
|
||||
const input = arcAdjusted(1000, 400, 600, 40);
|
||||
const result = adjustForBalloon(input, 0);
|
||||
expect(result).toBe(input); // identity for zero
|
||||
});
|
||||
|
||||
it('returns the input unchanged when ballooned is negative', () => {
|
||||
const input = arcAdjusted(1000, 400, 600, 40);
|
||||
const result = adjustForBalloon(input, -5);
|
||||
expect(result).toBe(input);
|
||||
});
|
||||
|
||||
it('subtracts ballooned from used, adds to free, sets optional fields', () => {
|
||||
const input = arcAdjusted(1000, 400, 600, 40);
|
||||
const result = adjustForBalloon(input, 200);
|
||||
expect(result.ballooned).toBe(200);
|
||||
expect(result.effectiveTotal).toBe(1000);
|
||||
expect(result.effectiveUsed).toBe(200); // 400 - 200
|
||||
expect(result.effectiveFree).toBe(800); // 600 + 200
|
||||
expect(result.effectiveUsagePercent).toBe(20); // 200 / 1000 * 100
|
||||
expect(result.balloonSource).toBe('linux_proc_meminfo');
|
||||
// Base fields unchanged.
|
||||
expect(result.total).toBe(1000);
|
||||
expect(result.used).toBe(400);
|
||||
expect(result.free).toBe(600);
|
||||
});
|
||||
|
||||
it('clamps effectiveUsed at 0 when balloon exceeds used', () => {
|
||||
const input = arcAdjusted(1000, 100, 900, 10);
|
||||
const result = adjustForBalloon(input, 5000);
|
||||
expect(result.effectiveUsed).toBe(0);
|
||||
expect(result.effectiveFree).toBe(1000);
|
||||
expect(result.effectiveUsagePercent).toBe(0);
|
||||
});
|
||||
|
||||
it('handles zero total gracefully', () => {
|
||||
const input = arcAdjusted(0, 0, 0, 0);
|
||||
const result = adjustForBalloon(input, 100);
|
||||
expect(result.effectiveUsagePercent).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getHostMemory balloon discovery', () => {
|
||||
it('returns the base ARC-adjusted shape when no meminfo is present', async () => {
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
const result = await getHostMemory();
|
||||
expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
|
||||
});
|
||||
|
||||
it('returns the base ARC-adjusted shape when Balloon is missing from meminfo', async () => {
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
arcFs.setRead(DEFAULT_MEMINFO_PATH, meminfoBody()); // no Balloon line
|
||||
const result = await getHostMemory();
|
||||
expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
|
||||
});
|
||||
|
||||
it('returns the base ARC-adjusted shape when Balloon is 0 kB', async () => {
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
arcFs.setRead(DEFAULT_MEMINFO_PATH, meminfoBody(0));
|
||||
const result = await getHostMemory();
|
||||
expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
|
||||
});
|
||||
|
||||
it('subtracts ballooned memory and sets optional fields', async () => {
|
||||
mockMem.mockResolvedValue(memSample(16000, 4000)); // 16 GB total, 4 GB available → 75% used
|
||||
// 4 GiB balloon = 4194304 kB
|
||||
arcFs.setRead(DEFAULT_MEMINFO_PATH, meminfoBody(4_194_304));
|
||||
const result = await getHostMemory();
|
||||
// ARC=0, available=4000: used=12000
|
||||
// ballooned=4194304*1024 = 4_294_967_296 bytes
|
||||
// effectiveUsed = 12000 - ballooned ≈ 7705 MB
|
||||
expect(result.used).toBe(12000);
|
||||
expect(typeof result.ballooned).toBe('number');
|
||||
expect(result.ballooned!).toBeGreaterThan(0);
|
||||
expect(result.effectiveUsed).toBeDefined();
|
||||
expect(result.effectiveFree).toBeDefined();
|
||||
expect(result.effectiveUsagePercent).toBeDefined();
|
||||
expect(result.effectiveUsed!).toBeLessThan(result.used);
|
||||
expect(result.balloonSource).toBe('linux_proc_meminfo');
|
||||
});
|
||||
|
||||
it('combines ARC reclaim and balloon adjustment', async () => {
|
||||
mockMem.mockResolvedValue(memSample(16000, 2000)); // 12.5% available
|
||||
arcFs.setRead(DEFAULT_ARC_PATH, arcstatsBody(5000, 1000)); // reclaimable ARC = 4000
|
||||
arcFs.setRead(DEFAULT_MEMINFO_PATH, meminfoBody(2_097_152)); // 2 GiB balloon
|
||||
const result = await getHostMemory();
|
||||
// ARC-adjusted: used = 16000 - (2000 + 4000) = 10000
|
||||
expect(result.used).toBe(10000);
|
||||
// Balloon-adjusted: effectiveUsed = 10000 - 2GiB
|
||||
expect(result.effectiveUsed).toBeDefined();
|
||||
expect(result.effectiveUsed!).toBeLessThan(result.used);
|
||||
});
|
||||
|
||||
it('prefers the meminfo override path', async () => {
|
||||
process.env.SENCHO_PROC_MEMINFO_PATH = '/custom/meminfo';
|
||||
mockMem.mockResolvedValue(memSample(16000, 4000));
|
||||
arcFs.setRead('/custom/meminfo', meminfoBody(4_194_304)); // 4 GiB balloon
|
||||
arcFs.setRead(DEFAULT_MEMINFO_PATH, meminfoBody(0)); // fixed path says no balloon
|
||||
const result = await getHostMemory();
|
||||
expect(result.ballooned).toBeGreaterThan(0); // override won
|
||||
});
|
||||
|
||||
it('falls through to a fixed meminfo path when the override is unreadable', async () => {
|
||||
process.env.SENCHO_PROC_MEMINFO_PATH = '/custom/meminfo';
|
||||
mockMem.mockResolvedValue(memSample(16000, 4000));
|
||||
arcFs.setReadError('/custom/meminfo', Object.assign(new Error('nope'), { code: 'ENOENT' }));
|
||||
arcFs.setRead(DEFAULT_MEMINFO_PATH, meminfoBody(2_097_152)); // 2 GiB
|
||||
const result = await getHostMemory();
|
||||
expect(result.ballooned).toBeGreaterThan(0); // fell through to fixed
|
||||
});
|
||||
|
||||
it('reads the host-mounted meminfo candidate and prefers it over /proc', async () => {
|
||||
mockMem.mockResolvedValue(memSample(16000, 4000));
|
||||
arcFs.setRead(MEMINFO_CANDIDATE_PATHS[0], meminfoBody(4_194_304)); // /host/proc: 4 GiB
|
||||
arcFs.setRead(MEMINFO_CANDIDATE_PATHS[1], meminfoBody(1_048_576)); // /proc: 1 GiB
|
||||
const result = await getHostMemory();
|
||||
// First candidate wins: 4 GiB balloon.
|
||||
expect(result.ballooned).toBeGreaterThan(0);
|
||||
expect(result.effectiveUsed).toBeDefined();
|
||||
});
|
||||
|
||||
it('skips an override path that is not a regular file', async () => {
|
||||
process.env.SENCHO_PROC_MEMINFO_PATH = '/custom/meminfo';
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
arcFs.setStat('/custom/meminfo', { isFile: false, size: 10 });
|
||||
arcFs.setRead('/custom/meminfo', meminfoBody(100));
|
||||
const result = await getHostMemory();
|
||||
expect(result.used).toBe(400); // override skipped, no meminfo on fixed
|
||||
});
|
||||
|
||||
it('skips an override path that exceeds the size bound', async () => {
|
||||
process.env.SENCHO_PROC_MEMINFO_PATH = '/custom/meminfo';
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
arcFs.setStat('/custom/meminfo', { isFile: true, size: 2 * 1024 * 1024 });
|
||||
arcFs.setRead('/custom/meminfo', meminfoBody(100));
|
||||
const result = await getHostMemory();
|
||||
expect(result.used).toBe(400);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['non-numeric value', 'Balloon: abc kB\n'],
|
||||
['negative value', 'Balloon: -100 kB\n'],
|
||||
['no kB suffix', 'Balloon: 100\n'],
|
||||
['wrong suffix', 'Balloon: 100 MB\n'],
|
||||
['extra token', 'Balloon: 100 kB extra\n'],
|
||||
])('treats a %s Balloon line as unusable and yields no balloon', async (_label, body) => {
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
arcFs.setRead(DEFAULT_MEMINFO_PATH, body);
|
||||
const result = await getHostMemory();
|
||||
expect(result).toEqual({ total: 1000, used: 400, free: 600, usagePercent: 40 });
|
||||
});
|
||||
|
||||
it('fails open (balloon 0) on a read error', async () => {
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
arcFs.setReadError(DEFAULT_MEMINFO_PATH, Object.assign(new Error('nope'), { code: 'EACCES' }));
|
||||
const result = await getHostMemory();
|
||||
expect(result.used).toBe(400);
|
||||
});
|
||||
|
||||
it('logs an unexpected meminfo read error once per code', async () => {
|
||||
mockMem.mockResolvedValue(memSample(1000, 600));
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
|
||||
// Expected fs error: silent fall-through.
|
||||
arcFs.setReadError(MEMINFO_CANDIDATE_PATHS[0], Object.assign(new Error('denied'), { code: 'EACCES' }));
|
||||
arcFs.setReadError(MEMINFO_CANDIDATE_PATHS[1], Object.assign(new Error('denied'), { code: 'EACCES' }));
|
||||
await getHostMemory();
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
|
||||
// Unexpected fs error: logged once. Use EBADF to avoid collision with
|
||||
// the ARC test suite's own EIO trigger (loggedErrorCodes is shared).
|
||||
arcFs.setReadError(MEMINFO_CANDIDATE_PATHS[0], Object.assign(new Error('badf'), { code: 'EBADF' }));
|
||||
arcFs.setReadError(MEMINFO_CANDIDATE_PATHS[1], Object.assign(new Error('badf'), { code: 'EBADF' }));
|
||||
await getHostMemory();
|
||||
await getHostMemory();
|
||||
const unexpectedLogs = warn.mock.calls.filter(([msg]) => String(msg).includes('EBADF'));
|
||||
expect(unexpectedLogs).toHaveLength(1);
|
||||
warn.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.SENCHO_ZFS_ARCSTATS_PATH;
|
||||
delete process.env.SENCHO_PROC_MEMINFO_PATH;
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user