feat: add ON/OFF toggle for host threshold alerts (#1456)

* feat: add ON/OFF toggle for host threshold alerts

Add host_alerts_enabled setting (default ON) as a master switch for CPU,
RAM, and disk host threshold evaluation. When OFF, the four threshold
controls in Settings > Host Alerts are disabled and MonitorService skips
the systeminformation calls and alert dispatch entirely, while clearing
stale suppression state so re-enabling starts fresh.

The dashboard Configuration Status card shows "Off" when host threshold
alerts are disabled. Crash capture, health gate, deploy guardrails,
stack alert rules, and the Docker janitor are all unaffected.

* fix: exit NumberChip edit mode when externally disabled

When the host threshold alerts master toggle is turned OFF while a
NumberChip is in edit mode, force-exit edit mode so the chip renders
the greyed-out button state consistently with the other chips.
This commit is contained in:
Anso
2026-06-25 16:05:16 -04:00
committed by GitHub
parent f1f64ec7f6
commit b7dd9dc1b0
19 changed files with 281 additions and 50 deletions
+101 -1
View File
@@ -2,7 +2,7 @@
* Unit tests for MonitorService — alert state machine, metric calculations,
* cleanup delegation, global settings evaluation, and concurrency guards.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
// ── Hoisted mocks ──────────────────────────────────────────────────────
@@ -370,6 +370,106 @@ describe('MonitorService - evaluateGlobalSettings', () => {
});
});
describe('MonitorService - host_alerts_enabled toggle', () => {
beforeEach(() => {
mockGetGlobalSettings.mockReturnValue({});
mockDispatchAlert.mockClear();
mockCurrentLoad.mockClear();
mockMem.mockClear();
mockFsSize.mockClear();
});
afterEach(() => {
// Restore default mock implementations so version-check state does not
// leak into sibling describe blocks (F-11 suppression, version check).
mockGetLatestVersion.mockResolvedValue(null);
mockGetSenchoVersion.mockReturnValue(null);
});
it('skips host metrics and dispatch when disabled', async () => {
mockCurrentLoad.mockResolvedValue({ currentLoad: 75 });
const svc = MonitorService.getInstance();
await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '0', host_cpu_limit: '50' });
expect(mockDispatchAlert).not.toHaveBeenCalled();
expect(mockCurrentLoad).not.toHaveBeenCalled();
expect(mockMem).not.toHaveBeenCalled();
expect(mockFsSize).not.toHaveBeenCalled();
});
it('does not call systeminformation when disabled', async () => {
mockMem.mockResolvedValue(memSample(15e9)); // ~94% - would breach
const svc = MonitorService.getInstance();
await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '0', host_ram_limit: '80' });
expect(mockCurrentLoad).not.toHaveBeenCalled();
expect(mockMem).not.toHaveBeenCalled();
expect(mockFsSize).not.toHaveBeenCalled();
expect(mockDispatchAlert).not.toHaveBeenCalled();
});
it('clears persisted suppression state when disabled', async () => {
const store: Record<string, string> = {};
mockGetSystemState.mockImplementation((key: string) => store[key] ?? null);
mockSetSystemState.mockImplementation((key: string, value: string) => { store[key] = value; });
// Simulate a prior breach that set the persisted suppression timestamp.
store['last_host_cpu_alert_ts'] = String(Date.now());
store['last_host_ram_alert_ts'] = String(Date.now());
store['last_host_disk_alert_ts'] = String(Date.now());
const svc = MonitorService.getInstance();
await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '0' });
expect(store['last_host_cpu_alert_ts']).toBe('0');
expect(store['last_host_ram_alert_ts']).toBe('0');
expect(store['last_host_disk_alert_ts']).toBe('0');
});
it('re-enable fires fresh without suppressed suffix', async () => {
const store: Record<string, string> = {};
mockGetSystemState.mockImplementation((key: string) => store[key] ?? null);
mockSetSystemState.mockImplementation((key: string, value: string) => { store[key] = value; });
mockMem.mockResolvedValue(memSample(15e9)); // ~94% - breaches 80% limit
const svc = MonitorService.getInstance();
// Step 1: breach while enabled — fires alert and seeds suppression state.
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'));
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Suppressed'));
mockDispatchAlert.mockClear();
// Step 2: disable — clears suppression state and skips dispatch.
await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '0', host_ram_limit: '80' });
expect(mockDispatchAlert).not.toHaveBeenCalled();
expect(store['last_host_ram_alert_ts']).toBe('0');
// Step 3: re-enable while still breaching — fires fresh, no "Suppressed" suffix.
await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '1', host_ram_limit: '80' });
const calls = mockDispatchAlert.mock.calls.filter(
(c: unknown[]) => c[0] === 'warning' && c[1] === 'monitor_alert',
);
expect(calls.length).toBe(1);
const msg = calls[0] as string[];
expect(msg[2]).toContain('Memory');
expect(msg[2]).not.toContain('Suppressed');
});
it('still runs version check when disabled', async () => {
mockGetSenchoVersion.mockReturnValue('0.45.0');
mockGetLatestVersion.mockResolvedValue('0.46.0');
const svc = MonitorService.getInstance();
(svc as any).lastVersionCheckAt = 0;
await (svc as any).evaluateGlobalSettings({ host_alerts_enabled: '0' });
expect(mockGetLatestVersion).toHaveBeenCalled();
});
});
// Crash + healthcheck detection now lives in DockerEventService (event-driven).
// Tests for those flows live in docker-event-service.test.ts.
@@ -337,6 +337,57 @@ describe('env_block_deploy_on_missing_required setting', () => {
});
});
describe('host_alerts_enabled toggle', () => {
it('seeds to "1" (on) in a fresh database', () => {
expect(DatabaseService.getInstance().getGlobalSettings().host_alerts_enabled).toBe('1');
});
it('is exposed through the settings GET projection', async () => {
const res = await request(app).get('/api/settings').set('Cookie', adminCookie);
expect(res.status).toBe(200);
expect(res.body.host_alerts_enabled).toBeDefined();
});
it('accepts a single-key POST write and persists it', async () => {
const res = await request(app)
.post('/api/settings')
.set('Cookie', adminCookie)
.send({ key: 'host_alerts_enabled', value: '0' });
expect(res.status).toBe(200);
expect(DatabaseService.getInstance().getGlobalSettings().host_alerts_enabled).toBe('0');
DatabaseService.getInstance().updateGlobalSetting('host_alerts_enabled', '1');
});
it('accepts a bulk PATCH alongside another key', async () => {
const res = await request(app)
.patch('/api/settings')
.set('Cookie', adminCookie)
.send({ host_alerts_enabled: '0', host_cpu_limit: 75 });
expect(res.status).toBe(200);
const settings = DatabaseService.getInstance().getGlobalSettings();
expect(settings.host_alerts_enabled).toBe('0');
expect(settings.host_cpu_limit).toBe('75');
DatabaseService.getInstance().updateGlobalSetting('host_alerts_enabled', '1');
DatabaseService.getInstance().updateGlobalSetting('host_cpu_limit', '90');
});
it('rejects non-enum values with 400', async () => {
const res = await request(app)
.post('/api/settings')
.set('Cookie', adminCookie)
.send({ key: 'host_alerts_enabled', value: '2' });
expect(res.status).toBe(400);
});
it('rejects viewer write with 403', async () => {
const res = await request(app)
.post('/api/settings')
.set('Cookie', viewerCookie)
.send({ key: 'host_alerts_enabled', value: '0' });
expect(res.status).toBe(403);
});
});
describe('PATCH /api/settings (bulk update)', () => {
it('rejects unauthenticated requests with 401', async () => {
const res = await request(app).patch('/api/settings').send({ host_cpu_limit: 50 });
+3
View File
@@ -37,6 +37,7 @@ export interface ConfigurationStatus {
diskLimit: number;
dockerJanitorGb: number;
globalCrash: boolean;
hostAlertsEnabled: boolean;
};
backup: {
provider: 'disabled' | 'sencho' | 'custom';
@@ -80,6 +81,7 @@ export function buildLocalConfigurationStatus(
const diskLimit = parseInt(settings['host_disk_limit'] ?? '90', 10);
const dockerJanitorGb = parseFloat(settings['docker_janitor_gb'] ?? '5');
const globalCrash = settings['global_crash'] === '1';
const hostAlertsEnabled = settings['host_alerts_enabled'] !== '0';
const cloudSvc = CloudBackupService.getInstance();
const cloudProvider = cloudSvc.getProvider();
@@ -140,6 +142,7 @@ export function buildLocalConfigurationStatus(
diskLimit,
dockerJanitorGb,
globalCrash,
hostAlertsEnabled,
},
backup: {
// Cloud Backup has a per-provider tier: Custom S3 is open to every
+2
View File
@@ -14,6 +14,7 @@ const ALLOWED_SETTING_KEYS = new Set([
'host_cpu_limit',
'host_ram_limit',
'host_disk_limit',
'host_alerts_enabled',
'host_alert_suppression_mins',
'docker_janitor_gb',
'global_crash',
@@ -42,6 +43,7 @@ const SettingsPatchSchema = z.object({
host_cpu_limit: z.coerce.number().int().min(1).max(100).transform(String),
host_ram_limit: z.coerce.number().int().min(1).max(100).transform(String),
host_disk_limit: z.coerce.number().int().min(1).max(100).transform(String),
host_alerts_enabled: z.enum(['0', '1']),
host_alert_suppression_mins: z.coerce.number().int().min(1).max(1440).transform(String),
docker_janitor_gb: z.coerce.number().min(0).transform(String),
global_crash: z.enum(['0', '1']),
+1
View File
@@ -1564,6 +1564,7 @@ export class DatabaseService {
stmt.run('host_ram_limit', '90');
stmt.run('host_disk_limit', '90');
stmt.run('host_alert_suppression_mins', '60');
stmt.run('host_alerts_enabled', '1');
stmt.run('global_crash', '1');
stmt.run('docker_janitor_gb', '5');
stmt.run('developer_mode', '0');
@@ -26,6 +26,7 @@ const SAFE_SETTING_KEYS = [
'host_cpu_limit',
'host_ram_limit',
'host_disk_limit',
'host_alerts_enabled',
'host_alert_suppression_mins',
'docker_janitor_gb',
'global_crash',
+44 -36
View File
@@ -283,46 +283,54 @@ export class MonitorService {
const suppressionMs = suppressionMin * 60 * 1000;
// 1. Host Limits — fetch CPU, RAM, disk concurrently
try {
const [currentLoad, mem, fsSize] = await Promise.all([
withTimeout(si.currentLoad(), STATS_TIMEOUT_MS, 'host CPU stats'),
withTimeout(si.mem(), STATS_TIMEOUT_MS, 'host RAM stats'),
withTimeout(si.fsSize(), STATS_TIMEOUT_MS, 'host disk stats'),
]);
if (settings['host_alerts_enabled'] !== '0') {
try {
const [currentLoad, mem, fsSize] = await Promise.all([
withTimeout(si.currentLoad(), STATS_TIMEOUT_MS, 'host CPU stats'),
withTimeout(si.mem(), STATS_TIMEOUT_MS, 'host RAM stats'),
withTimeout(si.fsSize(), STATS_TIMEOUT_MS, 'host disk stats'),
]);
const cpuUsage = currentLoad.currentLoad;
const cpuLimit = parseFloat(settings['host_cpu_limit']);
if (!isNaN(cpuLimit) && cpuLimit > 0 && cpuUsage > cpuLimit) {
await this.dispatchHostMetricAlert('cpu', 'warning', suppressionMs,
`Host CPU utilization is critically high: ${cpuUsage.toFixed(1)}% (Threshold: ${cpuLimit}%)`);
} else {
this.clearHostMetricSuppression('cpu');
}
// Key off mem.active (the real working set), not mem.used: on Linux/BSD/macOS
// mem.used counts reclaimable page cache and reads ~99% on a busy host, which
// would fire spurious host-memory alerts.
const ramUsage = (mem.active / mem.total) * 100;
const ramLimit = parseFloat(settings['host_ram_limit']);
if (!isNaN(ramLimit) && ramLimit > 0 && ramUsage > ramLimit) {
await this.dispatchHostMetricAlert('ram', 'warning', suppressionMs,
`Host Memory utilization is critically high: ${ramUsage.toFixed(1)}% (Threshold: ${ramLimit}%)`);
} else {
this.clearHostMetricSuppression('ram');
}
const mainDisk = fsSize.find(fs => fs.mount === '/' || fs.mount === 'C:') || fsSize[0];
if (mainDisk) {
const diskLimit = parseFloat(settings['host_disk_limit']);
if (!isNaN(diskLimit) && diskLimit > 0 && mainDisk.use > diskLimit) {
await this.dispatchHostMetricAlert('disk', 'warning', suppressionMs,
`Host Disk space utilization is critically high: ${mainDisk.use.toFixed(1)}% (Threshold: ${diskLimit}%)`);
const cpuUsage = currentLoad.currentLoad;
const cpuLimit = parseFloat(settings['host_cpu_limit']);
if (!isNaN(cpuLimit) && cpuLimit > 0 && cpuUsage > cpuLimit) {
await this.dispatchHostMetricAlert('cpu', 'warning', suppressionMs,
`Host CPU utilization is critically high: ${cpuUsage.toFixed(1)}% (Threshold: ${cpuLimit}%)`);
} else {
this.clearHostMetricSuppression('disk');
this.clearHostMetricSuppression('cpu');
}
// Key off mem.active (the real working set), not mem.used: on Linux/BSD/macOS
// mem.used counts reclaimable page cache and reads ~99% on a busy host, which
// would fire spurious host-memory alerts.
const ramUsage = (mem.active / mem.total) * 100;
const ramLimit = parseFloat(settings['host_ram_limit']);
if (!isNaN(ramLimit) && ramLimit > 0 && ramUsage > ramLimit) {
await this.dispatchHostMetricAlert('ram', 'warning', suppressionMs,
`Host Memory utilization is critically high: ${ramUsage.toFixed(1)}% (Threshold: ${ramLimit}%)`);
} else {
this.clearHostMetricSuppression('ram');
}
const mainDisk = fsSize.find(fs => fs.mount === '/' || fs.mount === 'C:') || fsSize[0];
if (mainDisk) {
const diskLimit = parseFloat(settings['host_disk_limit']);
if (!isNaN(diskLimit) && diskLimit > 0 && mainDisk.use > diskLimit) {
await this.dispatchHostMetricAlert('disk', 'warning', suppressionMs,
`Host Disk space utilization is critically high: ${mainDisk.use.toFixed(1)}% (Threshold: ${diskLimit}%)`);
} else {
this.clearHostMetricSuppression('disk');
}
}
} catch (e) {
console.error('Error checking host limits in watchdog', e);
}
} catch (e) {
console.error('Error checking host limits in watchdog', e);
} else {
// Host threshold alerts are disabled: clear any stale suppression
// state so re-enabling starts fresh (no "Suppressed N" carryover).
this.clearHostMetricSuppression('cpu');
this.clearHostMetricSuppression('ram');
this.clearHostMetricSuppression('disk');
}
// 2. (Removed) Container crash + healthcheck detection moved to