gsadmin/sencho
1
0
Fork 0
mirror of https://github.com/Studio-Saelix/sencho.git synced 2026-08-11 11:16:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(dashboard): harden real-time dashboard with bug fixes and design compliance (#517)

Browse Source 
* fix(dashboard): harden real-time dashboard with bug fixes and design compliance

- Fix host alert spam: add 5-minute cooldown for CPU/RAM/disk threshold
  alerts, preventing duplicate notifications every 30s during sustained
  breaches. Extract shared dispatchWithCooldown helper (also used by
  Docker janitor alerts).
- Fix memory metric inflation: subtract filesystem cache from stored
  memory_mb values, matching the existing calculateMemoryPercent logic.
- Fix crash detection reliability: replace fragile 'seconds ago' string
  matching with a tracked Set of alerted container IDs. Containers are
  only alerted once per crash event, with automatic cleanup when they
  start running again or after a 1-hour TTL.
- Fix health status bar: exited containers now trigger 'degraded' state
  independently of unread error notifications.
- Fix CPU chart Y-axis: auto-scale when aggregate container CPU exceeds
  100% instead of silently clipping at the hardcoded domain ceiling.
- Fix grammar: 'actives' to 'active' in container count label.
- Add shadow-card-bevel to all dashboard cards per design system.
- Update dashboard docs to reflect revised health status thresholds.

* test(dashboard): update monitor service tests for new alert signatures

- Add container Id fields to crash detection test fixtures
- Update host alert assertions to match dispatchWithCooldown 3-arg call
- Fix unhealthy container test to use State: 'unhealthy' instead of
  State: 'running' (running containers are now skipped in crash detect)
This commit is contained in:
Anso
2026-04-12 04:25:41 -04:00
committed by GitHub
parent c7f041957c
commit 9db97107aa
9 changed files with 98 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/__tests__/monitor-service.test.ts
+10 -7
View File
@@ -251,7 +251,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
const svc = MonitorService.getInstance();
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' });
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('CPU'));
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('CPU'), undefined);
});
it('does not dispatch when CPU below threshold', async () => {
@@ -260,7 +260,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
const svc = MonitorService.getInstance();
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' });
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'));
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'), undefined);
});
it('dispatches RAM warning when over threshold', async () => {
@@ -269,7 +269,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
const svc = MonitorService.getInstance();
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Memory'));
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Memory'), undefined);
});
it('dispatches disk warning when over threshold', async () => {
@@ -278,7 +278,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
const svc = MonitorService.getInstance();
await (svc as any).evaluateGlobalSettings({ host_disk_limit: '90' });
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Disk'));
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Disk'), undefined);
});
it('skips host limits when threshold is 0 or NaN', async () => {
@@ -286,10 +286,10 @@ describe('MonitorService - evaluateGlobalSettings', () => {
const svc = MonitorService.getInstance();
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '0' });
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'));
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'), undefined);
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: 'abc' });
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'));
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'), undefined);
});
});
@@ -299,6 +299,7 @@ describe('MonitorService - global crash detection', () => {
it('detects exited containers with non-intentional exit codes', async () => {
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetAllContainers.mockResolvedValue([{
Id: 'crash-1',
State: 'exited',
Status: 'Exited (1) 5 seconds ago',
Names: ['/my-container'],
@@ -317,6 +318,7 @@ describe('MonitorService - global crash detection', () => {
for (const code of intentionalExits) {
mockDispatchAlert.mockClear();
mockGetAllContainers.mockResolvedValue([{
Id: `safe-${code}`,
State: 'exited',
Status: `Exited (${code}) 5 seconds ago`,
Names: ['/safe-container'],
@@ -332,7 +334,8 @@ describe('MonitorService - global crash detection', () => {
it('detects unhealthy containers', async () => {
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetAllContainers.mockResolvedValue([{
State: 'running',
Id: 'sick-1',
State: 'unhealthy',
Status: 'Up 2 hours (unhealthy)',
Names: ['/sick-container'],
}]);
backend/src/services/MonitorService.ts
+70 -28
View File
@@ -30,6 +30,13 @@ interface AlertState {
breachStartedAt: number; // timestamp when the rule first breached
}
const HOST_ALERT_KEYS = {
cpu: 'last_host_cpu_alert_ts',
ram: 'last_host_ram_alert_ts',
disk: 'last_host_disk_alert_ts',
janitor: 'last_janitor_alert_timestamp',
} as const;
export class MonitorService {
private static instance: MonitorService;
private intervalId: NodeJS.Timeout | null = null;
@@ -39,6 +46,11 @@ export class MonitorService {
// key: rule_id, value: AlertState
private activeBreaches = new Map<number, AlertState>();
// Track containers that have already been alerted as crashed to avoid
// duplicate alerts. key: containerId, value: timestamp when alerted.
private alertedCrashes = new Map<string, number>();
private static readonly CRASH_ALERT_TTL_MS = 60 * 60 * 1000; // 1 hour
private constructor() { }
public static getInstance(): MonitorService {
@@ -86,6 +98,7 @@ export class MonitorService {
private async evaluateGlobalSettings(settings: Record<string, string>) {
const notifier = NotificationService.getInstance();
const HOST_ALERT_COOLDOWN_MS = 5 * 60 * 1000; // 5 minutes between repeat alerts
// 1. Host Limits
try {
@@ -93,14 +106,16 @@ export class MonitorService {
const cpuUsage = currentLoad.currentLoad;
const cpuLimit = parseFloat(settings['host_cpu_limit']);
if (!isNaN(cpuLimit) && cpuLimit > 0 && cpuUsage > cpuLimit) {
await notifier.dispatchAlert('warning', `Host CPU utilization is critically high: ${cpuUsage.toFixed(1)}% (Threshold: ${cpuLimit}%)`);
await this.dispatchWithCooldown(HOST_ALERT_KEYS.cpu, HOST_ALERT_COOLDOWN_MS, 'warning',
`Host CPU utilization is critically high: ${cpuUsage.toFixed(1)}% (Threshold: ${cpuLimit}%)`);
}
const mem = await si.mem();
const ramUsage = (mem.used / mem.total) * 100;
const ramLimit = parseFloat(settings['host_ram_limit']);
if (!isNaN(ramLimit) && ramLimit > 0 && ramUsage > ramLimit) {
await notifier.dispatchAlert('warning', `Host Memory utilization is critically high: ${ramUsage.toFixed(1)}% (Threshold: ${ramLimit}%)`);
await this.dispatchWithCooldown(HOST_ALERT_KEYS.ram, HOST_ALERT_COOLDOWN_MS, 'warning',
`Host Memory utilization is critically high: ${ramUsage.toFixed(1)}% (Threshold: ${ramLimit}%)`);
}
const fsSize = await si.fsSize();
@@ -108,7 +123,8 @@ export class MonitorService {
if (mainDisk) {
const diskLimit = parseFloat(settings['host_disk_limit']);
if (!isNaN(diskLimit) && diskLimit > 0 && mainDisk.use > diskLimit) {
await notifier.dispatchAlert('warning', `Host Disk space utilization is critically high: ${mainDisk.use.toFixed(1)}% (Threshold: ${diskLimit}%)`);
await this.dispatchWithCooldown(HOST_ALERT_KEYS.disk, HOST_ALERT_COOLDOWN_MS, 'warning',
`Host Disk space utilization is critically high: ${mainDisk.use.toFixed(1)}% (Threshold: ${diskLimit}%)`);
}
}
} catch (e) {
@@ -117,36 +133,55 @@ export class MonitorService {
// 2. Global Crash Detect
if (settings['global_crash'] === '1') {
// Prune expired entries from the crash tracker
const now = Date.now();
for (const [id, ts] of this.alertedCrashes) {
if (now - ts > MonitorService.CRASH_ALERT_TTL_MS) this.alertedCrashes.delete(id);
}
try {
const nodes = DatabaseService.getInstance().getNodes();
const runningIds = new Set<string>();
for (const node of nodes) {
if (!node.id) continue;
// Remote nodes run their own MonitorService locally - skip direct Docker access
// Remote nodes run their own MonitorService locally
if (node.type === 'remote') continue;
try {
const docker = DockerController.getInstance(node.id);
const containers = await docker.getAllContainers();
for (const c of containers) {
if (c.State === 'exited' || String(c.Status).includes('unhealthy')) {
const containerStack = c.Labels?.['com.docker.compose.project'] || undefined;
if (c.State === 'exited') {
if (c.Status.includes('seconds ago')) {
const match = c.Status.match(/Exited \((\d+)\)/i);
const exitCode = match ? parseInt(match[1], 10) : null;
const intentionalExitCodes = [0, 137, 143, 255];
if (exitCode !== null && !intentionalExitCodes.includes(exitCode)) {
await notifier.dispatchAlert('error', `[Node: ${node.name}] Container Crash Detected: ${c.Names[0]} exited unexpectedly (Code: ${exitCode}).`, containerStack);
}
}
} else if (String(c.Status).includes('unhealthy')) {
await notifier.dispatchAlert('error', `[Node: ${node.name}] Healthcheck Failed: Container ${c.Names[0]} is unhealthy.`, containerStack);
if (c.State === 'running') {
runningIds.add(c.Id);
continue;
}
// Skip containers already alerted
if (this.alertedCrashes.has(c.Id)) continue;
const containerStack = c.Labels?.['com.docker.compose.project'] || undefined;
if (c.State === 'exited') {
const match = c.Status.match(/Exited \((\d+)\)/i);
const exitCode = match ? parseInt(match[1], 10) : null;
const intentionalExitCodes = [0, 137, 143, 255];
if (exitCode !== null && !intentionalExitCodes.includes(exitCode)) {
await notifier.dispatchAlert('error', `[Node: ${node.name}] Container Crash Detected: ${c.Names[0]} exited unexpectedly (Code: ${exitCode}).`, containerStack);
this.alertedCrashes.set(c.Id, now);
}
} else if (String(c.Status).includes('unhealthy')) {
await notifier.dispatchAlert('error', `[Node: ${node.name}] Healthcheck Failed: Container ${c.Names[0]} is unhealthy.`, containerStack);
this.alertedCrashes.set(c.Id, now);
}
}
} catch (err) {
console.error(`Error checking crashes on node ${node.name}`, err);
}
}
// Clear crash tracking for containers that are running again
for (const id of this.alertedCrashes.keys()) {
if (runningIds.has(id)) this.alertedCrashes.delete(id);
}
} catch (e) {
console.error('Error checking global crashes', e);
}
@@ -187,18 +222,11 @@ export class MonitorService {
}
const reclaimGb = totalReclaimableBytes / (1024 * 1024 * 1024);
// Only trigger once every while? To avoid spamming, we just check if it's over limit
// Let's ensure we only spam once per limit breach. We can use a local static variable.
const LAST_JANITOR_ALERT_KEY = 'last_janitor_alert_timestamp';
const lastAlertRaw = DatabaseService.getInstance().getSystemState(LAST_JANITOR_ALERT_KEY);
const lastAlert = parseInt(lastAlertRaw || '0', 10);
const janitorCooldown = 24 * 60 * 60 * 1000; // 24 hours cooldown for janitor
const JANITOR_COOLDOWN_MS = 24 * 60 * 60 * 1000; // 24 hours
if (reclaimGb >= janitorLimitGb) {
if (Date.now() - lastAlert > janitorCooldown) {
await notifier.dispatchAlert('info', `Your system has accumulated ${reclaimGb.toFixed(1)} GB of unused Docker data. Consider using the Janitor tool.`);
DatabaseService.getInstance().setSystemState(LAST_JANITOR_ALERT_KEY, Date.now().toString());
}
await this.dispatchWithCooldown(HOST_ALERT_KEYS.janitor, JANITOR_COOLDOWN_MS, 'info',
`Your system has accumulated ${reclaimGb.toFixed(1)} GB of unused Docker data. Consider using the Janitor tool.`);
}
}
} catch (e) {
@@ -224,10 +252,11 @@ export class MonitorService {
const rawStats = await docker.getContainerStatsStream(container.Id);
const stats = JSON.parse(rawStats);
const usedMemory = (stats.memory_stats?.usage || 0) - (stats.memory_stats?.stats?.cache || 0);
const metrics = {
cpu_percent: this.calculateCpuPercent(stats),
memory_percent: this.calculateMemoryPercent(stats),
memory_mb: (stats.memory_stats?.usage || 0) / (1024 * 1024),
memory_mb: Math.max(0, usedMemory) / (1024 * 1024),
net_rx: this.calculateNetwork(stats, 'rx'),
net_tx: this.calculateNetwork(stats, 'tx'),
restart_count: 0 // Simplification since ContainerInfo doesn't have it natively
@@ -334,6 +363,19 @@ export class MonitorService {
}
}
/** Dispatch an alert only if the cooldown period has elapsed since the last alert for this key. */
private async dispatchWithCooldown(
stateKey: string, cooldownMs: number,
severity: 'info' | 'warning' | 'error', message: string, stack?: string,
): Promise<void> {
const db = DatabaseService.getInstance();
const last = parseInt(db.getSystemState(stateKey) || '0', 10);
if (Date.now() - last > cooldownMs) {
await NotificationService.getInstance().dispatchAlert(severity, message, stack);
db.setSystemState(stateKey, Date.now().toString());
}
}
private calculateCpuPercent(stats: any): number {
let cpuPercent = 0.0;
if (!stats?.cpu_stats?.cpu_usage || !stats?.precpu_stats?.cpu_usage) return 0.0;