diff --git a/backend/src/__tests__/monitor-service.test.ts b/backend/src/__tests__/monitor-service.test.ts index 77054dc0..b077c5b1 100644 --- a/backend/src/__tests__/monitor-service.test.ts +++ b/backend/src/__tests__/monitor-service.test.ts @@ -251,7 +251,7 @@ describe('MonitorService - evaluateGlobalSettings', () => { const svc = MonitorService.getInstance(); await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' }); - expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('CPU')); + expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('CPU'), undefined); }); it('does not dispatch when CPU below threshold', async () => { @@ -260,7 +260,7 @@ describe('MonitorService - evaluateGlobalSettings', () => { const svc = MonitorService.getInstance(); await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' }); - expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU')); + expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'), undefined); }); it('dispatches RAM warning when over threshold', async () => { @@ -269,7 +269,7 @@ describe('MonitorService - evaluateGlobalSettings', () => { const svc = MonitorService.getInstance(); await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' }); - expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Memory')); + expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Memory'), undefined); }); it('dispatches disk warning when over threshold', async () => { @@ -278,7 +278,7 @@ describe('MonitorService - evaluateGlobalSettings', () => { const svc = MonitorService.getInstance(); await (svc as any).evaluateGlobalSettings({ host_disk_limit: '90' }); - expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Disk')); + expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Disk'), undefined); }); it('skips host limits when threshold is 0 or NaN', async () => { @@ -286,10 +286,10 @@ describe('MonitorService - evaluateGlobalSettings', () => { const svc = MonitorService.getInstance(); await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '0' }); - expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU')); + expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'), undefined); await (svc as any).evaluateGlobalSettings({ host_cpu_limit: 'abc' }); - expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU')); + expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'), undefined); }); }); @@ -299,6 +299,7 @@ describe('MonitorService - global crash detection', () => { it('detects exited containers with non-intentional exit codes', async () => { mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); mockGetAllContainers.mockResolvedValue([{ + Id: 'crash-1', State: 'exited', Status: 'Exited (1) 5 seconds ago', Names: ['/my-container'], @@ -317,6 +318,7 @@ describe('MonitorService - global crash detection', () => { for (const code of intentionalExits) { mockDispatchAlert.mockClear(); mockGetAllContainers.mockResolvedValue([{ + Id: `safe-${code}`, State: 'exited', Status: `Exited (${code}) 5 seconds ago`, Names: ['/safe-container'], @@ -332,7 +334,8 @@ describe('MonitorService - global crash detection', () => { it('detects unhealthy containers', async () => { mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); mockGetAllContainers.mockResolvedValue([{ - State: 'running', + Id: 'sick-1', + State: 'unhealthy', Status: 'Up 2 hours (unhealthy)', Names: ['/sick-container'], }]); diff --git a/backend/src/services/MonitorService.ts b/backend/src/services/MonitorService.ts index 30d9b392..11365a76 100644 --- a/backend/src/services/MonitorService.ts +++ b/backend/src/services/MonitorService.ts @@ -30,6 +30,13 @@ interface AlertState { breachStartedAt: number; // timestamp when the rule first breached } +const HOST_ALERT_KEYS = { + cpu: 'last_host_cpu_alert_ts', + ram: 'last_host_ram_alert_ts', + disk: 'last_host_disk_alert_ts', + janitor: 'last_janitor_alert_timestamp', +} as const; + export class MonitorService { private static instance: MonitorService; private intervalId: NodeJS.Timeout | null = null; @@ -39,6 +46,11 @@ export class MonitorService { // key: rule_id, value: AlertState private activeBreaches = new Map(); + // Track containers that have already been alerted as crashed to avoid + // duplicate alerts. key: containerId, value: timestamp when alerted. + private alertedCrashes = new Map(); + private static readonly CRASH_ALERT_TTL_MS = 60 * 60 * 1000; // 1 hour + private constructor() { } public static getInstance(): MonitorService { @@ -86,6 +98,7 @@ export class MonitorService { private async evaluateGlobalSettings(settings: Record) { const notifier = NotificationService.getInstance(); + const HOST_ALERT_COOLDOWN_MS = 5 * 60 * 1000; // 5 minutes between repeat alerts // 1. Host Limits try { @@ -93,14 +106,16 @@ export class MonitorService { const cpuUsage = currentLoad.currentLoad; const cpuLimit = parseFloat(settings['host_cpu_limit']); if (!isNaN(cpuLimit) && cpuLimit > 0 && cpuUsage > cpuLimit) { - await notifier.dispatchAlert('warning', `Host CPU utilization is critically high: ${cpuUsage.toFixed(1)}% (Threshold: ${cpuLimit}%)`); + await this.dispatchWithCooldown(HOST_ALERT_KEYS.cpu, HOST_ALERT_COOLDOWN_MS, 'warning', + `Host CPU utilization is critically high: ${cpuUsage.toFixed(1)}% (Threshold: ${cpuLimit}%)`); } const mem = await si.mem(); const ramUsage = (mem.used / mem.total) * 100; const ramLimit = parseFloat(settings['host_ram_limit']); if (!isNaN(ramLimit) && ramLimit > 0 && ramUsage > ramLimit) { - await notifier.dispatchAlert('warning', `Host Memory utilization is critically high: ${ramUsage.toFixed(1)}% (Threshold: ${ramLimit}%)`); + await this.dispatchWithCooldown(HOST_ALERT_KEYS.ram, HOST_ALERT_COOLDOWN_MS, 'warning', + `Host Memory utilization is critically high: ${ramUsage.toFixed(1)}% (Threshold: ${ramLimit}%)`); } const fsSize = await si.fsSize(); @@ -108,7 +123,8 @@ export class MonitorService { if (mainDisk) { const diskLimit = parseFloat(settings['host_disk_limit']); if (!isNaN(diskLimit) && diskLimit > 0 && mainDisk.use > diskLimit) { - await notifier.dispatchAlert('warning', `Host Disk space utilization is critically high: ${mainDisk.use.toFixed(1)}% (Threshold: ${diskLimit}%)`); + await this.dispatchWithCooldown(HOST_ALERT_KEYS.disk, HOST_ALERT_COOLDOWN_MS, 'warning', + `Host Disk space utilization is critically high: ${mainDisk.use.toFixed(1)}% (Threshold: ${diskLimit}%)`); } } } catch (e) { @@ -117,36 +133,55 @@ export class MonitorService { // 2. Global Crash Detect if (settings['global_crash'] === '1') { + // Prune expired entries from the crash tracker + const now = Date.now(); + for (const [id, ts] of this.alertedCrashes) { + if (now - ts > MonitorService.CRASH_ALERT_TTL_MS) this.alertedCrashes.delete(id); + } + try { const nodes = DatabaseService.getInstance().getNodes(); + const runningIds = new Set(); + for (const node of nodes) { if (!node.id) continue; - // Remote nodes run their own MonitorService locally - skip direct Docker access + // Remote nodes run their own MonitorService locally if (node.type === 'remote') continue; try { const docker = DockerController.getInstance(node.id); const containers = await docker.getAllContainers(); for (const c of containers) { - if (c.State === 'exited' || String(c.Status).includes('unhealthy')) { - const containerStack = c.Labels?.['com.docker.compose.project'] || undefined; - if (c.State === 'exited') { - if (c.Status.includes('seconds ago')) { - const match = c.Status.match(/Exited \((\d+)\)/i); - const exitCode = match ? parseInt(match[1], 10) : null; - const intentionalExitCodes = [0, 137, 143, 255]; - if (exitCode !== null && !intentionalExitCodes.includes(exitCode)) { - await notifier.dispatchAlert('error', `[Node: ${node.name}] Container Crash Detected: ${c.Names[0]} exited unexpectedly (Code: ${exitCode}).`, containerStack); - } - } - } else if (String(c.Status).includes('unhealthy')) { - await notifier.dispatchAlert('error', `[Node: ${node.name}] Healthcheck Failed: Container ${c.Names[0]} is unhealthy.`, containerStack); + if (c.State === 'running') { + runningIds.add(c.Id); + continue; + } + // Skip containers already alerted + if (this.alertedCrashes.has(c.Id)) continue; + + const containerStack = c.Labels?.['com.docker.compose.project'] || undefined; + + if (c.State === 'exited') { + const match = c.Status.match(/Exited \((\d+)\)/i); + const exitCode = match ? parseInt(match[1], 10) : null; + const intentionalExitCodes = [0, 137, 143, 255]; + if (exitCode !== null && !intentionalExitCodes.includes(exitCode)) { + await notifier.dispatchAlert('error', `[Node: ${node.name}] Container Crash Detected: ${c.Names[0]} exited unexpectedly (Code: ${exitCode}).`, containerStack); + this.alertedCrashes.set(c.Id, now); } + } else if (String(c.Status).includes('unhealthy')) { + await notifier.dispatchAlert('error', `[Node: ${node.name}] Healthcheck Failed: Container ${c.Names[0]} is unhealthy.`, containerStack); + this.alertedCrashes.set(c.Id, now); } } } catch (err) { console.error(`Error checking crashes on node ${node.name}`, err); } } + + // Clear crash tracking for containers that are running again + for (const id of this.alertedCrashes.keys()) { + if (runningIds.has(id)) this.alertedCrashes.delete(id); + } } catch (e) { console.error('Error checking global crashes', e); } @@ -187,18 +222,11 @@ export class MonitorService { } const reclaimGb = totalReclaimableBytes / (1024 * 1024 * 1024); - // Only trigger once every while? To avoid spamming, we just check if it's over limit - // Let's ensure we only spam once per limit breach. We can use a local static variable. - const LAST_JANITOR_ALERT_KEY = 'last_janitor_alert_timestamp'; - const lastAlertRaw = DatabaseService.getInstance().getSystemState(LAST_JANITOR_ALERT_KEY); - const lastAlert = parseInt(lastAlertRaw || '0', 10); - const janitorCooldown = 24 * 60 * 60 * 1000; // 24 hours cooldown for janitor + const JANITOR_COOLDOWN_MS = 24 * 60 * 60 * 1000; // 24 hours if (reclaimGb >= janitorLimitGb) { - if (Date.now() - lastAlert > janitorCooldown) { - await notifier.dispatchAlert('info', `Your system has accumulated ${reclaimGb.toFixed(1)} GB of unused Docker data. Consider using the Janitor tool.`); - DatabaseService.getInstance().setSystemState(LAST_JANITOR_ALERT_KEY, Date.now().toString()); - } + await this.dispatchWithCooldown(HOST_ALERT_KEYS.janitor, JANITOR_COOLDOWN_MS, 'info', + `Your system has accumulated ${reclaimGb.toFixed(1)} GB of unused Docker data. Consider using the Janitor tool.`); } } } catch (e) { @@ -224,10 +252,11 @@ export class MonitorService { const rawStats = await docker.getContainerStatsStream(container.Id); const stats = JSON.parse(rawStats); + const usedMemory = (stats.memory_stats?.usage || 0) - (stats.memory_stats?.stats?.cache || 0); const metrics = { cpu_percent: this.calculateCpuPercent(stats), memory_percent: this.calculateMemoryPercent(stats), - memory_mb: (stats.memory_stats?.usage || 0) / (1024 * 1024), + memory_mb: Math.max(0, usedMemory) / (1024 * 1024), net_rx: this.calculateNetwork(stats, 'rx'), net_tx: this.calculateNetwork(stats, 'tx'), restart_count: 0 // Simplification since ContainerInfo doesn't have it natively @@ -334,6 +363,19 @@ export class MonitorService { } } + /** Dispatch an alert only if the cooldown period has elapsed since the last alert for this key. */ + private async dispatchWithCooldown( + stateKey: string, cooldownMs: number, + severity: 'info' | 'warning' | 'error', message: string, stack?: string, + ): Promise { + const db = DatabaseService.getInstance(); + const last = parseInt(db.getSystemState(stateKey) || '0', 10); + if (Date.now() - last > cooldownMs) { + await NotificationService.getInstance().dispatchAlert(severity, message, stack); + db.setSystemState(stateKey, Date.now().toString()); + } + } + private calculateCpuPercent(stats: any): number { let cpuPercent = 0.0; if (!stats?.cpu_stats?.cpu_usage || !stats?.precpu_stats?.cpu_usage) return 0.0; diff --git a/docs/features/dashboard.mdx b/docs/features/dashboard.mdx index 80ca16ae..e4447863 100644 --- a/docs/features/dashboard.mdx +++ b/docs/features/dashboard.mdx @@ -16,8 +16,8 @@ The top bar provides an at-a-glance health assessment for the active node. Sench | Status | Meaning | |--------|---------| | **Healthy** | All systems nominal. No resources above warning thresholds, no unread errors. | -| **Degraded** | At least one resource is above 80%, or there are unread error alerts. | -| **Critical** | At least one resource is above 90%, or there are exited containers with unread errors. | +| **Degraded** | At least one resource is above 80%, there are exited containers, or there are unread error alerts. | +| **Critical** | At least one resource is above 90%, or there are exited containers combined with unread errors. | The bar also shows the active node name, the number of running containers, and the current alert count. diff --git a/docs/images/dashboard/dashboard-overview.png b/docs/images/dashboard/dashboard-overview.png index 137c9499..61114649 100644 Binary files a/docs/images/dashboard/dashboard-overview.png and b/docs/images/dashboard/dashboard-overview.png differ diff --git a/frontend/src/components/dashboard/HealthStatusBar.tsx b/frontend/src/components/dashboard/HealthStatusBar.tsx index 35fc0f1b..f25c6c1e 100644 --- a/frontend/src/components/dashboard/HealthStatusBar.tsx +++ b/frontend/src/components/dashboard/HealthStatusBar.tsx @@ -38,13 +38,13 @@ function deriveHealth(stats: Stats, systemStats: SystemStats | null, notificatio if (disk >= 90) reasons.push(`Disk at ${disk.toFixed(1)}%`); else if (disk >= 80) reasons.push(`Disk at ${disk.toFixed(1)}%`); - if (stats.exited > 0 && unreadErrors > 0) reasons.push(`${stats.exited} exited container${stats.exited !== 1 ? 's' : ''}`); - else if (unreadErrors > 0) reasons.push(`${unreadErrors} unread error${unreadErrors !== 1 ? 's' : ''}`); + if (stats.exited > 0) reasons.push(`${stats.exited} exited container${stats.exited !== 1 ? 's' : ''}`); + if (unreadErrors > 0) reasons.push(`${unreadErrors} unread error${unreadErrors !== 1 ? 's' : ''}`); if (cpu >= 90 || ram >= 90 || disk >= 90 || (stats.exited > 0 && unreadErrors > 0)) { return { level: 'critical', reasons }; } - if (cpu >= 80 || ram >= 80 || disk >= 80 || unreadErrors > 0) { + if (cpu >= 80 || ram >= 80 || disk >= 80 || stats.exited > 0 || unreadErrors > 0) { return { level: 'degraded', reasons }; } return { level: 'healthy', reasons: ['All systems nominal'] }; @@ -65,7 +65,7 @@ export function HealthStatusBar({ stats, systemStats, notifications, activeNodeN const unreadAlerts = notifications.filter(n => !n.is_read).length; return ( - +
{/* Health badge */}
diff --git a/frontend/src/components/dashboard/HistoricalCharts.tsx b/frontend/src/components/dashboard/HistoricalCharts.tsx index b0b10df5..4654b950 100644 --- a/frontend/src/components/dashboard/HistoricalCharts.tsx +++ b/frontend/src/components/dashboard/HistoricalCharts.tsx @@ -45,7 +45,7 @@ export function HistoricalCharts({ metrics, systemStats }: HistoricalChartsProps return (
- + @@ -59,7 +59,7 @@ export function HistoricalCharts({ metrics, systemStats }: HistoricalChartsProps - `${Number(val).toFixed(0)}%`} domain={[0, 100]} tick={{ fill: 'var(--chart-tick)', fontSize: 11 }} /> + `${Number(val).toFixed(0)}%`} domain={[0, (dataMax: number) => Math.max(100, Math.ceil(dataMax / 10) * 10)]} tick={{ fill: 'var(--chart-tick)', fontSize: 11 }} /> } /> @@ -73,7 +73,7 @@ export function HistoricalCharts({ metrics, systemStats }: HistoricalChartsProps - + diff --git a/frontend/src/components/dashboard/RecentAlerts.tsx b/frontend/src/components/dashboard/RecentAlerts.tsx index 979a62c9..bcdaa1c9 100644 --- a/frontend/src/components/dashboard/RecentAlerts.tsx +++ b/frontend/src/components/dashboard/RecentAlerts.tsx @@ -57,7 +57,7 @@ export function RecentAlerts({ notifications, nodes, onCleared }: RecentAlertsPr }; return ( - +
Recent Alerts diff --git a/frontend/src/components/dashboard/ResourceGauges.tsx b/frontend/src/components/dashboard/ResourceGauges.tsx index 0fffd4b3..64426ec6 100644 --- a/frontend/src/components/dashboard/ResourceGauges.tsx +++ b/frontend/src/components/dashboard/ResourceGauges.tsx @@ -52,7 +52,7 @@ export function ResourceGauges({ stats, systemStats }: ResourceGaugesProps) { return (
{/* CPU */} - + CPU @@ -69,7 +69,7 @@ export function ResourceGauges({ stats, systemStats }: ResourceGaugesProps) { {/* RAM */} - + Memory @@ -86,7 +86,7 @@ export function ResourceGauges({ stats, systemStats }: ResourceGaugesProps) { {/* Disk */} - + Disk @@ -103,7 +103,7 @@ export function ResourceGauges({ stats, systemStats }: ResourceGaugesProps) { {/* Containers */} - + Containers @@ -113,7 +113,7 @@ export function ResourceGauges({ stats, systemStats }: ResourceGaugesProps) { {stats.active} - {stats.active === 1 ? 'active' : 'actives'} + active
@@ -141,7 +141,7 @@ export function ResourceGauges({ stats, systemStats }: ResourceGaugesProps) { {/* Network */} - + Network diff --git a/frontend/src/components/dashboard/StackHealthTable.tsx b/frontend/src/components/dashboard/StackHealthTable.tsx index 64732a0d..23b55494 100644 --- a/frontend/src/components/dashboard/StackHealthTable.tsx +++ b/frontend/src/components/dashboard/StackHealthTable.tsx @@ -78,7 +78,7 @@ export function StackHealthTable({ stackStatuses, metrics, onNavigateToStack }: if (Object.keys(stackStatuses).length === 0) { return ( - +
@@ -90,7 +90,7 @@ export function StackHealthTable({ stackStatuses, metrics, onNavigateToStack }: } return ( - +
Stack Health