fix(alerts): harden with security fixes, design compliance, and test coverage (#570)

* fix(alerts): harden with security fixes, design compliance, and test coverage

Add authMiddleware to all alert endpoints, validate notification test
dispatch inputs, fix restart_count metric via Docker inspect, correct
network metric units, replace any types with DockerContainerStats
interface, add webhook timeouts and dispatch error tracking.

Frontend: migrate Select to Combobox, add ScrollArea and delete
confirmation AlertDialog, fix icon strokeWidth to 1.5.

Add update availability notifications for both Sencho version updates
(6-hour check in MonitorService) and stack image updates (state
transition detection in ImageUpdateService). Extract shared version
fetch logic into utils/version-check.ts.

Add diagnostic logging gated behind developer_mode for MonitorService
breach state machine and NotificationService dispatch routing.

Tests: 24 new alert API integration tests, restart_count and version
check unit tests (688 total passing). Docs updated with HTTPS
requirement, update notifications section, and troubleshooting guide.

* fix(alerts): remove unused TEST_USERNAME import in alerts-api tests
This commit is contained in:
Anso
2026-04-13 21:40:53 -04:00
committed by GitHub
parent 4cf6949e9f
commit 0a94df318a
11 changed files with 963 additions and 282 deletions
+9 -2
View File
@@ -655,6 +655,8 @@ export class DatabaseService {
);
CREATE INDEX IF NOT EXISTS idx_notification_routes_priority ON notification_routes(priority);
`);
// Track external dispatch errors on notification records
try { this.db.prepare('ALTER TABLE notification_history ADD COLUMN dispatch_error TEXT').run(); } catch { /* already exists */ }
}
// --- Agents ---
@@ -796,11 +798,11 @@ export class DatabaseService {
}
}
public addStackAlert(alert: StackAlert): void {
public addStackAlert(alert: StackAlert): StackAlert {
const stmt = this.db.prepare(
'INSERT INTO stack_alerts (stack_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?)'
);
stmt.run(
const result = stmt.run(
alert.stack_name,
alert.metric,
alert.operator,
@@ -809,6 +811,7 @@ export class DatabaseService {
alert.cooldown_mins,
alert.last_fired_at || 0
);
return this.db.prepare('SELECT * FROM stack_alerts WHERE id = ?').get(result.lastInsertRowid) as StackAlert;
}
public deleteStackAlert(id: number): void {
@@ -866,6 +869,10 @@ export class DatabaseService {
stmt.run();
}
public updateNotificationDispatchError(id: number, error: string): void {
this.db.prepare('UPDATE notification_history SET dispatch_error = ? WHERE id = ?').run(error, id);
}
// --- Container Metrics ---
public addContainerMetric(metric: Omit<any, 'id'>): void {
+7
View File
@@ -1028,6 +1028,13 @@ class DockerController {
return typeof stats === 'string' ? stats : JSON.stringify(stats);
}
/** Return the cumulative restart count for a container via inspect(). */
public async getContainerRestartCount(containerId: string): Promise<number> {
const container = this.docker.getContainer(containerId);
const info = await container.inspect();
return info.RestartCount ?? 0;
}
/**
* Exec into a container with full session isolation.
* All state (exec instance, stream) lives in this closure - no singleton traps.
+32 -6
View File
@@ -7,6 +7,7 @@ import { DatabaseService } from './DatabaseService';
import { FileSystemService } from './FileSystemService';
import { RegistryService } from './RegistryService';
import { NodeRegistry } from './NodeRegistry';
import { NotificationService } from './NotificationService';
import { isDebugEnabled } from '../utils/debug';
// ─── Image ref parsing ────────────────────────────────────────────────────────
@@ -284,7 +285,7 @@ export class ImageUpdateService {
for (const node of db.getNodes()) {
if (node.type !== 'local' || !node.id) continue;
try {
await this.checkNode(node.id, db);
await this.checkNode(node.id, node.name, db);
} catch (e) {
console.error(`[ImageUpdateService] Error on node ${node.name}:`, e);
}
@@ -297,7 +298,7 @@ export class ImageUpdateService {
}
}
private async checkNode(nodeId: number, db: DatabaseService) {
private async checkNode(nodeId: number, nodeName: string, db: DatabaseService) {
const docker = DockerController.getInstance(nodeId);
const fs = FileSystemService.getInstance(nodeId);
const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId));
@@ -373,20 +374,45 @@ export class ImageUpdateService {
await sleep(ImageUpdateService.INTER_IMAGE_DELAY_MS);
}
// Read previous state to detect new updates for notifications
const previousState = db.getStackUpdateStatus(nodeId);
// Write status for ALL stacks (including those with no pullable images)
const now = Date.now();
let updatesFound = 0;
const newlyUpdated: string[] = [];
for (const [stackName, images] of stackImages) {
const hasUpdate = Array.from(images).some(img => imageUpdateMap.get(img)?.hasUpdate === true);
if (hasUpdate) updatesFound++;
if (hasUpdate) {
updatesFound++;
// Notify only on state transition: was false/absent, now true
if (!previousState[stackName]) {
newlyUpdated.push(stackName);
}
}
db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now);
}
// Dispatch notifications for stacks that newly have updates
if (newlyUpdated.length > 0) {
const notifier = NotificationService.getInstance();
for (const stackName of newlyUpdated) {
try {
await notifier.dispatchAlert(
'info',
`[Node: ${nodeName}] Stack "${stackName}" has image updates available.`,
stackName,
);
} catch (e) {
console.error(`[ImageUpdateService] Failed to dispatch update notification for "${stackName}":`, e);
}
}
}
console.log(`[ImageUpdateService] Node ${nodeId}: checked ${allImages.size} image(s), ${updatesFound} stack(s) with updates`);
// Prune stale entries for stacks no longer on disk
const existing = db.getStackUpdateStatus(nodeId);
for (const staleStack of Object.keys(existing)) {
// Prune stale entries for stacks no longer on disk (reuse previousState to avoid extra DB read)
for (const staleStack of Object.keys(previousState)) {
if (!stackImages.has(staleStack)) {
db.clearStackUpdateStatus(nodeId, staleStack);
}
+83 -9
View File
@@ -1,9 +1,13 @@
import si from 'systeminformation';
import { exec } from 'child_process';
import { promisify } from 'util';
import semver from 'semver';
import DockerController from './DockerController';
import { DatabaseService } from './DatabaseService';
import { NotificationService } from './NotificationService';
import { isValidVersion } from './CapabilityRegistry';
import { fetchLatestSenchoVersion } from '../utils/version-check';
import { isDebugEnabled } from '../utils/debug';
const execAsync = promisify(exec);
@@ -12,8 +16,8 @@ const getMetricDetails = (metric: string): { name: string, unit: string } => {
case 'cpu_percent': return { name: 'CPU usage', unit: '%' };
case 'memory_percent': return { name: 'Memory usage', unit: '%' };
case 'memory_mb': return { name: 'Memory allocation', unit: ' MB' };
case 'net_rx': return { name: 'Inbound network traffic', unit: ' MB/s' };
case 'net_tx': return { name: 'Outbound network traffic', unit: ' MB/s' };
case 'net_rx': return { name: 'Inbound network traffic', unit: ' MB' };
case 'net_tx': return { name: 'Outbound network traffic', unit: ' MB' };
case 'restart_count': return { name: 'Restart count', unit: ' restarts' };
default: return { name: metric, unit: '' };
}
@@ -26,6 +30,25 @@ const getOperatorPhrase = (operator: string): string => {
return `triggered the operator ${operator}`;
};
/** Shape of the JSON returned by Docker container stats (stream: false). */
interface DockerContainerStats {
cpu_stats?: {
cpu_usage?: { total_usage: number; percpu_usage?: number[] };
system_cpu_usage?: number;
online_cpus?: number;
};
precpu_stats?: {
cpu_usage?: { total_usage: number };
system_cpu_usage?: number;
};
memory_stats?: {
usage?: number;
limit?: number;
stats?: { cache?: number };
};
networks?: Record<string, { rx_bytes: number; tx_bytes: number }>;
}
interface AlertState {
breachStartedAt: number; // timestamp when the rule first breached
}
@@ -51,6 +74,10 @@ export class MonitorService {
private alertedCrashes = new Map<string, number>();
private static readonly CRASH_ALERT_TTL_MS = 60 * 60 * 1000; // 1 hour
// Sencho version check cooldown (6 hours between external API calls)
private lastVersionCheckAt = 0;
private static readonly VERSION_CHECK_INTERVAL_MS = 6 * 60 * 60 * 1000;
private constructor() { }
public static getInstance(): MonitorService {
@@ -62,6 +89,7 @@ export class MonitorService {
public start() {
if (this.intervalId) return;
if (isDebugEnabled()) console.log('[Monitor:diag] Starting evaluation loop (30s interval)');
// Run every 30 seconds
this.intervalId = setInterval(() => {
@@ -76,6 +104,7 @@ export class MonitorService {
if (this.intervalId) {
clearInterval(this.intervalId);
this.intervalId = null;
if (isDebugEnabled()) console.log('[Monitor:diag] Evaluation loop stopped');
}
}
@@ -232,12 +261,43 @@ export class MonitorService {
} catch (e) {
console.error('Error checking docker janitor limits', e);
}
// 4. Sencho version update check (runs once per VERSION_CHECK_INTERVAL_MS)
if (Date.now() - this.lastVersionCheckAt > MonitorService.VERSION_CHECK_INTERVAL_MS) {
this.lastVersionCheckAt = Date.now();
try {
const currentVersion = process.env.npm_package_version || '0.0.0';
const latest = await fetchLatestSenchoVersion();
if (isValidVersion(latest) && isValidVersion(currentVersion) && semver.gt(latest, currentVersion)) {
const db = DatabaseService.getInstance();
const stateKey = 'last_sencho_update_notified_version';
const lastNotified = db.getSystemState(stateKey) || '';
if (lastNotified !== latest) {
const notifier = NotificationService.getInstance();
await notifier.dispatchAlert('info',
`Sencho ${latest} is available (currently running ${currentVersion}). Visit the Fleet dashboard to update.`);
db.setSystemState(stateKey, latest);
}
}
} catch (e) {
// Network errors are expected; do not spam logs
if (isDebugEnabled()) console.debug('[Monitor:diag] Sencho version check failed:', e);
}
}
}
private async evaluateStackAlerts(db: DatabaseService) {
const alerts = db.getStackAlerts();
const nodes = db.getNodes();
// Pre-group alerts by stack name to avoid O(containers * alerts) scanning
const alertsByStack = new Map<string, typeof alerts>();
for (const a of alerts) {
const list = alertsByStack.get(a.stack_name);
if (list) list.push(a);
else alertsByStack.set(a.stack_name, [a]);
}
for (const node of nodes) {
if (!node.id) continue;
// Remote nodes are self-monitoring - skip direct Docker access
@@ -250,16 +310,24 @@ export class MonitorService {
try {
const rawStats = await docker.getContainerStatsStream(container.Id);
const stats = JSON.parse(rawStats);
const stats: DockerContainerStats = JSON.parse(rawStats);
const usedMemory = (stats.memory_stats?.usage || 0) - (stats.memory_stats?.stats?.cache || 0);
// Only fetch restart count when at least one rule for this stack uses it
const stackAlerts = alertsByStack.get(stackName) || [];
const needsRestartCount = stackAlerts.some(a => a.metric === 'restart_count');
const restartCount = needsRestartCount
? await docker.getContainerRestartCount(container.Id)
: 0;
const metrics = {
cpu_percent: this.calculateCpuPercent(stats),
memory_percent: this.calculateMemoryPercent(stats),
memory_mb: Math.max(0, usedMemory) / (1024 * 1024),
net_rx: this.calculateNetwork(stats, 'rx'),
net_tx: this.calculateNetwork(stats, 'tx'),
restart_count: 0 // Simplification since ContainerInfo doesn't have it natively
restart_count: restartCount,
};
db.addContainerMetric({
@@ -272,7 +340,6 @@ export class MonitorService {
timestamp: Date.now()
});
const stackAlerts = alerts.filter(a => a.stack_name === stackName);
for (const rule of stackAlerts) {
const ruleId = rule.id!;
const currentValue = metrics[rule.metric as keyof typeof metrics];
@@ -284,6 +351,7 @@ export class MonitorService {
if (isBreaching) {
if (!this.activeBreaches.has(ruleId)) {
this.activeBreaches.set(ruleId, { breachStartedAt: Date.now() });
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}"`);
}
const breachState = this.activeBreaches.get(ruleId)!;
@@ -305,6 +373,7 @@ export class MonitorService {
const message = `[Node: ${node.name}] The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
if (isDebugEnabled()) console.log(`[Monitor:diag] Duration met for rule ${ruleId}, dispatching alert`);
await NotificationService.getInstance().dispatchAlert(
'warning',
message,
@@ -313,11 +382,14 @@ export class MonitorService {
// Update last fired
db.updateStackAlertLastFired(ruleId, Date.now());
} else if (isDebugEnabled()) {
console.log(`[Monitor:diag] Cooldown active for rule ${ruleId}: ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`);
}
}
} else {
// Rule isn't breaching anymore, reset tracker
if (this.activeBreaches.has(ruleId)) {
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}"`);
this.activeBreaches.delete(ruleId);
}
}
@@ -347,6 +419,7 @@ export class MonitorService {
db.cleanupOldNotifications(isNaN(retentionDays) ? 30 : retentionDays);
const auditRetentionDays = parseInt(settings['audit_retention_days'] || '90', 10);
db.cleanupOldAuditLogs(isNaN(auditRetentionDays) ? 90 : auditRetentionDays);
if (isDebugEnabled()) console.log(`[Monitor:diag] Cleanup: metrics ${isNaN(retentionHours) ? 24 : retentionHours}h, notifications ${isNaN(retentionDays) ? 30 : retentionDays}d, audit ${isNaN(auditRetentionDays) ? 90 : auditRetentionDays}d`);
} catch (e) {
console.error('MonitorService: failed to cleanup old data', e);
}
@@ -376,7 +449,7 @@ export class MonitorService {
}
}
private calculateCpuPercent(stats: any): number {
private calculateCpuPercent(stats: DockerContainerStats): number {
let cpuPercent = 0.0;
if (!stats?.cpu_stats?.cpu_usage || !stats?.precpu_stats?.cpu_usage) return 0.0;
@@ -390,7 +463,7 @@ export class MonitorService {
return cpuPercent;
}
private calculateMemoryPercent(stats: any): number {
private calculateMemoryPercent(stats: DockerContainerStats): number {
if (!stats?.memory_stats?.usage || !stats?.memory_stats?.limit) return 0.0;
const used_memory = stats.memory_stats.usage - (stats.memory_stats.stats?.cache || 0);
@@ -401,11 +474,12 @@ export class MonitorService {
return 0.0;
}
private calculateNetwork(stats: any, direction: 'rx' | 'tx'): number {
private calculateNetwork(stats: DockerContainerStats, direction: 'rx' | 'tx'): number {
let bytes = 0;
if (stats.networks) {
const key = direction === 'rx' ? 'rx_bytes' : 'tx_bytes';
for (const iface in stats.networks) {
bytes += stats.networks[iface][`${direction}_bytes`];
bytes += stats.networks[iface][key];
}
}
return bytes / (1024 * 1024); // Return in MB
+60 -6
View File
@@ -1,4 +1,12 @@
import { DatabaseService, NotificationHistory } from './DatabaseService';
import { isDebugEnabled } from '../utils/debug';
import { getErrorMessage } from '../utils/errors';
/** Webhook timeout: 10 seconds per external dispatch call. */
const WEBHOOK_TIMEOUT_MS = 10_000;
/** Valid notification channel types for defense-in-depth validation. */
const ALLOWED_CHANNEL_TYPES = new Set(['discord', 'slack', 'webhook']);
export class NotificationService {
private static instance: NotificationService;
@@ -21,6 +29,16 @@ export class NotificationService {
this.broadcaster = fn;
}
/**
* Dispatch an alert: log to history, push via WebSocket, and route to
* external channels.
*
* Routing uses two tiers that coexist intentionally:
* - notification_routes (Admiral tier): per-stack pattern-based routing
* with priority ordering. If any route matches, global agents are skipped.
* - agents table (all tiers): global fallback channels used when no
* notification_routes match or when no stackName is provided.
*/
public async dispatchAlert(level: 'info' | 'warning' | 'error', message: string, stackName?: string) {
// 1. Log to history and get the full inserted record (with id)
const notification = this.dbService.addNotificationHistory({
@@ -35,16 +53,26 @@ export class NotificationService {
}
// 3. Check notification routing rules if a stack context is available
const errors: string[] = [];
if (stackName) {
const routes = this.dbService.getEnabledNotificationRoutes();
const matched = routes.filter(r => r.stack_patterns.includes(stackName));
if (matched.length > 0) {
if (isDebugEnabled()) console.log(`[Notify:diag] Matched ${matched.length} route(s) for stack "${stackName}"`);
await Promise.allSettled(
matched.map(route =>
this.sendToChannel(route.channel_type, route.channel_url, level, message)
.catch(error => console.error(`Failed to dispatch notification via route "${route.name}":`, error))
.then(() => {
if (isDebugEnabled()) console.log(`[Notify:diag] Dispatched ${level} via route "${route.name}" (${route.channel_type})`);
})
.catch(error => {
console.error(`Failed to dispatch notification via route "${route.name}":`, error);
errors.push(`Route "${route.name}": ${getErrorMessage(error, String(error))}`);
})
)
);
this.recordDispatchErrors(notification.id!, errors);
return;
}
}
@@ -52,16 +80,35 @@ export class NotificationService {
// 4. Fall back to global agents
const agents = this.dbService.getEnabledAgents();
if (agents.length === 0) {
console.log('No active notification agents found. Skipping external dispatch.');
if (isDebugEnabled()) console.log('[Notify:diag] No routes or agents matched; skipping external dispatch');
return;
}
if (isDebugEnabled()) console.log(`[Notify:diag] Falling back to ${agents.length} global agent(s)`);
await Promise.allSettled(
agents.map(agent =>
this.sendToChannel(agent.type, agent.url, level, message)
.catch(error => console.error(`Failed to dispatch notification to ${agent.type}:`, error))
.then(() => {
if (isDebugEnabled()) console.log(`[Notify:diag] Dispatched ${level} via global agent (${agent.type})`);
})
.catch(error => {
console.error(`Failed to dispatch notification to ${agent.type}:`, error);
errors.push(`${agent.type}: ${getErrorMessage(error, String(error))}`);
})
)
);
this.recordDispatchErrors(notification.id!, errors);
}
/** Persist dispatch errors to the notification record for user visibility. */
private recordDispatchErrors(notificationId: number, errors: string[]) {
if (errors.length > 0) {
try {
this.dbService.updateNotificationDispatchError(notificationId, errors.join('; '));
} catch (e) {
console.error('[Notify] Failed to record dispatch error:', e);
}
}
}
private async sendToChannel(type: string, url: string, level: 'info' | 'warning' | 'error', message: string): Promise<void> {
@@ -71,10 +118,14 @@ export class NotificationService {
await this.sendSlackWebhook(url, level, message);
} else if (type === 'webhook') {
await this.sendCustomWebhook(url, level, message);
} else {
throw new Error(`Unsupported channel type: ${type}`);
}
}
public async testDispatch(type: 'discord' | 'slack' | 'webhook', url: string) {
if (!ALLOWED_CHANNEL_TYPES.has(type)) throw new Error(`Invalid notification type: ${type}`);
if (!url || !url.startsWith('https://')) throw new Error('URL must use HTTPS');
await this.sendToChannel(type, url, 'info', '🔌 Test Notification from Sencho!');
}
@@ -97,7 +148,8 @@ export class NotificationService {
const response = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload)
body: JSON.stringify(payload),
signal: AbortSignal.timeout(WEBHOOK_TIMEOUT_MS),
});
if (!response.ok) {
@@ -119,7 +171,8 @@ export class NotificationService {
const response = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload)
body: JSON.stringify(payload),
signal: AbortSignal.timeout(WEBHOOK_TIMEOUT_MS),
});
if (!response.ok) {
@@ -138,7 +191,8 @@ export class NotificationService {
const response = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload)
body: JSON.stringify(payload),
signal: AbortSignal.timeout(WEBHOOK_TIMEOUT_MS),
});
if (!response.ok) {