mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-14 04:33:40 +00:00
feat(notifications): add structured category enum to dispatcher and history (#774)
Introduce a NotificationCategory string-literal union (11 values) and thread it through dispatchAlert as a required second argument. All callers (DockerEventService, AutoHealService, ImageUpdateService, MonitorService, PolicyEnforcement, policyGate, SchedulerService, imageUpdates route) pass an explicit category at every call site, giving TypeScript compile-time enforcement that no new emit site can be added without choosing a category. DatabaseService gains an idempotent migration that adds a nullable category TEXT column to notification_history; existing rows keep category=NULL (displayed as Uncategorized in the UI). The getNotificationHistory method accepts an optional category filter that is forwarded from the GET /api/notifications/history route via a ?category= query param. NotificationPanel gains a category Select dropdown so users can filter history by category. The frontend types mirror the backend union so API responses are type-safe end-to-end. All 75 test files (1410 tests) updated to the new 4-arg dispatchAlert signature and passing.
This commit is contained in:
Anso
@@ -242,9 +242,9 @@ export class AutoHealService {
|
||||
NotificationService.getInstance()
|
||||
.dispatchAlert(
|
||||
'info',
|
||||
'autoheal_triggered',
|
||||
`Auto-Heal: Restarted ${containerName} on stack ${policy.stack_name} after being unhealthy for ${policy.unhealthy_duration_mins} minute(s).`,
|
||||
policy.stack_name,
|
||||
containerName,
|
||||
{ stackName: policy.stack_name, containerName },
|
||||
)
|
||||
.catch(err => console.error('[AutoHeal] notification dispatch failed:', err));
|
||||
} catch (err) {
|
||||
@@ -266,9 +266,9 @@ export class AutoHealService {
|
||||
NotificationService.getInstance()
|
||||
.dispatchAlert(
|
||||
'warning',
|
||||
'autoheal_triggered',
|
||||
`Auto-Heal: Failed to restart ${containerName} on stack ${policy.stack_name}. Error: ${errorMsg}`,
|
||||
policy.stack_name,
|
||||
containerName,
|
||||
{ stackName: policy.stack_name, containerName },
|
||||
)
|
||||
.catch(e => console.error('[AutoHeal] notification dispatch failed:', e));
|
||||
|
||||
@@ -299,8 +299,9 @@ export class AutoHealService {
|
||||
NotificationService.getInstance()
|
||||
.dispatchAlert(
|
||||
'warning',
|
||||
'autoheal_triggered',
|
||||
`Auto-Heal: Policy for ${policy.stack_name}${policy.service_name ? '/' + policy.service_name : ''} has been auto-disabled after ${failures} consecutive failures.`,
|
||||
policy.stack_name,
|
||||
{ stackName: policy.stack_name },
|
||||
)
|
||||
.catch(e => console.error('[AutoHeal] notification dispatch failed:', e));
|
||||
}
|
||||
|
||||
@@ -196,6 +196,7 @@ export interface SSOConfig {
|
||||
export interface NotificationHistory {
|
||||
id?: number;
|
||||
level: 'info' | 'warning' | 'error';
|
||||
category?: string;
|
||||
message: string;
|
||||
timestamp: number;
|
||||
is_read: boolean;
|
||||
@@ -489,6 +490,7 @@ export class DatabaseService {
|
||||
this.migrateSecretMisconfigColumns();
|
||||
this.migrateAgentsAndNotificationsNodeId();
|
||||
this.migratePolicyEvaluationColumn();
|
||||
this.migrateNotificationCategory();
|
||||
}
|
||||
|
||||
public static getInstance(): DatabaseService {
|
||||
@@ -1202,6 +1204,14 @@ export class DatabaseService {
|
||||
}
|
||||
}
|
||||
|
||||
private migrateNotificationCategory(): void {
|
||||
try {
|
||||
this.db.prepare('ALTER TABLE notification_history ADD COLUMN category TEXT').run();
|
||||
} catch {
|
||||
// column already present
|
||||
}
|
||||
}
|
||||
|
||||
// --- Agents ---
|
||||
|
||||
public getAgents(nodeId: number): Agent[] {
|
||||
@@ -1457,19 +1467,23 @@ export class DatabaseService {
|
||||
|
||||
// --- Notification History ---
|
||||
|
||||
public getNotificationHistory(nodeId: number, limit = 50): NotificationHistory[] {
|
||||
const stmt = this.db.prepare('SELECT * FROM notification_history WHERE node_id = ? ORDER BY timestamp DESC LIMIT ?');
|
||||
return stmt.all(nodeId, limit).map((row: any) => ({
|
||||
public getNotificationHistory(nodeId: number, limit = 50, category?: string): NotificationHistory[] {
|
||||
const sql = category
|
||||
? 'SELECT * FROM notification_history WHERE node_id = ? AND category = ? ORDER BY timestamp DESC LIMIT ?'
|
||||
: 'SELECT * FROM notification_history WHERE node_id = ? ORDER BY timestamp DESC LIMIT ?';
|
||||
const args: (number | string)[] = category ? [nodeId, category, limit] : [nodeId, limit];
|
||||
return this.db.prepare(sql).all(...args).map((row: any) => ({
|
||||
...row,
|
||||
is_read: row.is_read === 1,
|
||||
stack_name: row.stack_name ?? undefined,
|
||||
container_name: row.container_name ?? undefined,
|
||||
category: row.category ?? undefined,
|
||||
}));
|
||||
}
|
||||
|
||||
public addNotificationHistory(nodeId: number, notification: Omit<NotificationHistory, 'id' | 'is_read'>): NotificationHistory {
|
||||
const stmt = this.db.prepare(
|
||||
'INSERT INTO notification_history (node_id, level, message, timestamp, is_read, stack_name, container_name) VALUES (?, ?, ?, ?, 0, ?, ?)'
|
||||
'INSERT INTO notification_history (node_id, level, message, timestamp, is_read, stack_name, container_name, category) VALUES (?, ?, ?, ?, 0, ?, ?, ?)'
|
||||
);
|
||||
const result = stmt.run(
|
||||
nodeId,
|
||||
@@ -1477,7 +1491,8 @@ export class DatabaseService {
|
||||
notification.message,
|
||||
notification.timestamp,
|
||||
notification.stack_name ?? null,
|
||||
notification.container_name ?? null
|
||||
notification.container_name ?? null,
|
||||
notification.category ?? null,
|
||||
);
|
||||
|
||||
this.db.prepare(`
|
||||
@@ -1490,6 +1505,7 @@ export class DatabaseService {
|
||||
return {
|
||||
id: result.lastInsertRowid as number,
|
||||
level: notification.level,
|
||||
category: notification.category,
|
||||
message: notification.message,
|
||||
timestamp: notification.timestamp,
|
||||
is_read: false,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import Docker from 'dockerode';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { NotificationService } from './NotificationService';
|
||||
import { NotificationCategory, NotificationService } from './NotificationService';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import {
|
||||
classifyDie,
|
||||
@@ -187,7 +187,7 @@ export class DockerEventService {
|
||||
this.status = 'connected';
|
||||
|
||||
if (this.disconnectedNoticeEmitted) {
|
||||
await this.emitInfo(`Reconnected to Docker daemon.`);
|
||||
await this.emitInfo('system', `Reconnected to Docker daemon.`);
|
||||
this.disconnectedNoticeEmitted = false;
|
||||
}
|
||||
|
||||
@@ -237,7 +237,7 @@ export class DockerEventService {
|
||||
|
||||
if (!this.disconnectedNoticeEmitted) {
|
||||
this.disconnectedNoticeEmitted = true;
|
||||
void this.emitWarning(`Lost connection to Docker daemon; monitoring paused.`);
|
||||
void this.emitWarning('system', `Lost connection to Docker daemon; monitoring paused.`);
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) {
|
||||
@@ -310,6 +310,7 @@ export class DockerEventService {
|
||||
|
||||
if (exitRatio >= MASS_EVENT_THRESHOLD) {
|
||||
await this.emitInfo(
|
||||
'system',
|
||||
`Docker daemon interruption detected: ${newlyExited.length} containers exited during connection gap.`
|
||||
);
|
||||
} else {
|
||||
@@ -450,6 +451,7 @@ export class DockerEventService {
|
||||
const name = state.name ?? id.slice(0, 12);
|
||||
const stackName = state.stackName;
|
||||
void this.emitError(
|
||||
'monitor_alert',
|
||||
`Healthcheck failed: ${name} is unhealthy.`,
|
||||
stackName,
|
||||
state.name,
|
||||
@@ -562,7 +564,7 @@ export class DockerEventService {
|
||||
// rate-suppressed alerts don't silently lock out the next real crash.
|
||||
if (state) state.lastCrashAlertAt = Date.now();
|
||||
|
||||
await this.emitError(message, info.stackName, info.name);
|
||||
await this.emitError('monitor_alert', message, info.stackName, info.name);
|
||||
}
|
||||
|
||||
private isCrashAlertsEnabled(): boolean {
|
||||
@@ -607,6 +609,7 @@ export class DockerEventService {
|
||||
this.suppressedCount = 0;
|
||||
if (count > 0) {
|
||||
void this.emitWarning(
|
||||
'monitor_alert',
|
||||
`${count} additional containers crashed in the last minute.`,
|
||||
);
|
||||
}
|
||||
@@ -624,6 +627,7 @@ export class DockerEventService {
|
||||
if (this.parseErrorCount > PARSE_ERROR_THRESHOLD && !this.parseWarningEmitted) {
|
||||
this.parseWarningEmitted = true;
|
||||
void this.emitWarning(
|
||||
'system',
|
||||
`Received malformed Docker event payloads. Monitoring continues but some events may be skipped.`,
|
||||
);
|
||||
}
|
||||
@@ -668,16 +672,16 @@ export class DockerEventService {
|
||||
// Notification wrappers (prefix with node name for multi-node clarity)
|
||||
// ========================================================================
|
||||
|
||||
private async emitError(message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('error', this.prefix(message), stackName, containerName);
|
||||
private async emitError(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('error', category, this.prefix(message), { stackName, containerName });
|
||||
}
|
||||
|
||||
private async emitWarning(message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('warning', this.prefix(message), stackName, containerName);
|
||||
private async emitWarning(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('warning', category, this.prefix(message), { stackName, containerName });
|
||||
}
|
||||
|
||||
private async emitInfo(message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('info', this.prefix(message), stackName, containerName);
|
||||
private async emitInfo(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('info', category, this.prefix(message), { stackName, containerName });
|
||||
}
|
||||
|
||||
private prefix(message: string): string {
|
||||
|
||||
@@ -275,8 +275,9 @@ export class ImageUpdateService {
|
||||
try {
|
||||
await notifier.dispatchAlert(
|
||||
'info',
|
||||
'image_update_available',
|
||||
`[Node: ${nodeName}] Stack "${stackName}" has image updates available.`,
|
||||
stackName,
|
||||
{ stackName },
|
||||
);
|
||||
} catch (e) {
|
||||
console.error(`[ImageUpdateService] Failed to dispatch update notification for "${stackName}":`, e);
|
||||
@@ -286,6 +287,7 @@ export class ImageUpdateService {
|
||||
try {
|
||||
db.addNotificationHistory(NodeRegistry.getInstance().getDefaultNodeId(), {
|
||||
level: 'error',
|
||||
category: 'system',
|
||||
message: `[Node: ${nodeName}] Failed to notify about image updates for stack "${stackName}": ${getErrorMessage(e, String(e))}`,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
@@ -135,7 +135,7 @@ export class MonitorService {
|
||||
const cpuUsage = currentLoad.currentLoad;
|
||||
const cpuLimit = parseFloat(settings['host_cpu_limit']);
|
||||
if (!isNaN(cpuLimit) && cpuLimit > 0 && cpuUsage > cpuLimit) {
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.cpu, HOST_ALERT_COOLDOWN_MS, 'warning',
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.cpu, HOST_ALERT_COOLDOWN_MS, 'warning', 'monitor_alert',
|
||||
`Host CPU utilization is critically high: ${cpuUsage.toFixed(1)}% (Threshold: ${cpuLimit}%)`);
|
||||
}
|
||||
|
||||
@@ -143,7 +143,7 @@ export class MonitorService {
|
||||
const ramUsage = (mem.used / mem.total) * 100;
|
||||
const ramLimit = parseFloat(settings['host_ram_limit']);
|
||||
if (!isNaN(ramLimit) && ramLimit > 0 && ramUsage > ramLimit) {
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.ram, HOST_ALERT_COOLDOWN_MS, 'warning',
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.ram, HOST_ALERT_COOLDOWN_MS, 'warning', 'monitor_alert',
|
||||
`Host Memory utilization is critically high: ${ramUsage.toFixed(1)}% (Threshold: ${ramLimit}%)`);
|
||||
}
|
||||
|
||||
@@ -152,7 +152,7 @@ export class MonitorService {
|
||||
if (mainDisk) {
|
||||
const diskLimit = parseFloat(settings['host_disk_limit']);
|
||||
if (!isNaN(diskLimit) && diskLimit > 0 && mainDisk.use > diskLimit) {
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.disk, HOST_ALERT_COOLDOWN_MS, 'warning',
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.disk, HOST_ALERT_COOLDOWN_MS, 'warning', 'monitor_alert',
|
||||
`Host Disk space utilization is critically high: ${mainDisk.use.toFixed(1)}% (Threshold: ${diskLimit}%)`);
|
||||
}
|
||||
}
|
||||
@@ -213,7 +213,7 @@ export class MonitorService {
|
||||
const registry = NodeRegistry.getInstance();
|
||||
const localNode = registry.getNode(registry.getDefaultNodeId());
|
||||
const nodeLabel = localNode?.name ?? 'this node';
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.janitor, JANITOR_COOLDOWN_MS, 'info',
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.janitor, JANITOR_COOLDOWN_MS, 'info', 'system',
|
||||
`Node "${nodeLabel}" has accumulated ${reclaimGb.toFixed(1)} GB of unused Docker data. Consider using the Janitor tool.`);
|
||||
}
|
||||
}
|
||||
@@ -290,7 +290,7 @@ export class MonitorService {
|
||||
|
||||
try {
|
||||
const notifier = NotificationService.getInstance();
|
||||
await notifier.dispatchAlert('info',
|
||||
await notifier.dispatchAlert('info', 'system',
|
||||
`Sencho ${latest} is available (currently running ${currentVersion}). Visit the Fleet dashboard to update.`);
|
||||
db.setSystemState(stateKey, latest);
|
||||
if (isDebugEnabled()) console.debug(`[Monitor:diag] Dispatched version notification: ${currentVersion} -> ${latest}`);
|
||||
@@ -391,8 +391,9 @@ export class MonitorService {
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Duration met for rule ${ruleId}, dispatching alert`);
|
||||
await NotificationService.getInstance().dispatchAlert(
|
||||
'warning',
|
||||
'monitor_alert',
|
||||
message,
|
||||
rule.stack_name
|
||||
{ stackName: rule.stack_name },
|
||||
);
|
||||
|
||||
// Update last fired
|
||||
@@ -454,12 +455,13 @@ export class MonitorService {
|
||||
/** Dispatch an alert only if the cooldown period has elapsed since the last alert for this key. */
|
||||
private async dispatchWithCooldown(
|
||||
stateKey: string, cooldownMs: number,
|
||||
severity: 'info' | 'warning' | 'error', message: string, stack?: string,
|
||||
severity: 'info' | 'warning' | 'error', category: import('./NotificationService').NotificationCategory,
|
||||
message: string, stack?: string,
|
||||
): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const last = parseInt(db.getSystemState(stateKey) || '0', 10);
|
||||
if (Date.now() - last > cooldownMs) {
|
||||
await NotificationService.getInstance().dispatchAlert(severity, message, stack);
|
||||
await NotificationService.getInstance().dispatchAlert(severity, category, message, { stackName: stack });
|
||||
db.setSystemState(stateKey, Date.now().toString());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,19 @@ import { NodeRegistry } from './NodeRegistry';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
|
||||
export type NotificationCategory =
|
||||
| 'deploy_success'
|
||||
| 'deploy_failure'
|
||||
| 'stack_started'
|
||||
| 'stack_stopped'
|
||||
| 'stack_restarted'
|
||||
| 'image_update_available'
|
||||
| 'image_update_applied'
|
||||
| 'autoheal_triggered'
|
||||
| 'monitor_alert'
|
||||
| 'scan_finding'
|
||||
| 'system';
|
||||
|
||||
/** Webhook timeout: 10 seconds per external dispatch call. */
|
||||
const WEBHOOK_TIMEOUT_MS = 10_000;
|
||||
|
||||
@@ -83,16 +96,18 @@ export class NotificationService {
|
||||
*/
|
||||
public async dispatchAlert(
|
||||
level: 'info' | 'warning' | 'error',
|
||||
category: NotificationCategory,
|
||||
message: string,
|
||||
stackName?: string,
|
||||
containerName?: string,
|
||||
options?: { stackName?: string; containerName?: string },
|
||||
) {
|
||||
const { stackName, containerName } = options ?? {};
|
||||
// Internal writes use the middleware default so they share a row key
|
||||
// with user-initiated requests; otherwise the UI and monitors split
|
||||
// between different node_id buckets.
|
||||
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
const notification = this.dbService.addNotificationHistory(localNodeId, {
|
||||
level,
|
||||
category,
|
||||
message,
|
||||
timestamp: Date.now(),
|
||||
stack_name: stackName,
|
||||
@@ -105,7 +120,7 @@ export class NotificationService {
|
||||
// 3. Check notification routing rules if a stack context is available
|
||||
const errors: string[] = [];
|
||||
|
||||
if (stackName) {
|
||||
if (stackName !== undefined) {
|
||||
const routes = this.dbService.getEnabledNotificationRoutes();
|
||||
const matched = routes.filter(r => r.stack_patterns.includes(stackName));
|
||||
if (matched.length > 0) {
|
||||
|
||||
@@ -60,8 +60,9 @@ export async function enforcePolicyPreDeploy(
|
||||
if (!svc.isTrivyAvailable()) {
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
'warning',
|
||||
'scan_finding',
|
||||
`Pre-deploy scan for "${stackName}" skipped: Trivy not installed on this node`,
|
||||
stackName,
|
||||
{ stackName },
|
||||
);
|
||||
return { ok: true, bypassed: false, policy, violations: [], trivyMissing: true };
|
||||
}
|
||||
|
||||
@@ -89,6 +89,7 @@ export class SchedulerService {
|
||||
await trivy.detectTrivy();
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
'system',
|
||||
`Trivy updated from v${previous} to v${check.latest}`,
|
||||
);
|
||||
db.updateGlobalSetting('trivy_last_notified_version', check.latest);
|
||||
@@ -100,6 +101,7 @@ export class SchedulerService {
|
||||
if (lastNotified === check.latest) return;
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
'system',
|
||||
`Trivy update available: v${check.latest} (currently v${check.current ?? 'unknown'})`,
|
||||
);
|
||||
db.updateGlobalSetting('trivy_last_notified_version', check.latest);
|
||||
@@ -159,9 +161,9 @@ export class SchedulerService {
|
||||
* Fire a notification without awaiting completion, catching any promise
|
||||
* rejection so the scheduler never crashes on a failed dispatch.
|
||||
*/
|
||||
private safeDispatch(level: 'info' | 'warning' | 'error', message: string, stackName?: string): void {
|
||||
private safeDispatch(level: 'info' | 'warning' | 'error', category: import('./NotificationService').NotificationCategory, message: string, stackName?: string): void {
|
||||
NotificationService.getInstance()
|
||||
.dispatchAlert(level, message, stackName)
|
||||
.dispatchAlert(level, category, message, { stackName })
|
||||
.catch(err => console.error('[SchedulerService] Notification dispatch failed:', getErrorMessage(err, 'unknown error')));
|
||||
}
|
||||
|
||||
@@ -314,12 +316,14 @@ export class SchedulerService {
|
||||
}
|
||||
this.safeDispatch(
|
||||
scanLevel,
|
||||
'scan_finding',
|
||||
`Scheduled scan "${task.name}" completed: ${output}`,
|
||||
task.target_id ?? undefined
|
||||
);
|
||||
} else if (task.last_status === 'failure') {
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
'system',
|
||||
`Scheduled task "${task.name}" (${task.action}) recovered successfully`,
|
||||
task.target_id ?? undefined
|
||||
);
|
||||
@@ -355,6 +359,7 @@ export class SchedulerService {
|
||||
console.error(`[SchedulerService] Task "${task.name}" (id=${task.id}) failed:`, errMsg);
|
||||
this.safeDispatch(
|
||||
'error',
|
||||
'system',
|
||||
`Scheduled task "${task.name}" (${task.action}) failed: ${errMsg}`,
|
||||
task.target_id ?? undefined
|
||||
);
|
||||
@@ -647,6 +652,7 @@ export class SchedulerService {
|
||||
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
'image_update_applied',
|
||||
`Auto-update: stack "${stackName}" updated with new images`,
|
||||
stackName
|
||||
);
|
||||
@@ -684,6 +690,7 @@ export class SchedulerService {
|
||||
for (const v of summary.violations ?? []) {
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
'warning',
|
||||
'scan_finding',
|
||||
`Policy "${v.policyName}" violated by ${v.imageRef}: ${v.severity} exceeds ${v.maxSeverity}`,
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user