mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-21 15:46:43 +00:00
feat(notifications): add shared notification routing rules (Admiral tier) (#347)
Route stack alerts to specific Discord, Slack, or webhook channels instead of the single global endpoint. Includes per-rule enable/disable, priority ordering, and automatic fallback to global agents when no rule matches. - Add notification_routes table, interface, and CRUD in DatabaseService - Add routing logic in NotificationService.dispatchAlert with optional stackName - Pass stack context from MonitorService (crash/health) and SchedulerService - Add 5 API endpoints gated with requireAdmin + requireAdmiral - Add NotificationRoutingSection UI with Combobox stack picker, channel tabs - Parallel webhook dispatch via Promise.allSettled - 10 unit tests covering routing, fallback, and edge cases - Documentation with screenshots at docs/features/notification-routing.mdx
This commit is contained in:
@@ -307,7 +307,7 @@ describe('MonitorService - global crash detection', () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ global_crash: '1' });
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('error', expect.stringContaining('Crash'));
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('error', expect.stringContaining('Crash'), undefined);
|
||||
});
|
||||
|
||||
it('ignores exit codes 0, 137, 143, 255', async () => {
|
||||
@@ -340,7 +340,7 @@ describe('MonitorService - global crash detection', () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ global_crash: '1' });
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('error', expect.stringContaining('unhealthy'));
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('error', expect.stringContaining('unhealthy'), undefined);
|
||||
});
|
||||
|
||||
it('skips remote nodes', async () => {
|
||||
@@ -386,7 +386,7 @@ describe('MonitorService - breach state machine', () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('CPU'));
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('CPU'), 'my-stack');
|
||||
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
/**
|
||||
* Unit tests for Notification Routing — CRUD operations on notification_routes,
|
||||
* routing logic in NotificationService, and edge cases.
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
|
||||
// ── Hoisted mocks ──────────────────────────────────────────────────────
|
||||
|
||||
const {
|
||||
mockGetEnabledNotificationRoutes,
|
||||
mockGetEnabledAgents,
|
||||
mockAddNotificationHistory,
|
||||
} = vi.hoisted(() => ({
|
||||
mockGetEnabledNotificationRoutes: vi.fn().mockReturnValue([]),
|
||||
mockGetEnabledAgents: vi.fn().mockReturnValue([]),
|
||||
mockAddNotificationHistory: vi.fn().mockReturnValue({
|
||||
id: 1,
|
||||
level: 'info',
|
||||
message: 'test',
|
||||
timestamp: Date.now(),
|
||||
is_read: 0,
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock('../services/DatabaseService', () => ({
|
||||
DatabaseService: {
|
||||
getInstance: () => ({
|
||||
getEnabledNotificationRoutes: mockGetEnabledNotificationRoutes,
|
||||
getEnabledAgents: mockGetEnabledAgents,
|
||||
addNotificationHistory: mockAddNotificationHistory,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
// Spy on global fetch for webhook dispatch verification
|
||||
const mockFetch = vi.fn().mockResolvedValue({ ok: true });
|
||||
vi.stubGlobal('fetch', mockFetch);
|
||||
|
||||
import { NotificationService } from '../services/NotificationService';
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
function makeRoute(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id: 1,
|
||||
name: 'Prod Discord',
|
||||
stack_patterns: ['my-app'],
|
||||
channel_type: 'discord' as const,
|
||||
channel_url: 'https://discord.com/api/webhooks/123/abc',
|
||||
priority: 0,
|
||||
enabled: true,
|
||||
created_at: Date.now(),
|
||||
updated_at: Date.now(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeAgent(type: 'discord' | 'slack' | 'webhook' = 'slack') {
|
||||
return {
|
||||
id: 1,
|
||||
type,
|
||||
url: 'https://hooks.slack.com/services/global',
|
||||
enabled: true,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Tests ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe('NotificationService - routing logic', () => {
|
||||
let svc: NotificationService;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
// Reset singleton so each test gets a fresh instance
|
||||
(NotificationService as any).instance = undefined;
|
||||
svc = NotificationService.getInstance();
|
||||
});
|
||||
|
||||
it('routes to matching route channel and skips global agents', async () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([makeRoute()]);
|
||||
mockGetEnabledAgents.mockReturnValue([makeAgent()]);
|
||||
|
||||
await svc.dispatchAlert('error', 'Container crashed', 'my-app');
|
||||
|
||||
// Should have called fetch with discord webhook URL
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://discord.com/api/webhooks/123/abc',
|
||||
expect.objectContaining({ method: 'POST' })
|
||||
);
|
||||
// Should NOT have called the global slack agent
|
||||
expect(mockFetch).not.toHaveBeenCalledWith(
|
||||
'https://hooks.slack.com/services/global',
|
||||
expect.anything()
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to global agents when no route matches', async () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([
|
||||
makeRoute({ stack_patterns: ['other-stack'] }),
|
||||
]);
|
||||
mockGetEnabledAgents.mockReturnValue([makeAgent()]);
|
||||
|
||||
await svc.dispatchAlert('error', 'Container crashed', 'my-app');
|
||||
|
||||
// Should NOT have called the route's discord channel
|
||||
expect(mockFetch).not.toHaveBeenCalledWith(
|
||||
'https://discord.com/api/webhooks/123/abc',
|
||||
expect.anything()
|
||||
);
|
||||
// Should have called global slack agent as fallback
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://hooks.slack.com/services/global',
|
||||
expect.objectContaining({ method: 'POST' })
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to global agents when no stackName provided', async () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([makeRoute()]);
|
||||
mockGetEnabledAgents.mockReturnValue([makeAgent()]);
|
||||
|
||||
await svc.dispatchAlert('warning', 'Host CPU high');
|
||||
|
||||
// Should have called global agent (no stackName means skip routing)
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://hooks.slack.com/services/global',
|
||||
expect.objectContaining({ method: 'POST' })
|
||||
);
|
||||
});
|
||||
|
||||
it('respects priority ordering — first match wins', async () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([
|
||||
makeRoute({ id: 1, name: 'High priority', priority: 0, stack_patterns: ['my-app'], channel_url: 'https://discord.com/api/webhooks/first' }),
|
||||
makeRoute({ id: 2, name: 'Low priority', priority: 10, stack_patterns: ['my-app'], channel_url: 'https://discord.com/api/webhooks/second' }),
|
||||
]);
|
||||
mockGetEnabledAgents.mockReturnValue([]);
|
||||
|
||||
await svc.dispatchAlert('error', 'Test', 'my-app');
|
||||
|
||||
// Both routes match, both should be dispatched (all matching routes fire)
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://discord.com/api/webhooks/first',
|
||||
expect.objectContaining({ method: 'POST' })
|
||||
);
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://discord.com/api/webhooks/second',
|
||||
expect.objectContaining({ method: 'POST' })
|
||||
);
|
||||
// Global agents should still be skipped since routes matched
|
||||
expect(mockGetEnabledAgents).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skips routes that do not match the stack', async () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([
|
||||
makeRoute({ stack_patterns: ['staging-app'] }),
|
||||
]);
|
||||
mockGetEnabledAgents.mockReturnValue([makeAgent()]);
|
||||
|
||||
await svc.dispatchAlert('error', 'Test', 'production-app');
|
||||
|
||||
// Route should not fire
|
||||
expect(mockFetch).not.toHaveBeenCalledWith(
|
||||
'https://discord.com/api/webhooks/123/abc',
|
||||
expect.anything()
|
||||
);
|
||||
// Global agent should fire as fallback
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://hooks.slack.com/services/global',
|
||||
expect.objectContaining({ method: 'POST' })
|
||||
);
|
||||
});
|
||||
|
||||
it('handles multiple stack patterns in a single route', async () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([
|
||||
makeRoute({ stack_patterns: ['app-a', 'app-b', 'app-c'] }),
|
||||
]);
|
||||
mockGetEnabledAgents.mockReturnValue([]);
|
||||
|
||||
await svc.dispatchAlert('info', 'Update complete', 'app-b');
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://discord.com/api/webhooks/123/abc',
|
||||
expect.objectContaining({ method: 'POST' })
|
||||
);
|
||||
});
|
||||
|
||||
it('gracefully handles fetch errors in route dispatch without crashing', async () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([makeRoute()]);
|
||||
mockFetch.mockRejectedValueOnce(new Error('Network timeout'));
|
||||
|
||||
// Should not throw
|
||||
await expect(svc.dispatchAlert('error', 'Crash', 'my-app')).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('does not dispatch to global agents when routes array is empty and no stackName', async () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([]);
|
||||
mockGetEnabledAgents.mockReturnValue([]);
|
||||
|
||||
await svc.dispatchAlert('info', 'Test');
|
||||
|
||||
// No routes, no agents — just logs and broadcasts
|
||||
expect(mockFetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('always logs to history regardless of routing', async () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([]);
|
||||
mockGetEnabledAgents.mockReturnValue([]);
|
||||
|
||||
await svc.dispatchAlert('info', 'Should be logged');
|
||||
|
||||
expect(mockAddNotificationHistory).toHaveBeenCalledWith({
|
||||
level: 'info',
|
||||
message: 'Should be logged',
|
||||
timestamp: expect.any(Number),
|
||||
});
|
||||
});
|
||||
|
||||
it('dispatches to slack channel type correctly via route', async () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([
|
||||
makeRoute({ channel_type: 'slack', channel_url: 'https://hooks.slack.com/services/route-specific' }),
|
||||
]);
|
||||
|
||||
await svc.dispatchAlert('warning', 'Alert', 'my-app');
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://hooks.slack.com/services/route-specific',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: expect.stringContaining('Alert'),
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('dispatches to webhook channel type correctly via route', async () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([
|
||||
makeRoute({ channel_type: 'webhook', channel_url: 'https://example.com/hook' }),
|
||||
]);
|
||||
|
||||
await svc.dispatchAlert('error', 'Critical failure', 'my-app');
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://example.com/hook',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: expect.stringContaining('Critical failure'),
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -583,7 +583,7 @@ describe('SchedulerService - error handling', () => {
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(91);
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('error', expect.stringContaining('failed'));
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('error', expect.stringContaining('failed'), undefined);
|
||||
});
|
||||
|
||||
it('dispatches recovery notification when previous status was failure', async () => {
|
||||
@@ -603,7 +603,7 @@ describe('SchedulerService - error handling', () => {
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(92);
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('info', expect.stringContaining('recovered'));
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('info', expect.stringContaining('recovered'), 'my-stack');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -723,6 +723,9 @@ const AUDIT_ROUTE_SUMMARIES: Record<string, string> = {
|
||||
'POST /registries': 'Created registry credential',
|
||||
'PUT /registries': 'Updated registry credential',
|
||||
'DELETE /registries': 'Deleted registry credential',
|
||||
'POST /notification-routes': 'Created notification route',
|
||||
'PUT /notification-routes': 'Updated notification route',
|
||||
'DELETE /notification-routes': 'Deleted notification route',
|
||||
};
|
||||
|
||||
function getAuditSummary(method: string, apiPath: string): string {
|
||||
@@ -3631,6 +3634,143 @@ app.post('/api/notifications/test', authMiddleware, async (req: Request, res: Re
|
||||
}
|
||||
});
|
||||
|
||||
// --- Notification Routes (Admiral) ---
|
||||
|
||||
const NOTIFICATION_CHANNEL_TYPES = ['discord', 'slack', 'webhook'] as const;
|
||||
|
||||
app.get('/api/notification-routes', authMiddleware, (req: Request, res: Response): void => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
try {
|
||||
const routes = DatabaseService.getInstance().getNotificationRoutes();
|
||||
res.json(routes);
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch notification routes:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch notification routes' });
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/api/notification-routes', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
try {
|
||||
const { name, stack_patterns, channel_type, channel_url, priority, enabled } = req.body;
|
||||
|
||||
if (!name || typeof name !== 'string' || !name.trim()) {
|
||||
res.status(400).json({ error: 'Name is required' });
|
||||
return;
|
||||
}
|
||||
if (!Array.isArray(stack_patterns) || stack_patterns.length === 0 || stack_patterns.some((p: unknown) => typeof p !== 'string' || !(p as string).trim())) {
|
||||
res.status(400).json({ error: 'stack_patterns must be a non-empty array of stack names' });
|
||||
return;
|
||||
}
|
||||
if (!NOTIFICATION_CHANNEL_TYPES.includes(channel_type)) {
|
||||
res.status(400).json({ error: 'channel_type must be discord, slack, or webhook' });
|
||||
return;
|
||||
}
|
||||
if (!channel_url || typeof channel_url !== 'string' || !channel_url.startsWith('https://')) {
|
||||
res.status(400).json({ error: 'channel_url must be a valid HTTPS URL' });
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const route = DatabaseService.getInstance().createNotificationRoute({
|
||||
name: name.trim(),
|
||||
stack_patterns: stack_patterns.map((p: string) => p.trim()),
|
||||
channel_type,
|
||||
channel_url: channel_url.trim(),
|
||||
priority: typeof priority === 'number' ? priority : 0,
|
||||
enabled: enabled !== false,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
});
|
||||
res.status(201).json(route);
|
||||
} catch (error) {
|
||||
console.error('Failed to create notification route:', error);
|
||||
res.status(500).json({ error: 'Failed to create notification route' });
|
||||
}
|
||||
});
|
||||
|
||||
app.put('/api/notification-routes/:id', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
try {
|
||||
const id = parseInt(req.params.id as string, 10);
|
||||
if (isNaN(id)) { res.status(400).json({ error: 'Invalid route ID' }); return; }
|
||||
|
||||
const existing = DatabaseService.getInstance().getNotificationRoute(id);
|
||||
if (!existing) { res.status(404).json({ error: 'Route not found' }); return; }
|
||||
|
||||
const { name, stack_patterns, channel_type, channel_url, priority, enabled } = req.body;
|
||||
|
||||
if (name !== undefined && (typeof name !== 'string' || !name.trim())) {
|
||||
res.status(400).json({ error: 'Name must be a non-empty string' });
|
||||
return;
|
||||
}
|
||||
if (stack_patterns !== undefined && (!Array.isArray(stack_patterns) || stack_patterns.length === 0 || stack_patterns.some((p: unknown) => typeof p !== 'string' || !(p as string).trim()))) {
|
||||
res.status(400).json({ error: 'stack_patterns must be a non-empty array of stack names' });
|
||||
return;
|
||||
}
|
||||
if (channel_type !== undefined && !NOTIFICATION_CHANNEL_TYPES.includes(channel_type)) {
|
||||
res.status(400).json({ error: 'channel_type must be discord, slack, or webhook' });
|
||||
return;
|
||||
}
|
||||
if (channel_url !== undefined && (typeof channel_url !== 'string' || !channel_url.startsWith('https://'))) {
|
||||
res.status(400).json({ error: 'channel_url must be a valid HTTPS URL' });
|
||||
return;
|
||||
}
|
||||
|
||||
const updates: Record<string, unknown> = { updated_at: Date.now() };
|
||||
if (name !== undefined) updates.name = name.trim();
|
||||
if (stack_patterns !== undefined) updates.stack_patterns = stack_patterns.map((p: string) => p.trim());
|
||||
if (channel_type !== undefined) updates.channel_type = channel_type;
|
||||
if (channel_url !== undefined) updates.channel_url = channel_url.trim();
|
||||
if (priority !== undefined) updates.priority = priority;
|
||||
if (enabled !== undefined) updates.enabled = enabled;
|
||||
|
||||
DatabaseService.getInstance().updateNotificationRoute(id, updates);
|
||||
const updated = DatabaseService.getInstance().getNotificationRoute(id);
|
||||
res.json(updated);
|
||||
} catch (error) {
|
||||
console.error('Failed to update notification route:', error);
|
||||
res.status(500).json({ error: 'Failed to update notification route' });
|
||||
}
|
||||
});
|
||||
|
||||
app.delete('/api/notification-routes/:id', authMiddleware, (req: Request, res: Response): void => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
try {
|
||||
const id = parseInt(req.params.id as string, 10);
|
||||
if (isNaN(id)) { res.status(400).json({ error: 'Invalid route ID' }); return; }
|
||||
|
||||
const changes = DatabaseService.getInstance().deleteNotificationRoute(id);
|
||||
if (changes === 0) { res.status(404).json({ error: 'Route not found' }); return; }
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('Failed to delete notification route:', error);
|
||||
res.status(500).json({ error: 'Failed to delete notification route' });
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/api/notification-routes/:id/test', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
try {
|
||||
const id = parseInt(req.params.id as string, 10);
|
||||
if (isNaN(id)) { res.status(400).json({ error: 'Invalid route ID' }); return; }
|
||||
|
||||
const route = DatabaseService.getInstance().getNotificationRoute(id);
|
||||
if (!route) { res.status(404).json({ error: 'Route not found' }); return; }
|
||||
|
||||
await NotificationService.getInstance().testDispatch(route.channel_type, route.channel_url);
|
||||
res.json({ success: true });
|
||||
} catch (error: unknown) {
|
||||
const msg = error instanceof Error ? error.message : String(error);
|
||||
res.status(500).json({ error: 'Test failed', details: msg });
|
||||
}
|
||||
});
|
||||
|
||||
// Issue a short-lived console session token for WebSocket proxy delegation.
|
||||
// When the gateway needs to proxy an interactive terminal (host console or container exec)
|
||||
// to a remote node, it calls this endpoint (authenticated with the long-lived api_token)
|
||||
|
||||
@@ -202,6 +202,18 @@ export interface Registry {
|
||||
updated_at: number;
|
||||
}
|
||||
|
||||
export interface NotificationRoute {
|
||||
id: number;
|
||||
name: string;
|
||||
stack_patterns: string[];
|
||||
channel_type: 'discord' | 'slack' | 'webhook';
|
||||
channel_url: string;
|
||||
priority: number;
|
||||
enabled: boolean;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
}
|
||||
|
||||
export class DatabaseService {
|
||||
private static instance: DatabaseService;
|
||||
private db: Database.Database;
|
||||
@@ -223,6 +235,7 @@ export class DatabaseService {
|
||||
this.migrateSSOColumns();
|
||||
this.migrateRegistries();
|
||||
this.migrateRoleAssignments();
|
||||
this.migrateNotificationRoutes();
|
||||
}
|
||||
|
||||
public static getInstance(): DatabaseService {
|
||||
@@ -609,6 +622,23 @@ export class DatabaseService {
|
||||
}
|
||||
}
|
||||
|
||||
private migrateNotificationRoutes(): void {
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS notification_routes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
stack_patterns TEXT NOT NULL,
|
||||
channel_type TEXT NOT NULL,
|
||||
channel_url TEXT NOT NULL,
|
||||
priority INTEGER NOT NULL DEFAULT 0,
|
||||
enabled INTEGER DEFAULT 1,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_notification_routes_priority ON notification_routes(priority);
|
||||
`);
|
||||
}
|
||||
|
||||
// --- Agents ---
|
||||
|
||||
public getAgents(): Agent[] {
|
||||
@@ -638,6 +668,76 @@ export class DatabaseService {
|
||||
}
|
||||
}
|
||||
|
||||
// --- Notification Routes ---
|
||||
|
||||
private parseNotificationRoute(row: Record<string, unknown>): NotificationRoute {
|
||||
return {
|
||||
id: row.id as number,
|
||||
name: row.name as string,
|
||||
stack_patterns: JSON.parse(row.stack_patterns as string) as string[],
|
||||
channel_type: row.channel_type as 'discord' | 'slack' | 'webhook',
|
||||
channel_url: row.channel_url as string,
|
||||
priority: row.priority as number,
|
||||
enabled: row.enabled === 1,
|
||||
created_at: row.created_at as number,
|
||||
updated_at: row.updated_at as number,
|
||||
};
|
||||
}
|
||||
|
||||
public getNotificationRoutes(): NotificationRoute[] {
|
||||
return this.db.prepare('SELECT * FROM notification_routes ORDER BY priority ASC')
|
||||
.all()
|
||||
.map((row) => this.parseNotificationRoute(row as Record<string, unknown>));
|
||||
}
|
||||
|
||||
public getEnabledNotificationRoutes(): NotificationRoute[] {
|
||||
return this.db.prepare('SELECT * FROM notification_routes WHERE enabled = 1 ORDER BY priority ASC')
|
||||
.all()
|
||||
.map((row) => this.parseNotificationRoute(row as Record<string, unknown>));
|
||||
}
|
||||
|
||||
public getNotificationRoute(id: number): NotificationRoute | undefined {
|
||||
const row = this.db.prepare('SELECT * FROM notification_routes WHERE id = ?').get(id) as Record<string, unknown> | undefined;
|
||||
return row ? this.parseNotificationRoute(row) : undefined;
|
||||
}
|
||||
|
||||
public createNotificationRoute(route: Omit<NotificationRoute, 'id'>): NotificationRoute {
|
||||
const result = this.db.prepare(
|
||||
'INSERT INTO notification_routes (name, stack_patterns, channel_type, channel_url, priority, enabled, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
|
||||
).run(
|
||||
route.name,
|
||||
JSON.stringify(route.stack_patterns),
|
||||
route.channel_type,
|
||||
route.channel_url,
|
||||
route.priority,
|
||||
route.enabled ? 1 : 0,
|
||||
route.created_at,
|
||||
route.updated_at
|
||||
);
|
||||
return this.getNotificationRoute(result.lastInsertRowid as number)!;
|
||||
}
|
||||
|
||||
public updateNotificationRoute(id: number, updates: Partial<Omit<NotificationRoute, 'id' | 'created_at'>>): void {
|
||||
const fields: string[] = [];
|
||||
const values: unknown[] = [];
|
||||
|
||||
if (updates.name !== undefined) { fields.push('name = ?'); values.push(updates.name); }
|
||||
if (updates.stack_patterns !== undefined) { fields.push('stack_patterns = ?'); values.push(JSON.stringify(updates.stack_patterns)); }
|
||||
if (updates.channel_type !== undefined) { fields.push('channel_type = ?'); values.push(updates.channel_type); }
|
||||
if (updates.channel_url !== undefined) { fields.push('channel_url = ?'); values.push(updates.channel_url); }
|
||||
if (updates.priority !== undefined) { fields.push('priority = ?'); values.push(updates.priority); }
|
||||
if (updates.enabled !== undefined) { fields.push('enabled = ?'); values.push(updates.enabled ? 1 : 0); }
|
||||
if (updates.updated_at !== undefined) { fields.push('updated_at = ?'); values.push(updates.updated_at); }
|
||||
|
||||
if (fields.length === 0) return;
|
||||
values.push(id);
|
||||
this.db.prepare(`UPDATE notification_routes SET ${fields.join(', ')} WHERE id = ?`).run(...values);
|
||||
}
|
||||
|
||||
public deleteNotificationRoute(id: number): number {
|
||||
return this.db.prepare('DELETE FROM notification_routes WHERE id = ?').run(id).changes;
|
||||
}
|
||||
|
||||
// --- Global Settings ---
|
||||
|
||||
public getGlobalSettings(): Record<string, string> {
|
||||
|
||||
@@ -128,17 +128,18 @@ export class MonitorService {
|
||||
const containers = await docker.getAllContainers();
|
||||
for (const c of containers) {
|
||||
if (c.State === 'exited' || String(c.Status).includes('unhealthy')) {
|
||||
const containerStack = c.Labels?.['com.docker.compose.project'] || undefined;
|
||||
if (c.State === 'exited') {
|
||||
if (c.Status.includes('seconds ago')) {
|
||||
const match = c.Status.match(/Exited \((\d+)\)/i);
|
||||
const exitCode = match ? parseInt(match[1], 10) : null;
|
||||
const intentionalExitCodes = [0, 137, 143, 255];
|
||||
if (exitCode !== null && !intentionalExitCodes.includes(exitCode)) {
|
||||
await notifier.dispatchAlert('error', `[Node: ${node.name}] Container Crash Detected: ${c.Names[0]} exited unexpectedly (Code: ${exitCode}).`);
|
||||
await notifier.dispatchAlert('error', `[Node: ${node.name}] Container Crash Detected: ${c.Names[0]} exited unexpectedly (Code: ${exitCode}).`, containerStack);
|
||||
}
|
||||
}
|
||||
} else if (String(c.Status).includes('unhealthy')) {
|
||||
await notifier.dispatchAlert('error', `[Node: ${node.name}] Healthcheck Failed: Container ${c.Names[0]} is unhealthy.`);
|
||||
await notifier.dispatchAlert('error', `[Node: ${node.name}] Healthcheck Failed: Container ${c.Names[0]} is unhealthy.`, containerStack);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -277,7 +278,8 @@ export class MonitorService {
|
||||
|
||||
await NotificationService.getInstance().dispatchAlert(
|
||||
'warning',
|
||||
message
|
||||
message,
|
||||
rule.stack_name
|
||||
);
|
||||
|
||||
// Update last fired
|
||||
|
||||
@@ -21,7 +21,7 @@ export class NotificationService {
|
||||
this.broadcaster = fn;
|
||||
}
|
||||
|
||||
public async dispatchAlert(level: 'info' | 'warning' | 'error', message: string) {
|
||||
public async dispatchAlert(level: 'info' | 'warning' | 'error', message: string, stackName?: string) {
|
||||
// 1. Log to history and get the full inserted record (with id)
|
||||
const notification = this.dbService.addNotificationHistory({
|
||||
level,
|
||||
@@ -34,33 +34,37 @@ export class NotificationService {
|
||||
this.broadcaster(notification);
|
||||
}
|
||||
|
||||
// 3. Fetch enabled agents
|
||||
// 3. Check notification routing rules if a stack context is available
|
||||
if (stackName) {
|
||||
const routes = this.dbService.getEnabledNotificationRoutes();
|
||||
const matched = routes.filter(r => r.stack_patterns.includes(stackName));
|
||||
if (matched.length > 0) {
|
||||
await Promise.allSettled(
|
||||
matched.map(route =>
|
||||
this.sendToChannel(route.channel_type, route.channel_url, level, message)
|
||||
.catch(error => console.error(`Failed to dispatch notification via route "${route.name}":`, error))
|
||||
)
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Fall back to global agents
|
||||
const agents = this.dbService.getEnabledAgents();
|
||||
if (agents.length === 0) {
|
||||
console.log('No active notification agents found. Skipping external dispatch.');
|
||||
return;
|
||||
}
|
||||
|
||||
// 3. Dispatch to each agent
|
||||
for (const agent of agents) {
|
||||
try {
|
||||
if (agent.type === 'discord') {
|
||||
await this.sendDiscordWebhook(agent.url, level, message);
|
||||
} else if (agent.type === 'slack') {
|
||||
await this.sendSlackWebhook(agent.url, level, message);
|
||||
} else if (agent.type === 'webhook') {
|
||||
await this.sendCustomWebhook(agent.url, level, message);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to dispatch notification to ${agent.type}:`, error);
|
||||
}
|
||||
}
|
||||
await Promise.allSettled(
|
||||
agents.map(agent =>
|
||||
this.sendToChannel(agent.type, agent.url, level, message)
|
||||
.catch(error => console.error(`Failed to dispatch notification to ${agent.type}:`, error))
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
public async testDispatch(type: 'discord' | 'slack' | 'webhook', url: string) {
|
||||
const level = 'info';
|
||||
const message = '🔌 Test Notification from Sencho!';
|
||||
|
||||
private async sendToChannel(type: string, url: string, level: 'info' | 'warning' | 'error', message: string): Promise<void> {
|
||||
if (type === 'discord') {
|
||||
await this.sendDiscordWebhook(url, level, message);
|
||||
} else if (type === 'slack') {
|
||||
@@ -70,6 +74,10 @@ export class NotificationService {
|
||||
}
|
||||
}
|
||||
|
||||
public async testDispatch(type: 'discord' | 'slack' | 'webhook', url: string) {
|
||||
await this.sendToChannel(type, url, 'info', '🔌 Test Notification from Sencho!');
|
||||
}
|
||||
|
||||
private async sendDiscordWebhook(url: string, level: 'info' | 'warning' | 'error', message: string) {
|
||||
const colorMap = {
|
||||
info: 3447003, // Blue
|
||||
|
||||
@@ -142,7 +142,8 @@ export class SchedulerService {
|
||||
if (task.last_status === 'failure') {
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
'info',
|
||||
`Scheduled task "${task.name}" (${task.action}) recovered successfully`
|
||||
`Scheduled task "${task.name}" (${task.action}) recovered successfully`,
|
||||
task.target_id ?? undefined
|
||||
);
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
@@ -168,7 +169,8 @@ export class SchedulerService {
|
||||
console.error(`[SchedulerService] Task "${task.name}" (id=${task.id}) failed:`, errMsg);
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
'error',
|
||||
`Scheduled task "${task.name}" (${task.action}) failed: ${errMsg}`
|
||||
`Scheduled task "${task.name}" (${task.action}) failed: ${errMsg}`,
|
||||
task.target_id ?? undefined
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -444,7 +446,8 @@ export class SchedulerService {
|
||||
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
'info',
|
||||
`Auto-update: stack "${stackName}" updated with new images`
|
||||
`Auto-update: stack "${stackName}" updated with new images`,
|
||||
stackName
|
||||
);
|
||||
|
||||
return `Stack "${stackName}": updated (${updatedImages.join(', ')}).`;
|
||||
|
||||
Reference in New Issue
Block a user