mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-07-26 11:49:16 +00:00
feat(notifications): add structured category enum to dispatcher and history (#774)
Introduce a NotificationCategory string-literal union (11 values) and thread it through dispatchAlert as a required second argument. All callers (DockerEventService, AutoHealService, ImageUpdateService, MonitorService, PolicyEnforcement, policyGate, SchedulerService, imageUpdates route) pass an explicit category at every call site, giving TypeScript compile-time enforcement that no new emit site can be added without choosing a category. DatabaseService gains an idempotent migration that adds a nullable category TEXT column to notification_history; existing rows keep category=NULL (displayed as Uncategorized in the UI). The getNotificationHistory method accepts an optional category filter that is forwarded from the GET /api/notifications/history route via a ?category= query param. NotificationPanel gains a category Select dropdown so users can filter history by category. The frontend types mirror the backend union so API responses are type-safe end-to-end. All 75 test files (1410 tests) updated to the new 4-arg dispatchAlert signature and passing.
This commit is contained in:
@@ -124,9 +124,9 @@ describe('DockerEventService - die classification', () => {
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
'monitor_alert',
|
||||
expect.stringContaining('Container Crash Detected'),
|
||||
undefined,
|
||||
'web',
|
||||
expect.objectContaining({ containerName: 'web' }),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -149,7 +149,7 @@ describe('DockerEventService - die classification', () => {
|
||||
await vi.advanceTimersByTimeAsync(600);
|
||||
|
||||
const crashCall = mockDispatchAlert.mock.calls.find(c =>
|
||||
typeof c[1] === 'string' && c[1].includes('Crash'));
|
||||
typeof c[2] === 'string' && c[2].includes('Crash'));
|
||||
expect(crashCall).toBeUndefined();
|
||||
});
|
||||
|
||||
@@ -174,7 +174,7 @@ describe('DockerEventService - die classification', () => {
|
||||
await vi.advanceTimersByTimeAsync(400); // total > 500ms
|
||||
|
||||
const crashCall = mockDispatchAlert.mock.calls.find(c =>
|
||||
typeof c[1] === 'string' && c[1].includes('Crash'));
|
||||
typeof c[2] === 'string' && c[2].includes('Crash'));
|
||||
expect(crashCall).toBeUndefined();
|
||||
});
|
||||
|
||||
@@ -192,9 +192,9 @@ describe('DockerEventService - die classification', () => {
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
'monitor_alert',
|
||||
expect.stringContaining('OOM Kill'),
|
||||
undefined,
|
||||
'hog',
|
||||
expect.objectContaining({ containerName: 'hog' }),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -225,9 +225,9 @@ describe('DockerEventService - die classification', () => {
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
'monitor_alert',
|
||||
expect.stringContaining('Healthcheck failed'),
|
||||
undefined,
|
||||
'api',
|
||||
expect.objectContaining({ containerName: 'api' }),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -291,15 +291,15 @@ describe('DockerEventService - rate limiting', () => {
|
||||
await vi.advanceTimersByTimeAsync(600);
|
||||
|
||||
const crashCalls = mockDispatchAlert.mock.calls.filter(c =>
|
||||
typeof c[1] === 'string' && c[1].includes('Crash'));
|
||||
typeof c[2] === 'string' && c[2].includes('Crash'));
|
||||
expect(crashCalls).toHaveLength(20);
|
||||
|
||||
// After the rate window, a summary warning fires.
|
||||
await vi.advanceTimersByTimeAsync(61_000);
|
||||
const summaryCalls = mockDispatchAlert.mock.calls.filter(c =>
|
||||
typeof c[1] === 'string' && c[1].includes('additional containers crashed'));
|
||||
typeof c[2] === 'string' && c[2].includes('additional containers crashed'));
|
||||
expect(summaryCalls).toHaveLength(1);
|
||||
expect(summaryCalls[0][1]).toContain('2 additional');
|
||||
expect(summaryCalls[0][2]).toContain('2 additional');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -320,9 +320,9 @@ describe('DockerEventService - malformed payloads', () => {
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
'monitor_alert',
|
||||
expect.stringContaining('Container Crash Detected'),
|
||||
undefined,
|
||||
'ok',
|
||||
expect.objectContaining({ containerName: 'ok' }),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -375,7 +375,7 @@ describe('DockerEventService - reconciliation', () => {
|
||||
await Promise.resolve();
|
||||
|
||||
const massCall = mockDispatchAlert.mock.calls.find(c =>
|
||||
typeof c[1] === 'string' && c[1].includes('daemon interruption'));
|
||||
typeof c[2] === 'string' && c[2].includes('daemon interruption'));
|
||||
expect(massCall).toBeDefined();
|
||||
});
|
||||
|
||||
@@ -415,9 +415,9 @@ describe('DockerEventService - reconciliation', () => {
|
||||
await Promise.resolve();
|
||||
|
||||
const crashCall = mockDispatchAlert.mock.calls.find(c =>
|
||||
typeof c[1] === 'string' && c[1].includes('crashed-app'));
|
||||
typeof c[2] === 'string' && c[2].includes('crashed-app'));
|
||||
expect(crashCall).toBeDefined();
|
||||
expect(crashCall?.[2]).toBe('my-stack');
|
||||
expect(crashCall?.[3]).toMatchObject({ stackName: 'my-stack' });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -443,8 +443,8 @@ describe('DockerEventService - reconnect', () => {
|
||||
|
||||
const warn = mockDispatchAlert.mock.calls.find(c => c[0] === 'warning');
|
||||
const info = mockDispatchAlert.mock.calls.find(c => c[0] === 'info');
|
||||
expect(warn?.[1]).toContain('Lost connection');
|
||||
expect(info?.[1]).toContain('Reconnected');
|
||||
expect(warn?.[2]).toContain('Lost connection');
|
||||
expect(info?.[2]).toContain('Reconnected');
|
||||
});
|
||||
|
||||
it('shutdown cancels pending reconnect', async () => {
|
||||
@@ -504,9 +504,9 @@ describe('DockerEventService - hardening', () => {
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
'monitor_alert',
|
||||
expect.stringContaining('Container Crash Detected'),
|
||||
undefined,
|
||||
'app',
|
||||
expect.objectContaining({ containerName: 'app' }),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -532,9 +532,9 @@ describe('DockerEventService - hardening', () => {
|
||||
await Promise.resolve();
|
||||
|
||||
const oomCall = mockDispatchAlert.mock.calls.find(c =>
|
||||
typeof c[1] === 'string' && c[1].includes('OOM Kill'));
|
||||
typeof c[2] === 'string' && c[2].includes('OOM Kill'));
|
||||
const crashCall = mockDispatchAlert.mock.calls.find(c =>
|
||||
typeof c[1] === 'string' && c[1].includes('Crash Detected'));
|
||||
typeof c[2] === 'string' && c[2].includes('Crash Detected'));
|
||||
expect(oomCall).toBeDefined();
|
||||
expect(crashCall).toBeUndefined();
|
||||
});
|
||||
@@ -558,9 +558,9 @@ describe('DockerEventService - hardening', () => {
|
||||
// Inspect failed, so classification stays as the original 'crash'.
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
'monitor_alert',
|
||||
expect.stringContaining('Container Crash Detected'),
|
||||
undefined,
|
||||
'ephemeral',
|
||||
expect.objectContaining({ containerName: 'ephemeral' }),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -585,9 +585,9 @@ describe('DockerEventService - hardening', () => {
|
||||
await vi.advanceTimersByTimeAsync(700);
|
||||
|
||||
const crashCalls = mockDispatchAlert.mock.calls.filter(c =>
|
||||
typeof c[1] === 'string' && c[1].includes('Crash Detected'));
|
||||
typeof c[2] === 'string' && c[2].includes('Crash Detected'));
|
||||
expect(crashCalls).toHaveLength(1);
|
||||
expect(crashCalls[0][1]).toContain('Code: 2');
|
||||
expect(crashCalls[0][2]).toContain('Code: 2');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -385,8 +385,9 @@ services:
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'info',
|
||||
'image_update_available',
|
||||
expect.stringContaining('stackA'),
|
||||
'stackA',
|
||||
{ stackName: 'stackA' },
|
||||
);
|
||||
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(1, 'stackA', true, expect.any(Number));
|
||||
});
|
||||
@@ -413,7 +414,7 @@ services:
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
||||
const dispatched = mockDispatchAlert.mock.calls.map(call => call[2]);
|
||||
const dispatched = mockDispatchAlert.mock.calls.map(call => (call[3] as any)?.stackName);
|
||||
expect(dispatched).toEqual(expect.arrayContaining(['stackA', 'stackB']));
|
||||
expect(mockSetSystemState).toHaveBeenCalledWith('image_update_notifications_backfilled', '1');
|
||||
|
||||
|
||||
@@ -274,7 +274,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' });
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('CPU'), undefined);
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: undefined });
|
||||
});
|
||||
|
||||
it('does not dispatch when CPU below threshold', async () => {
|
||||
@@ -283,7 +283,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' });
|
||||
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'), undefined);
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: undefined });
|
||||
});
|
||||
|
||||
it('dispatches RAM warning when over threshold', async () => {
|
||||
@@ -292,7 +292,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' });
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Memory'), undefined);
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Memory'), { stackName: undefined });
|
||||
});
|
||||
|
||||
it('dispatches disk warning when over threshold', async () => {
|
||||
@@ -301,7 +301,7 @@ describe('MonitorService - evaluateGlobalSettings', () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ host_disk_limit: '90' });
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Disk'), undefined);
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Disk'), { stackName: undefined });
|
||||
});
|
||||
|
||||
it('skips host limits when threshold is 0 or NaN', async () => {
|
||||
@@ -309,10 +309,10 @@ describe('MonitorService - evaluateGlobalSettings', () => {
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '0' });
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'), undefined);
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: undefined });
|
||||
|
||||
await (svc as any).evaluateGlobalSettings({ host_cpu_limit: 'abc' });
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'), undefined);
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: undefined });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -352,7 +352,7 @@ describe('MonitorService - breach state machine', () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('CPU'), 'my-stack');
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: 'my-stack' });
|
||||
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
|
||||
});
|
||||
|
||||
@@ -526,7 +526,7 @@ describe('MonitorService - restart_count metric', () => {
|
||||
|
||||
expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1');
|
||||
// restart_count=5 > threshold=3, should fire
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Restart count'), 'my-stack');
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Restart count'), { stackName: 'my-stack' });
|
||||
});
|
||||
|
||||
it('skips Docker inspect when no restart_count rules exist', async () => {
|
||||
@@ -545,7 +545,7 @@ describe('MonitorService - restart_count metric', () => {
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1');
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('Restart count'), expect.anything());
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Restart count'), expect.anything());
|
||||
});
|
||||
});
|
||||
|
||||
@@ -577,9 +577,9 @@ describe('MonitorService - Sencho version check', () => {
|
||||
(svc as any).lastVersionCheckAt = 0;
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('info', expect.stringContaining('0.46.0'));
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'system', expect.stringContaining('0.46.0'));
|
||||
// Message must include the real running version, not "0.0.0".
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('info', expect.stringContaining('currently running 0.45.0'));
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'system', expect.stringContaining('currently running 0.45.0'));
|
||||
expect(mockSetSystemState).toHaveBeenCalledWith('last_sencho_update_notified_version', '0.46.0');
|
||||
});
|
||||
|
||||
@@ -593,7 +593,7 @@ describe('MonitorService - Sencho version check', () => {
|
||||
(svc as any).lastVersionCheckAt = 0;
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', expect.stringContaining('0.46.0'));
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'system', expect.stringContaining('0.46.0'));
|
||||
});
|
||||
|
||||
it('handles version check failure gracefully', async () => {
|
||||
@@ -605,7 +605,7 @@ describe('MonitorService - Sencho version check', () => {
|
||||
|
||||
// Should not throw
|
||||
await expect((svc as any).evaluate()).resolves.toBeUndefined();
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', expect.stringContaining('available'));
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'system', expect.stringContaining('available'));
|
||||
});
|
||||
|
||||
it('respects the 6-hour cooldown interval', async () => {
|
||||
@@ -632,7 +632,7 @@ describe('MonitorService - Sencho version check', () => {
|
||||
(svc as any).lastVersionCheckAt = 0;
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', expect.stringContaining('0.46.0'));
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'system', expect.stringContaining('0.46.0'));
|
||||
expect(mockSetSystemState).not.toHaveBeenCalledWith('last_sencho_update_notified_version', expect.anything());
|
||||
// Should not have even attempted the lookup.
|
||||
expect(mockGetLatestVersion).not.toHaveBeenCalled();
|
||||
@@ -675,7 +675,7 @@ describe('MonitorService - Sencho version check', () => {
|
||||
expect(mockGetLatestVersion).not.toHaveBeenCalled();
|
||||
// Exactly one dispatch across both evals.
|
||||
const availabilityDispatches = mockDispatchAlert.mock.calls.filter(
|
||||
(args: unknown[]) => typeof args[1] === 'string' && args[1].includes('available'),
|
||||
(args: unknown[]) => typeof args[2] === 'string' && args[2].includes('available'),
|
||||
);
|
||||
expect(availabilityDispatches).toHaveLength(1);
|
||||
});
|
||||
@@ -691,7 +691,7 @@ describe('MonitorService - Sencho version check', () => {
|
||||
(svc as any).lastVersionCheckAt = 0;
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('info', expect.stringContaining('0.47.0'));
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'system', expect.stringContaining('0.47.0'));
|
||||
expect(store.last_sencho_update_notified_version).toBe('0.47.0');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -94,7 +94,7 @@ describe('NotificationService - routing logic', () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([makeRoute()]);
|
||||
mockGetEnabledAgents.mockReturnValue([makeAgent()]);
|
||||
|
||||
await svc.dispatchAlert('error', 'Container crashed', 'my-app');
|
||||
await svc.dispatchAlert('error', 'monitor_alert', 'Container crashed', { stackName: 'my-app' });
|
||||
|
||||
// Should have called fetch with discord webhook URL
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
@@ -114,7 +114,7 @@ describe('NotificationService - routing logic', () => {
|
||||
]);
|
||||
mockGetEnabledAgents.mockReturnValue([makeAgent()]);
|
||||
|
||||
await svc.dispatchAlert('error', 'Container crashed', 'my-app');
|
||||
await svc.dispatchAlert('error', 'monitor_alert', 'Container crashed', { stackName: 'my-app' });
|
||||
|
||||
// Should NOT have called the route's discord channel
|
||||
expect(mockFetch).not.toHaveBeenCalledWith(
|
||||
@@ -132,7 +132,7 @@ describe('NotificationService - routing logic', () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([makeRoute()]);
|
||||
mockGetEnabledAgents.mockReturnValue([makeAgent()]);
|
||||
|
||||
await svc.dispatchAlert('warning', 'Host CPU high');
|
||||
await svc.dispatchAlert('warning', 'monitor_alert', 'Host CPU high');
|
||||
|
||||
// Should have called global agent (no stackName means skip routing)
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
@@ -148,7 +148,7 @@ describe('NotificationService - routing logic', () => {
|
||||
]);
|
||||
mockGetEnabledAgents.mockReturnValue([]);
|
||||
|
||||
await svc.dispatchAlert('error', 'Test', 'my-app');
|
||||
await svc.dispatchAlert('error', 'monitor_alert', 'Test', { stackName: 'my-app' });
|
||||
|
||||
// Both routes match, both should be dispatched (all matching routes fire)
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
@@ -169,7 +169,7 @@ describe('NotificationService - routing logic', () => {
|
||||
]);
|
||||
mockGetEnabledAgents.mockReturnValue([makeAgent()]);
|
||||
|
||||
await svc.dispatchAlert('error', 'Test', 'production-app');
|
||||
await svc.dispatchAlert('error', 'monitor_alert', 'Test', { stackName: 'production-app' });
|
||||
|
||||
// Route should not fire
|
||||
expect(mockFetch).not.toHaveBeenCalledWith(
|
||||
@@ -189,7 +189,7 @@ describe('NotificationService - routing logic', () => {
|
||||
]);
|
||||
mockGetEnabledAgents.mockReturnValue([]);
|
||||
|
||||
await svc.dispatchAlert('info', 'Update complete', 'app-b');
|
||||
await svc.dispatchAlert('info', 'image_update_applied', 'Update complete', { stackName: 'app-b' });
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://discord.com/api/webhooks/123/abc',
|
||||
@@ -202,14 +202,14 @@ describe('NotificationService - routing logic', () => {
|
||||
mockFetch.mockRejectedValueOnce(new Error('Network timeout'));
|
||||
|
||||
// Should not throw
|
||||
await expect(svc.dispatchAlert('error', 'Crash', 'my-app')).resolves.toBeUndefined();
|
||||
await expect(svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' })).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('does not dispatch to global agents when routes array is empty and no stackName', async () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([]);
|
||||
mockGetEnabledAgents.mockReturnValue([]);
|
||||
|
||||
await svc.dispatchAlert('info', 'Test');
|
||||
await svc.dispatchAlert('info', 'system', 'Test');
|
||||
|
||||
// No routes, no agents — just logs and broadcasts
|
||||
expect(mockFetch).not.toHaveBeenCalled();
|
||||
@@ -219,10 +219,11 @@ describe('NotificationService - routing logic', () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([]);
|
||||
mockGetEnabledAgents.mockReturnValue([]);
|
||||
|
||||
await svc.dispatchAlert('info', 'Should be logged');
|
||||
await svc.dispatchAlert('info', 'system', 'Should be logged');
|
||||
|
||||
expect(mockAddNotificationHistory).toHaveBeenCalledWith(1, {
|
||||
level: 'info',
|
||||
category: 'system',
|
||||
message: 'Should be logged',
|
||||
timestamp: expect.any(Number),
|
||||
stack_name: undefined,
|
||||
@@ -234,10 +235,11 @@ describe('NotificationService - routing logic', () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([]);
|
||||
mockGetEnabledAgents.mockReturnValue([]);
|
||||
|
||||
await svc.dispatchAlert('warning', 'Restarted', 'my-app', 'my-app-web-1');
|
||||
await svc.dispatchAlert('warning', 'autoheal_triggered', 'Restarted', { stackName: 'my-app', containerName: 'my-app-web-1' });
|
||||
|
||||
expect(mockAddNotificationHistory).toHaveBeenCalledWith(1, {
|
||||
level: 'warning',
|
||||
category: 'autoheal_triggered',
|
||||
message: 'Restarted',
|
||||
timestamp: expect.any(Number),
|
||||
stack_name: 'my-app',
|
||||
@@ -250,7 +252,7 @@ describe('NotificationService - routing logic', () => {
|
||||
makeRoute({ channel_type: 'slack', channel_url: 'https://hooks.slack.com/services/route-specific' }),
|
||||
]);
|
||||
|
||||
await svc.dispatchAlert('warning', 'Alert', 'my-app');
|
||||
await svc.dispatchAlert('warning', 'monitor_alert', 'Alert', { stackName: 'my-app' });
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://hooks.slack.com/services/route-specific',
|
||||
@@ -266,7 +268,7 @@ describe('NotificationService - routing logic', () => {
|
||||
makeRoute({ channel_type: 'webhook', channel_url: 'https://example.com/hook' }),
|
||||
]);
|
||||
|
||||
await svc.dispatchAlert('error', 'Critical failure', 'my-app');
|
||||
await svc.dispatchAlert('error', 'monitor_alert', 'Critical failure', { stackName: 'my-app' });
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'https://example.com/hook',
|
||||
@@ -281,7 +283,7 @@ describe('NotificationService - routing logic', () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([makeRoute()]);
|
||||
mockFetch.mockRejectedValueOnce(new Error('Connection refused'));
|
||||
|
||||
await svc.dispatchAlert('error', 'Test', 'my-app');
|
||||
await svc.dispatchAlert('error', 'monitor_alert', 'Test', { stackName: 'my-app' });
|
||||
|
||||
expect(mockUpdateNotificationDispatchError).toHaveBeenCalledWith(
|
||||
1, // notification id from mock
|
||||
@@ -294,7 +296,7 @@ describe('NotificationService - routing logic', () => {
|
||||
mockGetEnabledAgents.mockReturnValue([makeAgent()]);
|
||||
mockFetch.mockRejectedValueOnce(new Error('Timeout'));
|
||||
|
||||
await svc.dispatchAlert('warning', 'Host alert');
|
||||
await svc.dispatchAlert('warning', 'monitor_alert', 'Host alert');
|
||||
|
||||
expect(mockUpdateNotificationDispatchError).toHaveBeenCalledWith(
|
||||
1,
|
||||
@@ -306,7 +308,7 @@ describe('NotificationService - routing logic', () => {
|
||||
mockGetEnabledNotificationRoutes.mockReturnValue([makeRoute()]);
|
||||
mockFetch.mockResolvedValueOnce({ ok: true });
|
||||
|
||||
await svc.dispatchAlert('info', 'All good', 'my-app');
|
||||
await svc.dispatchAlert('info', 'system', 'All good', { stackName: 'my-app' });
|
||||
|
||||
expect(mockUpdateNotificationDispatchError).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -161,7 +161,8 @@ describe('enforcePolicyPreDeploy', () => {
|
||||
expect(result.violations).toEqual([]);
|
||||
expect(notificationStub.dispatchAlert).toHaveBeenCalledTimes(1);
|
||||
expect(notificationStub.dispatchAlert.mock.calls[0][0]).toBe('warning');
|
||||
expect(notificationStub.dispatchAlert.mock.calls[0][1]).toContain('Trivy not installed');
|
||||
expect(notificationStub.dispatchAlert.mock.calls[0][1]).toBe('scan_finding');
|
||||
expect(notificationStub.dispatchAlert.mock.calls[0][2]).toContain('Trivy not installed');
|
||||
expect(composeStub.listStackImages).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
||||
@@ -162,11 +162,11 @@ describe('SchedulerService - scheduled scan policy alerts', () => {
|
||||
|
||||
const warningCalls = mockDispatchAlert.mock.calls.filter((c) => c[0] === 'warning');
|
||||
expect(warningCalls).toHaveLength(2);
|
||||
expect(warningCalls[0][1]).toContain('prod-high-gate');
|
||||
expect(warningCalls[0][1]).toContain('nginx:1.14');
|
||||
expect(warningCalls[0][1]).toContain('CRITICAL');
|
||||
expect(warningCalls[0][1]).toContain('HIGH');
|
||||
expect(warningCalls[1][1]).toContain('redis:6');
|
||||
expect(warningCalls[0][2]).toContain('prod-high-gate');
|
||||
expect(warningCalls[0][2]).toContain('nginx:1.14');
|
||||
expect(warningCalls[0][2]).toContain('CRITICAL');
|
||||
expect(warningCalls[0][2]).toContain('HIGH');
|
||||
expect(warningCalls[1][2]).toContain('redis:6');
|
||||
});
|
||||
|
||||
it('does not dispatch any policy alert when no violations occur', async () => {
|
||||
@@ -177,7 +177,7 @@ describe('SchedulerService - scheduled scan policy alerts', () => {
|
||||
await svc.triggerTask(300);
|
||||
|
||||
const warningCalls = mockDispatchAlert.mock.calls.filter(
|
||||
(c) => c[0] === 'warning' && typeof c[1] === 'string' && c[1].includes('Policy'),
|
||||
(c) => c[0] === 'warning' && typeof c[2] === 'string' && c[2].includes('Policy'),
|
||||
);
|
||||
expect(warningCalls).toHaveLength(0);
|
||||
});
|
||||
|
||||
@@ -870,7 +870,7 @@ describe('SchedulerService - error handling', () => {
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(91);
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('error', expect.stringContaining('failed'), undefined);
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('error', 'system', expect.stringContaining('failed'), { stackName: undefined });
|
||||
});
|
||||
|
||||
it('dispatches recovery notification when previous status was failure', async () => {
|
||||
@@ -890,7 +890,7 @@ describe('SchedulerService - error handling', () => {
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(92);
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('info', expect.stringContaining('recovered'), 'my-stack');
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'system', expect.stringContaining('recovered'), { stackName: 'my-stack' });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -946,13 +946,15 @@ describe('SchedulerService - scheduled scan notifications', () => {
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'info',
|
||||
'scan_finding',
|
||||
expect.stringContaining('nightly-scan'),
|
||||
undefined,
|
||||
{ stackName: undefined },
|
||||
);
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'info',
|
||||
'scan_finding',
|
||||
expect.stringContaining('Scanned 3 image(s)'),
|
||||
undefined,
|
||||
{ stackName: undefined },
|
||||
);
|
||||
});
|
||||
|
||||
@@ -965,8 +967,9 @@ describe('SchedulerService - scheduled scan notifications', () => {
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'warning',
|
||||
'scan_finding',
|
||||
expect.stringContaining('2 failed'),
|
||||
undefined,
|
||||
{ stackName: undefined },
|
||||
);
|
||||
});
|
||||
|
||||
@@ -979,8 +982,9 @@ describe('SchedulerService - scheduled scan notifications', () => {
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'info',
|
||||
'scan_finding',
|
||||
expect.stringContaining('completed'),
|
||||
'web-stack',
|
||||
{ stackName: 'web-stack' },
|
||||
);
|
||||
});
|
||||
|
||||
@@ -998,8 +1002,9 @@ describe('SchedulerService - scheduled scan notifications', () => {
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'info',
|
||||
'scan_finding',
|
||||
expect.stringContaining('recovered-scan'),
|
||||
undefined,
|
||||
{ stackName: undefined },
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1032,8 +1037,9 @@ describe('SchedulerService - scheduled scan notifications', () => {
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
'system',
|
||||
expect.stringMatching(/failed.*Trivy/i),
|
||||
'payment-stack',
|
||||
{ stackName: 'payment-stack' },
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1046,7 +1052,7 @@ describe('SchedulerService - scheduled scan notifications', () => {
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(206);
|
||||
|
||||
const message = mockDispatchAlert.mock.calls[0][1] as string;
|
||||
const message = mockDispatchAlert.mock.calls[0][2] as string;
|
||||
expect(message).toContain('2 critical');
|
||||
expect(message).toContain('5 high');
|
||||
expect(message).toContain('10 medium');
|
||||
@@ -1061,8 +1067,9 @@ describe('SchedulerService - scheduled scan notifications', () => {
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'info',
|
||||
'scan_finding',
|
||||
expect.stringContaining('No images to scan'),
|
||||
undefined,
|
||||
{ stackName: undefined },
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1075,8 +1082,9 @@ describe('SchedulerService - scheduled scan notifications', () => {
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'info',
|
||||
'scan_finding',
|
||||
expect.stringContaining('All 12 image(s) already scanned recently'),
|
||||
undefined,
|
||||
{ stackName: undefined },
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1196,8 +1204,9 @@ describe('SchedulerService - invalid cron at execution time', () => {
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
'system',
|
||||
expect.stringContaining('failed'),
|
||||
undefined
|
||||
{ stackName: undefined },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -81,8 +81,9 @@ export async function triggerPostDeployScan(
|
||||
if (scan.critical_count > 0 || scan.high_count > 0) {
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
scan.critical_count > 0 ? 'error' : 'warning',
|
||||
'scan_finding',
|
||||
`Vulnerability scan for ${imageRef}: ${scan.critical_count} critical, ${scan.high_count} high`,
|
||||
stackName,
|
||||
{ stackName },
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -90,8 +91,9 @@ export async function triggerPostDeployScan(
|
||||
console.error(`[Security] Post-deploy scan failed for ${imageRef}:`, message);
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
'warning',
|
||||
'scan_finding',
|
||||
`Post-deploy scan failed for ${imageRef} (${stackName}): ${message}`,
|
||||
stackName,
|
||||
{ stackName },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -284,7 +284,7 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
|
||||
);
|
||||
if (!autoUpdateGate.ok) {
|
||||
const blockedMsg = `Policy "${autoUpdateGate.policy?.name}" blocked auto-update: ${autoUpdateGate.violations.length} image(s) exceed ${autoUpdateGate.policy?.max_severity}`;
|
||||
NotificationService.getInstance().dispatchAlert('warning', blockedMsg, stackName);
|
||||
NotificationService.getInstance().dispatchAlert('warning', 'scan_finding', blockedMsg, { stackName });
|
||||
results.push(`Stack "${stackName}": ${blockedMsg}`);
|
||||
continue;
|
||||
}
|
||||
@@ -294,8 +294,9 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
|
||||
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
'info',
|
||||
'image_update_applied',
|
||||
`Auto-update: stack "${stackName}" updated with new images`,
|
||||
stackName,
|
||||
{ stackName },
|
||||
);
|
||||
|
||||
results.push(`Stack "${stackName}": updated (${updatedImages.join(', ')}).`);
|
||||
|
||||
@@ -25,7 +25,8 @@ export const notificationsRouter = Router();
|
||||
notificationsRouter.get('/', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
try {
|
||||
const nodeId = req.nodeId ?? 0;
|
||||
const history = DatabaseService.getInstance().getNotificationHistory(nodeId);
|
||||
const category = typeof req.query.category === 'string' ? req.query.category : undefined;
|
||||
const history = DatabaseService.getInstance().getNotificationHistory(nodeId, 50, category);
|
||||
res.json(history);
|
||||
} catch {
|
||||
res.status(500).json({ error: 'Failed to fetch notifications' });
|
||||
|
||||
@@ -242,9 +242,9 @@ export class AutoHealService {
|
||||
NotificationService.getInstance()
|
||||
.dispatchAlert(
|
||||
'info',
|
||||
'autoheal_triggered',
|
||||
`Auto-Heal: Restarted ${containerName} on stack ${policy.stack_name} after being unhealthy for ${policy.unhealthy_duration_mins} minute(s).`,
|
||||
policy.stack_name,
|
||||
containerName,
|
||||
{ stackName: policy.stack_name, containerName },
|
||||
)
|
||||
.catch(err => console.error('[AutoHeal] notification dispatch failed:', err));
|
||||
} catch (err) {
|
||||
@@ -266,9 +266,9 @@ export class AutoHealService {
|
||||
NotificationService.getInstance()
|
||||
.dispatchAlert(
|
||||
'warning',
|
||||
'autoheal_triggered',
|
||||
`Auto-Heal: Failed to restart ${containerName} on stack ${policy.stack_name}. Error: ${errorMsg}`,
|
||||
policy.stack_name,
|
||||
containerName,
|
||||
{ stackName: policy.stack_name, containerName },
|
||||
)
|
||||
.catch(e => console.error('[AutoHeal] notification dispatch failed:', e));
|
||||
|
||||
@@ -299,8 +299,9 @@ export class AutoHealService {
|
||||
NotificationService.getInstance()
|
||||
.dispatchAlert(
|
||||
'warning',
|
||||
'autoheal_triggered',
|
||||
`Auto-Heal: Policy for ${policy.stack_name}${policy.service_name ? '/' + policy.service_name : ''} has been auto-disabled after ${failures} consecutive failures.`,
|
||||
policy.stack_name,
|
||||
{ stackName: policy.stack_name },
|
||||
)
|
||||
.catch(e => console.error('[AutoHeal] notification dispatch failed:', e));
|
||||
}
|
||||
|
||||
@@ -196,6 +196,7 @@ export interface SSOConfig {
|
||||
export interface NotificationHistory {
|
||||
id?: number;
|
||||
level: 'info' | 'warning' | 'error';
|
||||
category?: string;
|
||||
message: string;
|
||||
timestamp: number;
|
||||
is_read: boolean;
|
||||
@@ -489,6 +490,7 @@ export class DatabaseService {
|
||||
this.migrateSecretMisconfigColumns();
|
||||
this.migrateAgentsAndNotificationsNodeId();
|
||||
this.migratePolicyEvaluationColumn();
|
||||
this.migrateNotificationCategory();
|
||||
}
|
||||
|
||||
public static getInstance(): DatabaseService {
|
||||
@@ -1202,6 +1204,14 @@ export class DatabaseService {
|
||||
}
|
||||
}
|
||||
|
||||
private migrateNotificationCategory(): void {
|
||||
try {
|
||||
this.db.prepare('ALTER TABLE notification_history ADD COLUMN category TEXT').run();
|
||||
} catch {
|
||||
// column already present
|
||||
}
|
||||
}
|
||||
|
||||
// --- Agents ---
|
||||
|
||||
public getAgents(nodeId: number): Agent[] {
|
||||
@@ -1457,19 +1467,23 @@ export class DatabaseService {
|
||||
|
||||
// --- Notification History ---
|
||||
|
||||
public getNotificationHistory(nodeId: number, limit = 50): NotificationHistory[] {
|
||||
const stmt = this.db.prepare('SELECT * FROM notification_history WHERE node_id = ? ORDER BY timestamp DESC LIMIT ?');
|
||||
return stmt.all(nodeId, limit).map((row: any) => ({
|
||||
public getNotificationHistory(nodeId: number, limit = 50, category?: string): NotificationHistory[] {
|
||||
const sql = category
|
||||
? 'SELECT * FROM notification_history WHERE node_id = ? AND category = ? ORDER BY timestamp DESC LIMIT ?'
|
||||
: 'SELECT * FROM notification_history WHERE node_id = ? ORDER BY timestamp DESC LIMIT ?';
|
||||
const args: (number | string)[] = category ? [nodeId, category, limit] : [nodeId, limit];
|
||||
return this.db.prepare(sql).all(...args).map((row: any) => ({
|
||||
...row,
|
||||
is_read: row.is_read === 1,
|
||||
stack_name: row.stack_name ?? undefined,
|
||||
container_name: row.container_name ?? undefined,
|
||||
category: row.category ?? undefined,
|
||||
}));
|
||||
}
|
||||
|
||||
public addNotificationHistory(nodeId: number, notification: Omit<NotificationHistory, 'id' | 'is_read'>): NotificationHistory {
|
||||
const stmt = this.db.prepare(
|
||||
'INSERT INTO notification_history (node_id, level, message, timestamp, is_read, stack_name, container_name) VALUES (?, ?, ?, ?, 0, ?, ?)'
|
||||
'INSERT INTO notification_history (node_id, level, message, timestamp, is_read, stack_name, container_name, category) VALUES (?, ?, ?, ?, 0, ?, ?, ?)'
|
||||
);
|
||||
const result = stmt.run(
|
||||
nodeId,
|
||||
@@ -1477,7 +1491,8 @@ export class DatabaseService {
|
||||
notification.message,
|
||||
notification.timestamp,
|
||||
notification.stack_name ?? null,
|
||||
notification.container_name ?? null
|
||||
notification.container_name ?? null,
|
||||
notification.category ?? null,
|
||||
);
|
||||
|
||||
this.db.prepare(`
|
||||
@@ -1490,6 +1505,7 @@ export class DatabaseService {
|
||||
return {
|
||||
id: result.lastInsertRowid as number,
|
||||
level: notification.level,
|
||||
category: notification.category,
|
||||
message: notification.message,
|
||||
timestamp: notification.timestamp,
|
||||
is_read: false,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import Docker from 'dockerode';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { NotificationService } from './NotificationService';
|
||||
import { NotificationCategory, NotificationService } from './NotificationService';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import {
|
||||
classifyDie,
|
||||
@@ -187,7 +187,7 @@ export class DockerEventService {
|
||||
this.status = 'connected';
|
||||
|
||||
if (this.disconnectedNoticeEmitted) {
|
||||
await this.emitInfo(`Reconnected to Docker daemon.`);
|
||||
await this.emitInfo('system', `Reconnected to Docker daemon.`);
|
||||
this.disconnectedNoticeEmitted = false;
|
||||
}
|
||||
|
||||
@@ -237,7 +237,7 @@ export class DockerEventService {
|
||||
|
||||
if (!this.disconnectedNoticeEmitted) {
|
||||
this.disconnectedNoticeEmitted = true;
|
||||
void this.emitWarning(`Lost connection to Docker daemon; monitoring paused.`);
|
||||
void this.emitWarning('system', `Lost connection to Docker daemon; monitoring paused.`);
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) {
|
||||
@@ -310,6 +310,7 @@ export class DockerEventService {
|
||||
|
||||
if (exitRatio >= MASS_EVENT_THRESHOLD) {
|
||||
await this.emitInfo(
|
||||
'system',
|
||||
`Docker daemon interruption detected: ${newlyExited.length} containers exited during connection gap.`
|
||||
);
|
||||
} else {
|
||||
@@ -450,6 +451,7 @@ export class DockerEventService {
|
||||
const name = state.name ?? id.slice(0, 12);
|
||||
const stackName = state.stackName;
|
||||
void this.emitError(
|
||||
'monitor_alert',
|
||||
`Healthcheck failed: ${name} is unhealthy.`,
|
||||
stackName,
|
||||
state.name,
|
||||
@@ -562,7 +564,7 @@ export class DockerEventService {
|
||||
// rate-suppressed alerts don't silently lock out the next real crash.
|
||||
if (state) state.lastCrashAlertAt = Date.now();
|
||||
|
||||
await this.emitError(message, info.stackName, info.name);
|
||||
await this.emitError('monitor_alert', message, info.stackName, info.name);
|
||||
}
|
||||
|
||||
private isCrashAlertsEnabled(): boolean {
|
||||
@@ -607,6 +609,7 @@ export class DockerEventService {
|
||||
this.suppressedCount = 0;
|
||||
if (count > 0) {
|
||||
void this.emitWarning(
|
||||
'monitor_alert',
|
||||
`${count} additional containers crashed in the last minute.`,
|
||||
);
|
||||
}
|
||||
@@ -624,6 +627,7 @@ export class DockerEventService {
|
||||
if (this.parseErrorCount > PARSE_ERROR_THRESHOLD && !this.parseWarningEmitted) {
|
||||
this.parseWarningEmitted = true;
|
||||
void this.emitWarning(
|
||||
'system',
|
||||
`Received malformed Docker event payloads. Monitoring continues but some events may be skipped.`,
|
||||
);
|
||||
}
|
||||
@@ -668,16 +672,16 @@ export class DockerEventService {
|
||||
// Notification wrappers (prefix with node name for multi-node clarity)
|
||||
// ========================================================================
|
||||
|
||||
private async emitError(message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('error', this.prefix(message), stackName, containerName);
|
||||
private async emitError(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('error', category, this.prefix(message), { stackName, containerName });
|
||||
}
|
||||
|
||||
private async emitWarning(message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('warning', this.prefix(message), stackName, containerName);
|
||||
private async emitWarning(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('warning', category, this.prefix(message), { stackName, containerName });
|
||||
}
|
||||
|
||||
private async emitInfo(message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('info', this.prefix(message), stackName, containerName);
|
||||
private async emitInfo(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('info', category, this.prefix(message), { stackName, containerName });
|
||||
}
|
||||
|
||||
private prefix(message: string): string {
|
||||
|
||||
@@ -275,8 +275,9 @@ export class ImageUpdateService {
|
||||
try {
|
||||
await notifier.dispatchAlert(
|
||||
'info',
|
||||
'image_update_available',
|
||||
`[Node: ${nodeName}] Stack "${stackName}" has image updates available.`,
|
||||
stackName,
|
||||
{ stackName },
|
||||
);
|
||||
} catch (e) {
|
||||
console.error(`[ImageUpdateService] Failed to dispatch update notification for "${stackName}":`, e);
|
||||
@@ -286,6 +287,7 @@ export class ImageUpdateService {
|
||||
try {
|
||||
db.addNotificationHistory(NodeRegistry.getInstance().getDefaultNodeId(), {
|
||||
level: 'error',
|
||||
category: 'system',
|
||||
message: `[Node: ${nodeName}] Failed to notify about image updates for stack "${stackName}": ${getErrorMessage(e, String(e))}`,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
@@ -135,7 +135,7 @@ export class MonitorService {
|
||||
const cpuUsage = currentLoad.currentLoad;
|
||||
const cpuLimit = parseFloat(settings['host_cpu_limit']);
|
||||
if (!isNaN(cpuLimit) && cpuLimit > 0 && cpuUsage > cpuLimit) {
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.cpu, HOST_ALERT_COOLDOWN_MS, 'warning',
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.cpu, HOST_ALERT_COOLDOWN_MS, 'warning', 'monitor_alert',
|
||||
`Host CPU utilization is critically high: ${cpuUsage.toFixed(1)}% (Threshold: ${cpuLimit}%)`);
|
||||
}
|
||||
|
||||
@@ -143,7 +143,7 @@ export class MonitorService {
|
||||
const ramUsage = (mem.used / mem.total) * 100;
|
||||
const ramLimit = parseFloat(settings['host_ram_limit']);
|
||||
if (!isNaN(ramLimit) && ramLimit > 0 && ramUsage > ramLimit) {
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.ram, HOST_ALERT_COOLDOWN_MS, 'warning',
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.ram, HOST_ALERT_COOLDOWN_MS, 'warning', 'monitor_alert',
|
||||
`Host Memory utilization is critically high: ${ramUsage.toFixed(1)}% (Threshold: ${ramLimit}%)`);
|
||||
}
|
||||
|
||||
@@ -152,7 +152,7 @@ export class MonitorService {
|
||||
if (mainDisk) {
|
||||
const diskLimit = parseFloat(settings['host_disk_limit']);
|
||||
if (!isNaN(diskLimit) && diskLimit > 0 && mainDisk.use > diskLimit) {
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.disk, HOST_ALERT_COOLDOWN_MS, 'warning',
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.disk, HOST_ALERT_COOLDOWN_MS, 'warning', 'monitor_alert',
|
||||
`Host Disk space utilization is critically high: ${mainDisk.use.toFixed(1)}% (Threshold: ${diskLimit}%)`);
|
||||
}
|
||||
}
|
||||
@@ -213,7 +213,7 @@ export class MonitorService {
|
||||
const registry = NodeRegistry.getInstance();
|
||||
const localNode = registry.getNode(registry.getDefaultNodeId());
|
||||
const nodeLabel = localNode?.name ?? 'this node';
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.janitor, JANITOR_COOLDOWN_MS, 'info',
|
||||
await this.dispatchWithCooldown(HOST_ALERT_KEYS.janitor, JANITOR_COOLDOWN_MS, 'info', 'system',
|
||||
`Node "${nodeLabel}" has accumulated ${reclaimGb.toFixed(1)} GB of unused Docker data. Consider using the Janitor tool.`);
|
||||
}
|
||||
}
|
||||
@@ -290,7 +290,7 @@ export class MonitorService {
|
||||
|
||||
try {
|
||||
const notifier = NotificationService.getInstance();
|
||||
await notifier.dispatchAlert('info',
|
||||
await notifier.dispatchAlert('info', 'system',
|
||||
`Sencho ${latest} is available (currently running ${currentVersion}). Visit the Fleet dashboard to update.`);
|
||||
db.setSystemState(stateKey, latest);
|
||||
if (isDebugEnabled()) console.debug(`[Monitor:diag] Dispatched version notification: ${currentVersion} -> ${latest}`);
|
||||
@@ -391,8 +391,9 @@ export class MonitorService {
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Duration met for rule ${ruleId}, dispatching alert`);
|
||||
await NotificationService.getInstance().dispatchAlert(
|
||||
'warning',
|
||||
'monitor_alert',
|
||||
message,
|
||||
rule.stack_name
|
||||
{ stackName: rule.stack_name },
|
||||
);
|
||||
|
||||
// Update last fired
|
||||
@@ -454,12 +455,13 @@ export class MonitorService {
|
||||
/** Dispatch an alert only if the cooldown period has elapsed since the last alert for this key. */
|
||||
private async dispatchWithCooldown(
|
||||
stateKey: string, cooldownMs: number,
|
||||
severity: 'info' | 'warning' | 'error', message: string, stack?: string,
|
||||
severity: 'info' | 'warning' | 'error', category: import('./NotificationService').NotificationCategory,
|
||||
message: string, stack?: string,
|
||||
): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const last = parseInt(db.getSystemState(stateKey) || '0', 10);
|
||||
if (Date.now() - last > cooldownMs) {
|
||||
await NotificationService.getInstance().dispatchAlert(severity, message, stack);
|
||||
await NotificationService.getInstance().dispatchAlert(severity, category, message, { stackName: stack });
|
||||
db.setSystemState(stateKey, Date.now().toString());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,19 @@ import { NodeRegistry } from './NodeRegistry';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
|
||||
export type NotificationCategory =
|
||||
| 'deploy_success'
|
||||
| 'deploy_failure'
|
||||
| 'stack_started'
|
||||
| 'stack_stopped'
|
||||
| 'stack_restarted'
|
||||
| 'image_update_available'
|
||||
| 'image_update_applied'
|
||||
| 'autoheal_triggered'
|
||||
| 'monitor_alert'
|
||||
| 'scan_finding'
|
||||
| 'system';
|
||||
|
||||
/** Webhook timeout: 10 seconds per external dispatch call. */
|
||||
const WEBHOOK_TIMEOUT_MS = 10_000;
|
||||
|
||||
@@ -83,16 +96,18 @@ export class NotificationService {
|
||||
*/
|
||||
public async dispatchAlert(
|
||||
level: 'info' | 'warning' | 'error',
|
||||
category: NotificationCategory,
|
||||
message: string,
|
||||
stackName?: string,
|
||||
containerName?: string,
|
||||
options?: { stackName?: string; containerName?: string },
|
||||
) {
|
||||
const { stackName, containerName } = options ?? {};
|
||||
// Internal writes use the middleware default so they share a row key
|
||||
// with user-initiated requests; otherwise the UI and monitors split
|
||||
// between different node_id buckets.
|
||||
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
const notification = this.dbService.addNotificationHistory(localNodeId, {
|
||||
level,
|
||||
category,
|
||||
message,
|
||||
timestamp: Date.now(),
|
||||
stack_name: stackName,
|
||||
@@ -105,7 +120,7 @@ export class NotificationService {
|
||||
// 3. Check notification routing rules if a stack context is available
|
||||
const errors: string[] = [];
|
||||
|
||||
if (stackName) {
|
||||
if (stackName !== undefined) {
|
||||
const routes = this.dbService.getEnabledNotificationRoutes();
|
||||
const matched = routes.filter(r => r.stack_patterns.includes(stackName));
|
||||
if (matched.length > 0) {
|
||||
|
||||
@@ -60,8 +60,9 @@ export async function enforcePolicyPreDeploy(
|
||||
if (!svc.isTrivyAvailable()) {
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
'warning',
|
||||
'scan_finding',
|
||||
`Pre-deploy scan for "${stackName}" skipped: Trivy not installed on this node`,
|
||||
stackName,
|
||||
{ stackName },
|
||||
);
|
||||
return { ok: true, bypassed: false, policy, violations: [], trivyMissing: true };
|
||||
}
|
||||
|
||||
@@ -89,6 +89,7 @@ export class SchedulerService {
|
||||
await trivy.detectTrivy();
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
'system',
|
||||
`Trivy updated from v${previous} to v${check.latest}`,
|
||||
);
|
||||
db.updateGlobalSetting('trivy_last_notified_version', check.latest);
|
||||
@@ -100,6 +101,7 @@ export class SchedulerService {
|
||||
if (lastNotified === check.latest) return;
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
'system',
|
||||
`Trivy update available: v${check.latest} (currently v${check.current ?? 'unknown'})`,
|
||||
);
|
||||
db.updateGlobalSetting('trivy_last_notified_version', check.latest);
|
||||
@@ -159,9 +161,9 @@ export class SchedulerService {
|
||||
* Fire a notification without awaiting completion, catching any promise
|
||||
* rejection so the scheduler never crashes on a failed dispatch.
|
||||
*/
|
||||
private safeDispatch(level: 'info' | 'warning' | 'error', message: string, stackName?: string): void {
|
||||
private safeDispatch(level: 'info' | 'warning' | 'error', category: import('./NotificationService').NotificationCategory, message: string, stackName?: string): void {
|
||||
NotificationService.getInstance()
|
||||
.dispatchAlert(level, message, stackName)
|
||||
.dispatchAlert(level, category, message, { stackName })
|
||||
.catch(err => console.error('[SchedulerService] Notification dispatch failed:', getErrorMessage(err, 'unknown error')));
|
||||
}
|
||||
|
||||
@@ -314,12 +316,14 @@ export class SchedulerService {
|
||||
}
|
||||
this.safeDispatch(
|
||||
scanLevel,
|
||||
'scan_finding',
|
||||
`Scheduled scan "${task.name}" completed: ${output}`,
|
||||
task.target_id ?? undefined
|
||||
);
|
||||
} else if (task.last_status === 'failure') {
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
'system',
|
||||
`Scheduled task "${task.name}" (${task.action}) recovered successfully`,
|
||||
task.target_id ?? undefined
|
||||
);
|
||||
@@ -355,6 +359,7 @@ export class SchedulerService {
|
||||
console.error(`[SchedulerService] Task "${task.name}" (id=${task.id}) failed:`, errMsg);
|
||||
this.safeDispatch(
|
||||
'error',
|
||||
'system',
|
||||
`Scheduled task "${task.name}" (${task.action}) failed: ${errMsg}`,
|
||||
task.target_id ?? undefined
|
||||
);
|
||||
@@ -647,6 +652,7 @@ export class SchedulerService {
|
||||
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
'image_update_applied',
|
||||
`Auto-update: stack "${stackName}" updated with new images`,
|
||||
stackName
|
||||
);
|
||||
@@ -684,6 +690,7 @@ export class SchedulerService {
|
||||
for (const v of summary.violations ?? []) {
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
'warning',
|
||||
'scan_finding',
|
||||
`Policy "${v.policyName}" violated by ${v.imageRef}: ${v.severity} exceeds ${v.maxSeverity}`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -20,12 +20,28 @@ import {
|
||||
SelectValue,
|
||||
} from '@/components/ui/select';
|
||||
import { cn } from '@/lib/utils';
|
||||
import type { NotificationItem } from './dashboard/types';
|
||||
import type { NotificationCategory, NotificationItem } from './dashboard/types';
|
||||
import type { Node } from '@/context/NodeContext';
|
||||
|
||||
const NODE_FILTER_ALL = 'all' as const;
|
||||
const CATEGORY_FILTER_ALL = 'all' as const;
|
||||
type NotifFilter = 'all' | 'unread' | 'alerts';
|
||||
type NodeFilter = typeof NODE_FILTER_ALL | number;
|
||||
type CategoryFilter = typeof CATEGORY_FILTER_ALL | NotificationCategory;
|
||||
|
||||
const CATEGORY_LABELS: Record<NotificationCategory, string> = {
|
||||
deploy_success: 'Deploy success',
|
||||
deploy_failure: 'Deploy failure',
|
||||
stack_started: 'Stack started',
|
||||
stack_stopped: 'Stack stopped',
|
||||
stack_restarted: 'Stack restarted',
|
||||
image_update_available: 'Update available',
|
||||
image_update_applied: 'Update applied',
|
||||
autoheal_triggered: 'Auto-heal',
|
||||
monitor_alert: 'Monitor alert',
|
||||
scan_finding: 'Scan finding',
|
||||
system: 'System',
|
||||
};
|
||||
|
||||
type LevelConfig = {
|
||||
icon: LucideIcon;
|
||||
@@ -90,11 +106,13 @@ function applyFilter(
|
||||
items: NotificationItem[],
|
||||
filter: NotifFilter,
|
||||
nodeFilter: NodeFilter,
|
||||
categoryFilter: CategoryFilter,
|
||||
): NotificationItem[] {
|
||||
let result = items;
|
||||
if (filter === 'unread') result = result.filter((n) => !n.is_read);
|
||||
else if (filter === 'alerts') result = result.filter((n) => n.level === 'warning' || n.level === 'error');
|
||||
if (nodeFilter !== NODE_FILTER_ALL) result = result.filter((n) => n.nodeId === nodeFilter);
|
||||
if (categoryFilter !== CATEGORY_FILTER_ALL) result = result.filter((n) => n.category === categoryFilter);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -117,6 +135,7 @@ export function NotificationPanel({
|
||||
}: NotificationPanelProps) {
|
||||
const [filter, setFilter] = useState<NotifFilter>('all');
|
||||
const [nodeFilter, setNodeFilter] = useState<NodeFilter>(NODE_FILTER_ALL);
|
||||
const [categoryFilter, setCategoryFilter] = useState<CategoryFilter>(CATEGORY_FILTER_ALL);
|
||||
const [open, setOpen] = useState(false);
|
||||
|
||||
const unreadCount = useMemo(
|
||||
@@ -141,8 +160,8 @@ export function NotificationPanel({
|
||||
: NODE_FILTER_ALL;
|
||||
|
||||
const filtered = useMemo(
|
||||
() => applyFilter(notifications, filter, effectiveNodeFilter),
|
||||
[notifications, filter, effectiveNodeFilter],
|
||||
() => applyFilter(notifications, filter, effectiveNodeFilter, categoryFilter),
|
||||
[notifications, filter, effectiveNodeFilter, categoryFilter],
|
||||
);
|
||||
const groups = useMemo(() => groupByDay(filtered), [filtered]);
|
||||
|
||||
@@ -234,12 +253,7 @@ export function NotificationPanel({
|
||||
</div>
|
||||
|
||||
{/* Filter segment */}
|
||||
<div
|
||||
className={cn(
|
||||
'flex items-center gap-2 border-t border-card-border/60 px-[var(--density-row-x)] py-[var(--density-row-y)]',
|
||||
showNodeFilter ? 'justify-between' : 'justify-end',
|
||||
)}
|
||||
>
|
||||
<div className="flex flex-wrap items-center gap-2 border-t border-card-border/60 px-[var(--density-row-x)] py-[var(--density-row-y)]">
|
||||
{showNodeFilter ? (
|
||||
<Select
|
||||
value={effectiveNodeFilter === NODE_FILTER_ALL ? NODE_FILTER_ALL : String(effectiveNodeFilter)}
|
||||
@@ -247,7 +261,7 @@ export function NotificationPanel({
|
||||
>
|
||||
<SelectTrigger
|
||||
aria-label="Filter by node"
|
||||
className="h-7 w-[140px] border-card-border bg-card px-2 font-mono text-[10px] uppercase tracking-[0.14em] text-stat-subtitle shadow-none focus:ring-0"
|
||||
className="h-7 w-[120px] border-card-border bg-card px-2 font-mono text-[10px] uppercase tracking-[0.14em] text-stat-subtitle shadow-none focus:ring-0"
|
||||
>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
@@ -263,12 +277,35 @@ export function NotificationPanel({
|
||||
</SelectContent>
|
||||
</Select>
|
||||
) : null}
|
||||
<SegmentedControl
|
||||
value={filter}
|
||||
options={filterOptions}
|
||||
onChange={setFilter}
|
||||
ariaLabel="Filter notifications"
|
||||
/>
|
||||
<Select
|
||||
value={categoryFilter}
|
||||
onValueChange={(v) => setCategoryFilter(v as CategoryFilter)}
|
||||
>
|
||||
<SelectTrigger
|
||||
aria-label="Filter by category"
|
||||
className="h-7 w-[130px] border-card-border bg-card px-2 font-mono text-[10px] uppercase tracking-[0.14em] text-stat-subtitle shadow-none focus:ring-0"
|
||||
>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value={CATEGORY_FILTER_ALL} className="font-mono text-[10px] uppercase tracking-[0.14em]">
|
||||
All types
|
||||
</SelectItem>
|
||||
{(Object.keys(CATEGORY_LABELS) as NotificationCategory[]).map((cat) => (
|
||||
<SelectItem key={cat} value={cat} className="font-mono text-[10px] uppercase tracking-[0.14em]">
|
||||
{CATEGORY_LABELS[cat]}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<div className="ml-auto">
|
||||
<SegmentedControl
|
||||
value={filter}
|
||||
options={filterOptions}
|
||||
onChange={setFilter}
|
||||
ariaLabel="Filter notifications"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Stream */}
|
||||
|
||||
@@ -43,9 +43,23 @@ export interface MetricPoint {
|
||||
net_tx_mb: number;
|
||||
}
|
||||
|
||||
export type NotificationCategory =
|
||||
| 'deploy_success'
|
||||
| 'deploy_failure'
|
||||
| 'stack_started'
|
||||
| 'stack_stopped'
|
||||
| 'stack_restarted'
|
||||
| 'image_update_available'
|
||||
| 'image_update_applied'
|
||||
| 'autoheal_triggered'
|
||||
| 'monitor_alert'
|
||||
| 'scan_finding'
|
||||
| 'system';
|
||||
|
||||
export interface NotificationItem {
|
||||
id: number;
|
||||
level: 'info' | 'warning' | 'error';
|
||||
category?: NotificationCategory | string;
|
||||
message: string;
|
||||
timestamp: number;
|
||||
is_read: number;
|
||||
|
||||
Reference in New Issue
Block a user