From 85842cc54788b453071890fb57764ab608de78a0 Mon Sep 17 00:00:00 2001 From: Anso Date: Thu, 23 Jul 2026 17:57:04 -0400 Subject: [PATCH] feat: add service-scoped stack alert rules (#1681) * feat: add service-scoped stack alert rules Stack alerts can target one Compose service or all services. Breach timers are per container and cooldowns are per service so a healthy sibling no longer clears another container's timer or silences a different service. * fix: gate remote scoped alert creates without losing the body Remote hops skip JSON parsing so the proxy stream stays pipeable, which left service_name invisible to the capability gate. Buffer POST /alerts bodies for inspection, fail closed when the remote lacks the capability, and rewrite the buffered bytes on forward. Restore alert-panel alt text to match the unchanged screenshot. * fix: bound remote alert body buffer and reject encoded JSON Cap proxied POST /alerts buffering at the local 100KB JSON limit with structured 413 cleanup, reject non-identity Content-Encoding with 415 so compressed scoped bodies cannot bypass the mixed-version gate, and cover oversized, chunked, and gzip regressions. * fix: harden service-scoped alert delete, cooldown, and proxy gates Reject non-digit alert ids, dual-write last_fired_at for rollback safety, gate cooldown on persisted notification history, fail-fast oversized proxy bodies with 413, and clarify Not in compose UI semantics. * test: expect dispatchAlert persisted result in crash-safety cases Update notification-routing assertions for the new { persisted } return shape so CI matches the cooldown-gating contract. --- .../AutoHealService.evaluate.test.ts | 2 +- backend/src/__tests__/alerts-api.test.ts | 148 +++++- .../__tests__/atomic-deploy-hardening.test.ts | 4 +- backend/src/__tests__/blueprints.test.ts | 8 +- .../src/__tests__/database-metrics.test.ts | 63 ++- .../__tests__/docker-event-service.test.ts | 2 +- .../fleet-sync-retry-service.test.ts | 2 +- .../src/__tests__/fleet-sync-service.test.ts | 2 +- .../__tests__/image-update-service.test.ts | 2 +- backend/src/__tests__/monitor-service.test.ts | 461 +++++++++++++++++- .../__tests__/notification-routing.test.ts | 8 +- .../proxy-service-scoped-alert-gate.test.ts | 256 ++++++++++ .../src/__tests__/scheduler-policy.test.ts | 2 +- .../src/__tests__/scheduler-service.test.ts | 2 +- .../service-scoped-update-routes.test.ts | 2 +- .../src/__tests__/stack-bulk-routes.test.ts | 2 +- .../__tests__/stack-docker-disconnect.test.ts | 2 +- .../stack-down-remove-volumes.test.ts | 2 +- .../__tests__/stack-op-lock-routes.test.ts | 2 +- .../stack-self-protected-routes.test.ts | 2 +- .../stacks-failure-notifications.test.ts | 2 +- backend/src/proxy/remoteNodeProxy.ts | 211 +++++++- backend/src/routes/alerts.ts | 43 +- backend/src/services/CapabilityRegistry.ts | 5 + backend/src/services/DatabaseService.ts | 81 ++- backend/src/services/DockerEventService.ts | 6 +- backend/src/services/MonitorService.ts | 165 +++++-- backend/src/services/NotificationService.ts | 16 +- docs/features/alerts-notifications.mdx | 21 +- .../src/components/StackAlertSheet.test.tsx | 248 ++++++++++ frontend/src/components/StackAlertSheet.tsx | 97 +++- frontend/src/lib/capabilities.ts | 2 + 32 files changed, 1736 insertions(+), 135 deletions(-) create mode 100644 backend/src/__tests__/proxy-service-scoped-alert-gate.test.ts create mode 100644 frontend/src/components/StackAlertSheet.test.tsx diff --git a/backend/src/__tests__/AutoHealService.evaluate.test.ts b/backend/src/__tests__/AutoHealService.evaluate.test.ts index a6085940..e9e530a8 100644 --- a/backend/src/__tests__/AutoHealService.evaluate.test.ts +++ b/backend/src/__tests__/AutoHealService.evaluate.test.ts @@ -55,7 +55,7 @@ beforeEach(() => { const db = DatabaseService.getInstance(); db.getDb().prepare('DELETE FROM auto_heal_history').run(); db.getDb().prepare('DELETE FROM auto_heal_policies').run(); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/alerts-api.test.ts b/backend/src/__tests__/alerts-api.test.ts index 53a3a600..693da057 100644 --- a/backend/src/__tests__/alerts-api.test.ts +++ b/backend/src/__tests__/alerts-api.test.ts @@ -57,8 +57,8 @@ describe('GET /api/alerts', () => { it('filters alerts by stackName query param', async () => { // Seed two alerts for different stacks const db = DatabaseService.getInstance(); - db.addStackAlert({ stack_name: 'web', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 }); - db.addStackAlert({ stack_name: 'api', metric: 'memory_percent', operator: '>', threshold: 90, duration_mins: 5, cooldown_mins: 60 }); + db.addStackAlert({ stack_name: 'web', service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 }); + db.addStackAlert({ stack_name: 'api', service_name: null, metric: 'memory_percent', operator: '>', threshold: 90, duration_mins: 5, cooldown_mins: 60 }); const res = await request(app) .get('/api/alerts?stackName=web') @@ -105,10 +105,101 @@ describe('POST /api/alerts', () => { expect(res.status).toBe(201); expect(res.body.id).toBeDefined(); expect(res.body.stack_name).toBe('new-stack'); + expect(res.body.service_name).toBeNull(); expect(res.body.metric).toBe('memory_percent'); expect(res.body.threshold).toBe(85); }); + it('persists a valid dotted service_name', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Cookie', authCookie) + .send({ + stack_name: 'svc-stack', + service_name: 'api.web', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }); + expect(res.status).toBe(201); + expect(res.body.service_name).toBe('api.web'); + }); + + it('normalizes empty service_name to null', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Cookie', authCookie) + .send({ + stack_name: 'empty-svc', + service_name: '', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }); + expect(res.status).toBe(201); + expect(res.body.service_name).toBeNull(); + }); + + it('rejects whitespace-only service_name', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Cookie', authCookie) + .send({ + stack_name: 'bad-svc', + service_name: ' ', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }); + expect(res.status).toBe(400); + }); + + it('rejects reserved _unlabeled service_name', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Cookie', authCookie) + .send({ + stack_name: 'bad-svc', + service_name: '_unlabeled', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }); + expect(res.status).toBe(400); + }); + + it('rejects service_name when the capability is disabled', async () => { + const { disableCapability, enableCapability, SERVICE_SCOPED_STACK_ALERT_CAPABILITY } = + await import('../services/CapabilityRegistry'); + disableCapability(SERVICE_SCOPED_STACK_ALERT_CAPABILITY); + try { + const res = await request(app) + .post('/api/alerts') + .set('Cookie', authCookie) + .send({ + stack_name: 'cap-stack', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }); + expect(res.status).toBe(400); + expect(res.body.code).toBe('capability_unavailable'); + } finally { + enableCapability(SERVICE_SCOPED_STACK_ALERT_CAPABILITY); + } + }); + it('validates required fields and returns 400 for missing data', async () => { const res = await request(app) .post('/api/alerts') @@ -202,6 +293,7 @@ describe('DELETE /api/alerts/:id', () => { // Create an alert to delete const created = DatabaseService.getInstance().addStackAlert({ stack_name: 'delete-me', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 90, @@ -216,6 +308,58 @@ describe('DELETE /api/alerts/:id', () => { expect(res.status).toBe(200); expect(res.body.success).toBe(true); }); + + it('rejects leading-junk ids like 1abc without deleting alert 1', async () => { + const created = DatabaseService.getInstance().addStackAlert({ + stack_name: 'strict-id-junk', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 90, + duration_mins: 0, + cooldown_mins: 0, + }); + expect(created.id).toBeDefined(); + + const res = await request(app) + .delete(`/api/alerts/${created.id}abc`) + .set('Cookie', authCookie); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Invalid alert id'); + expect(DatabaseService.getInstance().getStackAlerts('strict-id-junk').some((a) => a.id === created.id)).toBe(true); + }); + + it('rejects fractional ids like 2.5 without deleting alert 2', async () => { + const first = DatabaseService.getInstance().addStackAlert({ + stack_name: 'strict-id-fraction', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 0, + }); + const second = DatabaseService.getInstance().addStackAlert({ + stack_name: 'strict-id-fraction', + service_name: null, + metric: 'memory_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 0, + }); + expect(second.id).toBeDefined(); + + const res = await request(app) + .delete(`/api/alerts/${second.id}.5`) + .set('Cookie', authCookie); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Invalid alert id'); + const remaining = DatabaseService.getInstance().getStackAlerts('strict-id-fraction').map((a) => a.id); + expect(remaining).toEqual(expect.arrayContaining([first.id, second.id])); + }); }); // --- POST /api/notifications/test --- diff --git a/backend/src/__tests__/atomic-deploy-hardening.test.ts b/backend/src/__tests__/atomic-deploy-hardening.test.ts index 7814549a..9008e501 100644 --- a/backend/src/__tests__/atomic-deploy-hardening.test.ts +++ b/backend/src/__tests__/atomic-deploy-hardening.test.ts @@ -168,7 +168,7 @@ describe('Rollback notifications (M-2)', () => { mockTier('paid'); mockDeployStack.mockResolvedValue(undefined); const { NotificationService } = await import('../services/NotificationService'); - const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); const res = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(res.status).toBe(200); @@ -186,7 +186,7 @@ describe('Rollback notifications (M-2)', () => { mockTier('paid'); mockDeployStack.mockRejectedValueOnce(new Error('restore failed')); const { NotificationService } = await import('../services/NotificationService'); - const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); const res = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(res.status).toBe(500); diff --git a/backend/src/__tests__/blueprints.test.ts b/backend/src/__tests__/blueprints.test.ts index 935230af..337298e4 100644 --- a/backend/src/__tests__/blueprints.test.ts +++ b/backend/src/__tests__/blueprints.test.ts @@ -600,7 +600,7 @@ describe('BlueprintReconciler drift alert node wording', () => { it('suggest-mode local drift uses on this node', async () => { const { NotificationService } = await import('../services/NotificationService'); - const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); const nodeId = seedNode(); const bp = seedBlueprint({ name: 'drift-local', drift_mode: 'suggest', nodeIds: [nodeId] }); const node = DatabaseService.getInstance().getNode(nodeId)!; @@ -618,7 +618,7 @@ describe('BlueprintReconciler drift alert node wording', () => { it('suggest-mode remote drift keeps the authoritative node name', async () => { const { NotificationService } = await import('../services/NotificationService'); - const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); const nodeId = seedRemoteNode('sencho-test-02'); const bp = seedBlueprint({ name: 'drift-remote', drift_mode: 'suggest', nodeIds: [nodeId] }); const node = DatabaseService.getInstance().getNode(nodeId)!; @@ -636,7 +636,7 @@ describe('BlueprintReconciler drift alert node wording', () => { it('stateful marker-loss on local uses on this node', async () => { const { NotificationService } = await import('../services/NotificationService'); - const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue(null); const nodeId = seedNode(); const bp = seedBlueprint({ @@ -660,7 +660,7 @@ describe('BlueprintReconciler drift alert node wording', () => { it('enforce correction-failure on local uses on this node', async () => { const { NotificationService } = await import('../services/NotificationService'); - const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); vi.spyOn(BlueprintService.getInstance(), 'deployToNode').mockResolvedValue({ status: 'failed', error: 'compose up failed', diff --git a/backend/src/__tests__/database-metrics.test.ts b/backend/src/__tests__/database-metrics.test.ts index 522e0d87..0e4b37c5 100644 --- a/backend/src/__tests__/database-metrics.test.ts +++ b/backend/src/__tests__/database-metrics.test.ts @@ -324,6 +324,7 @@ describe('DatabaseService - stack alerts CRUD', () => { it('adds and retrieves stack alerts', () => { db.addStackAlert({ stack_name: 'alert-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -339,11 +340,27 @@ describe('DatabaseService - stack alerts CRUD', () => { expect(found.threshold).toBe(80); expect(found.duration_mins).toBe(5); expect(found.cooldown_mins).toBe(15); + expect(found.service_name).toBeNull(); + }); + + it('persists a named service_name', () => { + const created = db.addStackAlert({ + stack_name: 'scoped-stack', + service_name: 'api.web', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 15, + }); + expect(created.service_name).toBe('api.web'); + expect(db.getStackAlerts('scoped-stack')[0].service_name).toBe('api.web'); }); it('filters alerts by stack name', () => { db.addStackAlert({ stack_name: 'filter-stack-a', + service_name: null, metric: 'memory_mb', operator: '>=', threshold: 512, @@ -352,6 +369,7 @@ describe('DatabaseService - stack alerts CRUD', () => { }); db.addStackAlert({ stack_name: 'filter-stack-b', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 90, @@ -371,6 +389,7 @@ describe('DatabaseService - stack alerts CRUD', () => { it('updates last_fired_at timestamp', () => { db.addStackAlert({ stack_name: 'fired-stack', + service_name: null, metric: 'net_rx', operator: '>', threshold: 100, @@ -381,29 +400,55 @@ describe('DatabaseService - stack alerts CRUD', () => { const alerts = db.getStackAlerts('fired-stack'); const alert = alerts[0]; const fireTime = Date.now(); - db.updateStackAlertLastFired(alert.id, fireTime); + db.updateStackAlertLastFired(alert.id!, fireTime); const updated = db.getStackAlerts('fired-stack'); expect(updated[0].last_fired_at).toBe(fireTime); }); - it('deletes an alert by id', () => { - db.addStackAlert({ + it('upserts and reads distinct per-service cooldowns', () => { + const alert = db.addStackAlert({ + stack_name: 'cooldown-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 1, + cooldown_mins: 10, + }); + const id = alert.id!; + expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(false); + + db.upsertStackAlertServiceCooldown(id, 'api', 1000); + db.upsertStackAlertServiceCooldown(id, 'database', 2000); + + expect(db.getStackAlertServiceCooldown(id, 'api')).toBe(1000); + expect(db.getStackAlertServiceCooldown(id, 'database')).toBe(2000); + expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(true); + + db.upsertStackAlertServiceCooldown(id, 'api', 3000); + expect(db.getStackAlertServiceCooldown(id, 'api')).toBe(3000); + }); + + it('deletes an alert by id and removes child cooldown rows', () => { + const alert = db.addStackAlert({ stack_name: 'delete-stack', + service_name: null, metric: 'memory_percent', operator: '>', threshold: 95, duration_mins: 1, cooldown_mins: 5, }); + const id = alert.id!; + db.upsertStackAlertServiceCooldown(id, 'api', Date.now()); + expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(true); - const before = db.getStackAlerts('delete-stack'); - expect(before.length).toBe(1); + db.deleteStackAlert(id); - db.deleteStackAlert(before[0].id); - - const after = db.getStackAlerts('delete-stack'); - expect(after.length).toBe(0); + expect(db.getStackAlerts('delete-stack').length).toBe(0); + expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(false); + expect(db.getStackAlertServiceCooldown(id, 'api')).toBeNull(); }); }); diff --git a/backend/src/__tests__/docker-event-service.test.ts b/backend/src/__tests__/docker-event-service.test.ts index 2fb418e3..e06a97a8 100644 --- a/backend/src/__tests__/docker-event-service.test.ts +++ b/backend/src/__tests__/docker-event-service.test.ts @@ -27,7 +27,7 @@ const { mockGetDocker, mockIsOwnContainer, } = vi.hoisted(() => ({ - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockBroadcastEvent: vi.fn(), mockGetGlobalSettings: vi.fn().mockReturnValue({ global_crash: '1' }), mockGetEvents: vi.fn(), diff --git a/backend/src/__tests__/fleet-sync-retry-service.test.ts b/backend/src/__tests__/fleet-sync-retry-service.test.ts index b86c06c3..109c7acc 100644 --- a/backend/src/__tests__/fleet-sync-retry-service.test.ts +++ b/backend/src/__tests__/fleet-sync-retry-service.test.ts @@ -18,7 +18,7 @@ const { mockGetFleetSyncStatuses: vi.fn().mockReturnValue([]), mockGetSystemState: vi.fn().mockReturnValue(null), mockPushResourceToNode: vi.fn().mockResolvedValue(undefined), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), })); vi.mock('../services/DatabaseService', () => ({ diff --git a/backend/src/__tests__/fleet-sync-service.test.ts b/backend/src/__tests__/fleet-sync-service.test.ts index f30430d5..1787687c 100644 --- a/backend/src/__tests__/fleet-sync-service.test.ts +++ b/backend/src/__tests__/fleet-sync-service.test.ts @@ -40,7 +40,7 @@ const { mockGetSystemState: vi.fn().mockReturnValue(null), mockSetSystemState: vi.fn(), mockTransaction: vi.fn().mockImplementation((fn: () => unknown) => fn()), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockAxiosPost: vi.fn().mockResolvedValue({ data: { success: true } }), })); diff --git a/backend/src/__tests__/image-update-service.test.ts b/backend/src/__tests__/image-update-service.test.ts index 65a0b828..3645ba22 100644 --- a/backend/src/__tests__/image-update-service.test.ts +++ b/backend/src/__tests__/image-update-service.test.ts @@ -25,7 +25,7 @@ const { mockGetSystemState: vi.fn().mockReturnValue('1'), // default: backfilled mockSetSystemState: vi.fn(), mockAddNotificationHistory: vi.fn(), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockGetStacks: vi.fn().mockResolvedValue([]), mockGetStackContent: vi.fn().mockResolvedValue(''), mockGetEnvContent: vi.fn().mockRejectedValue(new Error('no env')), diff --git a/backend/src/__tests__/monitor-service.test.ts b/backend/src/__tests__/monitor-service.test.ts index 4d51b855..39003a15 100644 --- a/backend/src/__tests__/monitor-service.test.ts +++ b/backend/src/__tests__/monitor-service.test.ts @@ -9,7 +9,9 @@ import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsM const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContainerMetric, mockCleanupOldMetrics, mockCleanupOldNotifications, mockCleanupOldAuditLogs, - mockUpdateStackAlertLastFired, mockGetSystemState, mockSetSystemState, + mockUpdateStackAlertLastFired, mockGetStackAlertServiceCooldown, + mockHasAnyStackAlertServiceCooldown, mockUpsertStackAlertServiceCooldown, + mockGetSystemState, mockSetSystemState, mockPruneScanHistoryPerImage, mockDeleteScansByImageRef, mockGetRunningContainers, mockGetAllContainers, mockGetContainerStatsStream, mockGetContainerRestartCount, mockGetDiskUsage, mockGetImages, mockGetStacks, @@ -29,6 +31,9 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine mockCleanupOldNotifications: vi.fn(), mockCleanupOldAuditLogs: vi.fn(), mockUpdateStackAlertLastFired: vi.fn(), + mockGetStackAlertServiceCooldown: vi.fn().mockReturnValue(null), + mockHasAnyStackAlertServiceCooldown: vi.fn().mockReturnValue(false), + mockUpsertStackAlertServiceCooldown: vi.fn(), mockGetSystemState: vi.fn().mockReturnValue(null), mockSetSystemState: vi.fn(), mockPruneScanHistoryPerImage: vi.fn().mockReturnValue(0), @@ -43,7 +48,7 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine }), mockGetImages: vi.fn().mockResolvedValue([]), mockGetStacks: vi.fn().mockResolvedValue([]), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockCurrentLoad: vi.fn().mockResolvedValue({ currentLoad: 10 }), mockMem: vi.fn().mockResolvedValue({ total: 16e9, used: 4e9, active: 4e9, available: 12e9, free: 12e9, buffcache: 0 }), mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]), @@ -65,6 +70,9 @@ vi.mock('../services/DatabaseService', () => ({ cleanupOldNotifications: mockCleanupOldNotifications, cleanupOldAuditLogs: mockCleanupOldAuditLogs, updateStackAlertLastFired: mockUpdateStackAlertLastFired, + getStackAlertServiceCooldown: mockGetStackAlertServiceCooldown, + hasAnyStackAlertServiceCooldown: mockHasAnyStackAlertServiceCooldown, + upsertStackAlertServiceCooldown: mockUpsertStackAlertServiceCooldown, getSystemState: mockGetSystemState, setSystemState: mockSetSystemState, pruneScanHistoryPerImage: mockPruneScanHistoryPerImage, @@ -868,8 +876,12 @@ describe('MonitorService - breach state machine', () => { function setupAlertScenario(cpuPercent: number) { mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); mockGetRunningContainers.mockResolvedValue([{ - Id: 'c1', - Labels: { 'com.docker.compose.project': 'my-stack' }, + Id: 'c1abcdefghijk', + Names: ['/my-stack-api-1'], + Labels: { + 'com.docker.compose.project': 'my-stack', + 'com.docker.compose.service': 'api', + }, }]); mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ cpu_stats: { cpu_usage: { total_usage: 1000 + cpuPercent * 50 }, system_cpu_usage: 10000, online_cpus: 1 }, @@ -879,6 +891,7 @@ describe('MonitorService - breach state machine', () => { mockGetStackAlerts.mockReturnValue([{ id: 1, stack_name: 'my-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -887,6 +900,8 @@ describe('MonitorService - breach state machine', () => { last_fired_at: 0, }]); mockGetGlobalSettings.mockReturnValue({}); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); } it('fires alert when condition met and duration is 0', async () => { @@ -898,9 +913,10 @@ describe('MonitorService - breach state machine', () => { expect(mockDispatchAlert).toHaveBeenCalledWith( 'warning', 'monitor_alert', - 'The **CPU usage** for **my-stack** has exceeded your threshold of **80%** (Currently: 90%).', - { stackName: 'my-stack', actor: 'system:monitor' }, + 'The **CPU usage** for **api** in **my-stack** (container **my-stack-api-1**) has exceeded your threshold of **80%** (Currently: 90%).', + { stackName: 'my-stack', containerName: 'my-stack-api-1', actor: 'system:monitor' }, ); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number)); expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number)); }); @@ -913,12 +929,22 @@ describe('MonitorService - breach state machine', () => { expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU')); }); - it('respects cooldown after firing', async () => { + it('respects cooldown after firing via persisted service cooldown', async () => { + setupAlertScenario(90); + mockGetStackAlertServiceCooldown.mockReturnValue(Date.now() - 30 * 60 * 1000); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled(); + }); + + it('respects legacy last_fired_at floor when no child cooldown rows exist', async () => { setupAlertScenario(90); - // Simulate that alert was fired 30 minutes ago (within 60-min cooldown) mockGetStackAlerts.mockReturnValue([{ id: 1, stack_name: 'my-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -926,14 +952,94 @@ describe('MonitorService - breach state machine', () => { cooldown_mins: 60, last_fired_at: Date.now() - 30 * 60 * 1000, }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); const svc = MonitorService.getInstance(); await (svc as any).evaluate(); - expect(mockUpdateStackAlertLastFired).not.toHaveBeenCalled(); + expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled(); }); - it('resets breach state when condition clears', async () => { + it('fires after legacy last_fired_at cooldown expires and writes a child cooldown row', async () => { + setupAlertScenario(90); + mockGetStackAlerts.mockReturnValue([{ + id: 1, + stack_name: 'my-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 60, + last_fired_at: Date.now() - 90 * 60 * 1000, + }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number)); + expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number)); + }); + + it('does not advance cooldown when notification history is not persisted', async () => { + setupAlertScenario(90); + mockDispatchAlert.mockResolvedValueOnce({ persisted: false }); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled(); + expect(mockUpdateStackAlertLastFired).not.toHaveBeenCalled(); + + mockDispatchAlert.mockResolvedValueOnce({ persisted: true }); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number)); + expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number)); + }); + + it('drops in-memory breach timers when a rule is deleted', async () => { + const svc = MonitorService.getInstance(); + (svc as any).activeBreaches.set('55:gone-container', { breachStartedAt: Date.now() }); + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetRunningContainers.mockResolvedValue([]); + mockGetStackAlerts.mockReturnValue([]); + mockGetGlobalSettings.mockReturnValue({}); + + await (svc as any).evaluate(); + + expect((svc as any).activeBreaches.has('55:gone-container')).toBe(false); + }); + + it('names unlabeled containers as unknown service in the alert body', async () => { + setupAlertScenario(90); + mockGetRunningContainers.mockResolvedValue([{ + Id: 'orphan123456789', + Names: ['/orphan'], + Labels: { + 'com.docker.compose.project': 'my-stack', + }, + }]); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'warning', + 'monitor_alert', + expect.stringContaining('**unknown service**'), + expect.objectContaining({ stackName: 'my-stack', containerName: 'orphan' }), + ); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, '_unlabeled', expect.any(Number)); + }); + + it('resets breach state when condition clears for that container only', async () => { const svc = MonitorService.getInstance(); // First: breach starts @@ -941,6 +1047,7 @@ describe('MonitorService - breach state machine', () => { mockGetStackAlerts.mockReturnValue([{ id: 42, stack_name: 'my-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -949,13 +1056,14 @@ describe('MonitorService - breach state machine', () => { last_fired_at: 0, }]); await (svc as any).evaluate(); - expect((svc as any).activeBreaches.has(42)).toBe(true); + expect((svc as any).activeBreaches.has('42:c1abcdefghijk')).toBe(true); // Second: condition clears setupAlertScenario(10); mockGetStackAlerts.mockReturnValue([{ id: 42, stack_name: 'my-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -964,7 +1072,28 @@ describe('MonitorService - breach state machine', () => { last_fired_at: 0, }]); await (svc as any).evaluate(); - expect((svc as any).activeBreaches.has(42)).toBe(false); + expect((svc as any).activeBreaches.has('42:c1abcdefghijk')).toBe(false); + }); + + it('falls back to short container id when Names is absent', async () => { + setupAlertScenario(90); + mockGetRunningContainers.mockResolvedValue([{ + Id: 'abcdef1234567890', + Labels: { + 'com.docker.compose.project': 'my-stack', + 'com.docker.compose.service': 'api', + }, + }]); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'warning', + 'monitor_alert', + 'The **CPU usage** for **api** in **my-stack** (container **abcdef123456**) has exceeded your threshold of **80%** (Currently: 90%).', + { stackName: 'my-stack', containerName: 'abcdef123456', actor: 'system:monitor' }, + ); }); }); @@ -1074,7 +1203,12 @@ describe('MonitorService - restart_count metric', () => { expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1'); // restart_count=5 > threshold=3, should fire - expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Restart count'), { stackName: 'my-stack', actor: 'system:monitor' }); + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'warning', + 'monitor_alert', + expect.stringContaining('Restart count'), + { stackName: 'my-stack', containerName: 'c1', actor: 'system:monitor' }, + ); }); it('skips Docker inspect when no restart_count rules exist', async () => { @@ -1261,6 +1395,8 @@ describe('MonitorService - parallel container processing', () => { mockGetGlobalSettings.mockReturnValue({}); mockGetStackAlerts.mockReturnValue([]); mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); }); it('fans out per-container stats fetches in parallel (wall time ~ max not sum)', async () => { @@ -1311,12 +1447,15 @@ describe('MonitorService - parallel container processing', () => { }); it('dispatches a stack alert exactly once per cycle even when multiple containers in the stack breach', async () => { - // 5 containers in the same stack, all breaching the same rule. Without - // the per-cycle dedup, parallel workers race past the cooldown check - // and each fire dispatchAlert before any DB write lands. + // 5 replicas of the same Compose service, all breaching. Dedup is per + // rule+service, so only one fire should land this cycle. const containers = Array.from({ length: 5 }, (_, i) => ({ Id: `container-${i}`, - Labels: { 'com.docker.compose.project': 'shared-stack' }, + Names: [`/shared-stack-web-${i}`], + Labels: { + 'com.docker.compose.project': 'shared-stack', + 'com.docker.compose.service': 'web', + }, })); mockGetRunningContainers.mockResolvedValue(containers); mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ @@ -1328,6 +1467,7 @@ describe('MonitorService - parallel container processing', () => { mockGetStackAlerts.mockReturnValue([{ id: 7, stack_name: 'shared-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -1335,6 +1475,8 @@ describe('MonitorService - parallel container processing', () => { cooldown_mins: 60, last_fired_at: 0, }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); const svc = MonitorService.getInstance(); await (svc as any).evaluate(); @@ -1343,7 +1485,290 @@ describe('MonitorService - parallel container processing', () => { (args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'), ); expect(cpuDispatches).toHaveLength(1); - expect(mockUpdateStackAlertLastFired).toHaveBeenCalledTimes(1); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledTimes(1); + }); + + it('allows different services on an All-services rule to fire in the same cycle', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'api-1', + Names: ['/stack-api-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + { + Id: 'db-1', + Names: ['/stack-db-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' }, + }, + ]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + mockGetStackAlerts.mockReturnValue([{ + id: 8, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 60, + last_fired_at: 0, + }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); + mockGetGlobalSettings.mockReturnValue({}); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + const cpuDispatches = mockDispatchAlert.mock.calls.filter( + (args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'), + ); + expect(cpuDispatches).toHaveLength(2); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(8, 'api', expect.any(Number)); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(8, 'database', expect.any(Number)); + }); + + it('does not let a healthy sibling clear another container breach timer', async () => { + const svc = MonitorService.getInstance(); + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetStackAlerts.mockReturnValue([{ + id: 9, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 999, + cooldown_mins: 0, + last_fired_at: 0, + }]); + + const highCpu = JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + }); + const lowCpu = JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 1100 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + }); + + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'api-hot', + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + { + Id: 'db-cool', + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' }, + }, + ]); + mockGetContainerStatsStream.mockImplementation(async (id: string) => (id === 'api-hot' ? highCpu : lowCpu)); + + await (svc as any).evaluate(); + + expect((svc as any).activeBreaches.has('9:api-hot')).toBe(true); + expect((svc as any).activeBreaches.has('9:db-cool')).toBe(false); + }); + + it('ignores sibling services for a service-scoped rule', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'api-1', + Names: ['/stack-api-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + { + Id: 'db-1', + Names: ['/stack-db-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' }, + }, + ]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + mockGetStackAlerts.mockReturnValue([{ + id: 10, + stack_name: 'shared-stack', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 0, + last_fired_at: 0, + }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + expect(mockDispatchAlert.mock.calls[0][2]).toContain('**api**'); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(10, 'api', expect.any(Number)); + }); + + it('does not block service B after service A fires under new-schema cooldown', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'db-1', + Names: ['/stack-db-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' }, + }, + ]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + mockGetStackAlerts.mockReturnValue([{ + id: 11, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 60, + last_fired_at: Date.now() - 5 * 60 * 1000, // would block under old rule-wide semantics + }]); + // api already has a cooldown row; database does not, so database may still fire. + mockGetStackAlertServiceCooldown.mockImplementation((_id: number, service: string) => + service === 'api' ? Date.now() - 5 * 60 * 1000 : null, + ); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(true); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(11, 'database', expect.any(Number)); + }); + + it('honors persisted per-service cooldown after a fresh MonitorService instance', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'api-1', + Names: ['/stack-api-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + { + Id: 'api-2', + Names: ['/stack-api-2'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + { + Id: 'db-1', + Names: ['/stack-db-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' }, + }, + ]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + mockGetStackAlerts.mockReturnValue([{ + id: 13, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 60, + last_fired_at: 0, + }]); + mockGetStackAlertServiceCooldown.mockImplementation((_id: number, service: string) => + service === 'api' ? Date.now() - 5 * 60 * 1000 : null, + ); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(true); + + // beforeEach already cleared the singleton; this instance has empty maps. + const svc = MonitorService.getInstance(); + expect((svc as any).activeBreaches.size).toBe(0); + expect((svc as any).firedThisCycle.size).toBe(0); + + await (svc as any).evaluate(); + + const cpuDispatches = mockDispatchAlert.mock.calls.filter( + (args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'), + ); + expect(cpuDispatches).toHaveLength(1); + expect(cpuDispatches[0][2]).toContain('**database**'); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(13, 'database', expect.any(Number)); + expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalledWith(13, 'api', expect.any(Number)); + }); + + it('preserves breach timers when container enumeration fails', async () => { + const svc = MonitorService.getInstance(); + (svc as any).activeBreaches.set('12:still-running', { breachStartedAt: Date.now() }); + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetStackAlerts.mockReturnValue([{ + id: 12, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 999, + cooldown_mins: 0, + last_fired_at: 0, + }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetRunningContainers.mockRejectedValue(new Error('docker down')); + + await (svc as any).evaluate(); + + expect((svc as any).activeBreaches.has('12:still-running')).toBe(true); + }); + + it('removes breach timers for stopped containers after successful enumeration', async () => { + const svc = MonitorService.getInstance(); + (svc as any).activeBreaches.set('13:gone', { breachStartedAt: Date.now() }); + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetStackAlerts.mockReturnValue([{ + id: 13, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 999, + cooldown_mins: 0, + last_fired_at: 0, + }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'still-here', + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + ]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 1100 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + + await (svc as any).evaluate(); + + expect((svc as any).activeBreaches.has('13:gone')).toBe(false); }); it('caps simultaneous Docker calls at MAX_CONTAINER_CONCURRENCY', async () => { diff --git a/backend/src/__tests__/notification-routing.test.ts b/backend/src/__tests__/notification-routing.test.ts index 1bb7dc6a..08125b96 100644 --- a/backend/src/__tests__/notification-routing.test.ts +++ b/backend/src/__tests__/notification-routing.test.ts @@ -217,7 +217,7 @@ describe('NotificationService - routing logic', () => { mockFetch.mockRejectedValueOnce(new Error('Network timeout')); // Should not throw - await expect(svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' })).resolves.toBeUndefined(); + await expect(svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' })).resolves.toEqual({ persisted: true }); }); it('does not dispatch to global agents when routes array is empty and no stackName', async () => { @@ -553,7 +553,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', () await expect( svc.dispatchAlert('error', 'monitor_alert', 'Container crashed', { stackName: 'my-app' }), - ).resolves.toBeUndefined(); + ).resolves.toEqual({ persisted: false }); // No external dispatch and no routing read when there is no persisted row. expect(mockFetch).not.toHaveBeenCalled(); @@ -580,7 +580,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', () await expect( svc.dispatchAlert('error', 'monitor_alert', 'Container crashed', { stackName: 'my-app' }), - ).resolves.toBeUndefined(); + ).resolves.toEqual({ persisted: true }); // The history row was still written; only routing failed, and it was logged. expect(mockAddNotificationHistory).toHaveBeenCalledTimes(1); @@ -598,7 +598,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', () await expect( svc.dispatchAlert('info', 'system', 'Host rebooted'), - ).resolves.toBeUndefined(); + ).resolves.toEqual({ persisted: true }); expect(mockAddNotificationHistory).toHaveBeenCalledTimes(1); expect(consoleErrorSpy).toHaveBeenCalledWith('[Notify] dispatchAlert failed:', expect.any(Error)); diff --git a/backend/src/__tests__/proxy-service-scoped-alert-gate.test.ts b/backend/src/__tests__/proxy-service-scoped-alert-gate.test.ts new file mode 100644 index 00000000..614ab340 --- /dev/null +++ b/backend/src/__tests__/proxy-service-scoped-alert-gate.test.ts @@ -0,0 +1,256 @@ +/** + * Mixed-version gate for POST /api/alerts with a non-empty service_name. + * + * Remote-targeted requests skip express.json() so the raw stream stays + * pipeable. The hub must still buffer that body under the local JSON size + * cap, reject compressed bodies (cannot inspect safely), reject scoped + * creates when the remote lacks service-scoped-stack-alert, and forward + * unscoped identity-encoded JSON intact when allowed. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import http from 'http'; +import type { AddressInfo } from 'net'; +import zlib from 'zlib'; +import request from 'supertest'; +import jwt from 'jsonwebtoken'; +import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb'; + +let tmpDir: string; +let app: import('express').Express; +let listenServer: http.Server; +let listenPort: number; +let adminBearer: string; +let capServer: http.Server; +let noCapServer: http.Server; +let capNodeId: number; +let noCapNodeId: number; + +const noCapPaths: string[] = []; +const capPaths: string[] = []; +let lastCapBody: Buffer | null = null; +let lastNoCapBody: Buffer | null = null; + +function metaServer( + capabilities: string[], + seen: string[], + onBody: (body: Buffer) => void, +): http.Server { + return http.createServer((req, res) => { + if (req.url) seen.push(req.url); + const chunks: Buffer[] = []; + req.on('data', (chunk: Buffer) => chunks.push(chunk)); + req.on('end', () => { + if (!req.url?.startsWith('/api/meta')) { + onBody(Buffer.concat(chunks)); + } + res.writeHead(200, { 'Content-Type': 'application/json' }); + if (req.url?.startsWith('/api/meta')) { + res.end(JSON.stringify({ version: '0.93.0', capabilities })); + } else { + res.end(JSON.stringify({ id: 1, ok: true })); + } + }); + }); +} + +async function listen(server: http.Server): Promise { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return (server.address() as AddressInfo).port; +} + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance(); + + adminBearer = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' }); + + capServer = metaServer( + ['cross-node-rbac', 'service-scoped-stack-alert'], + capPaths, + (body) => { lastCapBody = body; }, + ); + noCapServer = metaServer( + ['cross-node-rbac'], + noCapPaths, + (body) => { lastNoCapBody = body; }, + ); + const capPort = await listen(capServer); + const noCapPort = await listen(noCapServer); + + capNodeId = db.addNode({ + name: 'alert-cap-remote', type: 'remote', mode: 'proxy', compose_dir: '/tmp', + is_default: false, api_url: `http://127.0.0.1:${capPort}`, api_token: 'cap-token', + }); + noCapNodeId = db.addNode({ + name: 'alert-nocap-remote', type: 'remote', mode: 'proxy', compose_dir: '/tmp', + is_default: false, api_url: `http://127.0.0.1:${noCapPort}`, api_token: 'nocap-token', + }); + + listenServer = http.createServer(app); + listenPort = await listen(listenServer); +}); + +afterAll(async () => { + await new Promise((resolve) => listenServer.close(() => resolve())); + await new Promise((resolve) => capServer.close(() => resolve())); + await new Promise((resolve) => noCapServer.close(() => resolve())); + cleanupTestDb(tmpDir); +}); + +beforeEach(() => { + noCapPaths.length = 0; + capPaths.length = 0; + lastNoCapBody = null; + lastCapBody = null; +}); + +const scopedPayload = { + stack_name: 'web', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, +}; + +const unscopedPayload = { + stack_name: 'web', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, +}; + +describe('remote proxy service-scoped alert gate', () => { + it('returns 400 for scoped alert create when remote lacks service-scoped-stack-alert', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Authorization', `Bearer ${adminBearer}`) + .set('x-node-id', String(noCapNodeId)) + .set('Content-Type', 'application/json') + .send(scopedPayload); + + expect(res.status).toBe(400); + expect(res.body.code).toBe('capability_unavailable'); + expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false); + expect(noCapPaths.some((p) => p.startsWith('/api/meta'))).toBe(true); + expect(lastNoCapBody).toBeNull(); + }); + + it('forwards unscoped alert JSON intact when remote lacks the capability', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Authorization', `Bearer ${adminBearer}`) + .set('x-node-id', String(noCapNodeId)) + .set('Content-Type', 'application/json') + .send(unscopedPayload); + + expect(res.status).toBe(200); + expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(true); + expect(lastNoCapBody).not.toBeNull(); + expect(JSON.parse(lastNoCapBody!.toString('utf-8'))).toEqual(unscopedPayload); + }); + + it('forwards scoped alert JSON intact when remote advertises the capability', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Authorization', `Bearer ${adminBearer}`) + .set('x-node-id', String(capNodeId)) + .set('Content-Type', 'application/json') + .send(scopedPayload); + + expect(res.status).toBe(200); + expect(capPaths.some((p) => p.includes('/api/alerts'))).toBe(true); + expect(lastCapBody).not.toBeNull(); + expect(JSON.parse(lastCapBody!.toString('utf-8'))).toEqual(scopedPayload); + }); + + it('rejects oversized identity JSON via Content-Length before upstream', async () => { + const huge = { + ...unscopedPayload, + stack_name: 'x'.repeat(120 * 1024), + }; + + const started = Date.now(); + const res = await request(app) + .post('/api/alerts') + .set('Authorization', `Bearer ${adminBearer}`) + .set('x-node-id', String(noCapNodeId)) + .set('Content-Type', 'application/json') + .send(huge); + const elapsedMs = Date.now() - started; + + expect(res.status).toBe(413); + expect(res.body.code).toBe('entity_too_large'); + expect(elapsedMs).toBeLessThan(2000); + expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false); + expect(lastNoCapBody).toBeNull(); + }); + + it('rejects chunked bodies once streamed bytes exceed the JSON limit', async () => { + const oversized = Buffer.from(JSON.stringify({ + ...unscopedPayload, + stack_name: 'y'.repeat(120 * 1024), + })); + + const result = await new Promise<{ status: number; body: { code?: string } }>((resolve, reject) => { + const req = http.request({ + hostname: '127.0.0.1', + port: listenPort, + method: 'POST', + path: '/api/alerts', + headers: { + Authorization: `Bearer ${adminBearer}`, + 'x-node-id': String(noCapNodeId), + 'Content-Type': 'application/json', + 'Transfer-Encoding': 'chunked', + }, + }, (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => { + const text = Buffer.concat(chunks).toString('utf-8'); + try { + resolve({ status: res.statusCode ?? 0, body: JSON.parse(text) as { code?: string } }); + } catch { + resolve({ status: res.statusCode ?? 0, body: {} }); + } + }); + }); + req.on('error', reject); + // Write in small chunks so the hub crosses the limit mid-stream. + const step = 16 * 1024; + for (let offset = 0; offset < oversized.length; offset += step) { + req.write(oversized.subarray(offset, Math.min(offset + step, oversized.length))); + } + req.end(); + }); + + expect(result.status).toBe(413); + expect(result.body.code).toBe('entity_too_large'); + expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false); + expect(lastNoCapBody).toBeNull(); + }); + + it('rejects gzip-encoded scoped JSON instead of forwarding as All-services', async () => { + const compressed = zlib.gzipSync(Buffer.from(JSON.stringify(scopedPayload), 'utf-8')); + + const res = await request(app) + .post('/api/alerts') + .set('Authorization', `Bearer ${adminBearer}`) + .set('x-node-id', String(noCapNodeId)) + .set('Content-Type', 'application/json') + .set('Content-Encoding', 'gzip') + .send(compressed); + + expect(res.status).toBe(415); + expect(res.body.code).toBe('encoding_unsupported'); + expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false); + expect(lastNoCapBody).toBeNull(); + }); +}); diff --git a/backend/src/__tests__/scheduler-policy.test.ts b/backend/src/__tests__/scheduler-policy.test.ts index 0edca7c4..0739c6af 100644 --- a/backend/src/__tests__/scheduler-policy.test.ts +++ b/backend/src/__tests__/scheduler-policy.test.ts @@ -34,7 +34,7 @@ const { mockMarkStaleRunsAsFailed: vi.fn().mockReturnValue(0), mockDeleteOldScans: vi.fn().mockReturnValue(0), mockGetTier: vi.fn().mockReturnValue('paid'), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockGetProxyTarget: vi.fn().mockReturnValue(null), mockIsTrivyAvailable: vi.fn().mockReturnValue(true), mockScanAllNodeImages: vi.fn(), diff --git a/backend/src/__tests__/scheduler-service.test.ts b/backend/src/__tests__/scheduler-service.test.ts index 18f8ae4f..f629f717 100644 --- a/backend/src/__tests__/scheduler-service.test.ts +++ b/backend/src/__tests__/scheduler-service.test.ts @@ -59,7 +59,7 @@ const { mockGetStackContent: vi.fn().mockResolvedValue(''), mockGetEnvContent: vi.fn().mockResolvedValue(''), mockCheckImage: vi.fn().mockResolvedValue({ hasUpdate: false }), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockGetProxyTarget: vi.fn().mockReturnValue(null), mockIsTrivyAvailable: vi.fn().mockReturnValue(true), mockScanAllNodeImages: vi.fn().mockResolvedValue({ diff --git a/backend/src/__tests__/service-scoped-update-routes.test.ts b/backend/src/__tests__/service-scoped-update-routes.test.ts index f2564257..5811e4e9 100644 --- a/backend/src/__tests__/service-scoped-update-routes.test.ts +++ b/backend/src/__tests__/service-scoped-update-routes.test.ts @@ -45,7 +45,7 @@ beforeAll(async () => { writeStack('web'); const { NotificationService } = await import('../services/NotificationService'); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stack-bulk-routes.test.ts b/backend/src/__tests__/stack-bulk-routes.test.ts index 3d9fe675..f62454c8 100644 --- a/backend/src/__tests__/stack-bulk-routes.test.ts +++ b/backend/src/__tests__/stack-bulk-routes.test.ts @@ -78,7 +78,7 @@ beforeAll(async () => { authCookie = await loginAsTestAdmin(app); const { NotificationService } = await import('../services/NotificationService'); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stack-docker-disconnect.test.ts b/backend/src/__tests__/stack-docker-disconnect.test.ts index 90dfcf8f..f1150bfa 100644 --- a/backend/src/__tests__/stack-docker-disconnect.test.ts +++ b/backend/src/__tests__/stack-docker-disconnect.test.ts @@ -90,7 +90,7 @@ beforeAll(async () => { authCookie = await loginAsTestAdmin(app); const { NotificationService } = await import('../services/NotificationService'); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stack-down-remove-volumes.test.ts b/backend/src/__tests__/stack-down-remove-volumes.test.ts index e390a3d5..117b8bb7 100644 --- a/backend/src/__tests__/stack-down-remove-volumes.test.ts +++ b/backend/src/__tests__/stack-down-remove-volumes.test.ts @@ -68,7 +68,7 @@ beforeAll(async () => { writeStack('web'); const { NotificationService } = await import('../services/NotificationService'); - dispatchAlertSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + dispatchAlertSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stack-op-lock-routes.test.ts b/backend/src/__tests__/stack-op-lock-routes.test.ts index 08a1fb45..4d76ea95 100644 --- a/backend/src/__tests__/stack-op-lock-routes.test.ts +++ b/backend/src/__tests__/stack-op-lock-routes.test.ts @@ -89,7 +89,7 @@ beforeAll(async () => { authCookie = await loginAsTestAdmin(app); const { NotificationService } = await import('../services/NotificationService'); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stack-self-protected-routes.test.ts b/backend/src/__tests__/stack-self-protected-routes.test.ts index 315a33e8..db7330d5 100644 --- a/backend/src/__tests__/stack-self-protected-routes.test.ts +++ b/backend/src/__tests__/stack-self-protected-routes.test.ts @@ -102,7 +102,7 @@ beforeAll(async () => { writeStack('web'); const { NotificationService } = await import('../services/NotificationService'); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stacks-failure-notifications.test.ts b/backend/src/__tests__/stacks-failure-notifications.test.ts index 168f0877..588e311b 100644 --- a/backend/src/__tests__/stacks-failure-notifications.test.ts +++ b/backend/src/__tests__/stacks-failure-notifications.test.ts @@ -132,7 +132,7 @@ beforeAll(async () => { const { NotificationService } = await import('../services/NotificationService'); dispatchAlertSpy = vi .spyOn(NotificationService.getInstance(), 'dispatchAlert') - .mockResolvedValue(undefined); + .mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/proxy/remoteNodeProxy.ts b/backend/src/proxy/remoteNodeProxy.ts index b359dd0c..e3a35185 100644 --- a/backend/src/proxy/remoteNodeProxy.ts +++ b/backend/src/proxy/remoteNodeProxy.ts @@ -5,7 +5,7 @@ import { PROXY_TIER_HEADER, PROXY_ROLE_HEADER, PROXY_DEPLOY_SOURCE_HEADER, PROXY import { LicenseService } from '../services/LicenseService'; import { isProxyExemptPath } from '../helpers/proxyExemptPaths'; import { remoteSupportsCrossNodeRbac, remoteAdvertisesCapability } from '../helpers/remoteCapabilities'; -import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY } from '../services/CapabilityRegistry'; +import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY, SERVICE_SCOPED_STACK_ALERT_CAPABILITY } from '../services/CapabilityRegistry'; import { getErrorMessage } from '../utils/errors'; import { DatabaseService } from '../services/DatabaseService'; import { redactSensitiveText } from '../utils/safeLog'; @@ -145,8 +145,19 @@ export function createRemoteProxyMiddleware(): RequestHandler { } // Body forwarding: conditionalJsonParser skips parsing for remote // requests (see middleware/jsonParser.ts), so req's raw stream is - // intact and http-proxy's req.pipe(proxyReq) forwards the body - // automatically. + // usually intact and http-proxy's req.pipe(proxyReq) forwards it. + // When a gate must inspect JSON (POST /alerts), we buffer into + // req.rawBody first; rewrite that buffer here because the stream is + // already consumed. + if (req.rawBody) { + proxyReq.removeHeader('Transfer-Encoding'); + proxyReq.removeHeader('Content-Length'); + if (!proxyReq.getHeader('Content-Type')) { + proxyReq.setHeader('Content-Type', 'application/json'); + } + proxyReq.setHeader('Content-Length', req.rawBody.length); + proxyReq.write(req.rawBody); + } }, proxyRes: (proxyRes, req) => { // Mark every response forwarded from a remote node with a sentinel @@ -246,7 +257,8 @@ export function createRemoteProxyMiddleware(): RequestHandler { } } - // Mixed-version RBAC gate (non-admin only). + // Mixed-version RBAC gate (non-admin only). Runs before alert body + // buffering so an unauthorized client cannot force unbounded memory use. if (req.user?.role !== 'admin') { const rbacSupported = await remoteSupportsCrossNodeRbac(req.nodeId); if (!rbacSupported) { @@ -257,6 +269,49 @@ export function createRemoteProxyMiddleware(): RequestHandler { } } + // POST /alerts bodies are not on req.body for remote hops (JSON parsing + // is skipped so the stream can be piped). Buffer once under the same + // 100 KB cap as express.json(), gate on service_name, then rewrite + // rawBody in on.proxyReq. Compressed bodies are rejected: the hub cannot + // inspect them, and treating parse failure as unscoped would bypass the + // mixed-version gate. + if (isAlertCreateRoute(req)) { + if (hasNonIdentityContentEncoding(req)) { + await drainRequestBody(req); + res.status(415).json({ + error: 'Compressed request bodies are not supported for remote alert creates', + code: 'encoding_unsupported', + }); + return; + } + try { + req.rawBody = await bufferRequestBody(req, ALERT_PROXY_BODY_LIMIT); + } catch (err) { + const status = Number((err as { status?: number }).status); + if (status === 413) { + console.error('[remoteNodeProxy] alert body rejected as too large:', err); + res.status(413).json({ error: 'Alert payload too large', code: 'entity_too_large' }); + return; + } + if (status === 400) { + console.error('[remoteNodeProxy] alert body incomplete:', err); + res.status(400).json({ error: 'Incomplete request body' }); + return; + } + throw err; + } + if (alertCreateHasScopedService(req.rawBody)) { + const supported = await remoteAdvertisesCapability(req.nodeId, SERVICE_SCOPED_STACK_ALERT_CAPABILITY); + if (!supported) { + res.status(400).json({ + error: 'Service-scoped alert rules are not supported on this node', + code: 'capability_unavailable', + }); + return; + } + } + } + req.proxyTarget = target; beginProxyTiming(req, res); proxy(req, res, next); @@ -283,3 +338,151 @@ function isServiceScopedUpdateRoute(req: Request): boolean { } return false; } + +/** POST /alerts (path is post-/api strip). */ +function isAlertCreateRoute(req: Request): boolean { + return req.method === 'POST' && /^\/alerts\/?$/.test(req.path); +} + +/** Same default as express.json(); remote alert creates must not exceed it. */ +const ALERT_PROXY_BODY_LIMIT = 100 * 1024; + +/** Max time to wait for leftover body bytes after a size/encoding reject. */ +const DRAIN_TIMEOUT_MS = 5_000; + +/** Error with HTTP status for the alert-body gate catch mapper. */ +function alertBodyError(message: string, status: number): Error { + return Object.assign(new Error(message), { status, expose: true }); +} + +/** True when Content-Encoding is present and not identity (gzip/deflate/br/…). */ +function hasNonIdentityContentEncoding(req: Request): boolean { + const raw = req.headers['content-encoding']; + if (raw == null) return false; + const value = Array.isArray(raw) ? raw.join(',') : raw; + return value.split(',').some((part) => { + const encoding = part.trim().toLowerCase(); + return encoding.length > 0 && encoding !== 'identity'; + }); +} + +/** True when the buffered JSON alert body targets a specific Compose service. */ +function alertCreateHasScopedService(rawBody: Buffer): boolean { + if (rawBody.length === 0) return false; + try { + const parsed = JSON.parse(rawBody.toString('utf-8')) as { service_name?: unknown }; + return typeof parsed.service_name === 'string' && parsed.service_name.trim() !== ''; + } catch { + // Identity-encoded non-JSON is forwarded as-is; the remote rejects it. + // Encoded bodies never reach here (rejected earlier). + return false; + } +} + +/** + * Consume remaining request bytes (or wait for abort/close) so the response + * can flush without leaving the socket half-open. Does not buffer into memory. + * Caps wait time so a stalled or endless chunked stream cannot hang the gate. + */ +function drainRequestBody(req: Request): Promise { + return new Promise((resolve) => { + if (req.readableEnded || req.destroyed) { + resolve(); + return; + } + let settled = false; + const done = () => { + if (settled) return; + settled = true; + clearTimeout(timer); + req.off('end', done); + req.off('error', done); + req.off('aborted', done); + req.off('close', done); + resolve(); + }; + // Bound hang time: destroy after timeout so mid-stream rejects still settle. + const timer = setTimeout(() => { + if (!req.destroyed) req.destroy(); + done(); + }, DRAIN_TIMEOUT_MS); + req.resume(); + req.once('end', done); + req.once('error', done); + req.once('aborted', done); + req.once('close', done); + }); +} + +/** + * Buffer the request so a capability gate can inspect JSON without leaving + * http-proxy with an already-ended empty stream. Enforces `limit` on both + * declared Content-Length and streamed accumulation. + */ +async function bufferRequestBody(req: Request, limit: number): Promise { + if (req.rawBody) { + if (req.rawBody.length > limit) { + throw alertBodyError('Alert payload too large', 413); + } + return req.rawBody; + } + if (req.readableEnded) return Buffer.alloc(0); + + const declared = Number.parseInt(String(req.headers['content-length'] ?? ''), 10); + if (Number.isFinite(declared) && declared > limit) { + // Reject immediately so the client gets a structured 413; drain leftover + // bytes in the background so the socket can close without holding the gate. + void drainRequestBody(req); + throw alertBodyError('Alert payload too large', 413); + } + + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let total = 0; + let settled = false; + + const settle = (fn: () => void) => { + if (settled) return; + settled = true; + cleanup(); + fn(); + }; + const finish = (buf: Buffer) => settle(() => resolve(buf)); + const fail = (err: Error) => settle(() => reject(err)); + + const onData = (chunk: Buffer) => { + const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + total += buf.length; + if (total > limit) { + chunks.length = 0; + // fail() removes listeners; drain leftover bytes so the socket can close. + fail(alertBodyError('Alert payload too large', 413)); + void drainRequestBody(req); + return; + } + chunks.push(buf); + }; + const onEnd = () => finish(Buffer.concat(chunks)); + const onError = (err: Error) => fail(err); + const onAborted = () => fail(alertBodyError('Client aborted request body', 400)); + const onClose = () => { + if (!settled && !req.readableEnded) { + fail(alertBodyError('Client closed request before body finished', 400)); + } + }; + + function cleanup(): void { + req.off('data', onData); + req.off('end', onEnd); + req.off('error', onError); + req.off('aborted', onAborted); + req.off('close', onClose); + } + + req.on('data', onData); + req.on('end', onEnd); + req.on('error', onError); + req.on('aborted', onAborted); + req.on('close', onClose); + }); +} diff --git a/backend/src/routes/alerts.ts b/backend/src/routes/alerts.ts index b6bea853..1b707c1a 100644 --- a/backend/src/routes/alerts.ts +++ b/backend/src/routes/alerts.ts @@ -3,9 +3,21 @@ import { z } from 'zod'; import { DatabaseService } from '../services/DatabaseService'; import { authMiddleware } from '../middleware/auth'; import { requireAdmin } from '../middleware/tierGates'; +import { isValidServiceName } from '../utils/validation'; +import { + getActiveCapabilities, + SERVICE_SCOPED_STACK_ALERT_CAPABILITY, +} from '../services/CapabilityRegistry'; const AlertCreateSchema = z.object({ stack_name: z.string().min(1).max(255), + service_name: z.preprocess( + (val) => (val === '' ? null : val), + z.string().max(255).nullable().optional().refine( + (val) => val == null || isValidServiceName(val), + { message: 'Invalid service name' }, + ), + ), metric: z.enum(['cpu_percent', 'memory_percent', 'memory_mb', 'net_rx', 'net_tx', 'restart_count']), operator: z.enum(['>', '>=', '<', '<=', '==']), threshold: z.number().min(0), @@ -22,7 +34,8 @@ alertsRouter.get('/', authMiddleware, async (req: Request, res: Response) => { const alerts = DatabaseService.getInstance().getStackAlerts(stackName); res.json(alerts); - } catch { + } catch (error) { + console.error('Failed to fetch alerts:', error); res.status(500).json({ error: 'Failed to fetch alerts' }); } }); @@ -34,8 +47,23 @@ alertsRouter.post('/', authMiddleware, async (req: Request, res: Response) => { res.status(400).json({ error: 'Invalid alert data', details: parsed.error.flatten().fieldErrors }); return; } + const { service_name, ...alertFields } = parsed.data; + const serviceName = service_name ?? null; + if ( + serviceName != null + && !getActiveCapabilities().includes(SERVICE_SCOPED_STACK_ALERT_CAPABILITY) + ) { + res.status(400).json({ + error: 'This node does not support service-scoped alert rules', + code: 'capability_unavailable', + }); + return; + } try { - const created = DatabaseService.getInstance().addStackAlert(parsed.data); + const created = DatabaseService.getInstance().addStackAlert({ + ...alertFields, + service_name: serviceName, + }); res.status(201).json(created); } catch (error) { console.error('Failed to add alert:', error); @@ -45,11 +73,18 @@ alertsRouter.post('/', authMiddleware, async (req: Request, res: Response) => { alertsRouter.delete('/:id', authMiddleware, async (req: Request, res: Response) => { if (!requireAdmin(req, res)) return; + // Reject leading-junk / fractional ids (parseInt("1abc") === 1, parseInt("2.5") === 2). + const rawId = String(req.params.id ?? ''); + const id = /^\d+$/.test(rawId) ? Number.parseInt(rawId, 10) : NaN; + if (!Number.isInteger(id) || id <= 0) { + res.status(400).json({ error: 'Invalid alert id' }); + return; + } try { - const id = parseInt(req.params.id as string, 10); DatabaseService.getInstance().deleteStackAlert(id); res.json({ success: true }); - } catch { + } catch (error) { + console.error('Failed to delete alert:', error); res.status(500).json({ error: 'Failed to delete alert' }); } }); diff --git a/backend/src/services/CapabilityRegistry.ts b/backend/src/services/CapabilityRegistry.ts index 578155a9..ba806c5d 100644 --- a/backend/src/services/CapabilityRegistry.ts +++ b/backend/src/services/CapabilityRegistry.ts @@ -59,6 +59,7 @@ export const CAPABILITIES = [ 'stack-down-remove-volumes', 'guided-external-network-preflight', 'service-scoped-update', + 'service-scoped-stack-alert', ] as const; /** @@ -88,6 +89,10 @@ export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes' /** Capability for the nested per-service update/restore routes and the `effective-services` model they read. */ export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability; +/** Capability for nullable `service_name` on stack alert rules and per-service cooldown evaluation. */ +export const SERVICE_SCOPED_STACK_ALERT_CAPABILITY = + 'service-scoped-stack-alert' as const satisfies Capability; + /** Returns true when the string is a usable semver version. */ export function isValidVersion(v: string | null | undefined): v is string { return !!v && v !== 'unknown' && v !== '0.0.0-dev' && !!semver.valid(v); diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 2d36dd49..9f66c424 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -101,6 +101,7 @@ function stringifyServicesJson(services: StackServiceStatus[], generation: numbe export interface StackAlert { id?: number; stack_name: string; + service_name: string | null; metric: string; operator: string; threshold: number; @@ -1062,6 +1063,7 @@ export class DatabaseService { this.migrateStackDossierHashes(); this.migrateGitSourceMultiFile(); this.migrateNodeUpdateSkips(); + this.migrateStackAlertServiceScope(); // Reset the cache once at end of constructor in case any migration // populated it via getGlobalSettings() and a subsequent migration @@ -1098,6 +1100,7 @@ export class DatabaseService { CREATE TABLE IF NOT EXISTS stack_alerts ( id INTEGER PRIMARY KEY AUTOINCREMENT, stack_name TEXT NOT NULL, + service_name TEXT, metric TEXT NOT NULL, operator TEXT NOT NULL, threshold REAL NOT NULL, @@ -1106,6 +1109,17 @@ export class DatabaseService { last_fired_at INTEGER DEFAULT 0 ); + -- FK cascade is declarative only: PRAGMA foreign_keys is never enabled + -- on this connection, so parent deletes must remove children explicitly + -- (see deleteStackAlert). + CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns ( + alert_id INTEGER NOT NULL, + service_name TEXT NOT NULL, + last_fired_at INTEGER NOT NULL, + PRIMARY KEY (alert_id, service_name), + FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE + ); + CREATE TABLE IF NOT EXISTS notification_history ( id INTEGER PRIMARY KEY AUTOINCREMENT, node_id INTEGER NOT NULL DEFAULT 0, @@ -2280,6 +2294,25 @@ export class DatabaseService { } } + private migrateStackAlertServiceScope(): void { + this.tryAddColumn('stack_alerts', 'service_name', 'TEXT'); + try { + // FK is not enforced (foreign_keys pragma off); deleteStackAlert removes children. + this.db.prepare(` + CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns ( + alert_id INTEGER NOT NULL, + service_name TEXT NOT NULL, + last_fired_at INTEGER NOT NULL, + PRIMARY KEY (alert_id, service_name), + FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE + ) + `).run(); + } catch (e) { + console.error('[DatabaseService] stack_alert_service_cooldowns migration failed:', (e as Error).message); + throw e; + } + } + private migrateScanPolicyFleetColumns(): void { this.tryAddColumn('scan_policies', 'node_identity', "TEXT NOT NULL DEFAULT ''"); this.tryAddColumn('scan_policies', 'replicated_from_control', 'INTEGER NOT NULL DEFAULT 0'); @@ -3075,22 +3108,19 @@ export class DatabaseService { // --- Stack Alerts --- public getStackAlerts(stackName?: string): StackAlert[] { - let stmt; if (stackName) { - stmt = this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?'); - return stmt.all(stackName) as StackAlert[]; - } else { - stmt = this.db.prepare('SELECT * FROM stack_alerts'); - return stmt.all() as StackAlert[]; + return this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?').all(stackName) as StackAlert[]; } + return this.db.prepare('SELECT * FROM stack_alerts').all() as StackAlert[]; } public addStackAlert(alert: StackAlert): StackAlert { const stmt = this.db.prepare( - 'INSERT INTO stack_alerts (stack_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?)' + 'INSERT INTO stack_alerts (stack_name, service_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)' ); const result = stmt.run( alert.stack_name, + alert.service_name ?? null, alert.metric, alert.operator, alert.threshold, @@ -3101,9 +3131,16 @@ export class DatabaseService { return this.db.prepare('SELECT * FROM stack_alerts WHERE id = ?').get(result.lastInsertRowid) as StackAlert; } + /** + * Delete an alert and its per-service cooldown rows. + * SQLite foreign_keys is not enabled here, so child rows are removed + * explicitly rather than relying on ON DELETE CASCADE. + */ public deleteStackAlert(id: number): void { - const stmt = this.db.prepare('DELETE FROM stack_alerts WHERE id = ?'); - stmt.run(id); + this.transaction(() => { + this.deleteStackAlertServiceCooldowns(id); + this.db.prepare('DELETE FROM stack_alerts WHERE id = ?').run(id); + }); } public updateStackAlertLastFired(id: number, timestamp: number): void { @@ -3111,6 +3148,32 @@ export class DatabaseService { stmt.run(timestamp, id); } + public getStackAlertServiceCooldown(alertId: number, serviceName: string): number | null { + const row = this.db.prepare( + 'SELECT last_fired_at FROM stack_alert_service_cooldowns WHERE alert_id = ? AND service_name = ?' + ).get(alertId, serviceName) as { last_fired_at: number } | undefined; + return row?.last_fired_at ?? null; + } + + public hasAnyStackAlertServiceCooldown(alertId: number): boolean { + const row = this.db.prepare( + 'SELECT 1 AS present FROM stack_alert_service_cooldowns WHERE alert_id = ? LIMIT 1' + ).get(alertId) as { present: number } | undefined; + return !!row; + } + + public upsertStackAlertServiceCooldown(alertId: number, serviceName: string, timestamp: number): void { + this.db.prepare(` + INSERT INTO stack_alert_service_cooldowns (alert_id, service_name, last_fired_at) + VALUES (?, ?, ?) + ON CONFLICT(alert_id, service_name) DO UPDATE SET last_fired_at = excluded.last_fired_at + `).run(alertId, serviceName, timestamp); + } + + public deleteStackAlertServiceCooldowns(alertId: number): void { + this.db.prepare('DELETE FROM stack_alert_service_cooldowns WHERE alert_id = ?').run(alertId); + } + // --- Auto-Heal Policies --- public getAutoHealPolicies(stackName?: string, nodeId?: number): AutoHealPolicy[] { diff --git a/backend/src/services/DockerEventService.ts b/backend/src/services/DockerEventService.ts index 1bcc9a3f..638b00ca 100644 --- a/backend/src/services/DockerEventService.ts +++ b/backend/src/services/DockerEventService.ts @@ -735,15 +735,15 @@ export class DockerEventService { } private async emitError(category: NotificationCategory, message: string, stackName?: string, containerName?: string, systemOnly = false): Promise { - return this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly)); + await this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly)); } private async emitWarning(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise { - return this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName)); + await this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName)); } private async emitInfo(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise { - return this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName)); + await this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName)); } // ======================================================================== diff --git a/backend/src/services/MonitorService.ts b/backend/src/services/MonitorService.ts index 4840b08c..4df2a575 100644 --- a/backend/src/services/MonitorService.ts +++ b/backend/src/services/MonitorService.ts @@ -32,6 +32,39 @@ const getOperatorPhrase = (operator: string): string => { return `triggered the operator ${operator}`; }; +/** Sentinel for containers without a Compose service label. Not a valid API target. */ +const UNLABELED_SERVICE_KEY = '_unlabeled'; + +function displayServiceName(serviceName: string): string { + return serviceName === UNLABELED_SERVICE_KEY ? 'unknown service' : serviceName; +} + +function parseAlertBreachKey(key: string): { ruleId: number; containerId: string } | null { + const sep = key.indexOf(':'); + if (sep < 0) return null; + const ruleId = Number(key.slice(0, sep)); + if (!Number.isFinite(ruleId)) return null; + return { ruleId, containerId: key.slice(sep + 1) }; +} + +/** Per-service cooldown timestamp, with pre-migration fallback to rule.last_fired_at. */ +function resolveStackAlertLastFired(rule: StackAlert, serviceName: string, db: DatabaseService): number { + const ruleId = rule.id!; + let lastFired = db.getStackAlertServiceCooldown(ruleId, serviceName); + if (lastFired == null && !db.hasAnyStackAlertServiceCooldown(ruleId) && (rule.last_fired_at || 0) > 0) { + lastFired = rule.last_fired_at || 0; + } + return lastFired || 0; +} + +function normalizeContainerName(container: { Id: string; Names?: string[] }): string { + const raw = container.Names?.[0]; + if (raw && raw.length > 0) { + return raw.startsWith('/') ? raw.slice(1) : raw; + } + return container.Id.slice(0, 12); +} + /** Shape of the JSON returned by Docker container stats (stream: false). */ interface DockerContainerStats { cpu_stats?: { @@ -153,21 +186,18 @@ export class MonitorService { private janitorConsecutiveTimeouts = 0; private janitorBreakerUntil = 0; - // Track the duration a specific stack alert rule has been in breach state - // key: rule_id, value: AlertState - private activeBreaches = new Map(); + // Track the duration a specific stack alert rule+container has been in breach + // key: `${ruleId}:${containerId}`, value: AlertState + private activeBreaches = new Map(); // Track previous network counters per container for rate calculation. // key: container_id, value: { rx bytes, tx bytes, sample timestamp } private previousNetworkStats = new Map(); - // Per-cycle dispatch dedup. Stack rules are shared across every container - // in the stack, so parallel processContainer calls can race past the - // cooldown check (DB write happens after the awaited dispatch) and fire - // the same alert N times on first breach. The synchronous check-and-add - // below is atomic in JS between awaits, so only the first worker wins. + // Per-cycle dispatch dedup keyed by rule+service so replicas of the same + // Compose service do not fire N times, while different services can. // Reset at the start of each evaluateStackAlerts call. - private firedThisCycle = new Set(); + private firedThisCycle = new Set(); // Crash and healthcheck detection live in DockerEventService (event-driven, // causal classification). MonitorService no longer polls for container @@ -604,6 +634,10 @@ export class MonitorService { else alertsByStack.set(a.stack_name, [a]); } + const activeRuleIds = new Set(alerts.map(a => a.id!)); + const allCurrentIds = new Set(); + let enumerationFailed = false; + for (const node of nodes) { if (!node.id) continue; // Remote nodes are self-monitoring - skip direct Docker access @@ -623,22 +657,39 @@ export class MonitorService { (container) => this.processContainer(node, container, alertsByStack, docker, db), ); + for (const c of containers) allCurrentIds.add(c.Id); + // Clean up stale network stats for containers on this node that no longer run if (this.previousNetworkStats.size > containers.length * 2) { - const currentIds = new Set(containers.map((c: { Id: string }) => c.Id)); + const nodeIds = new Set(containers.map((c: { Id: string }) => c.Id)); for (const key of this.previousNetworkStats.keys()) { - if (!currentIds.has(key)) this.previousNetworkStats.delete(key); + if (!nodeIds.has(key)) this.previousNetworkStats.delete(key); } } } catch (err) { + // Enumeration failed: preserve existing breach timers. + enumerationFailed = true; console.error(`Error fetching containers for node ${node.name}`, err); } } - // Clean up stale breach trackers for rules that have been deleted - const activeRuleIds = new Set(alerts.map(a => a.id!)); - for (const key of this.activeBreaches.keys()) { - if (!activeRuleIds.has(key)) { + // After successful enumeration(s), drop breach timers for containers + // that are no longer running. Skip when any local enumeration failed + // so a transient Docker error cannot reset duration timers. + if (!enumerationFailed) { + for (const key of [...this.activeBreaches.keys()]) { + const parsed = parseAlertBreachKey(key); + if (parsed && activeRuleIds.has(parsed.ruleId) && !allCurrentIds.has(parsed.containerId)) { + this.activeBreaches.delete(key); + } + } + } + + // Clean up in-memory breach trackers for rules that have been deleted. + // Persisted cooldowns are removed only via transactional deleteStackAlert. + for (const key of [...this.activeBreaches.keys()]) { + const parsed = parseAlertBreachKey(key); + if (parsed && !activeRuleIds.has(parsed.ruleId)) { this.activeBreaches.delete(key); } } @@ -666,12 +717,14 @@ export class MonitorService { */ private async processContainer( node: Node, - container: { Id: string; Labels?: Record }, + container: { Id: string; Names?: string[]; Labels?: Record }, alertsByStack: Map, docker: DockerController, db: DatabaseService, ): Promise { const stackName = container.Labels?.['com.docker.compose.project'] || 'system'; + const serviceName = container.Labels?.['com.docker.compose.service'] || UNLABELED_SERVICE_KEY; + const containerName = normalizeContainerName(container); try { const rawStats = await withTimeout( @@ -727,7 +780,11 @@ export class MonitorService { }); for (const rule of stackAlerts) { + if (rule.service_name && rule.service_name !== serviceName) continue; + const ruleId = rule.id!; + const breachKey = `${ruleId}:${container.Id}`; + const cooldownKey = `${ruleId}:${serviceName}`; const currentValue = metrics[rule.metric as keyof typeof metrics]; if (currentValue === undefined) continue; @@ -735,57 +792,75 @@ export class MonitorService { const isBreaching = this.evaluateCondition(currentValue, rule.operator, rule.threshold); if (isBreaching) { - if (!this.activeBreaches.has(ruleId)) { - this.activeBreaches.set(ruleId, { breachStartedAt: Date.now() }); - if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}"`); + if (!this.activeBreaches.has(breachKey)) { + this.activeBreaches.set(breachKey, { breachStartedAt: Date.now() }); + if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}" service "${serviceName}" container "${containerName}"`); } - const breachState = this.activeBreaches.get(ruleId)!; + const breachState = this.activeBreaches.get(breachKey)!; const durationMs = Date.now() - breachState.breachStartedAt; const requiredDurationMs = rule.duration_mins * 60 * 1000; if (durationMs >= requiredDurationMs) { - const timeSinceLastFired = Date.now() - (rule.last_fired_at || 0); + const timeSinceLastFired = Date.now() - resolveStackAlertLastFired(rule, serviceName, db); const requiredCooldownMs = rule.cooldown_mins * 60 * 1000; if (timeSinceLastFired >= requiredCooldownMs) { - // Claim this rule for the cycle before awaiting - // dispatch. The check-and-add is synchronous, so - // sibling workers evaluating the same shared rule - // see the claim and skip — preventing N-fire when - // multiple containers in one stack all breach. - if (this.firedThisCycle.has(ruleId)) { - if (isDebugEnabled()) console.log(`[Monitor:diag] Skipping duplicate dispatch for rule ${ruleId} (already fired this cycle by sibling container)`); - } else { - this.firedThisCycle.add(ruleId); + // Claim this rule+service for the cycle before awaiting + // dispatch so replicas of the same service skip. + if (this.firedThisCycle.has(cooldownKey)) { + if (isDebugEnabled()) console.log(`[Monitor:diag] Skipping duplicate dispatch for rule ${ruleId} service "${serviceName}" (already fired this cycle by sibling replica)`); + continue; + } - const { name: metricName, unit } = getMetricDetails(rule.metric); - const operatorPhrase = getOperatorPhrase(rule.operator); + this.firedThisCycle.add(cooldownKey); - const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue; - const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold; + const { name: metricName, unit } = getMetricDetails(rule.metric); + const operatorPhrase = getOperatorPhrase(rule.operator); - // Node-neutral body: the hub bell badge attributes remotes. - const message = `The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`; + const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue; + const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold; + const serviceLabel = displayServiceName(serviceName); - console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`); - await NotificationService.getInstance().dispatchAlert( + // Node-neutral body: the hub bell badge attributes remotes. + const message = `The **${metricName}** for **${serviceLabel}** in **${rule.stack_name}** (container **${containerName}**) ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`; + + try { + const { persisted } = await NotificationService.getInstance().dispatchAlert( 'warning', 'monitor_alert', message, - { stackName: rule.stack_name, actor: 'system:monitor' }, + { stackName: rule.stack_name, containerName, actor: 'system:monitor' }, + ); + if (!persisted) { + // History was not written; do not advance cooldown or we silence retries. + this.firedThisCycle.delete(cooldownKey); + console.error( + `[MonitorService] Alert history not persisted for rule ${ruleId} service "${serviceName}"; cooldown not advanced`, + ); + continue; + } + const firedAt = Date.now(); + // Dual-write: per-service row for current code, parent last_fired_at + // so a downgrade that only reads the parent column still has a floor. + db.upsertStackAlertServiceCooldown(ruleId, serviceName, firedAt); + db.updateStackAlertLastFired(ruleId, firedAt); + console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}" service "${serviceName}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`); + } catch (fireErr) { + this.firedThisCycle.delete(cooldownKey); + console.error( + `[MonitorService] Failed to fire alert rule ${ruleId} service "${serviceName}" container "${containerName}":`, + fireErr, ); - - db.updateStackAlertLastFired(ruleId, Date.now()); } } else if (isDebugEnabled()) { - console.log(`[Monitor:diag] Cooldown active for rule ${ruleId}: ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`); + console.log(`[Monitor:diag] Cooldown active for rule ${ruleId} service "${serviceName}": ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`); } } } else { - if (this.activeBreaches.has(ruleId)) { - if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}"`); - this.activeBreaches.delete(ruleId); + if (this.activeBreaches.has(breachKey)) { + if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}" container "${containerName}"`); + this.activeBreaches.delete(breachKey); } } } diff --git a/backend/src/services/NotificationService.ts b/backend/src/services/NotificationService.ts index 2a6ebbee..ba414ce0 100644 --- a/backend/src/services/NotificationService.ts +++ b/backend/src/services/NotificationService.ts @@ -181,7 +181,7 @@ export class NotificationService { category: NotificationCategory, message: string, options?: { stackName?: string; containerName?: string; actor?: string }, - ) { + ): Promise<{ persisted: boolean }> { const t0 = Date.now(); const { stackName, containerName, actor } = options ?? {}; @@ -191,6 +191,8 @@ export class NotificationService { // WebSocket broadcast can all throw on an unhealthy DB, which would // otherwise surface as an unhandledRejection and take the process down. // The whole body is wrapped so the worst case is a dropped notification. + // Callers that gate cooldowns on history use `persisted` (true only after the row write). + let wroteHistory = false; try { // Internal writes use the middleware default so they share a row key // with user-initiated requests; otherwise the UI and monitors split @@ -216,11 +218,12 @@ export class NotificationService { container_name: containerName, actor_username: actor ?? null, }); + wroteHistory = true; StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, true); } catch (err) { StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, false); console.error('[Notify] Failed to persist notification:', err); - return; + return { persisted: false }; } // Separate [StackActivity:diag] namespace from the [Notify:diag] lines // below so a single grep can pull every per-stack timeline write across @@ -273,7 +276,7 @@ export class NotificationService { } if (suppressExternal) { - return; + return { persisted: wroteHistory }; } // Resolve retry extras once for this dispatch (shared by all destinations). @@ -298,14 +301,14 @@ export class NotificationService { ) ); this.recordDispatchErrors(notification.id!, errors); - return; + return { persisted: wroteHistory }; } // 4. Fall back to this instance's agents (keyed by this instance's default node id). const agents = this.dbService.getEnabledAgents(localNodeId); if (agents.length === 0) { if (isDebugEnabled()) console.log('[Notify:diag] No routes or agents matched; skipping external dispatch'); - return; + return { persisted: wroteHistory }; } if (isDebugEnabled()) console.log(`[Notify:diag] Falling back to ${agents.length} global agent(s)`); @@ -322,8 +325,11 @@ export class NotificationService { ) ); this.recordDispatchErrors(notification.id!, errors); + return { persisted: wroteHistory }; } catch (err) { console.error('[Notify] dispatchAlert failed:', err); + // History may already be written; callers must not treat that as a miss. + return { persisted: wroteHistory }; } } diff --git a/docs/features/alerts-notifications.mdx b/docs/features/alerts-notifications.mdx index efa0acc8..ee67c6b3 100644 --- a/docs/features/alerts-notifications.mdx +++ b/docs/features/alerts-notifications.mdx @@ -1,10 +1,10 @@ --- title: Alerts & Notifications sidebarTitle: Alerts and notifications -description: Threshold and event alerts for your fleet, dispatched to Discord, Slack, Apprise, or any webhook, with per-stack rules and channel routing. +description: Threshold and event alerts for your fleet, dispatched to Discord, Slack, Apprise, or any webhook, with stack and service rules and channel routing. --- -Sencho watches each node it manages for container crashes, host pressure, scheduled-task results, and update availability, then surfaces every signal in two places: the in-app notification bell at the top of the shell and an external channel you configure. This page covers everything from configuring channels to writing per-stack threshold rules, routing alerts to dedicated channels with routing rules, and tuning retention. +Sencho watches each node it manages for container crashes, host pressure, scheduled-task results, and update availability, then surfaces every signal in two places: the in-app notification bell at the top of the shell and an external channel you configure. This page covers everything from configuring channels to writing stack and service threshold rules, routing alerts to dedicated channels with routing rules, and tuning retention. Settings · Notifications · Channels panel with NODE Local in the header, a Delivery retries row showing Extra attempts 0 and a Save retries button, Discord Slack Webhook and Apprise tabs with Apprise selected, Enabled off, an empty Apprise endpoint placeholder, and Test beside Save. @@ -37,7 +37,7 @@ Use the Generic Webhook tab when you have your own receiver: a Mattermost or Tea ```json { "level": "warning", - "message": "The **CPU usage** for **plex** has exceeded your threshold of **80%** (Currently: 91%).", + "message": "The **CPU usage** for **api** in **plex** (container **plex-api-1**) has exceeded your threshold of **80%** (Currently: 91%).", "timestamp": "2026-05-08T22:14:09.812Z", "source": "sencho" } @@ -161,9 +161,11 @@ Every alert Sencho dispatches carries a category that you can filter on in the b The four `blueprint_*` categories are accepted by routing rules but render as raw category strings in the bell because the frontend label map omits them. -## Per-stack alert rules +## Stack alert rules -Each stack carries its own set of threshold rules that fire when a metric stays above (or below) a value for a configurable window. Rules live on the node where the stack runs and are evaluated locally on a 30-second tick. +Each stack carries its own set of threshold rules that fire when a metric stays above (or below) a value for a configurable window. A rule can target **All services** in the stack or one Compose service. Metrics are evaluated per running container (not as a stack aggregate): each container has its own duration timer, and cooldown is tracked per Compose service so replicas of the same service share one silence window while different services can alert independently. + +Rules live on the node where the stack runs and are evaluated locally on a 30-second tick. Open the rules editor by right-clicking a stack in the sidebar and choosing **Alerts**, or by pressing `A` while focused on a stack. @@ -175,13 +177,14 @@ Open the rules editor by right-clicking a stack in the sidebar and choosing **Al | Field | Purpose | |-------|---------| +| **Service** | **All services** (default) or one Compose service from the stack. All services evaluates every matching container independently. | | **Metric** | The system resource or metric to monitor. | | **Operator** | Comparison: `Greater than`, `Greater or eq`, `Less than`, `Less or eq`, `Equals`. | | **Threshold** | A number ≥ 0. The unit follows the chosen metric. | -| **Duration (mins)** | How long the breach must persist before firing. Default `5`, range 0 to 1440. | -| **Cooldown (mins)** | Silence window between fires after a rule triggers. Default `60`, range 0 to 10080. | +| **Duration (mins)** | How long a container's breach must persist before firing. Default `5`, range 0 to 1440. | +| **Cooldown (mins)** | Silence window between fires for the same Compose service after a rule triggers. Default `60`, range 0 to 10080. | -Sencho tracks the start of each breach in memory; the rule fires only after the breach has lasted for the full **Duration**, and only if the previous fire is older than **Cooldown**. +Sencho tracks the start of each breach per container in memory; the rule fires only after that container's breach has lasted for the full **Duration**, and only if the previous fire for the same Compose service is older than **Cooldown**. The notification names the triggering service and container. Containers without a Compose service label are grouped under one shared cooldown key and appear as **unknown service** in the message; that label is not selectable as a rule target. A scoped rule whose service is no longer listed in the compose file shows **Not in compose** in the rules list; evaluation still matches running containers that carry that Compose service label. ### Available metrics @@ -441,7 +444,7 @@ Switching the active node tears down per-stack rule editors and reloads channel Check three things in order. First, the channel toggle in **Settings · Notifications · Channels** must be on; the kicker on each tab reads `enabled` or `off`. Second, Discord, Slack, and webhook URLs must use HTTPS (the form rejects plain `http://` for those channels); Apprise endpoints may use HTTP or HTTPS. Third, a routing rule with unconstrained Node plus empty Stacks, Labels, Categories, and Severity matchers will intercept every alert and skip the global channels. Use the per-channel **Test** button to issue a one-shot dispatch and watch your endpoint for the literal message `🔌 Test Notification from Sencho!` Sencho records the failure reason in `notification_history.dispatch_error` when delivery throws, so a row that appears in the bell with no follow-up at the endpoint usually means a 4xx or timeout at the receiver. - Three causes account for almost every case. First, the rule's **Duration** has not elapsed yet: the breach must persist for the full duration before the rule fires. Second, the rule is still in cooldown after a previous fire. Third, the panel's banner is not green: a remote-node banner means the rule was saved on a remote whose channels you may not have configured, and an amber `No notification channels configured` banner means the rule evaluates fine but Sencho has nowhere to send the alert. The evaluator runs on a 30-second tick, so expect up to 30 seconds of latency between the breach starting and the timer engaging. + Three causes account for almost every case. First, the rule's **Duration** has not elapsed yet: that container's breach must persist for the full duration before the rule fires (a healthy sibling service does not clear another container's timer). Second, the same Compose service is still in cooldown after a previous fire. Third, the panel's banner is not green: a remote-node banner means the rule was saved on a remote whose channels you may not have configured, and an amber `No notification channels configured` banner means the rule evaluates fine but Sencho has nowhere to send the alert. The evaluator runs on a 30-second tick, so expect up to 30 seconds of latency between the breach starting and the timer engaging. The Docker event service defers `die` classification by 500 ms to absorb out-of-order `kill` events from the daemon, then asks the lifecycle classifier whether the exit was intentional, clean (exit 0), a crash, or an OOM kill. If your stop happened far enough outside that window, or the daemon emitted the events without the kill marker the classifier looks for, the exit can be classified as a crash. The classifier favors avoiding silent crashes over avoiding noisy false positives. Compare the alert timestamp against your `docker compose down` time; entries within a second of each other are usually the same event seen from two angles. diff --git a/frontend/src/components/StackAlertSheet.test.tsx b/frontend/src/components/StackAlertSheet.test.tsx new file mode 100644 index 00000000..46f413cf --- /dev/null +++ b/frontend/src/components/StackAlertSheet.test.tsx @@ -0,0 +1,248 @@ +/** + * StackAlertSheet Alerts tab: service targeting, services-state machine, + * capability gating, and active-node reset. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, waitFor, fireEvent } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +const { nodeState } = vi.hoisted(() => ({ + nodeState: { + activeNode: { id: 1, type: 'local', name: 'Local' } as { id: number; type: string; name: string } | null, + activeNodeMeta: { + version: '1.0.0', + capabilities: ['service-scoped-stack-alert'], + } as { version: string; capabilities: string[] } | null, + }, +})); + +vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() })); +vi.mock('@/components/ui/toast-store', () => ({ + toast: { + error: vi.fn(), + success: vi.fn(), + warning: vi.fn(), + info: vi.fn(), + loading: vi.fn(), + dismiss: vi.fn(), + }, +})); +vi.mock('@/context/NodeContext', () => ({ + useNodes: () => ({ + activeNode: nodeState.activeNode, + activeNodeMeta: nodeState.activeNodeMeta, + hasCapability: (cap: string) => nodeState.activeNodeMeta?.capabilities.includes(cap) === true, + }), +})); +vi.mock('@/context/AuthContext', () => ({ + useAuth: () => ({ isAdmin: true }), +})); + +import { apiFetch } from '@/lib/api'; +import { toast } from '@/components/ui/toast-store'; +import { StackAlertSheet } from './StackAlertSheet'; + +const mockedFetch = apiFetch as unknown as ReturnType; + +function jsonRes(body: unknown, ok = true, status = ok ? 200 : 500) { + return { + ok, + status, + json: async () => body, + text: async () => '', + } as unknown as Response; +} + +beforeEach(() => { + nodeState.activeNode = { id: 1, type: 'local', name: 'Local' }; + nodeState.activeNodeMeta = { + version: '1.0.0', + capabilities: ['service-scoped-stack-alert'], + }; + mockedFetch.mockReset(); + vi.mocked(toast.success).mockReset(); + vi.mocked(toast.error).mockReset(); +}); + +function mockHappyPath(services: string[] = ['api', 'database'], alerts: unknown[] = []) { + mockedFetch.mockImplementation(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.includes('/agents')) { + return jsonRes([{ type: 'discord', enabled: true }]); + } + if (url.includes('/services')) { + return jsonRes(services); + } + if (url.startsWith('/alerts') && (!init || !init.method || init.method === 'GET')) { + return jsonRes(alerts); + } + if (url === '/alerts' && init?.method === 'POST') { + const body = JSON.parse(String(init.body)); + return jsonRes({ id: 99, ...body }, true, 201); + } + return jsonRes(null, false); + }); +} + +describe('StackAlertSheet Alerts tab', () => { + it('POSTs selected service_name when capability is present', async () => { + mockHappyPath(); + const user = userEvent.setup(); + render( {}} stackName="my-stack" />); + + await waitFor(() => expect(screen.getByText('Add Rule')).toBeInTheDocument()); + await waitFor(() => { + expect(mockedFetch.mock.calls.some(([url]) => String(url).includes('/services'))).toBe(true); + }); + + // Service combobox is the first one in the Add new rule form. + await waitFor(() => { + const serviceBox = screen.getAllByRole('combobox')[0]; + expect(serviceBox).not.toBeDisabled(); + }); + await user.click(screen.getAllByRole('combobox')[0]); + await user.click(await screen.findByRole('button', { name: 'api' })); + + fireEvent.change(screen.getByPlaceholderText('e.g. 90'), { target: { value: '80' } }); + await user.click(screen.getByText('Add Rule')); + + await waitFor(() => { + const post = mockedFetch.mock.calls.find( + ([url, init]) => String(url) === '/alerts' && (init as RequestInit | undefined)?.method === 'POST', + ); + expect(post).toBeDefined(); + const body = JSON.parse(String((post![1] as RequestInit).body)); + expect(body.service_name).toBe('api'); + expect(body.stack_name).toBe('my-stack'); + }); + }); + + it('POSTs service_name null for All services', async () => { + mockHappyPath(); + const user = userEvent.setup(); + render( {}} stackName="my-stack" />); + await waitFor(() => expect(screen.getByText('Add Rule')).toBeInTheDocument()); + + fireEvent.change(screen.getByPlaceholderText('e.g. 90'), { target: { value: '80' } }); + await user.click(screen.getByText('Add Rule')); + + await waitFor(() => { + const post = mockedFetch.mock.calls.find( + ([url, init]) => String(url) === '/alerts' && (init as RequestInit | undefined)?.method === 'POST', + ); + const body = JSON.parse(String((post![1] as RequestInit).body)); + expect(body.service_name).toBeNull(); + }); + }); + + it('shows Not in compose only after a successful services list', async () => { + mockHappyPath(['database'], [{ + id: 1, + stack_name: 'my-stack', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }]); + + render( {}} stackName="my-stack" />); + + await waitFor(() => { + expect(screen.getByText(/Not in compose/i)).toBeInTheDocument(); + }); + }); + + it('does not show Not in compose when services fetch fails', async () => { + mockedFetch.mockImplementation(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes('/agents')) return jsonRes([{ type: 'discord', enabled: true }]); + if (url.includes('/services')) return jsonRes(null, false, 500); + if (url.includes('/alerts')) { + return jsonRes([{ + id: 1, + stack_name: 'my-stack', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }]); + } + return jsonRes(null, false); + }); + + render( {}} stackName="my-stack" />); + + await waitFor(() => expect(screen.getByText('api')).toBeInTheDocument()); + expect(screen.queryByText(/Not in compose/i)).not.toBeInTheDocument(); + }); + + it('hides service selector when capability is missing and posts null', async () => { + nodeState.activeNodeMeta = { version: '0.90.0', capabilities: [] }; + mockHappyPath(); + const user = userEvent.setup(); + + render( {}} stackName="my-stack" />); + + await waitFor(() => expect(screen.getByText('Add Rule')).toBeInTheDocument()); + expect(screen.queryByText(/does not support service-scoped/i)).toBeInTheDocument(); + expect(screen.queryByRole('combobox', { name: /all services/i })).not.toBeInTheDocument(); + + fireEvent.change(screen.getByPlaceholderText('e.g. 90'), { target: { value: '80' } }); + await user.click(screen.getByText('Add Rule')); + + await waitFor(() => { + const post = mockedFetch.mock.calls.find( + ([url, init]) => String(url) === '/alerts' && (init as RequestInit | undefined)?.method === 'POST', + ); + const body = JSON.parse(String((post![1] as RequestInit).body)); + expect(body.service_name).toBeNull(); + }); + }); + + it('resets services state when active node changes', async () => { + let servicesCalls = 0; + mockedFetch.mockImplementation(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes('/agents')) return jsonRes([{ type: 'discord', enabled: true }]); + if (url.includes('/services')) { + servicesCalls += 1; + return jsonRes(servicesCalls === 1 ? ['api'] : ['worker']); + } + if (url.includes('/alerts')) { + return jsonRes([{ + id: 1, + stack_name: 'my-stack', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }]); + } + return jsonRes(null, false); + }); + + const { rerender } = render( + {}} stackName="my-stack" />, + ); + + await waitFor(() => expect(servicesCalls).toBe(1)); + await waitFor(() => expect(screen.queryByText(/Not in compose/i)).not.toBeInTheDocument()); + + nodeState.activeNode = { id: 2, type: 'remote', name: 'Remote' }; + nodeState.activeNodeMeta = { + version: '1.0.0', + capabilities: ['service-scoped-stack-alert'], + }; + rerender( {}} stackName="my-stack" />); + + await waitFor(() => expect(servicesCalls).toBe(2)); + // New node's list has only worker, so the api-targeted rule is missing. + await waitFor(() => expect(screen.getByText(/Not in compose/i)).toBeInTheDocument()); + }); +}); diff --git a/frontend/src/components/StackAlertSheet.tsx b/frontend/src/components/StackAlertSheet.tsx index ee665527..fbe70db0 100644 --- a/frontend/src/components/StackAlertSheet.tsx +++ b/frontend/src/components/StackAlertSheet.tsx @@ -20,12 +20,14 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/comp import { Trash2, HelpCircle, AlertTriangle, Info, CheckCircle2, Loader2, ChevronDown, ChevronUp } from 'lucide-react'; import { toast } from '@/components/ui/toast-store'; import { apiFetch } from '@/lib/api'; +import { SERVICE_SCOPED_STACK_ALERT_CAPABILITY } from '@/lib/capabilities'; import { useNodes } from '@/context/NodeContext'; import { useAuth } from '@/context/AuthContext'; interface StackAlert { id?: number; stack_name: string; + service_name: string | null; metric: string; operator: string; threshold: number; @@ -33,6 +35,11 @@ interface StackAlert { cooldown_mins: number; } +type ServicesState = + | { status: 'idle' | 'loading' } + | { status: 'success'; options: string[] } + | { status: 'error' }; + interface AutoHealPolicy { id?: number; node_id: number; @@ -161,8 +168,9 @@ export function StackAlertSheet({ open, onOpenChange, stackName, initialTab = 'a function AlertsTab({ stackName }: { stackName: string }) { const { isAdmin } = useAuth(); - const { activeNode } = useNodes(); + const { activeNode, activeNodeMeta } = useNodes(); const isRemote = activeNode?.type === 'remote'; + const canScopeService = activeNodeMeta?.capabilities.includes(SERVICE_SCOPED_STACK_ALERT_CAPABILITY) === true; const [alerts, setAlerts] = useState([]); const [isLoading, setIsLoading] = useState(false); @@ -172,7 +180,9 @@ function AlertsTab({ stackName }: { stackName: string }) { hasEnabled: false, enabledTypes: [], }); + const [servicesState, setServicesState] = useState({ status: 'idle' }); + const [service, setService] = useState(''); const [metric, setMetric] = useState('cpu_percent'); const [operator, setOperator] = useState('>'); const [threshold, setThreshold] = useState(''); @@ -183,7 +193,33 @@ function AlertsTab({ stackName }: { stackName: string }) { if (!stackName) return; fetchAlerts(); fetchAgentStatus(); - }, [stackName]); // eslint-disable-line react-hooks/exhaustive-deps + }, [stackName, activeNode?.id]); // eslint-disable-line react-hooks/exhaustive-deps + + useEffect(() => { + if (!stackName || !canScopeService) { + setServicesState({ status: 'idle' }); + setService(''); + return; + } + + let cancelled = false; + setServicesState({ status: 'loading' }); + setService(''); + + void (async () => { + try { + const res = await apiFetch(`/stacks/${encodeURIComponent(stackName)}/services`); + if (!res.ok) throw new Error(`services ${res.status}`); + const names = await res.json() as string[]; + if (!cancelled) setServicesState({ status: 'success', options: names }); + } catch (e) { + console.error('[StackAlertSheet] Failed to fetch stack services', e); + if (!cancelled) setServicesState({ status: 'error' }); + } + })(); + + return () => { cancelled = true; }; + }, [stackName, activeNode?.id, canScopeService]); const fetchAlerts = async () => { try { @@ -224,8 +260,10 @@ function AlertsTab({ stackName }: { stackName: string }) { return; } setIsLoading(true); + const scopedServiceName = canScopeService && service !== '' ? service : null; const newAlert = { stack_name: stackName, + service_name: scopedServiceName, metric, operator, threshold: parseFloat(threshold), @@ -240,6 +278,7 @@ function AlertsTab({ stackName }: { stackName: string }) { if (res.ok) { toast.success('Alert rule added.'); setThreshold(''); + setService(''); fetchAlerts(); } else { const err = await res.json().catch(() => ({})); @@ -272,6 +311,32 @@ function AlertsTab({ stackName }: { stackName: string }) { } }; + const serviceComboOptions = [ + { value: '', label: 'All services' }, + ...(servicesState.status === 'success' + ? servicesState.options.map(n => ({ value: n, label: n })) + : []), + ]; + + const renderTargetLabel = (alert: StackAlert) => { + if (!alert.service_name) { + return All services; + } + const missing = servicesState.status === 'success' + && !servicesState.options.includes(alert.service_name); + if (missing) { + return ( + + {alert.service_name} (Not in compose) + + ); + } + return {alert.service_name}; + }; + const renderAgentStatusBanner = () => { if (agentStatus.loading) { return ( @@ -355,7 +420,8 @@ function AlertsTab({ stackName }: { stackName: string }) { {metricLabels[alert.metric] || alert.metric} {alert.operator} {alert.threshold}
- Trigger after {alert.duration_mins}m • Cooldown {alert.cooldown_mins}m + {renderTargetLabel(alert)} + {' '}• Trigger after {alert.duration_mins}m • Cooldown {alert.cooldown_mins}m
{isAdmin && ( @@ -378,6 +444,31 @@ function AlertsTab({ stackName }: { stackName: string }) { {isAdmin && (
+ {canScopeService && ( +
+ + + {servicesState.status === 'error' && ( +

+ Could not load Compose services. The rule will target all services. +

+ )} +
+ )} + {!canScopeService && activeNodeMeta && ( +

+ This node does not support service-scoped alert rules yet. Update the node to target a specific service. +

+ )} +
diff --git a/frontend/src/lib/capabilities.ts b/frontend/src/lib/capabilities.ts index 76292bae..887ccedd 100644 --- a/frontend/src/lib/capabilities.ts +++ b/frontend/src/lib/capabilities.ts @@ -37,6 +37,7 @@ export const CAPABILITIES = [ 'stack-down-remove-volumes', 'guided-external-network-preflight', 'service-scoped-update', + 'service-scoped-stack-alert', ] as const; export type Capability = (typeof CAPABILITIES)[number]; @@ -50,3 +51,4 @@ export const HOST_CONSOLE_COMMUNITY_CAPABILITY = 'host-console-community' as con export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes' as const satisfies Capability; export const GUIDED_EXTERNAL_NETWORK_PREFLIGHT_CAPABILITY = 'guided-external-network-preflight' as const satisfies Capability; export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability; +export const SERVICE_SCOPED_STACK_ALERT_CAPABILITY = 'service-scoped-stack-alert' as const satisfies Capability;