feat: add service-scoped stack alert rules (#1681)

* feat: add service-scoped stack alert rules

Stack alerts can target one Compose service or all services. Breach timers
are per container and cooldowns are per service so a healthy sibling no
longer clears another container's timer or silences a different service.

* fix: gate remote scoped alert creates without losing the body

Remote hops skip JSON parsing so the proxy stream stays pipeable, which
left service_name invisible to the capability gate. Buffer POST /alerts
bodies for inspection, fail closed when the remote lacks the capability,
and rewrite the buffered bytes on forward. Restore alert-panel alt text
to match the unchanged screenshot.

* fix: bound remote alert body buffer and reject encoded JSON

Cap proxied POST /alerts buffering at the local 100KB JSON limit with
structured 413 cleanup, reject non-identity Content-Encoding with 415 so
compressed scoped bodies cannot bypass the mixed-version gate, and cover
oversized, chunked, and gzip regressions.

* fix: harden service-scoped alert delete, cooldown, and proxy gates

Reject non-digit alert ids, dual-write last_fired_at for rollback safety,
gate cooldown on persisted notification history, fail-fast oversized proxy
bodies with 413, and clarify Not in compose UI semantics.

* test: expect dispatchAlert persisted result in crash-safety cases

Update notification-routing assertions for the new { persisted } return
shape so CI matches the cooldown-gating contract.
This commit is contained in:
Anso
2026-07-23 17:57:04 -04:00
committed by GitHub
parent dd54a2e483
commit 85842cc547
32 changed files with 1736 additions and 135 deletions
@@ -55,7 +55,7 @@ beforeEach(() => {
const db = DatabaseService.getInstance();
db.getDb().prepare('DELETE FROM auto_heal_history').run();
db.getDb().prepare('DELETE FROM auto_heal_policies').run();
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue();
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
});
afterAll(() => {
+146 -2
View File
@@ -57,8 +57,8 @@ describe('GET /api/alerts', () => {
it('filters alerts by stackName query param', async () => {
// Seed two alerts for different stacks
const db = DatabaseService.getInstance();
db.addStackAlert({ stack_name: 'web', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
db.addStackAlert({ stack_name: 'api', metric: 'memory_percent', operator: '>', threshold: 90, duration_mins: 5, cooldown_mins: 60 });
db.addStackAlert({ stack_name: 'web', service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
db.addStackAlert({ stack_name: 'api', service_name: null, metric: 'memory_percent', operator: '>', threshold: 90, duration_mins: 5, cooldown_mins: 60 });
const res = await request(app)
.get('/api/alerts?stackName=web')
@@ -105,10 +105,101 @@ describe('POST /api/alerts', () => {
expect(res.status).toBe(201);
expect(res.body.id).toBeDefined();
expect(res.body.stack_name).toBe('new-stack');
expect(res.body.service_name).toBeNull();
expect(res.body.metric).toBe('memory_percent');
expect(res.body.threshold).toBe(85);
});
it('persists a valid dotted service_name', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({
stack_name: 'svc-stack',
service_name: 'api.web',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 60,
});
expect(res.status).toBe(201);
expect(res.body.service_name).toBe('api.web');
});
it('normalizes empty service_name to null', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({
stack_name: 'empty-svc',
service_name: '',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 60,
});
expect(res.status).toBe(201);
expect(res.body.service_name).toBeNull();
});
it('rejects whitespace-only service_name', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({
stack_name: 'bad-svc',
service_name: ' ',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 60,
});
expect(res.status).toBe(400);
});
it('rejects reserved _unlabeled service_name', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({
stack_name: 'bad-svc',
service_name: '_unlabeled',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 60,
});
expect(res.status).toBe(400);
});
it('rejects service_name when the capability is disabled', async () => {
const { disableCapability, enableCapability, SERVICE_SCOPED_STACK_ALERT_CAPABILITY } =
await import('../services/CapabilityRegistry');
disableCapability(SERVICE_SCOPED_STACK_ALERT_CAPABILITY);
try {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({
stack_name: 'cap-stack',
service_name: 'api',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 60,
});
expect(res.status).toBe(400);
expect(res.body.code).toBe('capability_unavailable');
} finally {
enableCapability(SERVICE_SCOPED_STACK_ALERT_CAPABILITY);
}
});
it('validates required fields and returns 400 for missing data', async () => {
const res = await request(app)
.post('/api/alerts')
@@ -202,6 +293,7 @@ describe('DELETE /api/alerts/:id', () => {
// Create an alert to delete
const created = DatabaseService.getInstance().addStackAlert({
stack_name: 'delete-me',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 90,
@@ -216,6 +308,58 @@ describe('DELETE /api/alerts/:id', () => {
expect(res.status).toBe(200);
expect(res.body.success).toBe(true);
});
it('rejects leading-junk ids like 1abc without deleting alert 1', async () => {
const created = DatabaseService.getInstance().addStackAlert({
stack_name: 'strict-id-junk',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 90,
duration_mins: 0,
cooldown_mins: 0,
});
expect(created.id).toBeDefined();
const res = await request(app)
.delete(`/api/alerts/${created.id}abc`)
.set('Cookie', authCookie);
expect(res.status).toBe(400);
expect(res.body.error).toBe('Invalid alert id');
expect(DatabaseService.getInstance().getStackAlerts('strict-id-junk').some((a) => a.id === created.id)).toBe(true);
});
it('rejects fractional ids like 2.5 without deleting alert 2', async () => {
const first = DatabaseService.getInstance().addStackAlert({
stack_name: 'strict-id-fraction',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 0,
cooldown_mins: 0,
});
const second = DatabaseService.getInstance().addStackAlert({
stack_name: 'strict-id-fraction',
service_name: null,
metric: 'memory_percent',
operator: '>',
threshold: 80,
duration_mins: 0,
cooldown_mins: 0,
});
expect(second.id).toBeDefined();
const res = await request(app)
.delete(`/api/alerts/${second.id}.5`)
.set('Cookie', authCookie);
expect(res.status).toBe(400);
expect(res.body.error).toBe('Invalid alert id');
const remaining = DatabaseService.getInstance().getStackAlerts('strict-id-fraction').map((a) => a.id);
expect(remaining).toEqual(expect.arrayContaining([first.id, second.id]));
});
});
// --- POST /api/notifications/test ---
@@ -168,7 +168,7 @@ describe('Rollback notifications (M-2)', () => {
mockTier('paid');
mockDeployStack.mockResolvedValue(undefined);
const { NotificationService } = await import('../services/NotificationService');
const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
const res = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie);
expect(res.status).toBe(200);
@@ -186,7 +186,7 @@ describe('Rollback notifications (M-2)', () => {
mockTier('paid');
mockDeployStack.mockRejectedValueOnce(new Error('restore failed'));
const { NotificationService } = await import('../services/NotificationService');
const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
const res = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie);
expect(res.status).toBe(500);
+4 -4
View File
@@ -600,7 +600,7 @@ describe('BlueprintReconciler drift alert node wording', () => {
it('suggest-mode local drift uses on this node', async () => {
const { NotificationService } = await import('../services/NotificationService');
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
const nodeId = seedNode();
const bp = seedBlueprint({ name: 'drift-local', drift_mode: 'suggest', nodeIds: [nodeId] });
const node = DatabaseService.getInstance().getNode(nodeId)!;
@@ -618,7 +618,7 @@ describe('BlueprintReconciler drift alert node wording', () => {
it('suggest-mode remote drift keeps the authoritative node name', async () => {
const { NotificationService } = await import('../services/NotificationService');
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
const nodeId = seedRemoteNode('sencho-test-02');
const bp = seedBlueprint({ name: 'drift-remote', drift_mode: 'suggest', nodeIds: [nodeId] });
const node = DatabaseService.getInstance().getNode(nodeId)!;
@@ -636,7 +636,7 @@ describe('BlueprintReconciler drift alert node wording', () => {
it('stateful marker-loss on local uses on this node', async () => {
const { NotificationService } = await import('../services/NotificationService');
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue(null);
const nodeId = seedNode();
const bp = seedBlueprint({
@@ -660,7 +660,7 @@ describe('BlueprintReconciler drift alert node wording', () => {
it('enforce correction-failure on local uses on this node', async () => {
const { NotificationService } = await import('../services/NotificationService');
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
vi.spyOn(BlueprintService.getInstance(), 'deployToNode').mockResolvedValue({
status: 'failed',
error: 'compose up failed',
+54 -9
View File
@@ -324,6 +324,7 @@ describe('DatabaseService - stack alerts CRUD', () => {
it('adds and retrieves stack alerts', () => {
db.addStackAlert({
stack_name: 'alert-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
@@ -339,11 +340,27 @@ describe('DatabaseService - stack alerts CRUD', () => {
expect(found.threshold).toBe(80);
expect(found.duration_mins).toBe(5);
expect(found.cooldown_mins).toBe(15);
expect(found.service_name).toBeNull();
});
it('persists a named service_name', () => {
const created = db.addStackAlert({
stack_name: 'scoped-stack',
service_name: 'api.web',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 15,
});
expect(created.service_name).toBe('api.web');
expect(db.getStackAlerts('scoped-stack')[0].service_name).toBe('api.web');
});
it('filters alerts by stack name', () => {
db.addStackAlert({
stack_name: 'filter-stack-a',
service_name: null,
metric: 'memory_mb',
operator: '>=',
threshold: 512,
@@ -352,6 +369,7 @@ describe('DatabaseService - stack alerts CRUD', () => {
});
db.addStackAlert({
stack_name: 'filter-stack-b',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 90,
@@ -371,6 +389,7 @@ describe('DatabaseService - stack alerts CRUD', () => {
it('updates last_fired_at timestamp', () => {
db.addStackAlert({
stack_name: 'fired-stack',
service_name: null,
metric: 'net_rx',
operator: '>',
threshold: 100,
@@ -381,29 +400,55 @@ describe('DatabaseService - stack alerts CRUD', () => {
const alerts = db.getStackAlerts('fired-stack');
const alert = alerts[0];
const fireTime = Date.now();
db.updateStackAlertLastFired(alert.id, fireTime);
db.updateStackAlertLastFired(alert.id!, fireTime);
const updated = db.getStackAlerts('fired-stack');
expect(updated[0].last_fired_at).toBe(fireTime);
});
it('deletes an alert by id', () => {
db.addStackAlert({
it('upserts and reads distinct per-service cooldowns', () => {
const alert = db.addStackAlert({
stack_name: 'cooldown-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 1,
cooldown_mins: 10,
});
const id = alert.id!;
expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(false);
db.upsertStackAlertServiceCooldown(id, 'api', 1000);
db.upsertStackAlertServiceCooldown(id, 'database', 2000);
expect(db.getStackAlertServiceCooldown(id, 'api')).toBe(1000);
expect(db.getStackAlertServiceCooldown(id, 'database')).toBe(2000);
expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(true);
db.upsertStackAlertServiceCooldown(id, 'api', 3000);
expect(db.getStackAlertServiceCooldown(id, 'api')).toBe(3000);
});
it('deletes an alert by id and removes child cooldown rows', () => {
const alert = db.addStackAlert({
stack_name: 'delete-stack',
service_name: null,
metric: 'memory_percent',
operator: '>',
threshold: 95,
duration_mins: 1,
cooldown_mins: 5,
});
const id = alert.id!;
db.upsertStackAlertServiceCooldown(id, 'api', Date.now());
expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(true);
const before = db.getStackAlerts('delete-stack');
expect(before.length).toBe(1);
db.deleteStackAlert(id);
db.deleteStackAlert(before[0].id);
const after = db.getStackAlerts('delete-stack');
expect(after.length).toBe(0);
expect(db.getStackAlerts('delete-stack').length).toBe(0);
expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(false);
expect(db.getStackAlertServiceCooldown(id, 'api')).toBeNull();
});
});
@@ -27,7 +27,7 @@ const {
mockGetDocker,
mockIsOwnContainer,
} = vi.hoisted(() => ({
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
mockBroadcastEvent: vi.fn(),
mockGetGlobalSettings: vi.fn().mockReturnValue({ global_crash: '1' }),
mockGetEvents: vi.fn(),
@@ -18,7 +18,7 @@ const {
mockGetFleetSyncStatuses: vi.fn().mockReturnValue([]),
mockGetSystemState: vi.fn().mockReturnValue(null),
mockPushResourceToNode: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
}));
vi.mock('../services/DatabaseService', () => ({
@@ -40,7 +40,7 @@ const {
mockGetSystemState: vi.fn().mockReturnValue(null),
mockSetSystemState: vi.fn(),
mockTransaction: vi.fn().mockImplementation((fn: () => unknown) => fn()),
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
mockAxiosPost: vi.fn().mockResolvedValue({ data: { success: true } }),
}));
@@ -25,7 +25,7 @@ const {
mockGetSystemState: vi.fn().mockReturnValue('1'), // default: backfilled
mockSetSystemState: vi.fn(),
mockAddNotificationHistory: vi.fn(),
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
mockGetStacks: vi.fn().mockResolvedValue([]),
mockGetStackContent: vi.fn().mockResolvedValue(''),
mockGetEnvContent: vi.fn().mockRejectedValue(new Error('no env')),
+443 -18
View File
@@ -9,7 +9,9 @@ import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsM
const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContainerMetric,
mockCleanupOldMetrics, mockCleanupOldNotifications, mockCleanupOldAuditLogs,
mockUpdateStackAlertLastFired, mockGetSystemState, mockSetSystemState,
mockUpdateStackAlertLastFired, mockGetStackAlertServiceCooldown,
mockHasAnyStackAlertServiceCooldown, mockUpsertStackAlertServiceCooldown,
mockGetSystemState, mockSetSystemState,
mockPruneScanHistoryPerImage, mockDeleteScansByImageRef,
mockGetRunningContainers, mockGetAllContainers, mockGetContainerStatsStream,
mockGetContainerRestartCount, mockGetDiskUsage, mockGetImages, mockGetStacks,
@@ -29,6 +31,9 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
mockCleanupOldNotifications: vi.fn(),
mockCleanupOldAuditLogs: vi.fn(),
mockUpdateStackAlertLastFired: vi.fn(),
mockGetStackAlertServiceCooldown: vi.fn().mockReturnValue(null),
mockHasAnyStackAlertServiceCooldown: vi.fn().mockReturnValue(false),
mockUpsertStackAlertServiceCooldown: vi.fn(),
mockGetSystemState: vi.fn().mockReturnValue(null),
mockSetSystemState: vi.fn(),
mockPruneScanHistoryPerImage: vi.fn().mockReturnValue(0),
@@ -43,7 +48,7 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
}),
mockGetImages: vi.fn().mockResolvedValue([]),
mockGetStacks: vi.fn().mockResolvedValue([]),
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
mockCurrentLoad: vi.fn().mockResolvedValue({ currentLoad: 10 }),
mockMem: vi.fn().mockResolvedValue({ total: 16e9, used: 4e9, active: 4e9, available: 12e9, free: 12e9, buffcache: 0 }),
mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]),
@@ -65,6 +70,9 @@ vi.mock('../services/DatabaseService', () => ({
cleanupOldNotifications: mockCleanupOldNotifications,
cleanupOldAuditLogs: mockCleanupOldAuditLogs,
updateStackAlertLastFired: mockUpdateStackAlertLastFired,
getStackAlertServiceCooldown: mockGetStackAlertServiceCooldown,
hasAnyStackAlertServiceCooldown: mockHasAnyStackAlertServiceCooldown,
upsertStackAlertServiceCooldown: mockUpsertStackAlertServiceCooldown,
getSystemState: mockGetSystemState,
setSystemState: mockSetSystemState,
pruneScanHistoryPerImage: mockPruneScanHistoryPerImage,
@@ -868,8 +876,12 @@ describe('MonitorService - breach state machine', () => {
function setupAlertScenario(cpuPercent: number) {
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetRunningContainers.mockResolvedValue([{
Id: 'c1',
Labels: { 'com.docker.compose.project': 'my-stack' },
Id: 'c1abcdefghijk',
Names: ['/my-stack-api-1'],
Labels: {
'com.docker.compose.project': 'my-stack',
'com.docker.compose.service': 'api',
},
}]);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 1000 + cpuPercent * 50 }, system_cpu_usage: 10000, online_cpus: 1 },
@@ -879,6 +891,7 @@ describe('MonitorService - breach state machine', () => {
mockGetStackAlerts.mockReturnValue([{
id: 1,
stack_name: 'my-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
@@ -887,6 +900,8 @@ describe('MonitorService - breach state machine', () => {
last_fired_at: 0,
}]);
mockGetGlobalSettings.mockReturnValue({});
mockGetStackAlertServiceCooldown.mockReturnValue(null);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
}
it('fires alert when condition met and duration is 0', async () => {
@@ -898,9 +913,10 @@ describe('MonitorService - breach state machine', () => {
expect(mockDispatchAlert).toHaveBeenCalledWith(
'warning',
'monitor_alert',
'The **CPU usage** for **my-stack** has exceeded your threshold of **80%** (Currently: 90%).',
{ stackName: 'my-stack', actor: 'system:monitor' },
'The **CPU usage** for **api** in **my-stack** (container **my-stack-api-1**) has exceeded your threshold of **80%** (Currently: 90%).',
{ stackName: 'my-stack', containerName: 'my-stack-api-1', actor: 'system:monitor' },
);
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number));
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
});
@@ -913,12 +929,22 @@ describe('MonitorService - breach state machine', () => {
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'));
});
it('respects cooldown after firing', async () => {
it('respects cooldown after firing via persisted service cooldown', async () => {
setupAlertScenario(90);
mockGetStackAlertServiceCooldown.mockReturnValue(Date.now() - 30 * 60 * 1000);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled();
});
it('respects legacy last_fired_at floor when no child cooldown rows exist', async () => {
setupAlertScenario(90);
// Simulate that alert was fired 30 minutes ago (within 60-min cooldown)
mockGetStackAlerts.mockReturnValue([{
id: 1,
stack_name: 'my-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
@@ -926,14 +952,94 @@ describe('MonitorService - breach state machine', () => {
cooldown_mins: 60,
last_fired_at: Date.now() - 30 * 60 * 1000,
}]);
mockGetStackAlertServiceCooldown.mockReturnValue(null);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockUpdateStackAlertLastFired).not.toHaveBeenCalled();
expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled();
});
it('resets breach state when condition clears', async () => {
it('fires after legacy last_fired_at cooldown expires and writes a child cooldown row', async () => {
setupAlertScenario(90);
mockGetStackAlerts.mockReturnValue([{
id: 1,
stack_name: 'my-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 0,
cooldown_mins: 60,
last_fired_at: Date.now() - 90 * 60 * 1000,
}]);
mockGetStackAlertServiceCooldown.mockReturnValue(null);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number));
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
});
it('does not advance cooldown when notification history is not persisted', async () => {
setupAlertScenario(90);
mockDispatchAlert.mockResolvedValueOnce({ persisted: false });
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled();
expect(mockUpdateStackAlertLastFired).not.toHaveBeenCalled();
mockDispatchAlert.mockResolvedValueOnce({ persisted: true });
await (svc as any).evaluate();
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number));
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
});
it('drops in-memory breach timers when a rule is deleted', async () => {
const svc = MonitorService.getInstance();
(svc as any).activeBreaches.set('55:gone-container', { breachStartedAt: Date.now() });
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetRunningContainers.mockResolvedValue([]);
mockGetStackAlerts.mockReturnValue([]);
mockGetGlobalSettings.mockReturnValue({});
await (svc as any).evaluate();
expect((svc as any).activeBreaches.has('55:gone-container')).toBe(false);
});
it('names unlabeled containers as unknown service in the alert body', async () => {
setupAlertScenario(90);
mockGetRunningContainers.mockResolvedValue([{
Id: 'orphan123456789',
Names: ['/orphan'],
Labels: {
'com.docker.compose.project': 'my-stack',
},
}]);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockDispatchAlert).toHaveBeenCalledWith(
'warning',
'monitor_alert',
expect.stringContaining('**unknown service**'),
expect.objectContaining({ stackName: 'my-stack', containerName: 'orphan' }),
);
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, '_unlabeled', expect.any(Number));
});
it('resets breach state when condition clears for that container only', async () => {
const svc = MonitorService.getInstance();
// First: breach starts
@@ -941,6 +1047,7 @@ describe('MonitorService - breach state machine', () => {
mockGetStackAlerts.mockReturnValue([{
id: 42,
stack_name: 'my-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
@@ -949,13 +1056,14 @@ describe('MonitorService - breach state machine', () => {
last_fired_at: 0,
}]);
await (svc as any).evaluate();
expect((svc as any).activeBreaches.has(42)).toBe(true);
expect((svc as any).activeBreaches.has('42:c1abcdefghijk')).toBe(true);
// Second: condition clears
setupAlertScenario(10);
mockGetStackAlerts.mockReturnValue([{
id: 42,
stack_name: 'my-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
@@ -964,7 +1072,28 @@ describe('MonitorService - breach state machine', () => {
last_fired_at: 0,
}]);
await (svc as any).evaluate();
expect((svc as any).activeBreaches.has(42)).toBe(false);
expect((svc as any).activeBreaches.has('42:c1abcdefghijk')).toBe(false);
});
it('falls back to short container id when Names is absent', async () => {
setupAlertScenario(90);
mockGetRunningContainers.mockResolvedValue([{
Id: 'abcdef1234567890',
Labels: {
'com.docker.compose.project': 'my-stack',
'com.docker.compose.service': 'api',
},
}]);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockDispatchAlert).toHaveBeenCalledWith(
'warning',
'monitor_alert',
'The **CPU usage** for **api** in **my-stack** (container **abcdef123456**) has exceeded your threshold of **80%** (Currently: 90%).',
{ stackName: 'my-stack', containerName: 'abcdef123456', actor: 'system:monitor' },
);
});
});
@@ -1074,7 +1203,12 @@ describe('MonitorService - restart_count metric', () => {
expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1');
// restart_count=5 > threshold=3, should fire
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Restart count'), { stackName: 'my-stack', actor: 'system:monitor' });
expect(mockDispatchAlert).toHaveBeenCalledWith(
'warning',
'monitor_alert',
expect.stringContaining('Restart count'),
{ stackName: 'my-stack', containerName: 'c1', actor: 'system:monitor' },
);
});
it('skips Docker inspect when no restart_count rules exist', async () => {
@@ -1261,6 +1395,8 @@ describe('MonitorService - parallel container processing', () => {
mockGetGlobalSettings.mockReturnValue({});
mockGetStackAlerts.mockReturnValue([]);
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetStackAlertServiceCooldown.mockReturnValue(null);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
});
it('fans out per-container stats fetches in parallel (wall time ~ max not sum)', async () => {
@@ -1311,12 +1447,15 @@ describe('MonitorService - parallel container processing', () => {
});
it('dispatches a stack alert exactly once per cycle even when multiple containers in the stack breach', async () => {
// 5 containers in the same stack, all breaching the same rule. Without
// the per-cycle dedup, parallel workers race past the cooldown check
// and each fire dispatchAlert before any DB write lands.
// 5 replicas of the same Compose service, all breaching. Dedup is per
// rule+service, so only one fire should land this cycle.
const containers = Array.from({ length: 5 }, (_, i) => ({
Id: `container-${i}`,
Labels: { 'com.docker.compose.project': 'shared-stack' },
Names: [`/shared-stack-web-${i}`],
Labels: {
'com.docker.compose.project': 'shared-stack',
'com.docker.compose.service': 'web',
},
}));
mockGetRunningContainers.mockResolvedValue(containers);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
@@ -1328,6 +1467,7 @@ describe('MonitorService - parallel container processing', () => {
mockGetStackAlerts.mockReturnValue([{
id: 7,
stack_name: 'shared-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
@@ -1335,6 +1475,8 @@ describe('MonitorService - parallel container processing', () => {
cooldown_mins: 60,
last_fired_at: 0,
}]);
mockGetStackAlertServiceCooldown.mockReturnValue(null);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
@@ -1343,7 +1485,290 @@ describe('MonitorService - parallel container processing', () => {
(args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'),
);
expect(cpuDispatches).toHaveLength(1);
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledTimes(1);
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledTimes(1);
});
it('allows different services on an All-services rule to fire in the same cycle', async () => {
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetRunningContainers.mockResolvedValue([
{
Id: 'api-1',
Names: ['/stack-api-1'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
},
{
Id: 'db-1',
Names: ['/stack-db-1'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
},
]);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
}));
mockGetStackAlerts.mockReturnValue([{
id: 8,
stack_name: 'shared-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 0,
cooldown_mins: 60,
last_fired_at: 0,
}]);
mockGetStackAlertServiceCooldown.mockReturnValue(null);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
mockGetGlobalSettings.mockReturnValue({});
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
const cpuDispatches = mockDispatchAlert.mock.calls.filter(
(args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'),
);
expect(cpuDispatches).toHaveLength(2);
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(8, 'api', expect.any(Number));
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(8, 'database', expect.any(Number));
});
it('does not let a healthy sibling clear another container breach timer', async () => {
const svc = MonitorService.getInstance();
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetGlobalSettings.mockReturnValue({});
mockGetStackAlerts.mockReturnValue([{
id: 9,
stack_name: 'shared-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 999,
cooldown_mins: 0,
last_fired_at: 0,
}]);
const highCpu = JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
});
const lowCpu = JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 1100 }, system_cpu_usage: 10000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
});
mockGetRunningContainers.mockResolvedValue([
{
Id: 'api-hot',
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
},
{
Id: 'db-cool',
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
},
]);
mockGetContainerStatsStream.mockImplementation(async (id: string) => (id === 'api-hot' ? highCpu : lowCpu));
await (svc as any).evaluate();
expect((svc as any).activeBreaches.has('9:api-hot')).toBe(true);
expect((svc as any).activeBreaches.has('9:db-cool')).toBe(false);
});
it('ignores sibling services for a service-scoped rule', async () => {
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetGlobalSettings.mockReturnValue({});
mockGetRunningContainers.mockResolvedValue([
{
Id: 'api-1',
Names: ['/stack-api-1'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
},
{
Id: 'db-1',
Names: ['/stack-db-1'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
},
]);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
}));
mockGetStackAlerts.mockReturnValue([{
id: 10,
stack_name: 'shared-stack',
service_name: 'api',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 0,
cooldown_mins: 0,
last_fired_at: 0,
}]);
mockGetStackAlertServiceCooldown.mockReturnValue(null);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
expect(mockDispatchAlert.mock.calls[0][2]).toContain('**api**');
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(10, 'api', expect.any(Number));
});
it('does not block service B after service A fires under new-schema cooldown', async () => {
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetGlobalSettings.mockReturnValue({});
mockGetRunningContainers.mockResolvedValue([
{
Id: 'db-1',
Names: ['/stack-db-1'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
},
]);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
}));
mockGetStackAlerts.mockReturnValue([{
id: 11,
stack_name: 'shared-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 0,
cooldown_mins: 60,
last_fired_at: Date.now() - 5 * 60 * 1000, // would block under old rule-wide semantics
}]);
// api already has a cooldown row; database does not, so database may still fire.
mockGetStackAlertServiceCooldown.mockImplementation((_id: number, service: string) =>
service === 'api' ? Date.now() - 5 * 60 * 1000 : null,
);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(true);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(11, 'database', expect.any(Number));
});
it('honors persisted per-service cooldown after a fresh MonitorService instance', async () => {
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetGlobalSettings.mockReturnValue({});
mockGetRunningContainers.mockResolvedValue([
{
Id: 'api-1',
Names: ['/stack-api-1'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
},
{
Id: 'api-2',
Names: ['/stack-api-2'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
},
{
Id: 'db-1',
Names: ['/stack-db-1'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
},
]);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
}));
mockGetStackAlerts.mockReturnValue([{
id: 13,
stack_name: 'shared-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 0,
cooldown_mins: 60,
last_fired_at: 0,
}]);
mockGetStackAlertServiceCooldown.mockImplementation((_id: number, service: string) =>
service === 'api' ? Date.now() - 5 * 60 * 1000 : null,
);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(true);
// beforeEach already cleared the singleton; this instance has empty maps.
const svc = MonitorService.getInstance();
expect((svc as any).activeBreaches.size).toBe(0);
expect((svc as any).firedThisCycle.size).toBe(0);
await (svc as any).evaluate();
const cpuDispatches = mockDispatchAlert.mock.calls.filter(
(args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'),
);
expect(cpuDispatches).toHaveLength(1);
expect(cpuDispatches[0][2]).toContain('**database**');
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(13, 'database', expect.any(Number));
expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalledWith(13, 'api', expect.any(Number));
});
it('preserves breach timers when container enumeration fails', async () => {
const svc = MonitorService.getInstance();
(svc as any).activeBreaches.set('12:still-running', { breachStartedAt: Date.now() });
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetStackAlerts.mockReturnValue([{
id: 12,
stack_name: 'shared-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 999,
cooldown_mins: 0,
last_fired_at: 0,
}]);
mockGetGlobalSettings.mockReturnValue({});
mockGetRunningContainers.mockRejectedValue(new Error('docker down'));
await (svc as any).evaluate();
expect((svc as any).activeBreaches.has('12:still-running')).toBe(true);
});
it('removes breach timers for stopped containers after successful enumeration', async () => {
const svc = MonitorService.getInstance();
(svc as any).activeBreaches.set('13:gone', { breachStartedAt: Date.now() });
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetStackAlerts.mockReturnValue([{
id: 13,
stack_name: 'shared-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 999,
cooldown_mins: 0,
last_fired_at: 0,
}]);
mockGetGlobalSettings.mockReturnValue({});
mockGetRunningContainers.mockResolvedValue([
{
Id: 'still-here',
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
},
]);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 1100 }, system_cpu_usage: 10000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
}));
await (svc as any).evaluate();
expect((svc as any).activeBreaches.has('13:gone')).toBe(false);
});
it('caps simultaneous Docker calls at MAX_CONTAINER_CONCURRENCY', async () => {
@@ -217,7 +217,7 @@ describe('NotificationService - routing logic', () => {
mockFetch.mockRejectedValueOnce(new Error('Network timeout'));
// Should not throw
await expect(svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' })).resolves.toBeUndefined();
await expect(svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' })).resolves.toEqual({ persisted: true });
});
it('does not dispatch to global agents when routes array is empty and no stackName', async () => {
@@ -553,7 +553,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', ()
await expect(
svc.dispatchAlert('error', 'monitor_alert', 'Container crashed', { stackName: 'my-app' }),
).resolves.toBeUndefined();
).resolves.toEqual({ persisted: false });
// No external dispatch and no routing read when there is no persisted row.
expect(mockFetch).not.toHaveBeenCalled();
@@ -580,7 +580,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', ()
await expect(
svc.dispatchAlert('error', 'monitor_alert', 'Container crashed', { stackName: 'my-app' }),
).resolves.toBeUndefined();
).resolves.toEqual({ persisted: true });
// The history row was still written; only routing failed, and it was logged.
expect(mockAddNotificationHistory).toHaveBeenCalledTimes(1);
@@ -598,7 +598,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', ()
await expect(
svc.dispatchAlert('info', 'system', 'Host rebooted'),
).resolves.toBeUndefined();
).resolves.toEqual({ persisted: true });
expect(mockAddNotificationHistory).toHaveBeenCalledTimes(1);
expect(consoleErrorSpy).toHaveBeenCalledWith('[Notify] dispatchAlert failed:', expect.any(Error));
@@ -0,0 +1,256 @@
/**
* Mixed-version gate for POST /api/alerts with a non-empty service_name.
*
* Remote-targeted requests skip express.json() so the raw stream stays
* pipeable. The hub must still buffer that body under the local JSON size
* cap, reject compressed bodies (cannot inspect safely), reject scoped
* creates when the remote lacks service-scoped-stack-alert, and forward
* unscoped identity-encoded JSON intact when allowed.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest';
import http from 'http';
import type { AddressInfo } from 'net';
import zlib from 'zlib';
import request from 'supertest';
import jwt from 'jsonwebtoken';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
let tmpDir: string;
let app: import('express').Express;
let listenServer: http.Server;
let listenPort: number;
let adminBearer: string;
let capServer: http.Server;
let noCapServer: http.Server;
let capNodeId: number;
let noCapNodeId: number;
const noCapPaths: string[] = [];
const capPaths: string[] = [];
let lastCapBody: Buffer | null = null;
let lastNoCapBody: Buffer | null = null;
function metaServer(
capabilities: string[],
seen: string[],
onBody: (body: Buffer) => void,
): http.Server {
return http.createServer((req, res) => {
if (req.url) seen.push(req.url);
const chunks: Buffer[] = [];
req.on('data', (chunk: Buffer) => chunks.push(chunk));
req.on('end', () => {
if (!req.url?.startsWith('/api/meta')) {
onBody(Buffer.concat(chunks));
}
res.writeHead(200, { 'Content-Type': 'application/json' });
if (req.url?.startsWith('/api/meta')) {
res.end(JSON.stringify({ version: '0.93.0', capabilities }));
} else {
res.end(JSON.stringify({ id: 1, ok: true }));
}
});
});
}
async function listen(server: http.Server): Promise<number> {
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
return (server.address() as AddressInfo).port;
}
beforeAll(async () => {
tmpDir = await setupTestDb();
({ app } = await import('../index'));
const { DatabaseService } = await import('../services/DatabaseService');
const db = DatabaseService.getInstance();
adminBearer = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
capServer = metaServer(
['cross-node-rbac', 'service-scoped-stack-alert'],
capPaths,
(body) => { lastCapBody = body; },
);
noCapServer = metaServer(
['cross-node-rbac'],
noCapPaths,
(body) => { lastNoCapBody = body; },
);
const capPort = await listen(capServer);
const noCapPort = await listen(noCapServer);
capNodeId = db.addNode({
name: 'alert-cap-remote', type: 'remote', mode: 'proxy', compose_dir: '/tmp',
is_default: false, api_url: `http://127.0.0.1:${capPort}`, api_token: 'cap-token',
});
noCapNodeId = db.addNode({
name: 'alert-nocap-remote', type: 'remote', mode: 'proxy', compose_dir: '/tmp',
is_default: false, api_url: `http://127.0.0.1:${noCapPort}`, api_token: 'nocap-token',
});
listenServer = http.createServer(app);
listenPort = await listen(listenServer);
});
afterAll(async () => {
await new Promise<void>((resolve) => listenServer.close(() => resolve()));
await new Promise<void>((resolve) => capServer.close(() => resolve()));
await new Promise<void>((resolve) => noCapServer.close(() => resolve()));
cleanupTestDb(tmpDir);
});
beforeEach(() => {
noCapPaths.length = 0;
capPaths.length = 0;
lastNoCapBody = null;
lastCapBody = null;
});
const scopedPayload = {
stack_name: 'web',
service_name: 'api',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 60,
};
const unscopedPayload = {
stack_name: 'web',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 60,
};
describe('remote proxy service-scoped alert gate', () => {
it('returns 400 for scoped alert create when remote lacks service-scoped-stack-alert', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Authorization', `Bearer ${adminBearer}`)
.set('x-node-id', String(noCapNodeId))
.set('Content-Type', 'application/json')
.send(scopedPayload);
expect(res.status).toBe(400);
expect(res.body.code).toBe('capability_unavailable');
expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false);
expect(noCapPaths.some((p) => p.startsWith('/api/meta'))).toBe(true);
expect(lastNoCapBody).toBeNull();
});
it('forwards unscoped alert JSON intact when remote lacks the capability', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Authorization', `Bearer ${adminBearer}`)
.set('x-node-id', String(noCapNodeId))
.set('Content-Type', 'application/json')
.send(unscopedPayload);
expect(res.status).toBe(200);
expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(true);
expect(lastNoCapBody).not.toBeNull();
expect(JSON.parse(lastNoCapBody!.toString('utf-8'))).toEqual(unscopedPayload);
});
it('forwards scoped alert JSON intact when remote advertises the capability', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Authorization', `Bearer ${adminBearer}`)
.set('x-node-id', String(capNodeId))
.set('Content-Type', 'application/json')
.send(scopedPayload);
expect(res.status).toBe(200);
expect(capPaths.some((p) => p.includes('/api/alerts'))).toBe(true);
expect(lastCapBody).not.toBeNull();
expect(JSON.parse(lastCapBody!.toString('utf-8'))).toEqual(scopedPayload);
});
it('rejects oversized identity JSON via Content-Length before upstream', async () => {
const huge = {
...unscopedPayload,
stack_name: 'x'.repeat(120 * 1024),
};
const started = Date.now();
const res = await request(app)
.post('/api/alerts')
.set('Authorization', `Bearer ${adminBearer}`)
.set('x-node-id', String(noCapNodeId))
.set('Content-Type', 'application/json')
.send(huge);
const elapsedMs = Date.now() - started;
expect(res.status).toBe(413);
expect(res.body.code).toBe('entity_too_large');
expect(elapsedMs).toBeLessThan(2000);
expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false);
expect(lastNoCapBody).toBeNull();
});
it('rejects chunked bodies once streamed bytes exceed the JSON limit', async () => {
const oversized = Buffer.from(JSON.stringify({
...unscopedPayload,
stack_name: 'y'.repeat(120 * 1024),
}));
const result = await new Promise<{ status: number; body: { code?: string } }>((resolve, reject) => {
const req = http.request({
hostname: '127.0.0.1',
port: listenPort,
method: 'POST',
path: '/api/alerts',
headers: {
Authorization: `Bearer ${adminBearer}`,
'x-node-id': String(noCapNodeId),
'Content-Type': 'application/json',
'Transfer-Encoding': 'chunked',
},
}, (res) => {
const chunks: Buffer[] = [];
res.on('data', (c: Buffer) => chunks.push(c));
res.on('end', () => {
const text = Buffer.concat(chunks).toString('utf-8');
try {
resolve({ status: res.statusCode ?? 0, body: JSON.parse(text) as { code?: string } });
} catch {
resolve({ status: res.statusCode ?? 0, body: {} });
}
});
});
req.on('error', reject);
// Write in small chunks so the hub crosses the limit mid-stream.
const step = 16 * 1024;
for (let offset = 0; offset < oversized.length; offset += step) {
req.write(oversized.subarray(offset, Math.min(offset + step, oversized.length)));
}
req.end();
});
expect(result.status).toBe(413);
expect(result.body.code).toBe('entity_too_large');
expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false);
expect(lastNoCapBody).toBeNull();
});
it('rejects gzip-encoded scoped JSON instead of forwarding as All-services', async () => {
const compressed = zlib.gzipSync(Buffer.from(JSON.stringify(scopedPayload), 'utf-8'));
const res = await request(app)
.post('/api/alerts')
.set('Authorization', `Bearer ${adminBearer}`)
.set('x-node-id', String(noCapNodeId))
.set('Content-Type', 'application/json')
.set('Content-Encoding', 'gzip')
.send(compressed);
expect(res.status).toBe(415);
expect(res.body.code).toBe('encoding_unsupported');
expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false);
expect(lastNoCapBody).toBeNull();
});
});
@@ -34,7 +34,7 @@ const {
mockMarkStaleRunsAsFailed: vi.fn().mockReturnValue(0),
mockDeleteOldScans: vi.fn().mockReturnValue(0),
mockGetTier: vi.fn().mockReturnValue('paid'),
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
mockGetProxyTarget: vi.fn().mockReturnValue(null),
mockIsTrivyAvailable: vi.fn().mockReturnValue(true),
mockScanAllNodeImages: vi.fn(),
@@ -59,7 +59,7 @@ const {
mockGetStackContent: vi.fn().mockResolvedValue(''),
mockGetEnvContent: vi.fn().mockResolvedValue(''),
mockCheckImage: vi.fn().mockResolvedValue({ hasUpdate: false }),
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
mockGetProxyTarget: vi.fn().mockReturnValue(null),
mockIsTrivyAvailable: vi.fn().mockReturnValue(true),
mockScanAllNodeImages: vi.fn().mockResolvedValue({
@@ -45,7 +45,7 @@ beforeAll(async () => {
writeStack('web');
const { NotificationService } = await import('../services/NotificationService');
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
});
afterAll(() => {
@@ -78,7 +78,7 @@ beforeAll(async () => {
authCookie = await loginAsTestAdmin(app);
const { NotificationService } = await import('../services/NotificationService');
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
});
afterAll(() => {
@@ -90,7 +90,7 @@ beforeAll(async () => {
authCookie = await loginAsTestAdmin(app);
const { NotificationService } = await import('../services/NotificationService');
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
});
afterAll(() => {
@@ -68,7 +68,7 @@ beforeAll(async () => {
writeStack('web');
const { NotificationService } = await import('../services/NotificationService');
dispatchAlertSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
dispatchAlertSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
});
afterAll(() => {
@@ -89,7 +89,7 @@ beforeAll(async () => {
authCookie = await loginAsTestAdmin(app);
const { NotificationService } = await import('../services/NotificationService');
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
});
afterAll(() => {
@@ -102,7 +102,7 @@ beforeAll(async () => {
writeStack('web');
const { NotificationService } = await import('../services/NotificationService');
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
});
afterAll(() => {
@@ -132,7 +132,7 @@ beforeAll(async () => {
const { NotificationService } = await import('../services/NotificationService');
dispatchAlertSpy = vi
.spyOn(NotificationService.getInstance(), 'dispatchAlert')
.mockResolvedValue(undefined);
.mockResolvedValue({ persisted: true });
});
afterAll(() => {
+207 -4
View File
@@ -5,7 +5,7 @@ import { PROXY_TIER_HEADER, PROXY_ROLE_HEADER, PROXY_DEPLOY_SOURCE_HEADER, PROXY
import { LicenseService } from '../services/LicenseService';
import { isProxyExemptPath } from '../helpers/proxyExemptPaths';
import { remoteSupportsCrossNodeRbac, remoteAdvertisesCapability } from '../helpers/remoteCapabilities';
import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY } from '../services/CapabilityRegistry';
import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY, SERVICE_SCOPED_STACK_ALERT_CAPABILITY } from '../services/CapabilityRegistry';
import { getErrorMessage } from '../utils/errors';
import { DatabaseService } from '../services/DatabaseService';
import { redactSensitiveText } from '../utils/safeLog';
@@ -145,8 +145,19 @@ export function createRemoteProxyMiddleware(): RequestHandler {
}
// Body forwarding: conditionalJsonParser skips parsing for remote
// requests (see middleware/jsonParser.ts), so req's raw stream is
// intact and http-proxy's req.pipe(proxyReq) forwards the body
// automatically.
// usually intact and http-proxy's req.pipe(proxyReq) forwards it.
// When a gate must inspect JSON (POST /alerts), we buffer into
// req.rawBody first; rewrite that buffer here because the stream is
// already consumed.
if (req.rawBody) {
proxyReq.removeHeader('Transfer-Encoding');
proxyReq.removeHeader('Content-Length');
if (!proxyReq.getHeader('Content-Type')) {
proxyReq.setHeader('Content-Type', 'application/json');
}
proxyReq.setHeader('Content-Length', req.rawBody.length);
proxyReq.write(req.rawBody);
}
},
proxyRes: (proxyRes, req) => {
// Mark every response forwarded from a remote node with a sentinel
@@ -246,7 +257,8 @@ export function createRemoteProxyMiddleware(): RequestHandler {
}
}
// Mixed-version RBAC gate (non-admin only).
// Mixed-version RBAC gate (non-admin only). Runs before alert body
// buffering so an unauthorized client cannot force unbounded memory use.
if (req.user?.role !== 'admin') {
const rbacSupported = await remoteSupportsCrossNodeRbac(req.nodeId);
if (!rbacSupported) {
@@ -257,6 +269,49 @@ export function createRemoteProxyMiddleware(): RequestHandler {
}
}
// POST /alerts bodies are not on req.body for remote hops (JSON parsing
// is skipped so the stream can be piped). Buffer once under the same
// 100 KB cap as express.json(), gate on service_name, then rewrite
// rawBody in on.proxyReq. Compressed bodies are rejected: the hub cannot
// inspect them, and treating parse failure as unscoped would bypass the
// mixed-version gate.
if (isAlertCreateRoute(req)) {
if (hasNonIdentityContentEncoding(req)) {
await drainRequestBody(req);
res.status(415).json({
error: 'Compressed request bodies are not supported for remote alert creates',
code: 'encoding_unsupported',
});
return;
}
try {
req.rawBody = await bufferRequestBody(req, ALERT_PROXY_BODY_LIMIT);
} catch (err) {
const status = Number((err as { status?: number }).status);
if (status === 413) {
console.error('[remoteNodeProxy] alert body rejected as too large:', err);
res.status(413).json({ error: 'Alert payload too large', code: 'entity_too_large' });
return;
}
if (status === 400) {
console.error('[remoteNodeProxy] alert body incomplete:', err);
res.status(400).json({ error: 'Incomplete request body' });
return;
}
throw err;
}
if (alertCreateHasScopedService(req.rawBody)) {
const supported = await remoteAdvertisesCapability(req.nodeId, SERVICE_SCOPED_STACK_ALERT_CAPABILITY);
if (!supported) {
res.status(400).json({
error: 'Service-scoped alert rules are not supported on this node',
code: 'capability_unavailable',
});
return;
}
}
}
req.proxyTarget = target;
beginProxyTiming(req, res);
proxy(req, res, next);
@@ -283,3 +338,151 @@ function isServiceScopedUpdateRoute(req: Request): boolean {
}
return false;
}
/** POST /alerts (path is post-/api strip). */
function isAlertCreateRoute(req: Request): boolean {
return req.method === 'POST' && /^\/alerts\/?$/.test(req.path);
}
/** Same default as express.json(); remote alert creates must not exceed it. */
const ALERT_PROXY_BODY_LIMIT = 100 * 1024;
/** Max time to wait for leftover body bytes after a size/encoding reject. */
const DRAIN_TIMEOUT_MS = 5_000;
/** Error with HTTP status for the alert-body gate catch mapper. */
function alertBodyError(message: string, status: number): Error {
return Object.assign(new Error(message), { status, expose: true });
}
/** True when Content-Encoding is present and not identity (gzip/deflate/br/…). */
function hasNonIdentityContentEncoding(req: Request): boolean {
const raw = req.headers['content-encoding'];
if (raw == null) return false;
const value = Array.isArray(raw) ? raw.join(',') : raw;
return value.split(',').some((part) => {
const encoding = part.trim().toLowerCase();
return encoding.length > 0 && encoding !== 'identity';
});
}
/** True when the buffered JSON alert body targets a specific Compose service. */
function alertCreateHasScopedService(rawBody: Buffer): boolean {
if (rawBody.length === 0) return false;
try {
const parsed = JSON.parse(rawBody.toString('utf-8')) as { service_name?: unknown };
return typeof parsed.service_name === 'string' && parsed.service_name.trim() !== '';
} catch {
// Identity-encoded non-JSON is forwarded as-is; the remote rejects it.
// Encoded bodies never reach here (rejected earlier).
return false;
}
}
/**
* Consume remaining request bytes (or wait for abort/close) so the response
* can flush without leaving the socket half-open. Does not buffer into memory.
* Caps wait time so a stalled or endless chunked stream cannot hang the gate.
*/
function drainRequestBody(req: Request): Promise<void> {
return new Promise((resolve) => {
if (req.readableEnded || req.destroyed) {
resolve();
return;
}
let settled = false;
const done = () => {
if (settled) return;
settled = true;
clearTimeout(timer);
req.off('end', done);
req.off('error', done);
req.off('aborted', done);
req.off('close', done);
resolve();
};
// Bound hang time: destroy after timeout so mid-stream rejects still settle.
const timer = setTimeout(() => {
if (!req.destroyed) req.destroy();
done();
}, DRAIN_TIMEOUT_MS);
req.resume();
req.once('end', done);
req.once('error', done);
req.once('aborted', done);
req.once('close', done);
});
}
/**
* Buffer the request so a capability gate can inspect JSON without leaving
* http-proxy with an already-ended empty stream. Enforces `limit` on both
* declared Content-Length and streamed accumulation.
*/
async function bufferRequestBody(req: Request, limit: number): Promise<Buffer> {
if (req.rawBody) {
if (req.rawBody.length > limit) {
throw alertBodyError('Alert payload too large', 413);
}
return req.rawBody;
}
if (req.readableEnded) return Buffer.alloc(0);
const declared = Number.parseInt(String(req.headers['content-length'] ?? ''), 10);
if (Number.isFinite(declared) && declared > limit) {
// Reject immediately so the client gets a structured 413; drain leftover
// bytes in the background so the socket can close without holding the gate.
void drainRequestBody(req);
throw alertBodyError('Alert payload too large', 413);
}
return new Promise((resolve, reject) => {
const chunks: Buffer[] = [];
let total = 0;
let settled = false;
const settle = (fn: () => void) => {
if (settled) return;
settled = true;
cleanup();
fn();
};
const finish = (buf: Buffer) => settle(() => resolve(buf));
const fail = (err: Error) => settle(() => reject(err));
const onData = (chunk: Buffer) => {
const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
total += buf.length;
if (total > limit) {
chunks.length = 0;
// fail() removes listeners; drain leftover bytes so the socket can close.
fail(alertBodyError('Alert payload too large', 413));
void drainRequestBody(req);
return;
}
chunks.push(buf);
};
const onEnd = () => finish(Buffer.concat(chunks));
const onError = (err: Error) => fail(err);
const onAborted = () => fail(alertBodyError('Client aborted request body', 400));
const onClose = () => {
if (!settled && !req.readableEnded) {
fail(alertBodyError('Client closed request before body finished', 400));
}
};
function cleanup(): void {
req.off('data', onData);
req.off('end', onEnd);
req.off('error', onError);
req.off('aborted', onAborted);
req.off('close', onClose);
}
req.on('data', onData);
req.on('end', onEnd);
req.on('error', onError);
req.on('aborted', onAborted);
req.on('close', onClose);
});
}
+39 -4
View File
@@ -3,9 +3,21 @@ import { z } from 'zod';
import { DatabaseService } from '../services/DatabaseService';
import { authMiddleware } from '../middleware/auth';
import { requireAdmin } from '../middleware/tierGates';
import { isValidServiceName } from '../utils/validation';
import {
getActiveCapabilities,
SERVICE_SCOPED_STACK_ALERT_CAPABILITY,
} from '../services/CapabilityRegistry';
const AlertCreateSchema = z.object({
stack_name: z.string().min(1).max(255),
service_name: z.preprocess(
(val) => (val === '' ? null : val),
z.string().max(255).nullable().optional().refine(
(val) => val == null || isValidServiceName(val),
{ message: 'Invalid service name' },
),
),
metric: z.enum(['cpu_percent', 'memory_percent', 'memory_mb', 'net_rx', 'net_tx', 'restart_count']),
operator: z.enum(['>', '>=', '<', '<=', '==']),
threshold: z.number().min(0),
@@ -22,7 +34,8 @@ alertsRouter.get('/', authMiddleware, async (req: Request, res: Response) => {
const alerts = DatabaseService.getInstance().getStackAlerts(stackName);
res.json(alerts);
} catch {
} catch (error) {
console.error('Failed to fetch alerts:', error);
res.status(500).json({ error: 'Failed to fetch alerts' });
}
});
@@ -34,8 +47,23 @@ alertsRouter.post('/', authMiddleware, async (req: Request, res: Response) => {
res.status(400).json({ error: 'Invalid alert data', details: parsed.error.flatten().fieldErrors });
return;
}
const { service_name, ...alertFields } = parsed.data;
const serviceName = service_name ?? null;
if (
serviceName != null
&& !getActiveCapabilities().includes(SERVICE_SCOPED_STACK_ALERT_CAPABILITY)
) {
res.status(400).json({
error: 'This node does not support service-scoped alert rules',
code: 'capability_unavailable',
});
return;
}
try {
const created = DatabaseService.getInstance().addStackAlert(parsed.data);
const created = DatabaseService.getInstance().addStackAlert({
...alertFields,
service_name: serviceName,
});
res.status(201).json(created);
} catch (error) {
console.error('Failed to add alert:', error);
@@ -45,11 +73,18 @@ alertsRouter.post('/', authMiddleware, async (req: Request, res: Response) => {
alertsRouter.delete('/:id', authMiddleware, async (req: Request, res: Response) => {
if (!requireAdmin(req, res)) return;
// Reject leading-junk / fractional ids (parseInt("1abc") === 1, parseInt("2.5") === 2).
const rawId = String(req.params.id ?? '');
const id = /^\d+$/.test(rawId) ? Number.parseInt(rawId, 10) : NaN;
if (!Number.isInteger(id) || id <= 0) {
res.status(400).json({ error: 'Invalid alert id' });
return;
}
try {
const id = parseInt(req.params.id as string, 10);
DatabaseService.getInstance().deleteStackAlert(id);
res.json({ success: true });
} catch {
} catch (error) {
console.error('Failed to delete alert:', error);
res.status(500).json({ error: 'Failed to delete alert' });
}
});
@@ -59,6 +59,7 @@ export const CAPABILITIES = [
'stack-down-remove-volumes',
'guided-external-network-preflight',
'service-scoped-update',
'service-scoped-stack-alert',
] as const;
/**
@@ -88,6 +89,10 @@ export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes'
/** Capability for the nested per-service update/restore routes and the `effective-services` model they read. */
export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability;
/** Capability for nullable `service_name` on stack alert rules and per-service cooldown evaluation. */
export const SERVICE_SCOPED_STACK_ALERT_CAPABILITY =
'service-scoped-stack-alert' as const satisfies Capability;
/** Returns true when the string is a usable semver version. */
export function isValidVersion(v: string | null | undefined): v is string {
return !!v && v !== 'unknown' && v !== '0.0.0-dev' && !!semver.valid(v);
+72 -9
View File
@@ -101,6 +101,7 @@ function stringifyServicesJson(services: StackServiceStatus[], generation: numbe
export interface StackAlert {
id?: number;
stack_name: string;
service_name: string | null;
metric: string;
operator: string;
threshold: number;
@@ -1062,6 +1063,7 @@ export class DatabaseService {
this.migrateStackDossierHashes();
this.migrateGitSourceMultiFile();
this.migrateNodeUpdateSkips();
this.migrateStackAlertServiceScope();
// Reset the cache once at end of constructor in case any migration
// populated it via getGlobalSettings() and a subsequent migration
@@ -1098,6 +1100,7 @@ export class DatabaseService {
CREATE TABLE IF NOT EXISTS stack_alerts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
stack_name TEXT NOT NULL,
service_name TEXT,
metric TEXT NOT NULL,
operator TEXT NOT NULL,
threshold REAL NOT NULL,
@@ -1106,6 +1109,17 @@ export class DatabaseService {
last_fired_at INTEGER DEFAULT 0
);
-- FK cascade is declarative only: PRAGMA foreign_keys is never enabled
-- on this connection, so parent deletes must remove children explicitly
-- (see deleteStackAlert).
CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns (
alert_id INTEGER NOT NULL,
service_name TEXT NOT NULL,
last_fired_at INTEGER NOT NULL,
PRIMARY KEY (alert_id, service_name),
FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS notification_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
node_id INTEGER NOT NULL DEFAULT 0,
@@ -2280,6 +2294,25 @@ export class DatabaseService {
}
}
private migrateStackAlertServiceScope(): void {
this.tryAddColumn('stack_alerts', 'service_name', 'TEXT');
try {
// FK is not enforced (foreign_keys pragma off); deleteStackAlert removes children.
this.db.prepare(`
CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns (
alert_id INTEGER NOT NULL,
service_name TEXT NOT NULL,
last_fired_at INTEGER NOT NULL,
PRIMARY KEY (alert_id, service_name),
FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE
)
`).run();
} catch (e) {
console.error('[DatabaseService] stack_alert_service_cooldowns migration failed:', (e as Error).message);
throw e;
}
}
private migrateScanPolicyFleetColumns(): void {
this.tryAddColumn('scan_policies', 'node_identity', "TEXT NOT NULL DEFAULT ''");
this.tryAddColumn('scan_policies', 'replicated_from_control', 'INTEGER NOT NULL DEFAULT 0');
@@ -3075,22 +3108,19 @@ export class DatabaseService {
// --- Stack Alerts ---
public getStackAlerts(stackName?: string): StackAlert[] {
let stmt;
if (stackName) {
stmt = this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?');
return stmt.all(stackName) as StackAlert[];
} else {
stmt = this.db.prepare('SELECT * FROM stack_alerts');
return stmt.all() as StackAlert[];
return this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?').all(stackName) as StackAlert[];
}
return this.db.prepare('SELECT * FROM stack_alerts').all() as StackAlert[];
}
public addStackAlert(alert: StackAlert): StackAlert {
const stmt = this.db.prepare(
'INSERT INTO stack_alerts (stack_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?)'
'INSERT INTO stack_alerts (stack_name, service_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
);
const result = stmt.run(
alert.stack_name,
alert.service_name ?? null,
alert.metric,
alert.operator,
alert.threshold,
@@ -3101,9 +3131,16 @@ export class DatabaseService {
return this.db.prepare('SELECT * FROM stack_alerts WHERE id = ?').get(result.lastInsertRowid) as StackAlert;
}
/**
* Delete an alert and its per-service cooldown rows.
* SQLite foreign_keys is not enabled here, so child rows are removed
* explicitly rather than relying on ON DELETE CASCADE.
*/
public deleteStackAlert(id: number): void {
const stmt = this.db.prepare('DELETE FROM stack_alerts WHERE id = ?');
stmt.run(id);
this.transaction(() => {
this.deleteStackAlertServiceCooldowns(id);
this.db.prepare('DELETE FROM stack_alerts WHERE id = ?').run(id);
});
}
public updateStackAlertLastFired(id: number, timestamp: number): void {
@@ -3111,6 +3148,32 @@ export class DatabaseService {
stmt.run(timestamp, id);
}
public getStackAlertServiceCooldown(alertId: number, serviceName: string): number | null {
const row = this.db.prepare(
'SELECT last_fired_at FROM stack_alert_service_cooldowns WHERE alert_id = ? AND service_name = ?'
).get(alertId, serviceName) as { last_fired_at: number } | undefined;
return row?.last_fired_at ?? null;
}
public hasAnyStackAlertServiceCooldown(alertId: number): boolean {
const row = this.db.prepare(
'SELECT 1 AS present FROM stack_alert_service_cooldowns WHERE alert_id = ? LIMIT 1'
).get(alertId) as { present: number } | undefined;
return !!row;
}
public upsertStackAlertServiceCooldown(alertId: number, serviceName: string, timestamp: number): void {
this.db.prepare(`
INSERT INTO stack_alert_service_cooldowns (alert_id, service_name, last_fired_at)
VALUES (?, ?, ?)
ON CONFLICT(alert_id, service_name) DO UPDATE SET last_fired_at = excluded.last_fired_at
`).run(alertId, serviceName, timestamp);
}
public deleteStackAlertServiceCooldowns(alertId: number): void {
this.db.prepare('DELETE FROM stack_alert_service_cooldowns WHERE alert_id = ?').run(alertId);
}
// --- Auto-Heal Policies ---
public getAutoHealPolicies(stackName?: string, nodeId?: number): AutoHealPolicy[] {
+3 -3
View File
@@ -735,15 +735,15 @@ export class DockerEventService {
}
private async emitError(category: NotificationCategory, message: string, stackName?: string, containerName?: string, systemOnly = false): Promise<void> {
return this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly));
await this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly));
}
private async emitWarning(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
return this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName));
await this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName));
}
private async emitInfo(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
return this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName));
await this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName));
}
// ========================================================================
+120 -45
View File
@@ -32,6 +32,39 @@ const getOperatorPhrase = (operator: string): string => {
return `triggered the operator ${operator}`;
};
/** Sentinel for containers without a Compose service label. Not a valid API target. */
const UNLABELED_SERVICE_KEY = '_unlabeled';
function displayServiceName(serviceName: string): string {
return serviceName === UNLABELED_SERVICE_KEY ? 'unknown service' : serviceName;
}
function parseAlertBreachKey(key: string): { ruleId: number; containerId: string } | null {
const sep = key.indexOf(':');
if (sep < 0) return null;
const ruleId = Number(key.slice(0, sep));
if (!Number.isFinite(ruleId)) return null;
return { ruleId, containerId: key.slice(sep + 1) };
}
/** Per-service cooldown timestamp, with pre-migration fallback to rule.last_fired_at. */
function resolveStackAlertLastFired(rule: StackAlert, serviceName: string, db: DatabaseService): number {
const ruleId = rule.id!;
let lastFired = db.getStackAlertServiceCooldown(ruleId, serviceName);
if (lastFired == null && !db.hasAnyStackAlertServiceCooldown(ruleId) && (rule.last_fired_at || 0) > 0) {
lastFired = rule.last_fired_at || 0;
}
return lastFired || 0;
}
function normalizeContainerName(container: { Id: string; Names?: string[] }): string {
const raw = container.Names?.[0];
if (raw && raw.length > 0) {
return raw.startsWith('/') ? raw.slice(1) : raw;
}
return container.Id.slice(0, 12);
}
/** Shape of the JSON returned by Docker container stats (stream: false). */
interface DockerContainerStats {
cpu_stats?: {
@@ -153,21 +186,18 @@ export class MonitorService {
private janitorConsecutiveTimeouts = 0;
private janitorBreakerUntil = 0;
// Track the duration a specific stack alert rule has been in breach state
// key: rule_id, value: AlertState
private activeBreaches = new Map<number, AlertState>();
// Track the duration a specific stack alert rule+container has been in breach
// key: `${ruleId}:${containerId}`, value: AlertState
private activeBreaches = new Map<string, AlertState>();
// Track previous network counters per container for rate calculation.
// key: container_id, value: { rx bytes, tx bytes, sample timestamp }
private previousNetworkStats = new Map<string, { rx: number; tx: number; ts: number }>();
// Per-cycle dispatch dedup. Stack rules are shared across every container
// in the stack, so parallel processContainer calls can race past the
// cooldown check (DB write happens after the awaited dispatch) and fire
// the same alert N times on first breach. The synchronous check-and-add
// below is atomic in JS between awaits, so only the first worker wins.
// Per-cycle dispatch dedup keyed by rule+service so replicas of the same
// Compose service do not fire N times, while different services can.
// Reset at the start of each evaluateStackAlerts call.
private firedThisCycle = new Set<number>();
private firedThisCycle = new Set<string>();
// Crash and healthcheck detection live in DockerEventService (event-driven,
// causal classification). MonitorService no longer polls for container
@@ -604,6 +634,10 @@ export class MonitorService {
else alertsByStack.set(a.stack_name, [a]);
}
const activeRuleIds = new Set(alerts.map(a => a.id!));
const allCurrentIds = new Set<string>();
let enumerationFailed = false;
for (const node of nodes) {
if (!node.id) continue;
// Remote nodes are self-monitoring - skip direct Docker access
@@ -623,22 +657,39 @@ export class MonitorService {
(container) => this.processContainer(node, container, alertsByStack, docker, db),
);
for (const c of containers) allCurrentIds.add(c.Id);
// Clean up stale network stats for containers on this node that no longer run
if (this.previousNetworkStats.size > containers.length * 2) {
const currentIds = new Set(containers.map((c: { Id: string }) => c.Id));
const nodeIds = new Set(containers.map((c: { Id: string }) => c.Id));
for (const key of this.previousNetworkStats.keys()) {
if (!currentIds.has(key)) this.previousNetworkStats.delete(key);
if (!nodeIds.has(key)) this.previousNetworkStats.delete(key);
}
}
} catch (err) {
// Enumeration failed: preserve existing breach timers.
enumerationFailed = true;
console.error(`Error fetching containers for node ${node.name}`, err);
}
}
// Clean up stale breach trackers for rules that have been deleted
const activeRuleIds = new Set(alerts.map(a => a.id!));
for (const key of this.activeBreaches.keys()) {
if (!activeRuleIds.has(key)) {
// After successful enumeration(s), drop breach timers for containers
// that are no longer running. Skip when any local enumeration failed
// so a transient Docker error cannot reset duration timers.
if (!enumerationFailed) {
for (const key of [...this.activeBreaches.keys()]) {
const parsed = parseAlertBreachKey(key);
if (parsed && activeRuleIds.has(parsed.ruleId) && !allCurrentIds.has(parsed.containerId)) {
this.activeBreaches.delete(key);
}
}
}
// Clean up in-memory breach trackers for rules that have been deleted.
// Persisted cooldowns are removed only via transactional deleteStackAlert.
for (const key of [...this.activeBreaches.keys()]) {
const parsed = parseAlertBreachKey(key);
if (parsed && !activeRuleIds.has(parsed.ruleId)) {
this.activeBreaches.delete(key);
}
}
@@ -666,12 +717,14 @@ export class MonitorService {
*/
private async processContainer(
node: Node,
container: { Id: string; Labels?: Record<string, string> },
container: { Id: string; Names?: string[]; Labels?: Record<string, string> },
alertsByStack: Map<string, StackAlert[]>,
docker: DockerController,
db: DatabaseService,
): Promise<void> {
const stackName = container.Labels?.['com.docker.compose.project'] || 'system';
const serviceName = container.Labels?.['com.docker.compose.service'] || UNLABELED_SERVICE_KEY;
const containerName = normalizeContainerName(container);
try {
const rawStats = await withTimeout(
@@ -727,7 +780,11 @@ export class MonitorService {
});
for (const rule of stackAlerts) {
if (rule.service_name && rule.service_name !== serviceName) continue;
const ruleId = rule.id!;
const breachKey = `${ruleId}:${container.Id}`;
const cooldownKey = `${ruleId}:${serviceName}`;
const currentValue = metrics[rule.metric as keyof typeof metrics];
if (currentValue === undefined) continue;
@@ -735,57 +792,75 @@ export class MonitorService {
const isBreaching = this.evaluateCondition(currentValue, rule.operator, rule.threshold);
if (isBreaching) {
if (!this.activeBreaches.has(ruleId)) {
this.activeBreaches.set(ruleId, { breachStartedAt: Date.now() });
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}"`);
if (!this.activeBreaches.has(breachKey)) {
this.activeBreaches.set(breachKey, { breachStartedAt: Date.now() });
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}" service "${serviceName}" container "${containerName}"`);
}
const breachState = this.activeBreaches.get(ruleId)!;
const breachState = this.activeBreaches.get(breachKey)!;
const durationMs = Date.now() - breachState.breachStartedAt;
const requiredDurationMs = rule.duration_mins * 60 * 1000;
if (durationMs >= requiredDurationMs) {
const timeSinceLastFired = Date.now() - (rule.last_fired_at || 0);
const timeSinceLastFired = Date.now() - resolveStackAlertLastFired(rule, serviceName, db);
const requiredCooldownMs = rule.cooldown_mins * 60 * 1000;
if (timeSinceLastFired >= requiredCooldownMs) {
// Claim this rule for the cycle before awaiting
// dispatch. The check-and-add is synchronous, so
// sibling workers evaluating the same shared rule
// see the claim and skip — preventing N-fire when
// multiple containers in one stack all breach.
if (this.firedThisCycle.has(ruleId)) {
if (isDebugEnabled()) console.log(`[Monitor:diag] Skipping duplicate dispatch for rule ${ruleId} (already fired this cycle by sibling container)`);
} else {
this.firedThisCycle.add(ruleId);
// Claim this rule+service for the cycle before awaiting
// dispatch so replicas of the same service skip.
if (this.firedThisCycle.has(cooldownKey)) {
if (isDebugEnabled()) console.log(`[Monitor:diag] Skipping duplicate dispatch for rule ${ruleId} service "${serviceName}" (already fired this cycle by sibling replica)`);
continue;
}
const { name: metricName, unit } = getMetricDetails(rule.metric);
const operatorPhrase = getOperatorPhrase(rule.operator);
this.firedThisCycle.add(cooldownKey);
const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue;
const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold;
const { name: metricName, unit } = getMetricDetails(rule.metric);
const operatorPhrase = getOperatorPhrase(rule.operator);
// Node-neutral body: the hub bell badge attributes remotes.
const message = `The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue;
const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold;
const serviceLabel = displayServiceName(serviceName);
console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`);
await NotificationService.getInstance().dispatchAlert(
// Node-neutral body: the hub bell badge attributes remotes.
const message = `The **${metricName}** for **${serviceLabel}** in **${rule.stack_name}** (container **${containerName}**) ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
try {
const { persisted } = await NotificationService.getInstance().dispatchAlert(
'warning',
'monitor_alert',
message,
{ stackName: rule.stack_name, actor: 'system:monitor' },
{ stackName: rule.stack_name, containerName, actor: 'system:monitor' },
);
if (!persisted) {
// History was not written; do not advance cooldown or we silence retries.
this.firedThisCycle.delete(cooldownKey);
console.error(
`[MonitorService] Alert history not persisted for rule ${ruleId} service "${serviceName}"; cooldown not advanced`,
);
continue;
}
const firedAt = Date.now();
// Dual-write: per-service row for current code, parent last_fired_at
// so a downgrade that only reads the parent column still has a floor.
db.upsertStackAlertServiceCooldown(ruleId, serviceName, firedAt);
db.updateStackAlertLastFired(ruleId, firedAt);
console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}" service "${serviceName}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`);
} catch (fireErr) {
this.firedThisCycle.delete(cooldownKey);
console.error(
`[MonitorService] Failed to fire alert rule ${ruleId} service "${serviceName}" container "${containerName}":`,
fireErr,
);
db.updateStackAlertLastFired(ruleId, Date.now());
}
} else if (isDebugEnabled()) {
console.log(`[Monitor:diag] Cooldown active for rule ${ruleId}: ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`);
console.log(`[Monitor:diag] Cooldown active for rule ${ruleId} service "${serviceName}": ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`);
}
}
} else {
if (this.activeBreaches.has(ruleId)) {
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}"`);
this.activeBreaches.delete(ruleId);
if (this.activeBreaches.has(breachKey)) {
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}" container "${containerName}"`);
this.activeBreaches.delete(breachKey);
}
}
}
+11 -5
View File
@@ -181,7 +181,7 @@ export class NotificationService {
category: NotificationCategory,
message: string,
options?: { stackName?: string; containerName?: string; actor?: string },
) {
): Promise<{ persisted: boolean }> {
const t0 = Date.now();
const { stackName, containerName, actor } = options ?? {};
@@ -191,6 +191,8 @@ export class NotificationService {
// WebSocket broadcast can all throw on an unhealthy DB, which would
// otherwise surface as an unhandledRejection and take the process down.
// The whole body is wrapped so the worst case is a dropped notification.
// Callers that gate cooldowns on history use `persisted` (true only after the row write).
let wroteHistory = false;
try {
// Internal writes use the middleware default so they share a row key
// with user-initiated requests; otherwise the UI and monitors split
@@ -216,11 +218,12 @@ export class NotificationService {
container_name: containerName,
actor_username: actor ?? null,
});
wroteHistory = true;
StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, true);
} catch (err) {
StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, false);
console.error('[Notify] Failed to persist notification:', err);
return;
return { persisted: false };
}
// Separate [StackActivity:diag] namespace from the [Notify:diag] lines
// below so a single grep can pull every per-stack timeline write across
@@ -273,7 +276,7 @@ export class NotificationService {
}
if (suppressExternal) {
return;
return { persisted: wroteHistory };
}
// Resolve retry extras once for this dispatch (shared by all destinations).
@@ -298,14 +301,14 @@ export class NotificationService {
)
);
this.recordDispatchErrors(notification.id!, errors);
return;
return { persisted: wroteHistory };
}
// 4. Fall back to this instance's agents (keyed by this instance's default node id).
const agents = this.dbService.getEnabledAgents(localNodeId);
if (agents.length === 0) {
if (isDebugEnabled()) console.log('[Notify:diag] No routes or agents matched; skipping external dispatch');
return;
return { persisted: wroteHistory };
}
if (isDebugEnabled()) console.log(`[Notify:diag] Falling back to ${agents.length} global agent(s)`);
@@ -322,8 +325,11 @@ export class NotificationService {
)
);
this.recordDispatchErrors(notification.id!, errors);
return { persisted: wroteHistory };
} catch (err) {
console.error('[Notify] dispatchAlert failed:', err);
// History may already be written; callers must not treat that as a miss.
return { persisted: wroteHistory };
}
}