mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-25 09:46:47 +00:00
feat: add service-scoped stack alert rules (#1681)
* feat: add service-scoped stack alert rules
Stack alerts can target one Compose service or all services. Breach timers
are per container and cooldowns are per service so a healthy sibling no
longer clears another container's timer or silences a different service.
* fix: gate remote scoped alert creates without losing the body
Remote hops skip JSON parsing so the proxy stream stays pipeable, which
left service_name invisible to the capability gate. Buffer POST /alerts
bodies for inspection, fail closed when the remote lacks the capability,
and rewrite the buffered bytes on forward. Restore alert-panel alt text
to match the unchanged screenshot.
* fix: bound remote alert body buffer and reject encoded JSON
Cap proxied POST /alerts buffering at the local 100KB JSON limit with
structured 413 cleanup, reject non-identity Content-Encoding with 415 so
compressed scoped bodies cannot bypass the mixed-version gate, and cover
oversized, chunked, and gzip regressions.
* fix: harden service-scoped alert delete, cooldown, and proxy gates
Reject non-digit alert ids, dual-write last_fired_at for rollback safety,
gate cooldown on persisted notification history, fail-fast oversized proxy
bodies with 413, and clarify Not in compose UI semantics.
* test: expect dispatchAlert persisted result in crash-safety cases
Update notification-routing assertions for the new { persisted } return
shape so CI matches the cooldown-gating contract.
This commit is contained in:
@@ -55,7 +55,7 @@ beforeEach(() => {
|
||||
const db = DatabaseService.getInstance();
|
||||
db.getDb().prepare('DELETE FROM auto_heal_history').run();
|
||||
db.getDb().prepare('DELETE FROM auto_heal_policies').run();
|
||||
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue();
|
||||
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
|
||||
@@ -57,8 +57,8 @@ describe('GET /api/alerts', () => {
|
||||
it('filters alerts by stackName query param', async () => {
|
||||
// Seed two alerts for different stacks
|
||||
const db = DatabaseService.getInstance();
|
||||
db.addStackAlert({ stack_name: 'web', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
|
||||
db.addStackAlert({ stack_name: 'api', metric: 'memory_percent', operator: '>', threshold: 90, duration_mins: 5, cooldown_mins: 60 });
|
||||
db.addStackAlert({ stack_name: 'web', service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
|
||||
db.addStackAlert({ stack_name: 'api', service_name: null, metric: 'memory_percent', operator: '>', threshold: 90, duration_mins: 5, cooldown_mins: 60 });
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/alerts?stackName=web')
|
||||
@@ -105,10 +105,101 @@ describe('POST /api/alerts', () => {
|
||||
expect(res.status).toBe(201);
|
||||
expect(res.body.id).toBeDefined();
|
||||
expect(res.body.stack_name).toBe('new-stack');
|
||||
expect(res.body.service_name).toBeNull();
|
||||
expect(res.body.metric).toBe('memory_percent');
|
||||
expect(res.body.threshold).toBe(85);
|
||||
});
|
||||
|
||||
it('persists a valid dotted service_name', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Cookie', authCookie)
|
||||
.send({
|
||||
stack_name: 'svc-stack',
|
||||
service_name: 'api.web',
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 5,
|
||||
cooldown_mins: 60,
|
||||
});
|
||||
expect(res.status).toBe(201);
|
||||
expect(res.body.service_name).toBe('api.web');
|
||||
});
|
||||
|
||||
it('normalizes empty service_name to null', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Cookie', authCookie)
|
||||
.send({
|
||||
stack_name: 'empty-svc',
|
||||
service_name: '',
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 5,
|
||||
cooldown_mins: 60,
|
||||
});
|
||||
expect(res.status).toBe(201);
|
||||
expect(res.body.service_name).toBeNull();
|
||||
});
|
||||
|
||||
it('rejects whitespace-only service_name', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Cookie', authCookie)
|
||||
.send({
|
||||
stack_name: 'bad-svc',
|
||||
service_name: ' ',
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 5,
|
||||
cooldown_mins: 60,
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it('rejects reserved _unlabeled service_name', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Cookie', authCookie)
|
||||
.send({
|
||||
stack_name: 'bad-svc',
|
||||
service_name: '_unlabeled',
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 5,
|
||||
cooldown_mins: 60,
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it('rejects service_name when the capability is disabled', async () => {
|
||||
const { disableCapability, enableCapability, SERVICE_SCOPED_STACK_ALERT_CAPABILITY } =
|
||||
await import('../services/CapabilityRegistry');
|
||||
disableCapability(SERVICE_SCOPED_STACK_ALERT_CAPABILITY);
|
||||
try {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Cookie', authCookie)
|
||||
.send({
|
||||
stack_name: 'cap-stack',
|
||||
service_name: 'api',
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 5,
|
||||
cooldown_mins: 60,
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.code).toBe('capability_unavailable');
|
||||
} finally {
|
||||
enableCapability(SERVICE_SCOPED_STACK_ALERT_CAPABILITY);
|
||||
}
|
||||
});
|
||||
|
||||
it('validates required fields and returns 400 for missing data', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
@@ -202,6 +293,7 @@ describe('DELETE /api/alerts/:id', () => {
|
||||
// Create an alert to delete
|
||||
const created = DatabaseService.getInstance().addStackAlert({
|
||||
stack_name: 'delete-me',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 90,
|
||||
@@ -216,6 +308,58 @@ describe('DELETE /api/alerts/:id', () => {
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.success).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects leading-junk ids like 1abc without deleting alert 1', async () => {
|
||||
const created = DatabaseService.getInstance().addStackAlert({
|
||||
stack_name: 'strict-id-junk',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 90,
|
||||
duration_mins: 0,
|
||||
cooldown_mins: 0,
|
||||
});
|
||||
expect(created.id).toBeDefined();
|
||||
|
||||
const res = await request(app)
|
||||
.delete(`/api/alerts/${created.id}abc`)
|
||||
.set('Cookie', authCookie);
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toBe('Invalid alert id');
|
||||
expect(DatabaseService.getInstance().getStackAlerts('strict-id-junk').some((a) => a.id === created.id)).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects fractional ids like 2.5 without deleting alert 2', async () => {
|
||||
const first = DatabaseService.getInstance().addStackAlert({
|
||||
stack_name: 'strict-id-fraction',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 0,
|
||||
cooldown_mins: 0,
|
||||
});
|
||||
const second = DatabaseService.getInstance().addStackAlert({
|
||||
stack_name: 'strict-id-fraction',
|
||||
service_name: null,
|
||||
metric: 'memory_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 0,
|
||||
cooldown_mins: 0,
|
||||
});
|
||||
expect(second.id).toBeDefined();
|
||||
|
||||
const res = await request(app)
|
||||
.delete(`/api/alerts/${second.id}.5`)
|
||||
.set('Cookie', authCookie);
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toBe('Invalid alert id');
|
||||
const remaining = DatabaseService.getInstance().getStackAlerts('strict-id-fraction').map((a) => a.id);
|
||||
expect(remaining).toEqual(expect.arrayContaining([first.id, second.id]));
|
||||
});
|
||||
});
|
||||
|
||||
// --- POST /api/notifications/test ---
|
||||
|
||||
@@ -168,7 +168,7 @@ describe('Rollback notifications (M-2)', () => {
|
||||
mockTier('paid');
|
||||
mockDeployStack.mockResolvedValue(undefined);
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
|
||||
|
||||
const res = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie);
|
||||
expect(res.status).toBe(200);
|
||||
@@ -186,7 +186,7 @@ describe('Rollback notifications (M-2)', () => {
|
||||
mockTier('paid');
|
||||
mockDeployStack.mockRejectedValueOnce(new Error('restore failed'));
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
|
||||
|
||||
const res = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie);
|
||||
expect(res.status).toBe(500);
|
||||
|
||||
@@ -600,7 +600,7 @@ describe('BlueprintReconciler drift alert node wording', () => {
|
||||
|
||||
it('suggest-mode local drift uses on this node', async () => {
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
|
||||
const nodeId = seedNode();
|
||||
const bp = seedBlueprint({ name: 'drift-local', drift_mode: 'suggest', nodeIds: [nodeId] });
|
||||
const node = DatabaseService.getInstance().getNode(nodeId)!;
|
||||
@@ -618,7 +618,7 @@ describe('BlueprintReconciler drift alert node wording', () => {
|
||||
|
||||
it('suggest-mode remote drift keeps the authoritative node name', async () => {
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
|
||||
const nodeId = seedRemoteNode('sencho-test-02');
|
||||
const bp = seedBlueprint({ name: 'drift-remote', drift_mode: 'suggest', nodeIds: [nodeId] });
|
||||
const node = DatabaseService.getInstance().getNode(nodeId)!;
|
||||
@@ -636,7 +636,7 @@ describe('BlueprintReconciler drift alert node wording', () => {
|
||||
|
||||
it('stateful marker-loss on local uses on this node', async () => {
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
|
||||
vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue(null);
|
||||
const nodeId = seedNode();
|
||||
const bp = seedBlueprint({
|
||||
@@ -660,7 +660,7 @@ describe('BlueprintReconciler drift alert node wording', () => {
|
||||
|
||||
it('enforce correction-failure on local uses on this node', async () => {
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
|
||||
vi.spyOn(BlueprintService.getInstance(), 'deployToNode').mockResolvedValue({
|
||||
status: 'failed',
|
||||
error: 'compose up failed',
|
||||
|
||||
@@ -324,6 +324,7 @@ describe('DatabaseService - stack alerts CRUD', () => {
|
||||
it('adds and retrieves stack alerts', () => {
|
||||
db.addStackAlert({
|
||||
stack_name: 'alert-stack',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
@@ -339,11 +340,27 @@ describe('DatabaseService - stack alerts CRUD', () => {
|
||||
expect(found.threshold).toBe(80);
|
||||
expect(found.duration_mins).toBe(5);
|
||||
expect(found.cooldown_mins).toBe(15);
|
||||
expect(found.service_name).toBeNull();
|
||||
});
|
||||
|
||||
it('persists a named service_name', () => {
|
||||
const created = db.addStackAlert({
|
||||
stack_name: 'scoped-stack',
|
||||
service_name: 'api.web',
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 5,
|
||||
cooldown_mins: 15,
|
||||
});
|
||||
expect(created.service_name).toBe('api.web');
|
||||
expect(db.getStackAlerts('scoped-stack')[0].service_name).toBe('api.web');
|
||||
});
|
||||
|
||||
it('filters alerts by stack name', () => {
|
||||
db.addStackAlert({
|
||||
stack_name: 'filter-stack-a',
|
||||
service_name: null,
|
||||
metric: 'memory_mb',
|
||||
operator: '>=',
|
||||
threshold: 512,
|
||||
@@ -352,6 +369,7 @@ describe('DatabaseService - stack alerts CRUD', () => {
|
||||
});
|
||||
db.addStackAlert({
|
||||
stack_name: 'filter-stack-b',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 90,
|
||||
@@ -371,6 +389,7 @@ describe('DatabaseService - stack alerts CRUD', () => {
|
||||
it('updates last_fired_at timestamp', () => {
|
||||
db.addStackAlert({
|
||||
stack_name: 'fired-stack',
|
||||
service_name: null,
|
||||
metric: 'net_rx',
|
||||
operator: '>',
|
||||
threshold: 100,
|
||||
@@ -381,29 +400,55 @@ describe('DatabaseService - stack alerts CRUD', () => {
|
||||
const alerts = db.getStackAlerts('fired-stack');
|
||||
const alert = alerts[0];
|
||||
const fireTime = Date.now();
|
||||
db.updateStackAlertLastFired(alert.id, fireTime);
|
||||
db.updateStackAlertLastFired(alert.id!, fireTime);
|
||||
|
||||
const updated = db.getStackAlerts('fired-stack');
|
||||
expect(updated[0].last_fired_at).toBe(fireTime);
|
||||
});
|
||||
|
||||
it('deletes an alert by id', () => {
|
||||
db.addStackAlert({
|
||||
it('upserts and reads distinct per-service cooldowns', () => {
|
||||
const alert = db.addStackAlert({
|
||||
stack_name: 'cooldown-stack',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 1,
|
||||
cooldown_mins: 10,
|
||||
});
|
||||
const id = alert.id!;
|
||||
expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(false);
|
||||
|
||||
db.upsertStackAlertServiceCooldown(id, 'api', 1000);
|
||||
db.upsertStackAlertServiceCooldown(id, 'database', 2000);
|
||||
|
||||
expect(db.getStackAlertServiceCooldown(id, 'api')).toBe(1000);
|
||||
expect(db.getStackAlertServiceCooldown(id, 'database')).toBe(2000);
|
||||
expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(true);
|
||||
|
||||
db.upsertStackAlertServiceCooldown(id, 'api', 3000);
|
||||
expect(db.getStackAlertServiceCooldown(id, 'api')).toBe(3000);
|
||||
});
|
||||
|
||||
it('deletes an alert by id and removes child cooldown rows', () => {
|
||||
const alert = db.addStackAlert({
|
||||
stack_name: 'delete-stack',
|
||||
service_name: null,
|
||||
metric: 'memory_percent',
|
||||
operator: '>',
|
||||
threshold: 95,
|
||||
duration_mins: 1,
|
||||
cooldown_mins: 5,
|
||||
});
|
||||
const id = alert.id!;
|
||||
db.upsertStackAlertServiceCooldown(id, 'api', Date.now());
|
||||
expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(true);
|
||||
|
||||
const before = db.getStackAlerts('delete-stack');
|
||||
expect(before.length).toBe(1);
|
||||
db.deleteStackAlert(id);
|
||||
|
||||
db.deleteStackAlert(before[0].id);
|
||||
|
||||
const after = db.getStackAlerts('delete-stack');
|
||||
expect(after.length).toBe(0);
|
||||
expect(db.getStackAlerts('delete-stack').length).toBe(0);
|
||||
expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(false);
|
||||
expect(db.getStackAlertServiceCooldown(id, 'api')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ const {
|
||||
mockGetDocker,
|
||||
mockIsOwnContainer,
|
||||
} = vi.hoisted(() => ({
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
|
||||
mockBroadcastEvent: vi.fn(),
|
||||
mockGetGlobalSettings: vi.fn().mockReturnValue({ global_crash: '1' }),
|
||||
mockGetEvents: vi.fn(),
|
||||
|
||||
@@ -18,7 +18,7 @@ const {
|
||||
mockGetFleetSyncStatuses: vi.fn().mockReturnValue([]),
|
||||
mockGetSystemState: vi.fn().mockReturnValue(null),
|
||||
mockPushResourceToNode: vi.fn().mockResolvedValue(undefined),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
|
||||
}));
|
||||
|
||||
vi.mock('../services/DatabaseService', () => ({
|
||||
|
||||
@@ -40,7 +40,7 @@ const {
|
||||
mockGetSystemState: vi.fn().mockReturnValue(null),
|
||||
mockSetSystemState: vi.fn(),
|
||||
mockTransaction: vi.fn().mockImplementation((fn: () => unknown) => fn()),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
|
||||
mockAxiosPost: vi.fn().mockResolvedValue({ data: { success: true } }),
|
||||
}));
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ const {
|
||||
mockGetSystemState: vi.fn().mockReturnValue('1'), // default: backfilled
|
||||
mockSetSystemState: vi.fn(),
|
||||
mockAddNotificationHistory: vi.fn(),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
|
||||
mockGetStacks: vi.fn().mockResolvedValue([]),
|
||||
mockGetStackContent: vi.fn().mockResolvedValue(''),
|
||||
mockGetEnvContent: vi.fn().mockRejectedValue(new Error('no env')),
|
||||
|
||||
@@ -9,7 +9,9 @@ import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsM
|
||||
|
||||
const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContainerMetric,
|
||||
mockCleanupOldMetrics, mockCleanupOldNotifications, mockCleanupOldAuditLogs,
|
||||
mockUpdateStackAlertLastFired, mockGetSystemState, mockSetSystemState,
|
||||
mockUpdateStackAlertLastFired, mockGetStackAlertServiceCooldown,
|
||||
mockHasAnyStackAlertServiceCooldown, mockUpsertStackAlertServiceCooldown,
|
||||
mockGetSystemState, mockSetSystemState,
|
||||
mockPruneScanHistoryPerImage, mockDeleteScansByImageRef,
|
||||
mockGetRunningContainers, mockGetAllContainers, mockGetContainerStatsStream,
|
||||
mockGetContainerRestartCount, mockGetDiskUsage, mockGetImages, mockGetStacks,
|
||||
@@ -29,6 +31,9 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
|
||||
mockCleanupOldNotifications: vi.fn(),
|
||||
mockCleanupOldAuditLogs: vi.fn(),
|
||||
mockUpdateStackAlertLastFired: vi.fn(),
|
||||
mockGetStackAlertServiceCooldown: vi.fn().mockReturnValue(null),
|
||||
mockHasAnyStackAlertServiceCooldown: vi.fn().mockReturnValue(false),
|
||||
mockUpsertStackAlertServiceCooldown: vi.fn(),
|
||||
mockGetSystemState: vi.fn().mockReturnValue(null),
|
||||
mockSetSystemState: vi.fn(),
|
||||
mockPruneScanHistoryPerImage: vi.fn().mockReturnValue(0),
|
||||
@@ -43,7 +48,7 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
|
||||
}),
|
||||
mockGetImages: vi.fn().mockResolvedValue([]),
|
||||
mockGetStacks: vi.fn().mockResolvedValue([]),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
|
||||
mockCurrentLoad: vi.fn().mockResolvedValue({ currentLoad: 10 }),
|
||||
mockMem: vi.fn().mockResolvedValue({ total: 16e9, used: 4e9, active: 4e9, available: 12e9, free: 12e9, buffcache: 0 }),
|
||||
mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]),
|
||||
@@ -65,6 +70,9 @@ vi.mock('../services/DatabaseService', () => ({
|
||||
cleanupOldNotifications: mockCleanupOldNotifications,
|
||||
cleanupOldAuditLogs: mockCleanupOldAuditLogs,
|
||||
updateStackAlertLastFired: mockUpdateStackAlertLastFired,
|
||||
getStackAlertServiceCooldown: mockGetStackAlertServiceCooldown,
|
||||
hasAnyStackAlertServiceCooldown: mockHasAnyStackAlertServiceCooldown,
|
||||
upsertStackAlertServiceCooldown: mockUpsertStackAlertServiceCooldown,
|
||||
getSystemState: mockGetSystemState,
|
||||
setSystemState: mockSetSystemState,
|
||||
pruneScanHistoryPerImage: mockPruneScanHistoryPerImage,
|
||||
@@ -868,8 +876,12 @@ describe('MonitorService - breach state machine', () => {
|
||||
function setupAlertScenario(cpuPercent: number) {
|
||||
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
||||
mockGetRunningContainers.mockResolvedValue([{
|
||||
Id: 'c1',
|
||||
Labels: { 'com.docker.compose.project': 'my-stack' },
|
||||
Id: 'c1abcdefghijk',
|
||||
Names: ['/my-stack-api-1'],
|
||||
Labels: {
|
||||
'com.docker.compose.project': 'my-stack',
|
||||
'com.docker.compose.service': 'api',
|
||||
},
|
||||
}]);
|
||||
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
|
||||
cpu_stats: { cpu_usage: { total_usage: 1000 + cpuPercent * 50 }, system_cpu_usage: 10000, online_cpus: 1 },
|
||||
@@ -879,6 +891,7 @@ describe('MonitorService - breach state machine', () => {
|
||||
mockGetStackAlerts.mockReturnValue([{
|
||||
id: 1,
|
||||
stack_name: 'my-stack',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
@@ -887,6 +900,8 @@ describe('MonitorService - breach state machine', () => {
|
||||
last_fired_at: 0,
|
||||
}]);
|
||||
mockGetGlobalSettings.mockReturnValue({});
|
||||
mockGetStackAlertServiceCooldown.mockReturnValue(null);
|
||||
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
|
||||
}
|
||||
|
||||
it('fires alert when condition met and duration is 0', async () => {
|
||||
@@ -898,9 +913,10 @@ describe('MonitorService - breach state machine', () => {
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'warning',
|
||||
'monitor_alert',
|
||||
'The **CPU usage** for **my-stack** has exceeded your threshold of **80%** (Currently: 90%).',
|
||||
{ stackName: 'my-stack', actor: 'system:monitor' },
|
||||
'The **CPU usage** for **api** in **my-stack** (container **my-stack-api-1**) has exceeded your threshold of **80%** (Currently: 90%).',
|
||||
{ stackName: 'my-stack', containerName: 'my-stack-api-1', actor: 'system:monitor' },
|
||||
);
|
||||
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number));
|
||||
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
|
||||
});
|
||||
|
||||
@@ -913,12 +929,22 @@ describe('MonitorService - breach state machine', () => {
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'));
|
||||
});
|
||||
|
||||
it('respects cooldown after firing', async () => {
|
||||
it('respects cooldown after firing via persisted service cooldown', async () => {
|
||||
setupAlertScenario(90);
|
||||
mockGetStackAlertServiceCooldown.mockReturnValue(Date.now() - 30 * 60 * 1000);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('respects legacy last_fired_at floor when no child cooldown rows exist', async () => {
|
||||
setupAlertScenario(90);
|
||||
// Simulate that alert was fired 30 minutes ago (within 60-min cooldown)
|
||||
mockGetStackAlerts.mockReturnValue([{
|
||||
id: 1,
|
||||
stack_name: 'my-stack',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
@@ -926,14 +952,94 @@ describe('MonitorService - breach state machine', () => {
|
||||
cooldown_mins: 60,
|
||||
last_fired_at: Date.now() - 30 * 60 * 1000,
|
||||
}]);
|
||||
mockGetStackAlertServiceCooldown.mockReturnValue(null);
|
||||
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockUpdateStackAlertLastFired).not.toHaveBeenCalled();
|
||||
expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('resets breach state when condition clears', async () => {
|
||||
it('fires after legacy last_fired_at cooldown expires and writes a child cooldown row', async () => {
|
||||
setupAlertScenario(90);
|
||||
mockGetStackAlerts.mockReturnValue([{
|
||||
id: 1,
|
||||
stack_name: 'my-stack',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 0,
|
||||
cooldown_mins: 60,
|
||||
last_fired_at: Date.now() - 90 * 60 * 1000,
|
||||
}]);
|
||||
mockGetStackAlertServiceCooldown.mockReturnValue(null);
|
||||
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number));
|
||||
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
|
||||
});
|
||||
|
||||
it('does not advance cooldown when notification history is not persisted', async () => {
|
||||
setupAlertScenario(90);
|
||||
mockDispatchAlert.mockResolvedValueOnce({ persisted: false });
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled();
|
||||
expect(mockUpdateStackAlertLastFired).not.toHaveBeenCalled();
|
||||
|
||||
mockDispatchAlert.mockResolvedValueOnce({ persisted: true });
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
||||
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number));
|
||||
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
|
||||
});
|
||||
|
||||
it('drops in-memory breach timers when a rule is deleted', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
(svc as any).activeBreaches.set('55:gone-container', { breachStartedAt: Date.now() });
|
||||
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
||||
mockGetRunningContainers.mockResolvedValue([]);
|
||||
mockGetStackAlerts.mockReturnValue([]);
|
||||
mockGetGlobalSettings.mockReturnValue({});
|
||||
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect((svc as any).activeBreaches.has('55:gone-container')).toBe(false);
|
||||
});
|
||||
|
||||
it('names unlabeled containers as unknown service in the alert body', async () => {
|
||||
setupAlertScenario(90);
|
||||
mockGetRunningContainers.mockResolvedValue([{
|
||||
Id: 'orphan123456789',
|
||||
Names: ['/orphan'],
|
||||
Labels: {
|
||||
'com.docker.compose.project': 'my-stack',
|
||||
},
|
||||
}]);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'warning',
|
||||
'monitor_alert',
|
||||
expect.stringContaining('**unknown service**'),
|
||||
expect.objectContaining({ stackName: 'my-stack', containerName: 'orphan' }),
|
||||
);
|
||||
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, '_unlabeled', expect.any(Number));
|
||||
});
|
||||
|
||||
it('resets breach state when condition clears for that container only', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
|
||||
// First: breach starts
|
||||
@@ -941,6 +1047,7 @@ describe('MonitorService - breach state machine', () => {
|
||||
mockGetStackAlerts.mockReturnValue([{
|
||||
id: 42,
|
||||
stack_name: 'my-stack',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
@@ -949,13 +1056,14 @@ describe('MonitorService - breach state machine', () => {
|
||||
last_fired_at: 0,
|
||||
}]);
|
||||
await (svc as any).evaluate();
|
||||
expect((svc as any).activeBreaches.has(42)).toBe(true);
|
||||
expect((svc as any).activeBreaches.has('42:c1abcdefghijk')).toBe(true);
|
||||
|
||||
// Second: condition clears
|
||||
setupAlertScenario(10);
|
||||
mockGetStackAlerts.mockReturnValue([{
|
||||
id: 42,
|
||||
stack_name: 'my-stack',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
@@ -964,7 +1072,28 @@ describe('MonitorService - breach state machine', () => {
|
||||
last_fired_at: 0,
|
||||
}]);
|
||||
await (svc as any).evaluate();
|
||||
expect((svc as any).activeBreaches.has(42)).toBe(false);
|
||||
expect((svc as any).activeBreaches.has('42:c1abcdefghijk')).toBe(false);
|
||||
});
|
||||
|
||||
it('falls back to short container id when Names is absent', async () => {
|
||||
setupAlertScenario(90);
|
||||
mockGetRunningContainers.mockResolvedValue([{
|
||||
Id: 'abcdef1234567890',
|
||||
Labels: {
|
||||
'com.docker.compose.project': 'my-stack',
|
||||
'com.docker.compose.service': 'api',
|
||||
},
|
||||
}]);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'warning',
|
||||
'monitor_alert',
|
||||
'The **CPU usage** for **api** in **my-stack** (container **abcdef123456**) has exceeded your threshold of **80%** (Currently: 90%).',
|
||||
{ stackName: 'my-stack', containerName: 'abcdef123456', actor: 'system:monitor' },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1074,7 +1203,12 @@ describe('MonitorService - restart_count metric', () => {
|
||||
|
||||
expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1');
|
||||
// restart_count=5 > threshold=3, should fire
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Restart count'), { stackName: 'my-stack', actor: 'system:monitor' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'warning',
|
||||
'monitor_alert',
|
||||
expect.stringContaining('Restart count'),
|
||||
{ stackName: 'my-stack', containerName: 'c1', actor: 'system:monitor' },
|
||||
);
|
||||
});
|
||||
|
||||
it('skips Docker inspect when no restart_count rules exist', async () => {
|
||||
@@ -1261,6 +1395,8 @@ describe('MonitorService - parallel container processing', () => {
|
||||
mockGetGlobalSettings.mockReturnValue({});
|
||||
mockGetStackAlerts.mockReturnValue([]);
|
||||
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
||||
mockGetStackAlertServiceCooldown.mockReturnValue(null);
|
||||
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
|
||||
});
|
||||
|
||||
it('fans out per-container stats fetches in parallel (wall time ~ max not sum)', async () => {
|
||||
@@ -1311,12 +1447,15 @@ describe('MonitorService - parallel container processing', () => {
|
||||
});
|
||||
|
||||
it('dispatches a stack alert exactly once per cycle even when multiple containers in the stack breach', async () => {
|
||||
// 5 containers in the same stack, all breaching the same rule. Without
|
||||
// the per-cycle dedup, parallel workers race past the cooldown check
|
||||
// and each fire dispatchAlert before any DB write lands.
|
||||
// 5 replicas of the same Compose service, all breaching. Dedup is per
|
||||
// rule+service, so only one fire should land this cycle.
|
||||
const containers = Array.from({ length: 5 }, (_, i) => ({
|
||||
Id: `container-${i}`,
|
||||
Labels: { 'com.docker.compose.project': 'shared-stack' },
|
||||
Names: [`/shared-stack-web-${i}`],
|
||||
Labels: {
|
||||
'com.docker.compose.project': 'shared-stack',
|
||||
'com.docker.compose.service': 'web',
|
||||
},
|
||||
}));
|
||||
mockGetRunningContainers.mockResolvedValue(containers);
|
||||
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
|
||||
@@ -1328,6 +1467,7 @@ describe('MonitorService - parallel container processing', () => {
|
||||
mockGetStackAlerts.mockReturnValue([{
|
||||
id: 7,
|
||||
stack_name: 'shared-stack',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
@@ -1335,6 +1475,8 @@ describe('MonitorService - parallel container processing', () => {
|
||||
cooldown_mins: 60,
|
||||
last_fired_at: 0,
|
||||
}]);
|
||||
mockGetStackAlertServiceCooldown.mockReturnValue(null);
|
||||
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
@@ -1343,7 +1485,290 @@ describe('MonitorService - parallel container processing', () => {
|
||||
(args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'),
|
||||
);
|
||||
expect(cpuDispatches).toHaveLength(1);
|
||||
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledTimes(1);
|
||||
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('allows different services on an All-services rule to fire in the same cycle', async () => {
|
||||
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
||||
mockGetRunningContainers.mockResolvedValue([
|
||||
{
|
||||
Id: 'api-1',
|
||||
Names: ['/stack-api-1'],
|
||||
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
|
||||
},
|
||||
{
|
||||
Id: 'db-1',
|
||||
Names: ['/stack-db-1'],
|
||||
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
|
||||
},
|
||||
]);
|
||||
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
|
||||
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
|
||||
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
||||
memory_stats: { usage: 100e6, limit: 1e9 },
|
||||
}));
|
||||
mockGetStackAlerts.mockReturnValue([{
|
||||
id: 8,
|
||||
stack_name: 'shared-stack',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 0,
|
||||
cooldown_mins: 60,
|
||||
last_fired_at: 0,
|
||||
}]);
|
||||
mockGetStackAlertServiceCooldown.mockReturnValue(null);
|
||||
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
|
||||
mockGetGlobalSettings.mockReturnValue({});
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
|
||||
const cpuDispatches = mockDispatchAlert.mock.calls.filter(
|
||||
(args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'),
|
||||
);
|
||||
expect(cpuDispatches).toHaveLength(2);
|
||||
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(8, 'api', expect.any(Number));
|
||||
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(8, 'database', expect.any(Number));
|
||||
});
|
||||
|
||||
it('does not let a healthy sibling clear another container breach timer', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
||||
mockGetGlobalSettings.mockReturnValue({});
|
||||
mockGetStackAlerts.mockReturnValue([{
|
||||
id: 9,
|
||||
stack_name: 'shared-stack',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 999,
|
||||
cooldown_mins: 0,
|
||||
last_fired_at: 0,
|
||||
}]);
|
||||
|
||||
const highCpu = JSON.stringify({
|
||||
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
|
||||
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
||||
memory_stats: { usage: 100e6, limit: 1e9 },
|
||||
});
|
||||
const lowCpu = JSON.stringify({
|
||||
cpu_stats: { cpu_usage: { total_usage: 1100 }, system_cpu_usage: 10000, online_cpus: 1 },
|
||||
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
||||
memory_stats: { usage: 100e6, limit: 1e9 },
|
||||
});
|
||||
|
||||
mockGetRunningContainers.mockResolvedValue([
|
||||
{
|
||||
Id: 'api-hot',
|
||||
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
|
||||
},
|
||||
{
|
||||
Id: 'db-cool',
|
||||
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
|
||||
},
|
||||
]);
|
||||
mockGetContainerStatsStream.mockImplementation(async (id: string) => (id === 'api-hot' ? highCpu : lowCpu));
|
||||
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect((svc as any).activeBreaches.has('9:api-hot')).toBe(true);
|
||||
expect((svc as any).activeBreaches.has('9:db-cool')).toBe(false);
|
||||
});
|
||||
|
||||
it('ignores sibling services for a service-scoped rule', async () => {
|
||||
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
||||
mockGetGlobalSettings.mockReturnValue({});
|
||||
mockGetRunningContainers.mockResolvedValue([
|
||||
{
|
||||
Id: 'api-1',
|
||||
Names: ['/stack-api-1'],
|
||||
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
|
||||
},
|
||||
{
|
||||
Id: 'db-1',
|
||||
Names: ['/stack-db-1'],
|
||||
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
|
||||
},
|
||||
]);
|
||||
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
|
||||
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
|
||||
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
||||
memory_stats: { usage: 100e6, limit: 1e9 },
|
||||
}));
|
||||
mockGetStackAlerts.mockReturnValue([{
|
||||
id: 10,
|
||||
stack_name: 'shared-stack',
|
||||
service_name: 'api',
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 0,
|
||||
cooldown_mins: 0,
|
||||
last_fired_at: 0,
|
||||
}]);
|
||||
mockGetStackAlertServiceCooldown.mockReturnValue(null);
|
||||
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
expect(mockDispatchAlert.mock.calls[0][2]).toContain('**api**');
|
||||
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(10, 'api', expect.any(Number));
|
||||
});
|
||||
|
||||
it('does not block service B after service A fires under new-schema cooldown', async () => {
|
||||
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
||||
mockGetGlobalSettings.mockReturnValue({});
|
||||
mockGetRunningContainers.mockResolvedValue([
|
||||
{
|
||||
Id: 'db-1',
|
||||
Names: ['/stack-db-1'],
|
||||
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
|
||||
},
|
||||
]);
|
||||
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
|
||||
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
|
||||
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
||||
memory_stats: { usage: 100e6, limit: 1e9 },
|
||||
}));
|
||||
mockGetStackAlerts.mockReturnValue([{
|
||||
id: 11,
|
||||
stack_name: 'shared-stack',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 0,
|
||||
cooldown_mins: 60,
|
||||
last_fired_at: Date.now() - 5 * 60 * 1000, // would block under old rule-wide semantics
|
||||
}]);
|
||||
// api already has a cooldown row; database does not, so database may still fire.
|
||||
mockGetStackAlertServiceCooldown.mockImplementation((_id: number, service: string) =>
|
||||
service === 'api' ? Date.now() - 5 * 60 * 1000 : null,
|
||||
);
|
||||
mockHasAnyStackAlertServiceCooldown.mockReturnValue(true);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(11, 'database', expect.any(Number));
|
||||
});
|
||||
|
||||
it('honors persisted per-service cooldown after a fresh MonitorService instance', async () => {
|
||||
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
||||
mockGetGlobalSettings.mockReturnValue({});
|
||||
mockGetRunningContainers.mockResolvedValue([
|
||||
{
|
||||
Id: 'api-1',
|
||||
Names: ['/stack-api-1'],
|
||||
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
|
||||
},
|
||||
{
|
||||
Id: 'api-2',
|
||||
Names: ['/stack-api-2'],
|
||||
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
|
||||
},
|
||||
{
|
||||
Id: 'db-1',
|
||||
Names: ['/stack-db-1'],
|
||||
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
|
||||
},
|
||||
]);
|
||||
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
|
||||
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
|
||||
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
||||
memory_stats: { usage: 100e6, limit: 1e9 },
|
||||
}));
|
||||
mockGetStackAlerts.mockReturnValue([{
|
||||
id: 13,
|
||||
stack_name: 'shared-stack',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 0,
|
||||
cooldown_mins: 60,
|
||||
last_fired_at: 0,
|
||||
}]);
|
||||
mockGetStackAlertServiceCooldown.mockImplementation((_id: number, service: string) =>
|
||||
service === 'api' ? Date.now() - 5 * 60 * 1000 : null,
|
||||
);
|
||||
mockHasAnyStackAlertServiceCooldown.mockReturnValue(true);
|
||||
|
||||
// beforeEach already cleared the singleton; this instance has empty maps.
|
||||
const svc = MonitorService.getInstance();
|
||||
expect((svc as any).activeBreaches.size).toBe(0);
|
||||
expect((svc as any).firedThisCycle.size).toBe(0);
|
||||
|
||||
await (svc as any).evaluate();
|
||||
|
||||
const cpuDispatches = mockDispatchAlert.mock.calls.filter(
|
||||
(args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'),
|
||||
);
|
||||
expect(cpuDispatches).toHaveLength(1);
|
||||
expect(cpuDispatches[0][2]).toContain('**database**');
|
||||
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(13, 'database', expect.any(Number));
|
||||
expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalledWith(13, 'api', expect.any(Number));
|
||||
});
|
||||
|
||||
it('preserves breach timers when container enumeration fails', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
(svc as any).activeBreaches.set('12:still-running', { breachStartedAt: Date.now() });
|
||||
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
||||
mockGetStackAlerts.mockReturnValue([{
|
||||
id: 12,
|
||||
stack_name: 'shared-stack',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 999,
|
||||
cooldown_mins: 0,
|
||||
last_fired_at: 0,
|
||||
}]);
|
||||
mockGetGlobalSettings.mockReturnValue({});
|
||||
mockGetRunningContainers.mockRejectedValue(new Error('docker down'));
|
||||
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect((svc as any).activeBreaches.has('12:still-running')).toBe(true);
|
||||
});
|
||||
|
||||
it('removes breach timers for stopped containers after successful enumeration', async () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
(svc as any).activeBreaches.set('13:gone', { breachStartedAt: Date.now() });
|
||||
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
||||
mockGetStackAlerts.mockReturnValue([{
|
||||
id: 13,
|
||||
stack_name: 'shared-stack',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 999,
|
||||
cooldown_mins: 0,
|
||||
last_fired_at: 0,
|
||||
}]);
|
||||
mockGetGlobalSettings.mockReturnValue({});
|
||||
mockGetRunningContainers.mockResolvedValue([
|
||||
{
|
||||
Id: 'still-here',
|
||||
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
|
||||
},
|
||||
]);
|
||||
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
|
||||
cpu_stats: { cpu_usage: { total_usage: 1100 }, system_cpu_usage: 10000, online_cpus: 1 },
|
||||
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
||||
memory_stats: { usage: 100e6, limit: 1e9 },
|
||||
}));
|
||||
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect((svc as any).activeBreaches.has('13:gone')).toBe(false);
|
||||
});
|
||||
|
||||
it('caps simultaneous Docker calls at MAX_CONTAINER_CONCURRENCY', async () => {
|
||||
|
||||
@@ -217,7 +217,7 @@ describe('NotificationService - routing logic', () => {
|
||||
mockFetch.mockRejectedValueOnce(new Error('Network timeout'));
|
||||
|
||||
// Should not throw
|
||||
await expect(svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' })).resolves.toBeUndefined();
|
||||
await expect(svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' })).resolves.toEqual({ persisted: true });
|
||||
});
|
||||
|
||||
it('does not dispatch to global agents when routes array is empty and no stackName', async () => {
|
||||
@@ -553,7 +553,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', ()
|
||||
|
||||
await expect(
|
||||
svc.dispatchAlert('error', 'monitor_alert', 'Container crashed', { stackName: 'my-app' }),
|
||||
).resolves.toBeUndefined();
|
||||
).resolves.toEqual({ persisted: false });
|
||||
|
||||
// No external dispatch and no routing read when there is no persisted row.
|
||||
expect(mockFetch).not.toHaveBeenCalled();
|
||||
@@ -580,7 +580,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', ()
|
||||
|
||||
await expect(
|
||||
svc.dispatchAlert('error', 'monitor_alert', 'Container crashed', { stackName: 'my-app' }),
|
||||
).resolves.toBeUndefined();
|
||||
).resolves.toEqual({ persisted: true });
|
||||
|
||||
// The history row was still written; only routing failed, and it was logged.
|
||||
expect(mockAddNotificationHistory).toHaveBeenCalledTimes(1);
|
||||
@@ -598,7 +598,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', ()
|
||||
|
||||
await expect(
|
||||
svc.dispatchAlert('info', 'system', 'Host rebooted'),
|
||||
).resolves.toBeUndefined();
|
||||
).resolves.toEqual({ persisted: true });
|
||||
|
||||
expect(mockAddNotificationHistory).toHaveBeenCalledTimes(1);
|
||||
expect(consoleErrorSpy).toHaveBeenCalledWith('[Notify] dispatchAlert failed:', expect.any(Error));
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
/**
|
||||
* Mixed-version gate for POST /api/alerts with a non-empty service_name.
|
||||
*
|
||||
* Remote-targeted requests skip express.json() so the raw stream stays
|
||||
* pipeable. The hub must still buffer that body under the local JSON size
|
||||
* cap, reject compressed bodies (cannot inspect safely), reject scoped
|
||||
* creates when the remote lacks service-scoped-stack-alert, and forward
|
||||
* unscoped identity-encoded JSON intact when allowed.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest';
|
||||
import http from 'http';
|
||||
import type { AddressInfo } from 'net';
|
||||
import zlib from 'zlib';
|
||||
import request from 'supertest';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
let listenServer: http.Server;
|
||||
let listenPort: number;
|
||||
let adminBearer: string;
|
||||
let capServer: http.Server;
|
||||
let noCapServer: http.Server;
|
||||
let capNodeId: number;
|
||||
let noCapNodeId: number;
|
||||
|
||||
const noCapPaths: string[] = [];
|
||||
const capPaths: string[] = [];
|
||||
let lastCapBody: Buffer | null = null;
|
||||
let lastNoCapBody: Buffer | null = null;
|
||||
|
||||
function metaServer(
|
||||
capabilities: string[],
|
||||
seen: string[],
|
||||
onBody: (body: Buffer) => void,
|
||||
): http.Server {
|
||||
return http.createServer((req, res) => {
|
||||
if (req.url) seen.push(req.url);
|
||||
const chunks: Buffer[] = [];
|
||||
req.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
req.on('end', () => {
|
||||
if (!req.url?.startsWith('/api/meta')) {
|
||||
onBody(Buffer.concat(chunks));
|
||||
}
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
if (req.url?.startsWith('/api/meta')) {
|
||||
res.end(JSON.stringify({ version: '0.93.0', capabilities }));
|
||||
} else {
|
||||
res.end(JSON.stringify({ id: 1, ok: true }));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function listen(server: http.Server): Promise<number> {
|
||||
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
|
||||
return (server.address() as AddressInfo).port;
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ app } = await import('../index'));
|
||||
const { DatabaseService } = await import('../services/DatabaseService');
|
||||
const db = DatabaseService.getInstance();
|
||||
|
||||
adminBearer = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
|
||||
|
||||
capServer = metaServer(
|
||||
['cross-node-rbac', 'service-scoped-stack-alert'],
|
||||
capPaths,
|
||||
(body) => { lastCapBody = body; },
|
||||
);
|
||||
noCapServer = metaServer(
|
||||
['cross-node-rbac'],
|
||||
noCapPaths,
|
||||
(body) => { lastNoCapBody = body; },
|
||||
);
|
||||
const capPort = await listen(capServer);
|
||||
const noCapPort = await listen(noCapServer);
|
||||
|
||||
capNodeId = db.addNode({
|
||||
name: 'alert-cap-remote', type: 'remote', mode: 'proxy', compose_dir: '/tmp',
|
||||
is_default: false, api_url: `http://127.0.0.1:${capPort}`, api_token: 'cap-token',
|
||||
});
|
||||
noCapNodeId = db.addNode({
|
||||
name: 'alert-nocap-remote', type: 'remote', mode: 'proxy', compose_dir: '/tmp',
|
||||
is_default: false, api_url: `http://127.0.0.1:${noCapPort}`, api_token: 'nocap-token',
|
||||
});
|
||||
|
||||
listenServer = http.createServer(app);
|
||||
listenPort = await listen(listenServer);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await new Promise<void>((resolve) => listenServer.close(() => resolve()));
|
||||
await new Promise<void>((resolve) => capServer.close(() => resolve()));
|
||||
await new Promise<void>((resolve) => noCapServer.close(() => resolve()));
|
||||
cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
noCapPaths.length = 0;
|
||||
capPaths.length = 0;
|
||||
lastNoCapBody = null;
|
||||
lastCapBody = null;
|
||||
});
|
||||
|
||||
const scopedPayload = {
|
||||
stack_name: 'web',
|
||||
service_name: 'api',
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 5,
|
||||
cooldown_mins: 60,
|
||||
};
|
||||
|
||||
const unscopedPayload = {
|
||||
stack_name: 'web',
|
||||
service_name: null,
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 80,
|
||||
duration_mins: 5,
|
||||
cooldown_mins: 60,
|
||||
};
|
||||
|
||||
describe('remote proxy service-scoped alert gate', () => {
|
||||
it('returns 400 for scoped alert create when remote lacks service-scoped-stack-alert', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Authorization', `Bearer ${adminBearer}`)
|
||||
.set('x-node-id', String(noCapNodeId))
|
||||
.set('Content-Type', 'application/json')
|
||||
.send(scopedPayload);
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.code).toBe('capability_unavailable');
|
||||
expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false);
|
||||
expect(noCapPaths.some((p) => p.startsWith('/api/meta'))).toBe(true);
|
||||
expect(lastNoCapBody).toBeNull();
|
||||
});
|
||||
|
||||
it('forwards unscoped alert JSON intact when remote lacks the capability', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Authorization', `Bearer ${adminBearer}`)
|
||||
.set('x-node-id', String(noCapNodeId))
|
||||
.set('Content-Type', 'application/json')
|
||||
.send(unscopedPayload);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(true);
|
||||
expect(lastNoCapBody).not.toBeNull();
|
||||
expect(JSON.parse(lastNoCapBody!.toString('utf-8'))).toEqual(unscopedPayload);
|
||||
});
|
||||
|
||||
it('forwards scoped alert JSON intact when remote advertises the capability', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Authorization', `Bearer ${adminBearer}`)
|
||||
.set('x-node-id', String(capNodeId))
|
||||
.set('Content-Type', 'application/json')
|
||||
.send(scopedPayload);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(capPaths.some((p) => p.includes('/api/alerts'))).toBe(true);
|
||||
expect(lastCapBody).not.toBeNull();
|
||||
expect(JSON.parse(lastCapBody!.toString('utf-8'))).toEqual(scopedPayload);
|
||||
});
|
||||
|
||||
it('rejects oversized identity JSON via Content-Length before upstream', async () => {
|
||||
const huge = {
|
||||
...unscopedPayload,
|
||||
stack_name: 'x'.repeat(120 * 1024),
|
||||
};
|
||||
|
||||
const started = Date.now();
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Authorization', `Bearer ${adminBearer}`)
|
||||
.set('x-node-id', String(noCapNodeId))
|
||||
.set('Content-Type', 'application/json')
|
||||
.send(huge);
|
||||
const elapsedMs = Date.now() - started;
|
||||
|
||||
expect(res.status).toBe(413);
|
||||
expect(res.body.code).toBe('entity_too_large');
|
||||
expect(elapsedMs).toBeLessThan(2000);
|
||||
expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false);
|
||||
expect(lastNoCapBody).toBeNull();
|
||||
});
|
||||
|
||||
it('rejects chunked bodies once streamed bytes exceed the JSON limit', async () => {
|
||||
const oversized = Buffer.from(JSON.stringify({
|
||||
...unscopedPayload,
|
||||
stack_name: 'y'.repeat(120 * 1024),
|
||||
}));
|
||||
|
||||
const result = await new Promise<{ status: number; body: { code?: string } }>((resolve, reject) => {
|
||||
const req = http.request({
|
||||
hostname: '127.0.0.1',
|
||||
port: listenPort,
|
||||
method: 'POST',
|
||||
path: '/api/alerts',
|
||||
headers: {
|
||||
Authorization: `Bearer ${adminBearer}`,
|
||||
'x-node-id': String(noCapNodeId),
|
||||
'Content-Type': 'application/json',
|
||||
'Transfer-Encoding': 'chunked',
|
||||
},
|
||||
}, (res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
res.on('data', (c: Buffer) => chunks.push(c));
|
||||
res.on('end', () => {
|
||||
const text = Buffer.concat(chunks).toString('utf-8');
|
||||
try {
|
||||
resolve({ status: res.statusCode ?? 0, body: JSON.parse(text) as { code?: string } });
|
||||
} catch {
|
||||
resolve({ status: res.statusCode ?? 0, body: {} });
|
||||
}
|
||||
});
|
||||
});
|
||||
req.on('error', reject);
|
||||
// Write in small chunks so the hub crosses the limit mid-stream.
|
||||
const step = 16 * 1024;
|
||||
for (let offset = 0; offset < oversized.length; offset += step) {
|
||||
req.write(oversized.subarray(offset, Math.min(offset + step, oversized.length)));
|
||||
}
|
||||
req.end();
|
||||
});
|
||||
|
||||
expect(result.status).toBe(413);
|
||||
expect(result.body.code).toBe('entity_too_large');
|
||||
expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false);
|
||||
expect(lastNoCapBody).toBeNull();
|
||||
});
|
||||
|
||||
it('rejects gzip-encoded scoped JSON instead of forwarding as All-services', async () => {
|
||||
const compressed = zlib.gzipSync(Buffer.from(JSON.stringify(scopedPayload), 'utf-8'));
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Authorization', `Bearer ${adminBearer}`)
|
||||
.set('x-node-id', String(noCapNodeId))
|
||||
.set('Content-Type', 'application/json')
|
||||
.set('Content-Encoding', 'gzip')
|
||||
.send(compressed);
|
||||
|
||||
expect(res.status).toBe(415);
|
||||
expect(res.body.code).toBe('encoding_unsupported');
|
||||
expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false);
|
||||
expect(lastNoCapBody).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -34,7 +34,7 @@ const {
|
||||
mockMarkStaleRunsAsFailed: vi.fn().mockReturnValue(0),
|
||||
mockDeleteOldScans: vi.fn().mockReturnValue(0),
|
||||
mockGetTier: vi.fn().mockReturnValue('paid'),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
|
||||
mockGetProxyTarget: vi.fn().mockReturnValue(null),
|
||||
mockIsTrivyAvailable: vi.fn().mockReturnValue(true),
|
||||
mockScanAllNodeImages: vi.fn(),
|
||||
|
||||
@@ -59,7 +59,7 @@ const {
|
||||
mockGetStackContent: vi.fn().mockResolvedValue(''),
|
||||
mockGetEnvContent: vi.fn().mockResolvedValue(''),
|
||||
mockCheckImage: vi.fn().mockResolvedValue({ hasUpdate: false }),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
|
||||
mockGetProxyTarget: vi.fn().mockReturnValue(null),
|
||||
mockIsTrivyAvailable: vi.fn().mockReturnValue(true),
|
||||
mockScanAllNodeImages: vi.fn().mockResolvedValue({
|
||||
|
||||
@@ -45,7 +45,7 @@ beforeAll(async () => {
|
||||
writeStack('web');
|
||||
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
|
||||
@@ -78,7 +78,7 @@ beforeAll(async () => {
|
||||
authCookie = await loginAsTestAdmin(app);
|
||||
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
|
||||
@@ -90,7 +90,7 @@ beforeAll(async () => {
|
||||
authCookie = await loginAsTestAdmin(app);
|
||||
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
|
||||
@@ -68,7 +68,7 @@ beforeAll(async () => {
|
||||
writeStack('web');
|
||||
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
dispatchAlertSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
dispatchAlertSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
|
||||
@@ -89,7 +89,7 @@ beforeAll(async () => {
|
||||
authCookie = await loginAsTestAdmin(app);
|
||||
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
|
||||
@@ -102,7 +102,7 @@ beforeAll(async () => {
|
||||
writeStack('web');
|
||||
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
|
||||
@@ -132,7 +132,7 @@ beforeAll(async () => {
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
dispatchAlertSpy = vi
|
||||
.spyOn(NotificationService.getInstance(), 'dispatchAlert')
|
||||
.mockResolvedValue(undefined);
|
||||
.mockResolvedValue({ persisted: true });
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
|
||||
@@ -5,7 +5,7 @@ import { PROXY_TIER_HEADER, PROXY_ROLE_HEADER, PROXY_DEPLOY_SOURCE_HEADER, PROXY
|
||||
import { LicenseService } from '../services/LicenseService';
|
||||
import { isProxyExemptPath } from '../helpers/proxyExemptPaths';
|
||||
import { remoteSupportsCrossNodeRbac, remoteAdvertisesCapability } from '../helpers/remoteCapabilities';
|
||||
import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY } from '../services/CapabilityRegistry';
|
||||
import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY, SERVICE_SCOPED_STACK_ALERT_CAPABILITY } from '../services/CapabilityRegistry';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { redactSensitiveText } from '../utils/safeLog';
|
||||
@@ -145,8 +145,19 @@ export function createRemoteProxyMiddleware(): RequestHandler {
|
||||
}
|
||||
// Body forwarding: conditionalJsonParser skips parsing for remote
|
||||
// requests (see middleware/jsonParser.ts), so req's raw stream is
|
||||
// intact and http-proxy's req.pipe(proxyReq) forwards the body
|
||||
// automatically.
|
||||
// usually intact and http-proxy's req.pipe(proxyReq) forwards it.
|
||||
// When a gate must inspect JSON (POST /alerts), we buffer into
|
||||
// req.rawBody first; rewrite that buffer here because the stream is
|
||||
// already consumed.
|
||||
if (req.rawBody) {
|
||||
proxyReq.removeHeader('Transfer-Encoding');
|
||||
proxyReq.removeHeader('Content-Length');
|
||||
if (!proxyReq.getHeader('Content-Type')) {
|
||||
proxyReq.setHeader('Content-Type', 'application/json');
|
||||
}
|
||||
proxyReq.setHeader('Content-Length', req.rawBody.length);
|
||||
proxyReq.write(req.rawBody);
|
||||
}
|
||||
},
|
||||
proxyRes: (proxyRes, req) => {
|
||||
// Mark every response forwarded from a remote node with a sentinel
|
||||
@@ -246,7 +257,8 @@ export function createRemoteProxyMiddleware(): RequestHandler {
|
||||
}
|
||||
}
|
||||
|
||||
// Mixed-version RBAC gate (non-admin only).
|
||||
// Mixed-version RBAC gate (non-admin only). Runs before alert body
|
||||
// buffering so an unauthorized client cannot force unbounded memory use.
|
||||
if (req.user?.role !== 'admin') {
|
||||
const rbacSupported = await remoteSupportsCrossNodeRbac(req.nodeId);
|
||||
if (!rbacSupported) {
|
||||
@@ -257,6 +269,49 @@ export function createRemoteProxyMiddleware(): RequestHandler {
|
||||
}
|
||||
}
|
||||
|
||||
// POST /alerts bodies are not on req.body for remote hops (JSON parsing
|
||||
// is skipped so the stream can be piped). Buffer once under the same
|
||||
// 100 KB cap as express.json(), gate on service_name, then rewrite
|
||||
// rawBody in on.proxyReq. Compressed bodies are rejected: the hub cannot
|
||||
// inspect them, and treating parse failure as unscoped would bypass the
|
||||
// mixed-version gate.
|
||||
if (isAlertCreateRoute(req)) {
|
||||
if (hasNonIdentityContentEncoding(req)) {
|
||||
await drainRequestBody(req);
|
||||
res.status(415).json({
|
||||
error: 'Compressed request bodies are not supported for remote alert creates',
|
||||
code: 'encoding_unsupported',
|
||||
});
|
||||
return;
|
||||
}
|
||||
try {
|
||||
req.rawBody = await bufferRequestBody(req, ALERT_PROXY_BODY_LIMIT);
|
||||
} catch (err) {
|
||||
const status = Number((err as { status?: number }).status);
|
||||
if (status === 413) {
|
||||
console.error('[remoteNodeProxy] alert body rejected as too large:', err);
|
||||
res.status(413).json({ error: 'Alert payload too large', code: 'entity_too_large' });
|
||||
return;
|
||||
}
|
||||
if (status === 400) {
|
||||
console.error('[remoteNodeProxy] alert body incomplete:', err);
|
||||
res.status(400).json({ error: 'Incomplete request body' });
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (alertCreateHasScopedService(req.rawBody)) {
|
||||
const supported = await remoteAdvertisesCapability(req.nodeId, SERVICE_SCOPED_STACK_ALERT_CAPABILITY);
|
||||
if (!supported) {
|
||||
res.status(400).json({
|
||||
error: 'Service-scoped alert rules are not supported on this node',
|
||||
code: 'capability_unavailable',
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
req.proxyTarget = target;
|
||||
beginProxyTiming(req, res);
|
||||
proxy(req, res, next);
|
||||
@@ -283,3 +338,151 @@ function isServiceScopedUpdateRoute(req: Request): boolean {
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** POST /alerts (path is post-/api strip). */
|
||||
function isAlertCreateRoute(req: Request): boolean {
|
||||
return req.method === 'POST' && /^\/alerts\/?$/.test(req.path);
|
||||
}
|
||||
|
||||
/** Same default as express.json(); remote alert creates must not exceed it. */
|
||||
const ALERT_PROXY_BODY_LIMIT = 100 * 1024;
|
||||
|
||||
/** Max time to wait for leftover body bytes after a size/encoding reject. */
|
||||
const DRAIN_TIMEOUT_MS = 5_000;
|
||||
|
||||
/** Error with HTTP status for the alert-body gate catch mapper. */
|
||||
function alertBodyError(message: string, status: number): Error {
|
||||
return Object.assign(new Error(message), { status, expose: true });
|
||||
}
|
||||
|
||||
/** True when Content-Encoding is present and not identity (gzip/deflate/br/…). */
|
||||
function hasNonIdentityContentEncoding(req: Request): boolean {
|
||||
const raw = req.headers['content-encoding'];
|
||||
if (raw == null) return false;
|
||||
const value = Array.isArray(raw) ? raw.join(',') : raw;
|
||||
return value.split(',').some((part) => {
|
||||
const encoding = part.trim().toLowerCase();
|
||||
return encoding.length > 0 && encoding !== 'identity';
|
||||
});
|
||||
}
|
||||
|
||||
/** True when the buffered JSON alert body targets a specific Compose service. */
|
||||
function alertCreateHasScopedService(rawBody: Buffer): boolean {
|
||||
if (rawBody.length === 0) return false;
|
||||
try {
|
||||
const parsed = JSON.parse(rawBody.toString('utf-8')) as { service_name?: unknown };
|
||||
return typeof parsed.service_name === 'string' && parsed.service_name.trim() !== '';
|
||||
} catch {
|
||||
// Identity-encoded non-JSON is forwarded as-is; the remote rejects it.
|
||||
// Encoded bodies never reach here (rejected earlier).
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Consume remaining request bytes (or wait for abort/close) so the response
|
||||
* can flush without leaving the socket half-open. Does not buffer into memory.
|
||||
* Caps wait time so a stalled or endless chunked stream cannot hang the gate.
|
||||
*/
|
||||
function drainRequestBody(req: Request): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (req.readableEnded || req.destroyed) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
let settled = false;
|
||||
const done = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
req.off('end', done);
|
||||
req.off('error', done);
|
||||
req.off('aborted', done);
|
||||
req.off('close', done);
|
||||
resolve();
|
||||
};
|
||||
// Bound hang time: destroy after timeout so mid-stream rejects still settle.
|
||||
const timer = setTimeout(() => {
|
||||
if (!req.destroyed) req.destroy();
|
||||
done();
|
||||
}, DRAIN_TIMEOUT_MS);
|
||||
req.resume();
|
||||
req.once('end', done);
|
||||
req.once('error', done);
|
||||
req.once('aborted', done);
|
||||
req.once('close', done);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Buffer the request so a capability gate can inspect JSON without leaving
|
||||
* http-proxy with an already-ended empty stream. Enforces `limit` on both
|
||||
* declared Content-Length and streamed accumulation.
|
||||
*/
|
||||
async function bufferRequestBody(req: Request, limit: number): Promise<Buffer> {
|
||||
if (req.rawBody) {
|
||||
if (req.rawBody.length > limit) {
|
||||
throw alertBodyError('Alert payload too large', 413);
|
||||
}
|
||||
return req.rawBody;
|
||||
}
|
||||
if (req.readableEnded) return Buffer.alloc(0);
|
||||
|
||||
const declared = Number.parseInt(String(req.headers['content-length'] ?? ''), 10);
|
||||
if (Number.isFinite(declared) && declared > limit) {
|
||||
// Reject immediately so the client gets a structured 413; drain leftover
|
||||
// bytes in the background so the socket can close without holding the gate.
|
||||
void drainRequestBody(req);
|
||||
throw alertBodyError('Alert payload too large', 413);
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
let total = 0;
|
||||
let settled = false;
|
||||
|
||||
const settle = (fn: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
fn();
|
||||
};
|
||||
const finish = (buf: Buffer) => settle(() => resolve(buf));
|
||||
const fail = (err: Error) => settle(() => reject(err));
|
||||
|
||||
const onData = (chunk: Buffer) => {
|
||||
const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
||||
total += buf.length;
|
||||
if (total > limit) {
|
||||
chunks.length = 0;
|
||||
// fail() removes listeners; drain leftover bytes so the socket can close.
|
||||
fail(alertBodyError('Alert payload too large', 413));
|
||||
void drainRequestBody(req);
|
||||
return;
|
||||
}
|
||||
chunks.push(buf);
|
||||
};
|
||||
const onEnd = () => finish(Buffer.concat(chunks));
|
||||
const onError = (err: Error) => fail(err);
|
||||
const onAborted = () => fail(alertBodyError('Client aborted request body', 400));
|
||||
const onClose = () => {
|
||||
if (!settled && !req.readableEnded) {
|
||||
fail(alertBodyError('Client closed request before body finished', 400));
|
||||
}
|
||||
};
|
||||
|
||||
function cleanup(): void {
|
||||
req.off('data', onData);
|
||||
req.off('end', onEnd);
|
||||
req.off('error', onError);
|
||||
req.off('aborted', onAborted);
|
||||
req.off('close', onClose);
|
||||
}
|
||||
|
||||
req.on('data', onData);
|
||||
req.on('end', onEnd);
|
||||
req.on('error', onError);
|
||||
req.on('aborted', onAborted);
|
||||
req.on('close', onClose);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3,9 +3,21 @@ import { z } from 'zod';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { authMiddleware } from '../middleware/auth';
|
||||
import { requireAdmin } from '../middleware/tierGates';
|
||||
import { isValidServiceName } from '../utils/validation';
|
||||
import {
|
||||
getActiveCapabilities,
|
||||
SERVICE_SCOPED_STACK_ALERT_CAPABILITY,
|
||||
} from '../services/CapabilityRegistry';
|
||||
|
||||
const AlertCreateSchema = z.object({
|
||||
stack_name: z.string().min(1).max(255),
|
||||
service_name: z.preprocess(
|
||||
(val) => (val === '' ? null : val),
|
||||
z.string().max(255).nullable().optional().refine(
|
||||
(val) => val == null || isValidServiceName(val),
|
||||
{ message: 'Invalid service name' },
|
||||
),
|
||||
),
|
||||
metric: z.enum(['cpu_percent', 'memory_percent', 'memory_mb', 'net_rx', 'net_tx', 'restart_count']),
|
||||
operator: z.enum(['>', '>=', '<', '<=', '==']),
|
||||
threshold: z.number().min(0),
|
||||
@@ -22,7 +34,8 @@ alertsRouter.get('/', authMiddleware, async (req: Request, res: Response) => {
|
||||
|
||||
const alerts = DatabaseService.getInstance().getStackAlerts(stackName);
|
||||
res.json(alerts);
|
||||
} catch {
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch alerts:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch alerts' });
|
||||
}
|
||||
});
|
||||
@@ -34,8 +47,23 @@ alertsRouter.post('/', authMiddleware, async (req: Request, res: Response) => {
|
||||
res.status(400).json({ error: 'Invalid alert data', details: parsed.error.flatten().fieldErrors });
|
||||
return;
|
||||
}
|
||||
const { service_name, ...alertFields } = parsed.data;
|
||||
const serviceName = service_name ?? null;
|
||||
if (
|
||||
serviceName != null
|
||||
&& !getActiveCapabilities().includes(SERVICE_SCOPED_STACK_ALERT_CAPABILITY)
|
||||
) {
|
||||
res.status(400).json({
|
||||
error: 'This node does not support service-scoped alert rules',
|
||||
code: 'capability_unavailable',
|
||||
});
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const created = DatabaseService.getInstance().addStackAlert(parsed.data);
|
||||
const created = DatabaseService.getInstance().addStackAlert({
|
||||
...alertFields,
|
||||
service_name: serviceName,
|
||||
});
|
||||
res.status(201).json(created);
|
||||
} catch (error) {
|
||||
console.error('Failed to add alert:', error);
|
||||
@@ -45,11 +73,18 @@ alertsRouter.post('/', authMiddleware, async (req: Request, res: Response) => {
|
||||
|
||||
alertsRouter.delete('/:id', authMiddleware, async (req: Request, res: Response) => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
// Reject leading-junk / fractional ids (parseInt("1abc") === 1, parseInt("2.5") === 2).
|
||||
const rawId = String(req.params.id ?? '');
|
||||
const id = /^\d+$/.test(rawId) ? Number.parseInt(rawId, 10) : NaN;
|
||||
if (!Number.isInteger(id) || id <= 0) {
|
||||
res.status(400).json({ error: 'Invalid alert id' });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const id = parseInt(req.params.id as string, 10);
|
||||
DatabaseService.getInstance().deleteStackAlert(id);
|
||||
res.json({ success: true });
|
||||
} catch {
|
||||
} catch (error) {
|
||||
console.error('Failed to delete alert:', error);
|
||||
res.status(500).json({ error: 'Failed to delete alert' });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -59,6 +59,7 @@ export const CAPABILITIES = [
|
||||
'stack-down-remove-volumes',
|
||||
'guided-external-network-preflight',
|
||||
'service-scoped-update',
|
||||
'service-scoped-stack-alert',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
@@ -88,6 +89,10 @@ export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes'
|
||||
/** Capability for the nested per-service update/restore routes and the `effective-services` model they read. */
|
||||
export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability;
|
||||
|
||||
/** Capability for nullable `service_name` on stack alert rules and per-service cooldown evaluation. */
|
||||
export const SERVICE_SCOPED_STACK_ALERT_CAPABILITY =
|
||||
'service-scoped-stack-alert' as const satisfies Capability;
|
||||
|
||||
/** Returns true when the string is a usable semver version. */
|
||||
export function isValidVersion(v: string | null | undefined): v is string {
|
||||
return !!v && v !== 'unknown' && v !== '0.0.0-dev' && !!semver.valid(v);
|
||||
|
||||
@@ -101,6 +101,7 @@ function stringifyServicesJson(services: StackServiceStatus[], generation: numbe
|
||||
export interface StackAlert {
|
||||
id?: number;
|
||||
stack_name: string;
|
||||
service_name: string | null;
|
||||
metric: string;
|
||||
operator: string;
|
||||
threshold: number;
|
||||
@@ -1062,6 +1063,7 @@ export class DatabaseService {
|
||||
this.migrateStackDossierHashes();
|
||||
this.migrateGitSourceMultiFile();
|
||||
this.migrateNodeUpdateSkips();
|
||||
this.migrateStackAlertServiceScope();
|
||||
|
||||
// Reset the cache once at end of constructor in case any migration
|
||||
// populated it via getGlobalSettings() and a subsequent migration
|
||||
@@ -1098,6 +1100,7 @@ export class DatabaseService {
|
||||
CREATE TABLE IF NOT EXISTS stack_alerts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
stack_name TEXT NOT NULL,
|
||||
service_name TEXT,
|
||||
metric TEXT NOT NULL,
|
||||
operator TEXT NOT NULL,
|
||||
threshold REAL NOT NULL,
|
||||
@@ -1106,6 +1109,17 @@ export class DatabaseService {
|
||||
last_fired_at INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
-- FK cascade is declarative only: PRAGMA foreign_keys is never enabled
|
||||
-- on this connection, so parent deletes must remove children explicitly
|
||||
-- (see deleteStackAlert).
|
||||
CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns (
|
||||
alert_id INTEGER NOT NULL,
|
||||
service_name TEXT NOT NULL,
|
||||
last_fired_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (alert_id, service_name),
|
||||
FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS notification_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
node_id INTEGER NOT NULL DEFAULT 0,
|
||||
@@ -2280,6 +2294,25 @@ export class DatabaseService {
|
||||
}
|
||||
}
|
||||
|
||||
private migrateStackAlertServiceScope(): void {
|
||||
this.tryAddColumn('stack_alerts', 'service_name', 'TEXT');
|
||||
try {
|
||||
// FK is not enforced (foreign_keys pragma off); deleteStackAlert removes children.
|
||||
this.db.prepare(`
|
||||
CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns (
|
||||
alert_id INTEGER NOT NULL,
|
||||
service_name TEXT NOT NULL,
|
||||
last_fired_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (alert_id, service_name),
|
||||
FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE
|
||||
)
|
||||
`).run();
|
||||
} catch (e) {
|
||||
console.error('[DatabaseService] stack_alert_service_cooldowns migration failed:', (e as Error).message);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private migrateScanPolicyFleetColumns(): void {
|
||||
this.tryAddColumn('scan_policies', 'node_identity', "TEXT NOT NULL DEFAULT ''");
|
||||
this.tryAddColumn('scan_policies', 'replicated_from_control', 'INTEGER NOT NULL DEFAULT 0');
|
||||
@@ -3075,22 +3108,19 @@ export class DatabaseService {
|
||||
// --- Stack Alerts ---
|
||||
|
||||
public getStackAlerts(stackName?: string): StackAlert[] {
|
||||
let stmt;
|
||||
if (stackName) {
|
||||
stmt = this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?');
|
||||
return stmt.all(stackName) as StackAlert[];
|
||||
} else {
|
||||
stmt = this.db.prepare('SELECT * FROM stack_alerts');
|
||||
return stmt.all() as StackAlert[];
|
||||
return this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?').all(stackName) as StackAlert[];
|
||||
}
|
||||
return this.db.prepare('SELECT * FROM stack_alerts').all() as StackAlert[];
|
||||
}
|
||||
|
||||
public addStackAlert(alert: StackAlert): StackAlert {
|
||||
const stmt = this.db.prepare(
|
||||
'INSERT INTO stack_alerts (stack_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?)'
|
||||
'INSERT INTO stack_alerts (stack_name, service_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
|
||||
);
|
||||
const result = stmt.run(
|
||||
alert.stack_name,
|
||||
alert.service_name ?? null,
|
||||
alert.metric,
|
||||
alert.operator,
|
||||
alert.threshold,
|
||||
@@ -3101,9 +3131,16 @@ export class DatabaseService {
|
||||
return this.db.prepare('SELECT * FROM stack_alerts WHERE id = ?').get(result.lastInsertRowid) as StackAlert;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an alert and its per-service cooldown rows.
|
||||
* SQLite foreign_keys is not enabled here, so child rows are removed
|
||||
* explicitly rather than relying on ON DELETE CASCADE.
|
||||
*/
|
||||
public deleteStackAlert(id: number): void {
|
||||
const stmt = this.db.prepare('DELETE FROM stack_alerts WHERE id = ?');
|
||||
stmt.run(id);
|
||||
this.transaction(() => {
|
||||
this.deleteStackAlertServiceCooldowns(id);
|
||||
this.db.prepare('DELETE FROM stack_alerts WHERE id = ?').run(id);
|
||||
});
|
||||
}
|
||||
|
||||
public updateStackAlertLastFired(id: number, timestamp: number): void {
|
||||
@@ -3111,6 +3148,32 @@ export class DatabaseService {
|
||||
stmt.run(timestamp, id);
|
||||
}
|
||||
|
||||
public getStackAlertServiceCooldown(alertId: number, serviceName: string): number | null {
|
||||
const row = this.db.prepare(
|
||||
'SELECT last_fired_at FROM stack_alert_service_cooldowns WHERE alert_id = ? AND service_name = ?'
|
||||
).get(alertId, serviceName) as { last_fired_at: number } | undefined;
|
||||
return row?.last_fired_at ?? null;
|
||||
}
|
||||
|
||||
public hasAnyStackAlertServiceCooldown(alertId: number): boolean {
|
||||
const row = this.db.prepare(
|
||||
'SELECT 1 AS present FROM stack_alert_service_cooldowns WHERE alert_id = ? LIMIT 1'
|
||||
).get(alertId) as { present: number } | undefined;
|
||||
return !!row;
|
||||
}
|
||||
|
||||
public upsertStackAlertServiceCooldown(alertId: number, serviceName: string, timestamp: number): void {
|
||||
this.db.prepare(`
|
||||
INSERT INTO stack_alert_service_cooldowns (alert_id, service_name, last_fired_at)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(alert_id, service_name) DO UPDATE SET last_fired_at = excluded.last_fired_at
|
||||
`).run(alertId, serviceName, timestamp);
|
||||
}
|
||||
|
||||
public deleteStackAlertServiceCooldowns(alertId: number): void {
|
||||
this.db.prepare('DELETE FROM stack_alert_service_cooldowns WHERE alert_id = ?').run(alertId);
|
||||
}
|
||||
|
||||
// --- Auto-Heal Policies ---
|
||||
|
||||
public getAutoHealPolicies(stackName?: string, nodeId?: number): AutoHealPolicy[] {
|
||||
|
||||
@@ -735,15 +735,15 @@ export class DockerEventService {
|
||||
}
|
||||
|
||||
private async emitError(category: NotificationCategory, message: string, stackName?: string, containerName?: string, systemOnly = false): Promise<void> {
|
||||
return this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly));
|
||||
await this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly));
|
||||
}
|
||||
|
||||
private async emitWarning(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName));
|
||||
await this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName));
|
||||
}
|
||||
|
||||
private async emitInfo(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName));
|
||||
await this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName));
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
|
||||
@@ -32,6 +32,39 @@ const getOperatorPhrase = (operator: string): string => {
|
||||
return `triggered the operator ${operator}`;
|
||||
};
|
||||
|
||||
/** Sentinel for containers without a Compose service label. Not a valid API target. */
|
||||
const UNLABELED_SERVICE_KEY = '_unlabeled';
|
||||
|
||||
function displayServiceName(serviceName: string): string {
|
||||
return serviceName === UNLABELED_SERVICE_KEY ? 'unknown service' : serviceName;
|
||||
}
|
||||
|
||||
function parseAlertBreachKey(key: string): { ruleId: number; containerId: string } | null {
|
||||
const sep = key.indexOf(':');
|
||||
if (sep < 0) return null;
|
||||
const ruleId = Number(key.slice(0, sep));
|
||||
if (!Number.isFinite(ruleId)) return null;
|
||||
return { ruleId, containerId: key.slice(sep + 1) };
|
||||
}
|
||||
|
||||
/** Per-service cooldown timestamp, with pre-migration fallback to rule.last_fired_at. */
|
||||
function resolveStackAlertLastFired(rule: StackAlert, serviceName: string, db: DatabaseService): number {
|
||||
const ruleId = rule.id!;
|
||||
let lastFired = db.getStackAlertServiceCooldown(ruleId, serviceName);
|
||||
if (lastFired == null && !db.hasAnyStackAlertServiceCooldown(ruleId) && (rule.last_fired_at || 0) > 0) {
|
||||
lastFired = rule.last_fired_at || 0;
|
||||
}
|
||||
return lastFired || 0;
|
||||
}
|
||||
|
||||
function normalizeContainerName(container: { Id: string; Names?: string[] }): string {
|
||||
const raw = container.Names?.[0];
|
||||
if (raw && raw.length > 0) {
|
||||
return raw.startsWith('/') ? raw.slice(1) : raw;
|
||||
}
|
||||
return container.Id.slice(0, 12);
|
||||
}
|
||||
|
||||
/** Shape of the JSON returned by Docker container stats (stream: false). */
|
||||
interface DockerContainerStats {
|
||||
cpu_stats?: {
|
||||
@@ -153,21 +186,18 @@ export class MonitorService {
|
||||
private janitorConsecutiveTimeouts = 0;
|
||||
private janitorBreakerUntil = 0;
|
||||
|
||||
// Track the duration a specific stack alert rule has been in breach state
|
||||
// key: rule_id, value: AlertState
|
||||
private activeBreaches = new Map<number, AlertState>();
|
||||
// Track the duration a specific stack alert rule+container has been in breach
|
||||
// key: `${ruleId}:${containerId}`, value: AlertState
|
||||
private activeBreaches = new Map<string, AlertState>();
|
||||
|
||||
// Track previous network counters per container for rate calculation.
|
||||
// key: container_id, value: { rx bytes, tx bytes, sample timestamp }
|
||||
private previousNetworkStats = new Map<string, { rx: number; tx: number; ts: number }>();
|
||||
|
||||
// Per-cycle dispatch dedup. Stack rules are shared across every container
|
||||
// in the stack, so parallel processContainer calls can race past the
|
||||
// cooldown check (DB write happens after the awaited dispatch) and fire
|
||||
// the same alert N times on first breach. The synchronous check-and-add
|
||||
// below is atomic in JS between awaits, so only the first worker wins.
|
||||
// Per-cycle dispatch dedup keyed by rule+service so replicas of the same
|
||||
// Compose service do not fire N times, while different services can.
|
||||
// Reset at the start of each evaluateStackAlerts call.
|
||||
private firedThisCycle = new Set<number>();
|
||||
private firedThisCycle = new Set<string>();
|
||||
|
||||
// Crash and healthcheck detection live in DockerEventService (event-driven,
|
||||
// causal classification). MonitorService no longer polls for container
|
||||
@@ -604,6 +634,10 @@ export class MonitorService {
|
||||
else alertsByStack.set(a.stack_name, [a]);
|
||||
}
|
||||
|
||||
const activeRuleIds = new Set(alerts.map(a => a.id!));
|
||||
const allCurrentIds = new Set<string>();
|
||||
let enumerationFailed = false;
|
||||
|
||||
for (const node of nodes) {
|
||||
if (!node.id) continue;
|
||||
// Remote nodes are self-monitoring - skip direct Docker access
|
||||
@@ -623,22 +657,39 @@ export class MonitorService {
|
||||
(container) => this.processContainer(node, container, alertsByStack, docker, db),
|
||||
);
|
||||
|
||||
for (const c of containers) allCurrentIds.add(c.Id);
|
||||
|
||||
// Clean up stale network stats for containers on this node that no longer run
|
||||
if (this.previousNetworkStats.size > containers.length * 2) {
|
||||
const currentIds = new Set(containers.map((c: { Id: string }) => c.Id));
|
||||
const nodeIds = new Set(containers.map((c: { Id: string }) => c.Id));
|
||||
for (const key of this.previousNetworkStats.keys()) {
|
||||
if (!currentIds.has(key)) this.previousNetworkStats.delete(key);
|
||||
if (!nodeIds.has(key)) this.previousNetworkStats.delete(key);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
// Enumeration failed: preserve existing breach timers.
|
||||
enumerationFailed = true;
|
||||
console.error(`Error fetching containers for node ${node.name}`, err);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up stale breach trackers for rules that have been deleted
|
||||
const activeRuleIds = new Set(alerts.map(a => a.id!));
|
||||
for (const key of this.activeBreaches.keys()) {
|
||||
if (!activeRuleIds.has(key)) {
|
||||
// After successful enumeration(s), drop breach timers for containers
|
||||
// that are no longer running. Skip when any local enumeration failed
|
||||
// so a transient Docker error cannot reset duration timers.
|
||||
if (!enumerationFailed) {
|
||||
for (const key of [...this.activeBreaches.keys()]) {
|
||||
const parsed = parseAlertBreachKey(key);
|
||||
if (parsed && activeRuleIds.has(parsed.ruleId) && !allCurrentIds.has(parsed.containerId)) {
|
||||
this.activeBreaches.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up in-memory breach trackers for rules that have been deleted.
|
||||
// Persisted cooldowns are removed only via transactional deleteStackAlert.
|
||||
for (const key of [...this.activeBreaches.keys()]) {
|
||||
const parsed = parseAlertBreachKey(key);
|
||||
if (parsed && !activeRuleIds.has(parsed.ruleId)) {
|
||||
this.activeBreaches.delete(key);
|
||||
}
|
||||
}
|
||||
@@ -666,12 +717,14 @@ export class MonitorService {
|
||||
*/
|
||||
private async processContainer(
|
||||
node: Node,
|
||||
container: { Id: string; Labels?: Record<string, string> },
|
||||
container: { Id: string; Names?: string[]; Labels?: Record<string, string> },
|
||||
alertsByStack: Map<string, StackAlert[]>,
|
||||
docker: DockerController,
|
||||
db: DatabaseService,
|
||||
): Promise<void> {
|
||||
const stackName = container.Labels?.['com.docker.compose.project'] || 'system';
|
||||
const serviceName = container.Labels?.['com.docker.compose.service'] || UNLABELED_SERVICE_KEY;
|
||||
const containerName = normalizeContainerName(container);
|
||||
|
||||
try {
|
||||
const rawStats = await withTimeout(
|
||||
@@ -727,7 +780,11 @@ export class MonitorService {
|
||||
});
|
||||
|
||||
for (const rule of stackAlerts) {
|
||||
if (rule.service_name && rule.service_name !== serviceName) continue;
|
||||
|
||||
const ruleId = rule.id!;
|
||||
const breachKey = `${ruleId}:${container.Id}`;
|
||||
const cooldownKey = `${ruleId}:${serviceName}`;
|
||||
const currentValue = metrics[rule.metric as keyof typeof metrics];
|
||||
|
||||
if (currentValue === undefined) continue;
|
||||
@@ -735,57 +792,75 @@ export class MonitorService {
|
||||
const isBreaching = this.evaluateCondition(currentValue, rule.operator, rule.threshold);
|
||||
|
||||
if (isBreaching) {
|
||||
if (!this.activeBreaches.has(ruleId)) {
|
||||
this.activeBreaches.set(ruleId, { breachStartedAt: Date.now() });
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}"`);
|
||||
if (!this.activeBreaches.has(breachKey)) {
|
||||
this.activeBreaches.set(breachKey, { breachStartedAt: Date.now() });
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}" service "${serviceName}" container "${containerName}"`);
|
||||
}
|
||||
|
||||
const breachState = this.activeBreaches.get(ruleId)!;
|
||||
const breachState = this.activeBreaches.get(breachKey)!;
|
||||
const durationMs = Date.now() - breachState.breachStartedAt;
|
||||
const requiredDurationMs = rule.duration_mins * 60 * 1000;
|
||||
|
||||
if (durationMs >= requiredDurationMs) {
|
||||
const timeSinceLastFired = Date.now() - (rule.last_fired_at || 0);
|
||||
const timeSinceLastFired = Date.now() - resolveStackAlertLastFired(rule, serviceName, db);
|
||||
const requiredCooldownMs = rule.cooldown_mins * 60 * 1000;
|
||||
|
||||
if (timeSinceLastFired >= requiredCooldownMs) {
|
||||
// Claim this rule for the cycle before awaiting
|
||||
// dispatch. The check-and-add is synchronous, so
|
||||
// sibling workers evaluating the same shared rule
|
||||
// see the claim and skip — preventing N-fire when
|
||||
// multiple containers in one stack all breach.
|
||||
if (this.firedThisCycle.has(ruleId)) {
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Skipping duplicate dispatch for rule ${ruleId} (already fired this cycle by sibling container)`);
|
||||
} else {
|
||||
this.firedThisCycle.add(ruleId);
|
||||
// Claim this rule+service for the cycle before awaiting
|
||||
// dispatch so replicas of the same service skip.
|
||||
if (this.firedThisCycle.has(cooldownKey)) {
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Skipping duplicate dispatch for rule ${ruleId} service "${serviceName}" (already fired this cycle by sibling replica)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const { name: metricName, unit } = getMetricDetails(rule.metric);
|
||||
const operatorPhrase = getOperatorPhrase(rule.operator);
|
||||
this.firedThisCycle.add(cooldownKey);
|
||||
|
||||
const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue;
|
||||
const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold;
|
||||
const { name: metricName, unit } = getMetricDetails(rule.metric);
|
||||
const operatorPhrase = getOperatorPhrase(rule.operator);
|
||||
|
||||
// Node-neutral body: the hub bell badge attributes remotes.
|
||||
const message = `The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
|
||||
const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue;
|
||||
const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold;
|
||||
const serviceLabel = displayServiceName(serviceName);
|
||||
|
||||
console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`);
|
||||
await NotificationService.getInstance().dispatchAlert(
|
||||
// Node-neutral body: the hub bell badge attributes remotes.
|
||||
const message = `The **${metricName}** for **${serviceLabel}** in **${rule.stack_name}** (container **${containerName}**) ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
|
||||
|
||||
try {
|
||||
const { persisted } = await NotificationService.getInstance().dispatchAlert(
|
||||
'warning',
|
||||
'monitor_alert',
|
||||
message,
|
||||
{ stackName: rule.stack_name, actor: 'system:monitor' },
|
||||
{ stackName: rule.stack_name, containerName, actor: 'system:monitor' },
|
||||
);
|
||||
if (!persisted) {
|
||||
// History was not written; do not advance cooldown or we silence retries.
|
||||
this.firedThisCycle.delete(cooldownKey);
|
||||
console.error(
|
||||
`[MonitorService] Alert history not persisted for rule ${ruleId} service "${serviceName}"; cooldown not advanced`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const firedAt = Date.now();
|
||||
// Dual-write: per-service row for current code, parent last_fired_at
|
||||
// so a downgrade that only reads the parent column still has a floor.
|
||||
db.upsertStackAlertServiceCooldown(ruleId, serviceName, firedAt);
|
||||
db.updateStackAlertLastFired(ruleId, firedAt);
|
||||
console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}" service "${serviceName}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`);
|
||||
} catch (fireErr) {
|
||||
this.firedThisCycle.delete(cooldownKey);
|
||||
console.error(
|
||||
`[MonitorService] Failed to fire alert rule ${ruleId} service "${serviceName}" container "${containerName}":`,
|
||||
fireErr,
|
||||
);
|
||||
|
||||
db.updateStackAlertLastFired(ruleId, Date.now());
|
||||
}
|
||||
} else if (isDebugEnabled()) {
|
||||
console.log(`[Monitor:diag] Cooldown active for rule ${ruleId}: ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`);
|
||||
console.log(`[Monitor:diag] Cooldown active for rule ${ruleId} service "${serviceName}": ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (this.activeBreaches.has(ruleId)) {
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}"`);
|
||||
this.activeBreaches.delete(ruleId);
|
||||
if (this.activeBreaches.has(breachKey)) {
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}" container "${containerName}"`);
|
||||
this.activeBreaches.delete(breachKey);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,7 +181,7 @@ export class NotificationService {
|
||||
category: NotificationCategory,
|
||||
message: string,
|
||||
options?: { stackName?: string; containerName?: string; actor?: string },
|
||||
) {
|
||||
): Promise<{ persisted: boolean }> {
|
||||
const t0 = Date.now();
|
||||
const { stackName, containerName, actor } = options ?? {};
|
||||
|
||||
@@ -191,6 +191,8 @@ export class NotificationService {
|
||||
// WebSocket broadcast can all throw on an unhealthy DB, which would
|
||||
// otherwise surface as an unhandledRejection and take the process down.
|
||||
// The whole body is wrapped so the worst case is a dropped notification.
|
||||
// Callers that gate cooldowns on history use `persisted` (true only after the row write).
|
||||
let wroteHistory = false;
|
||||
try {
|
||||
// Internal writes use the middleware default so they share a row key
|
||||
// with user-initiated requests; otherwise the UI and monitors split
|
||||
@@ -216,11 +218,12 @@ export class NotificationService {
|
||||
container_name: containerName,
|
||||
actor_username: actor ?? null,
|
||||
});
|
||||
wroteHistory = true;
|
||||
StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, true);
|
||||
} catch (err) {
|
||||
StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, false);
|
||||
console.error('[Notify] Failed to persist notification:', err);
|
||||
return;
|
||||
return { persisted: false };
|
||||
}
|
||||
// Separate [StackActivity:diag] namespace from the [Notify:diag] lines
|
||||
// below so a single grep can pull every per-stack timeline write across
|
||||
@@ -273,7 +276,7 @@ export class NotificationService {
|
||||
}
|
||||
|
||||
if (suppressExternal) {
|
||||
return;
|
||||
return { persisted: wroteHistory };
|
||||
}
|
||||
|
||||
// Resolve retry extras once for this dispatch (shared by all destinations).
|
||||
@@ -298,14 +301,14 @@ export class NotificationService {
|
||||
)
|
||||
);
|
||||
this.recordDispatchErrors(notification.id!, errors);
|
||||
return;
|
||||
return { persisted: wroteHistory };
|
||||
}
|
||||
|
||||
// 4. Fall back to this instance's agents (keyed by this instance's default node id).
|
||||
const agents = this.dbService.getEnabledAgents(localNodeId);
|
||||
if (agents.length === 0) {
|
||||
if (isDebugEnabled()) console.log('[Notify:diag] No routes or agents matched; skipping external dispatch');
|
||||
return;
|
||||
return { persisted: wroteHistory };
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) console.log(`[Notify:diag] Falling back to ${agents.length} global agent(s)`);
|
||||
@@ -322,8 +325,11 @@ export class NotificationService {
|
||||
)
|
||||
);
|
||||
this.recordDispatchErrors(notification.id!, errors);
|
||||
return { persisted: wroteHistory };
|
||||
} catch (err) {
|
||||
console.error('[Notify] dispatchAlert failed:', err);
|
||||
// History may already be written; callers must not treat that as a miss.
|
||||
return { persisted: wroteHistory };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user