From 93ae147ec1444a719c61348e238476d2b73f1309 Mon Sep 17 00:00:00 2001 From: Anso Date: Wed, 1 Apr 2026 22:43:35 -0400 Subject: [PATCH] test: expand backend test coverage for stability gaps (#329) Add 5 new test files (116 test cases) targeting previously untested service-layer logic: - database-metrics: metrics CRUD, minute-bucket aggregation, cleanup retention, notification auto-cap at 100, stack alerts CRUD, and stress tests with 1000+ metrics - monitor-service: CPU/memory/network calculation helpers, all 6 alert condition operators, breach state machine lifecycle, global crash detection, host limit thresholds, cleanup delegation, and isProcessing concurrency guard - docker-controller: validateApiData error detection, state-safe start/stop (304 handling), batch container removal with partial failures, disk usage calculation, resource classification (managed/unmanaged/system), orphan detection, and daemon unreachable error propagation - scheduler-service: cron parsing, license tier gating, concurrent task prevention via runningTasks Set, manual trigger, all 4 task types (restart/snapshot/prune/update), wildcard targets, error recording, recovery notifications, and cleanup - compose-service: subprocess spawn/exit handling, WebSocket output, deploy with health probe and atomic rollback, registry auth temp dir lifecycle, and downStack teardown --- backend/src/__tests__/compose-service.test.ts | 383 +++++++++++ .../src/__tests__/database-metrics.test.ts | 398 +++++++++++ .../src/__tests__/docker-controller.test.ts | 378 ++++++++++ backend/src/__tests__/monitor-service.test.ts | 521 ++++++++++++++ .../src/__tests__/scheduler-service.test.ts | 645 ++++++++++++++++++ 5 files changed, 2325 insertions(+) create mode 100644 backend/src/__tests__/compose-service.test.ts create mode 100644 backend/src/__tests__/database-metrics.test.ts create mode 100644 backend/src/__tests__/docker-controller.test.ts create mode 100644 backend/src/__tests__/monitor-service.test.ts create mode 100644 backend/src/__tests__/scheduler-service.test.ts diff --git a/backend/src/__tests__/compose-service.test.ts b/backend/src/__tests__/compose-service.test.ts new file mode 100644 index 00000000..c83f317a --- /dev/null +++ b/backend/src/__tests__/compose-service.test.ts @@ -0,0 +1,383 @@ +/** + * Unit tests for ComposeService — subprocess handling, deploy/rollback, + * registry auth temp dir management, and WebSocket output. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { EventEmitter } from 'events'; + +// ── Hoisted mocks ────────────────────────────────────────────────────── + +const { + mockSpawn, + mockGetContainersByStack, mockRemoveContainers, mockListContainers, + mockContainerInspect, mockContainerLogs, + mockGetRegistries, mockResolveDockerConfig, + mockBackupStackFiles, mockRestoreStackFiles, + mockMkdtempSync, mockWriteFileSync, mockUnlinkSync, mockRmdirSync, +} = vi.hoisted(() => ({ + mockSpawn: vi.fn(), + mockGetContainersByStack: vi.fn().mockResolvedValue([]), + mockRemoveContainers: vi.fn().mockResolvedValue([]), + mockListContainers: vi.fn().mockResolvedValue([]), + mockContainerInspect: vi.fn().mockResolvedValue({ State: { ExitCode: 0 } }), + mockContainerLogs: vi.fn().mockResolvedValue(Buffer.from('')), + mockGetRegistries: vi.fn().mockReturnValue([]), + mockResolveDockerConfig: vi.fn().mockResolvedValue({ auths: {} }), + mockBackupStackFiles: vi.fn().mockResolvedValue(undefined), + mockRestoreStackFiles: vi.fn().mockResolvedValue(undefined), + mockMkdtempSync: vi.fn().mockReturnValue('/tmp/sencho-docker-test'), + mockWriteFileSync: vi.fn(), + mockUnlinkSync: vi.fn(), + mockRmdirSync: vi.fn(), +})); + +vi.mock('child_process', () => ({ spawn: mockSpawn })); + +vi.mock('fs', () => ({ + default: { + mkdtempSync: (...args: unknown[]) => mockMkdtempSync(...args), + writeFileSync: (...args: unknown[]) => mockWriteFileSync(...args), + unlinkSync: (...args: unknown[]) => mockUnlinkSync(...args), + rmdirSync: (...args: unknown[]) => mockRmdirSync(...args), + }, + mkdtempSync: (...args: unknown[]) => mockMkdtempSync(...args), + writeFileSync: (...args: unknown[]) => mockWriteFileSync(...args), + unlinkSync: (...args: unknown[]) => mockUnlinkSync(...args), + rmdirSync: (...args: unknown[]) => mockRmdirSync(...args), +})); + +vi.mock('../services/NodeRegistry', () => ({ + NodeRegistry: { + getInstance: () => ({ + getDefaultNodeId: () => 1, + getComposeDir: () => '/test/compose', + }), + }, +})); + +vi.mock('../services/DockerController', () => ({ + default: { + getInstance: () => ({ + getContainersByStack: mockGetContainersByStack, + removeContainers: mockRemoveContainers, + getDocker: () => ({ + listContainers: mockListContainers, + getContainer: () => ({ + inspect: mockContainerInspect, + logs: mockContainerLogs, + }), + }), + }), + }, +})); + +vi.mock('../services/DatabaseService', () => ({ + DatabaseService: { + getInstance: () => ({ + getRegistries: mockGetRegistries, + }), + }, +})); + +vi.mock('../services/RegistryService', () => ({ + RegistryService: { + getInstance: () => ({ + resolveDockerConfig: mockResolveDockerConfig, + }), + }, +})); + +vi.mock('../services/FileSystemService', () => ({ + FileSystemService: { + getInstance: () => ({ + backupStackFiles: mockBackupStackFiles, + restoreStackFiles: mockRestoreStackFiles, + }), + }, +})); + +vi.mock('../services/LogFormatter', () => ({ + LogFormatter: { formatLine: (line: string) => line }, +})); + +import { ComposeService } from '../services/ComposeService'; + +/** Creates an EventEmitter that mimics a child_process spawn result */ +function createMockProcess() { + const proc = new EventEmitter() as EventEmitter & { + stdout: EventEmitter; + stderr: EventEmitter; + kill: ReturnType; + }; + proc.stdout = new EventEmitter(); + proc.stderr = new EventEmitter(); + proc.kill = vi.fn(); + return proc; +} + +/** Sets up mockSpawn to auto-close with exit code 0 on next tick */ +function setupAutoCloseSpawn(exitCode = 0) { + mockSpawn.mockImplementation(() => { + const proc = createMockProcess(); + // Emit close asynchronously (next microtask) + Promise.resolve().then(() => proc.emit('close', exitCode)); + return proc; + }); +} + +function createMockWs() { + return { + readyState: 1, + send: vi.fn(), + on: vi.fn(), + close: vi.fn(), + OPEN: 1, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers({ shouldAdvanceTime: true }); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +// ── runCommand ───────────────────────────────────────────────────────── + +describe('ComposeService - runCommand', () => { + it('spawns docker compose with the correct action', async () => { + const proc = createMockProcess(); + mockSpawn.mockReturnValue(proc); + + const svc = ComposeService.getInstance(1); + const promise = svc.runCommand('my-stack', 'restart'); + proc.emit('close', 0); + await promise; + + expect(mockSpawn).toHaveBeenCalledWith( + 'docker', + ['compose', 'restart'], + expect.objectContaining({ cwd: expect.stringContaining('my-stack') }) + ); + }); + + it('resolves on exit code 0', async () => { + const proc = createMockProcess(); + mockSpawn.mockReturnValue(proc); + + const svc = ComposeService.getInstance(1); + const promise = svc.runCommand('my-stack', 'start'); + proc.emit('close', 0); + + await expect(promise).resolves.toBeUndefined(); + }); + + it('rejects on non-zero exit code', async () => { + const proc = createMockProcess(); + mockSpawn.mockReturnValue(proc); + + const svc = ComposeService.getInstance(1); + const promise = svc.runCommand('my-stack', 'stop'); + proc.stderr.emit('data', Buffer.from('service not found')); + proc.emit('close', 1); + + await expect(promise).rejects.toThrow('service not found'); + }); + + it('sends output to WebSocket when provided', async () => { + const proc = createMockProcess(); + mockSpawn.mockReturnValue(proc); + const ws = createMockWs(); + + const svc = ComposeService.getInstance(1); + const promise = svc.runCommand('my-stack', 'restart', ws as any); + proc.stdout.emit('data', Buffer.from('Restarting...')); + proc.emit('close', 0); + await promise; + + expect(ws.send).toHaveBeenCalledWith('Restarting...'); + }); +}); + +// ── deployStack ──────────────────────────────────────────────────────── + +describe('ComposeService - deployStack', () => { + it('runs docker compose up -d --remove-orphans', async () => { + setupAutoCloseSpawn(); + mockListContainers.mockResolvedValue([]); + + const svc = ComposeService.getInstance(1); + const promise = svc.deployStack('my-stack'); + + // Advance past the 3s health probe timeout + await vi.advanceTimersByTimeAsync(3100); + await promise; + + expect(mockSpawn).toHaveBeenCalledWith( + 'docker', + ['compose', 'up', '-d', '--remove-orphans'], + expect.any(Object) + ); + }); + + it('creates backup when atomic=true', async () => { + setupAutoCloseSpawn(); + mockListContainers.mockResolvedValue([]); + + const svc = ComposeService.getInstance(1); + const promise = svc.deployStack('my-stack', undefined, true); + + await vi.advanceTimersByTimeAsync(3100); + await promise; + + expect(mockBackupStackFiles).toHaveBeenCalledWith('my-stack'); + }); + + it('throws CONTAINER_CRASHED when exited container has non-zero exit code', async () => { + setupAutoCloseSpawn(); + mockListContainers.mockResolvedValue([{ + Id: 'crashed-c1', + State: 'exited', + Labels: { 'com.docker.compose.project': 'my-stack' }, + }]); + mockContainerInspect.mockResolvedValue({ State: { ExitCode: 1 } }); + mockContainerLogs.mockResolvedValue(Buffer.from('Error: something failed')); + + const svc = ComposeService.getInstance(1); + // Attach catch handler immediately so rejection is never "unhandled" + const result = svc.deployStack('my-stack').then(() => null, (e: Error) => e); + + await vi.runAllTimersAsync(); + const error = await result; + expect(error).not.toBeNull(); + expect(error!.message).toContain('CONTAINER_CRASHED'); + }); + + it('rolls back on failure when atomic=true', async () => { + setupAutoCloseSpawn(); + mockListContainers.mockResolvedValue([{ + Id: 'crashed-c1', + State: 'exited', + Labels: { 'com.docker.compose.project': 'my-stack' }, + }]); + mockContainerInspect.mockResolvedValue({ State: { ExitCode: 1 } }); + mockContainerLogs.mockResolvedValue(Buffer.from('Error')); + + const svc = ComposeService.getInstance(1); + const result = svc.deployStack('my-stack', undefined, true).then(() => null, (e: Error) => e); + + await vi.runAllTimersAsync(); + const error = await result; + expect(error).not.toBeNull(); + expect(error!.message).toContain('CONTAINER_CRASHED'); + expect(mockRestoreStackFiles).toHaveBeenCalledWith('my-stack'); + }); + + it('does not roll back when atomic=false', async () => { + setupAutoCloseSpawn(); + mockListContainers.mockResolvedValue([{ + Id: 'crashed-c1', + State: 'exited', + }]); + mockContainerInspect.mockResolvedValue({ State: { ExitCode: 1 } }); + mockContainerLogs.mockResolvedValue(Buffer.from('Error')); + + const svc = ComposeService.getInstance(1); + const result = svc.deployStack('my-stack', undefined, false).then(() => null, (e: Error) => e); + + await vi.runAllTimersAsync(); + const error = await result; + expect(error).not.toBeNull(); + expect(error!.message).toContain('CONTAINER_CRASHED'); + expect(mockRestoreStackFiles).not.toHaveBeenCalled(); + }); +}); + +// ── withRegistryAuth ─────────────────────────────────────────────────── + +describe('ComposeService - withRegistryAuth', () => { + it('passes default env when no registries configured', async () => { + mockGetRegistries.mockReturnValue([]); + setupAutoCloseSpawn(); + mockListContainers.mockResolvedValue([]); + + const svc = ComposeService.getInstance(1); + const promise = svc.deployStack('my-stack'); + + await vi.advanceTimersByTimeAsync(3100); + await promise; + + expect(mockMkdtempSync).not.toHaveBeenCalled(); + }); + + it('creates temp config dir when registries exist', async () => { + mockGetRegistries.mockReturnValue([{ url: 'https://registry.example.com', username: 'user', password: 'pass' }]); + mockResolveDockerConfig.mockResolvedValue({ auths: { 'registry.example.com': { auth: 'dXNlcjpwYXNz' } } }); + setupAutoCloseSpawn(); + mockListContainers.mockResolvedValue([]); + + const svc = ComposeService.getInstance(1); + const promise = svc.deployStack('my-stack'); + + await vi.advanceTimersByTimeAsync(3100); + await promise; + + expect(mockMkdtempSync).toHaveBeenCalled(); + expect(mockWriteFileSync).toHaveBeenCalled(); + expect(mockUnlinkSync).toHaveBeenCalled(); + expect(mockRmdirSync).toHaveBeenCalled(); + }); + + it('cleans up temp dir even on command failure', async () => { + mockGetRegistries.mockReturnValue([{ url: 'https://registry.example.com' }]); + mockResolveDockerConfig.mockResolvedValue({ auths: {} }); + + // Make spawn fail + mockSpawn.mockImplementation(() => { + const proc = createMockProcess(); + Promise.resolve().then(() => { + proc.stderr.emit('data', Buffer.from('pull failed')); + proc.emit('close', 1); + }); + return proc; + }); + + const svc = ComposeService.getInstance(1); + const result = svc.deployStack('my-stack').then(() => null, (e: Error) => e); + + await vi.runAllTimersAsync(); + const error = await result; + expect(error).not.toBeNull(); + expect(mockUnlinkSync).toHaveBeenCalled(); + }); +}); + +// ── downStack ────────────────────────────────────────────────────────── + +describe('ComposeService - downStack', () => { + it('runs docker compose down with volumes and remove-orphans', async () => { + setupAutoCloseSpawn(); + + const svc = ComposeService.getInstance(1); + await svc.downStack('my-stack'); + + expect(mockSpawn).toHaveBeenCalledWith( + 'docker', + ['compose', 'down', '--volumes', '--remove-orphans'], + expect.any(Object) + ); + }); + + it('resolves even when command fails (throwOnError=false)', async () => { + mockSpawn.mockImplementation(() => { + const proc = createMockProcess(); + Promise.resolve().then(() => proc.emit('close', 1)); + return proc; + }); + + const svc = ComposeService.getInstance(1); + await expect(svc.downStack('my-stack')).resolves.toBeUndefined(); + }); +}); diff --git a/backend/src/__tests__/database-metrics.test.ts b/backend/src/__tests__/database-metrics.test.ts new file mode 100644 index 00000000..a89a33e7 --- /dev/null +++ b/backend/src/__tests__/database-metrics.test.ts @@ -0,0 +1,398 @@ +/** + * Integration tests for DatabaseService metrics, cleanup, notification cap, + * and stack alert CRUD. Uses a real temp SQLite database. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; + +let tmpDir: string; +let DatabaseService: any; +let db: any; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + DatabaseService = (await import('../services/DatabaseService')).DatabaseService; + db = DatabaseService.getInstance(); +}); + +afterAll(() => { + cleanupTestDb(tmpDir); +}); + +describe('DatabaseService - container metrics', () => { + it('stores and retrieves a metric', () => { + const now = Date.now(); + db.addContainerMetric({ + container_id: 'abc123', + stack_name: 'test-stack', + cpu_percent: 42.5, + memory_mb: 256, + net_rx_mb: 1.2, + net_tx_mb: 0.8, + timestamp: now, + }); + + const metrics = db.getContainerMetrics(1); + expect(metrics.length).toBeGreaterThanOrEqual(1); + const found = metrics.find((m: any) => m.container_id === 'abc123'); + expect(found).toBeDefined(); + expect(found.stack_name).toBe('test-stack'); + expect(found.cpu_percent).toBeCloseTo(42.5, 0); + }); + + it('aggregates metrics into minute buckets', () => { + const baseTime = Date.now(); + // Insert two metrics within the same minute + db.addContainerMetric({ + container_id: 'bucket-test', + stack_name: 'bucket-stack', + cpu_percent: 20, + memory_mb: 100, + net_rx_mb: 1, + net_tx_mb: 2, + timestamp: baseTime, + }); + db.addContainerMetric({ + container_id: 'bucket-test', + stack_name: 'bucket-stack', + cpu_percent: 40, + memory_mb: 200, + net_rx_mb: 3, + net_tx_mb: 4, + timestamp: baseTime + 5000, // 5 seconds later, same minute bucket + }); + + const metrics = db.getContainerMetrics(1); + const bucketMetrics = metrics.filter((m: any) => m.container_id === 'bucket-test'); + // Should be aggregated into 1 bucket (same minute) + expect(bucketMetrics.length).toBe(1); + // CPU and memory are averaged + expect(bucketMetrics[0].cpu_percent).toBeCloseTo(30, 0); + expect(bucketMetrics[0].memory_mb).toBeCloseTo(150, 0); + // Network uses MAX + expect(bucketMetrics[0].net_rx_mb).toBeCloseTo(3, 0); + expect(bucketMetrics[0].net_tx_mb).toBeCloseTo(4, 0); + }); + + it('filters out metrics older than hoursLookback', () => { + const oldTimestamp = Date.now() - 3 * 60 * 60 * 1000; // 3 hours ago + db.addContainerMetric({ + container_id: 'old-container', + stack_name: 'old-stack', + cpu_percent: 10, + memory_mb: 50, + net_rx_mb: 0.1, + net_tx_mb: 0.1, + timestamp: oldTimestamp, + }); + + // Look back only 1 hour — should not find it + const recent = db.getContainerMetrics(1); + const found = recent.find((m: any) => m.container_id === 'old-container'); + expect(found).toBeUndefined(); + + // Look back 4 hours — should find it + const wider = db.getContainerMetrics(4); + const foundWider = wider.find((m: any) => m.container_id === 'old-container'); + expect(foundWider).toBeDefined(); + }); + + it('handles multiple containers in the same stack', () => { + const now = Date.now(); + db.addContainerMetric({ + container_id: 'multi-1', + stack_name: 'multi-stack', + cpu_percent: 10, + memory_mb: 100, + net_rx_mb: 1, + net_tx_mb: 1, + timestamp: now, + }); + db.addContainerMetric({ + container_id: 'multi-2', + stack_name: 'multi-stack', + cpu_percent: 20, + memory_mb: 200, + net_rx_mb: 2, + net_tx_mb: 2, + timestamp: now, + }); + + const metrics = db.getContainerMetrics(1); + const stackMetrics = metrics.filter((m: any) => m.stack_name === 'multi-stack'); + // Two distinct containers, so two rows (different container_id in GROUP BY) + expect(stackMetrics.length).toBe(2); + }); +}); + +describe('DatabaseService - cleanupOldMetrics', () => { + it('deletes metrics older than specified hours', () => { + const oldTimestamp = Date.now() - 48 * 60 * 60 * 1000; // 48 hours ago + db.addContainerMetric({ + container_id: 'cleanup-target', + stack_name: 'cleanup-stack', + cpu_percent: 5, + memory_mb: 32, + net_rx_mb: 0, + net_tx_mb: 0, + timestamp: oldTimestamp, + }); + + db.cleanupOldMetrics(24); + + const all = db.getContainerMetrics(72); + const found = all.find((m: any) => m.container_id === 'cleanup-target'); + expect(found).toBeUndefined(); + }); + + it('retains metrics within the retention window', () => { + const recentTimestamp = Date.now() - 1 * 60 * 60 * 1000; // 1 hour ago + db.addContainerMetric({ + container_id: 'keep-me', + stack_name: 'keep-stack', + cpu_percent: 15, + memory_mb: 64, + net_rx_mb: 0.5, + net_tx_mb: 0.5, + timestamp: recentTimestamp, + }); + + db.cleanupOldMetrics(24); + + const metrics = db.getContainerMetrics(24); + const found = metrics.find((m: any) => m.container_id === 'keep-me'); + expect(found).toBeDefined(); + }); +}); + +describe('DatabaseService - cleanupOldNotifications', () => { + it('deletes notifications older than specified days and retains recent ones', () => { + const oldTimestamp = Date.now() - 60 * 24 * 60 * 60 * 1000; // 60 days ago + const recentTimestamp = Date.now() - 1 * 24 * 60 * 60 * 1000; // 1 day ago + + db.addNotificationHistory({ level: 'info', message: 'old notification', timestamp: oldTimestamp }); + db.addNotificationHistory({ level: 'info', message: 'recent notification', timestamp: recentTimestamp }); + + db.cleanupOldNotifications(30); + + const history = db.getNotificationHistory(200); + const old = history.find((n: any) => n.message === 'old notification'); + const recent = history.find((n: any) => n.message === 'recent notification'); + expect(old).toBeUndefined(); + expect(recent).toBeDefined(); + }); +}); + +describe('DatabaseService - cleanupOldAuditLogs', () => { + it('deletes audit logs older than specified days and retains recent ones', () => { + const oldTimestamp = Date.now() - 120 * 24 * 60 * 60 * 1000; // 120 days ago + const recentTimestamp = Date.now() - 10 * 24 * 60 * 60 * 1000; // 10 days ago + + db.insertAuditLog({ + timestamp: oldTimestamp, + username: 'admin', + method: 'GET', + path: '/api/old', + status_code: 200, + node_id: null, + ip_address: '127.0.0.1', + summary: 'old audit entry', + }); + db.insertAuditLog({ + timestamp: recentTimestamp, + username: 'admin', + method: 'POST', + path: '/api/recent', + status_code: 200, + node_id: null, + ip_address: '127.0.0.1', + summary: 'recent audit entry', + }); + + db.cleanupOldAuditLogs(90); + + const { entries } = db.getAuditLogs({ limit: 200, offset: 0 }); + const old = entries.find((e: any) => e.summary === 'old audit entry'); + const recent = entries.find((e: any) => e.summary === 'recent audit entry'); + expect(old).toBeUndefined(); + expect(recent).toBeDefined(); + }); +}); + +describe('DatabaseService - notification history cap', () => { + it('auto-prunes to 100 entries when adding notifications', () => { + // Insert 105 notifications + for (let i = 0; i < 105; i++) { + db.addNotificationHistory({ + level: 'info', + message: `cap-test-${i}`, + timestamp: Date.now() + i, + }); + } + + // The table should have at most 100 rows + const all = db.getNotificationHistory(200); + expect(all.length).toBeLessThanOrEqual(100); + }); + + it('keeps the most recent entries after pruning', () => { + // Clear all first + db.deleteAllNotifications(); + + for (let i = 0; i < 105; i++) { + db.addNotificationHistory({ + level: 'info', + message: `order-test-${i}`, + timestamp: Date.now() + i * 10, + }); + } + + const all = db.getNotificationHistory(200); + // The newest entries should survive (ordered DESC by timestamp) + expect(all[0].message).toContain('order-test-'); + // The oldest entries (0-4) should have been pruned + const oldest = all.find((n: any) => n.message === 'order-test-0'); + expect(oldest).toBeUndefined(); + }); +}); + +describe('DatabaseService - stack alerts CRUD', () => { + it('adds and retrieves stack alerts', () => { + db.addStackAlert({ + stack_name: 'alert-stack', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 15, + }); + + const alerts = db.getStackAlerts(); + const found = alerts.find((a: any) => a.stack_name === 'alert-stack'); + expect(found).toBeDefined(); + expect(found.metric).toBe('cpu_percent'); + expect(found.operator).toBe('>'); + expect(found.threshold).toBe(80); + expect(found.duration_mins).toBe(5); + expect(found.cooldown_mins).toBe(15); + }); + + it('filters alerts by stack name', () => { + db.addStackAlert({ + stack_name: 'filter-stack-a', + metric: 'memory_mb', + operator: '>=', + threshold: 512, + duration_mins: 1, + cooldown_mins: 10, + }); + db.addStackAlert({ + stack_name: 'filter-stack-b', + metric: 'cpu_percent', + operator: '>', + threshold: 90, + duration_mins: 2, + cooldown_mins: 5, + }); + + const alertsA = db.getStackAlerts('filter-stack-a'); + expect(alertsA.length).toBe(1); + expect(alertsA[0].stack_name).toBe('filter-stack-a'); + + const alertsB = db.getStackAlerts('filter-stack-b'); + expect(alertsB.length).toBe(1); + expect(alertsB[0].stack_name).toBe('filter-stack-b'); + }); + + it('updates last_fired_at timestamp', () => { + db.addStackAlert({ + stack_name: 'fired-stack', + metric: 'net_rx', + operator: '>', + threshold: 100, + duration_mins: 1, + cooldown_mins: 30, + }); + + const alerts = db.getStackAlerts('fired-stack'); + const alert = alerts[0]; + const fireTime = Date.now(); + db.updateStackAlertLastFired(alert.id, fireTime); + + const updated = db.getStackAlerts('fired-stack'); + expect(updated[0].last_fired_at).toBe(fireTime); + }); + + it('deletes an alert by id', () => { + db.addStackAlert({ + stack_name: 'delete-stack', + metric: 'memory_percent', + operator: '>', + threshold: 95, + duration_mins: 1, + cooldown_mins: 5, + }); + + const before = db.getStackAlerts('delete-stack'); + expect(before.length).toBe(1); + + db.deleteStackAlert(before[0].id); + + const after = db.getStackAlerts('delete-stack'); + expect(after.length).toBe(0); + }); +}); + +describe('DatabaseService - stress tests', () => { + it('handles 1000+ metrics and cleanup bounds growth', () => { + const now = Date.now(); + + // Insert 1200 metrics spread across 2 hours + for (let i = 0; i < 1200; i++) { + db.addContainerMetric({ + container_id: `stress-container-${i % 10}`, + stack_name: 'stress-stack', + cpu_percent: Math.random() * 100, + memory_mb: Math.random() * 1024, + net_rx_mb: Math.random() * 10, + net_tx_mb: Math.random() * 10, + timestamp: now - (i * 6000), // Every 6 seconds over ~2 hours + }); + } + + // Cleanup with 1 hour retention + db.cleanupOldMetrics(1); + + // Only metrics from last hour should remain (600 entries = 1 hour / 6 seconds) + const remaining = db.getContainerMetrics(2); + // Aggregated into minute buckets, so much fewer than 600 + expect(remaining.length).toBeLessThan(700); + // But should still have data from the retained window + expect(remaining.length).toBeGreaterThan(0); + }); + + it('aggregation remains correct at scale', () => { + const now = Date.now(); + const minuteBase = Math.floor(now / 60000) * 60000; // Start of current minute + + // Insert 50 metrics in the same minute for one container + for (let i = 0; i < 50; i++) { + db.addContainerMetric({ + container_id: 'agg-stress', + stack_name: 'agg-stack', + cpu_percent: 50, // Constant value so average should be 50 + memory_mb: 256, + net_rx_mb: i, // Increasing, MAX should be 49 + net_tx_mb: 0, + timestamp: minuteBase + i * 100, + }); + } + + const metrics = db.getContainerMetrics(1); + const aggMetrics = metrics.filter((m: any) => m.container_id === 'agg-stress'); + expect(aggMetrics.length).toBe(1); // All in one minute bucket + expect(aggMetrics[0].cpu_percent).toBeCloseTo(50, 0); + expect(aggMetrics[0].memory_mb).toBeCloseTo(256, 0); + expect(aggMetrics[0].net_rx_mb).toBe(49); + }); +}); diff --git a/backend/src/__tests__/docker-controller.test.ts b/backend/src/__tests__/docker-controller.test.ts new file mode 100644 index 00000000..c2d0c998 --- /dev/null +++ b/backend/src/__tests__/docker-controller.test.ts @@ -0,0 +1,378 @@ +/** + * Unit tests for DockerController — validateApiData, state-safe container ops, + * disk usage, classified resources, orphan detection, and error paths. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// ── Hoisted mocks ────────────────────────────────────────────────────── + +const { mockDocker } = vi.hoisted(() => { + const mockDocker = { + df: vi.fn(), + listImages: vi.fn().mockResolvedValue([]), + listVolumes: vi.fn().mockResolvedValue({ Volumes: [] }), + listNetworks: vi.fn().mockResolvedValue([]), + listContainers: vi.fn().mockResolvedValue([]), + getContainer: vi.fn(), + getImage: vi.fn(), + getVolume: vi.fn(), + getNetwork: vi.fn(), + pruneContainers: vi.fn().mockResolvedValue({ SpaceReclaimed: 0 }), + pruneImages: vi.fn().mockResolvedValue({ SpaceReclaimed: 0 }), + pruneNetworks: vi.fn().mockResolvedValue({}), + pruneVolumes: vi.fn().mockResolvedValue({ SpaceReclaimed: 0 }), + }; + return { mockDocker }; +}); + +vi.mock('../services/NodeRegistry', () => ({ + NodeRegistry: { + getInstance: () => ({ + getDocker: () => mockDocker, + getDefaultNodeId: () => 1, + }), + }, +})); + +// Prevent COMPOSE_DIR related issues +vi.mock('child_process', () => ({ + exec: vi.fn(), +})); +vi.mock('util', () => ({ + promisify: () => vi.fn(), +})); + +import DockerController from '../services/DockerController'; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +// ── validateApiData ──────────────────────────────────────────────────── + +describe('DockerController - validateApiData', () => { + it('throws when response is a string (HTML from wrong port)', async () => { + mockDocker.listImages.mockResolvedValue('Not Docker'); + + const dc = DockerController.getInstance(1); + await expect(dc.getImages()).rejects.toThrow('Invalid response from Docker API'); + }); + + it('passes through valid object data', async () => { + const imageData = [{ Id: 'sha256:abc', RepoTags: ['nginx:latest'], Size: 100 }]; + mockDocker.listImages.mockResolvedValue(imageData); + + const dc = DockerController.getInstance(1); + const result = await dc.getImages(); + expect(result).toEqual(imageData); + }); +}); + +// ── State-safe container operations ──────────────────────────────────── + +describe('DockerController - startContainer', () => { + it('starts a container successfully', async () => { + const mockStart = vi.fn().mockResolvedValue(undefined); + mockDocker.getContainer.mockReturnValue({ start: mockStart }); + + const dc = DockerController.getInstance(1); + await dc.startContainer('abc123'); + + expect(mockStart).toHaveBeenCalled(); + }); + + it('silently ignores 304 already-started error', async () => { + const mockStart = vi.fn().mockRejectedValue({ statusCode: 304 }); + mockDocker.getContainer.mockReturnValue({ start: mockStart }); + + const dc = DockerController.getInstance(1); + await expect(dc.startContainer('abc123')).resolves.toBeUndefined(); + }); + + it('rethrows other errors', async () => { + const mockStart = vi.fn().mockRejectedValue(new Error('container not found')); + mockDocker.getContainer.mockReturnValue({ start: mockStart }); + + const dc = DockerController.getInstance(1); + await expect(dc.startContainer('abc123')).rejects.toThrow('container not found'); + }); +}); + +describe('DockerController - stopContainer', () => { + it('stops a container successfully', async () => { + const mockStop = vi.fn().mockResolvedValue(undefined); + mockDocker.getContainer.mockReturnValue({ stop: mockStop }); + + const dc = DockerController.getInstance(1); + await dc.stopContainer('abc123'); + + expect(mockStop).toHaveBeenCalled(); + }); + + it('silently ignores 304 already-stopped error', async () => { + const mockStop = vi.fn().mockRejectedValue({ statusCode: 304 }); + mockDocker.getContainer.mockReturnValue({ stop: mockStop }); + + const dc = DockerController.getInstance(1); + await expect(dc.stopContainer('abc123')).resolves.toBeUndefined(); + }); + + it('rethrows other errors', async () => { + const err = new Error('permission denied'); + const mockStop = vi.fn().mockRejectedValue(err); + mockDocker.getContainer.mockReturnValue({ stop: mockStop }); + + const dc = DockerController.getInstance(1); + await expect(dc.stopContainer('abc123')).rejects.toThrow('permission denied'); + }); +}); + +// ── removeContainers ─────────────────────────────────────────────────── + +describe('DockerController - removeContainers', () => { + it('removes multiple containers and returns results', async () => { + const mockRemove = vi.fn().mockResolvedValue(undefined); + mockDocker.getContainer.mockReturnValue({ remove: mockRemove }); + + const dc = DockerController.getInstance(1); + const results = await dc.removeContainers(['c1', 'c2']); + + expect(results).toEqual([ + { id: 'c1', success: true }, + { id: 'c2', success: true }, + ]); + }); + + it('returns failure result for containers that cannot be removed', async () => { + let callCount = 0; + mockDocker.getContainer.mockImplementation(() => ({ + remove: vi.fn().mockImplementation(() => { + callCount++; + if (callCount === 2) throw new Error('in use'); + return Promise.resolve(); + }), + })); + + const dc = DockerController.getInstance(1); + const results = await dc.removeContainers(['c1', 'c2']); + + expect(results[0]).toEqual({ id: 'c1', success: true }); + expect(results[1]).toMatchObject({ id: 'c2', success: false, error: 'in use' }); + }); +}); + +// ── getDiskUsage ─────────────────────────────────────────────────────── + +describe('DockerController - getDiskUsage', () => { + it('calculates reclaimable space correctly', async () => { + mockDocker.df.mockResolvedValue({ + Images: [ + { Id: 'img1', Containers: 0, Size: 500 }, // reclaimable (unused) + { Id: 'img2', Containers: 1, Size: 300 }, // not reclaimable (in use) + ], + Containers: [ + { State: 'running', SizeRw: 100 }, // not reclaimable (running) + { State: 'exited', SizeRw: 200 }, // reclaimable (stopped) + ], + Volumes: [ + { UsageData: { RefCount: 0, Size: 400 } }, // reclaimable (unused) + { UsageData: { RefCount: 1, Size: 300 } }, // not reclaimable (in use) + ], + }); + + const dc = DockerController.getInstance(1); + const usage = await dc.getDiskUsage(); + + expect(usage.reclaimableImages).toBe(500); + expect(usage.reclaimableContainers).toBe(200); + expect(usage.reclaimableVolumes).toBe(400); + }); + + it('handles empty arrays gracefully', async () => { + mockDocker.df.mockResolvedValue({ + Images: [], + Containers: [], + Volumes: [], + }); + + const dc = DockerController.getInstance(1); + const usage = await dc.getDiskUsage(); + + expect(usage.reclaimableImages).toBe(0); + expect(usage.reclaimableContainers).toBe(0); + expect(usage.reclaimableVolumes).toBe(0); + }); + + it('handles missing fields gracefully', async () => { + mockDocker.df.mockResolvedValue({}); + + const dc = DockerController.getInstance(1); + const usage = await dc.getDiskUsage(); + + expect(usage.reclaimableImages).toBe(0); + expect(usage.reclaimableContainers).toBe(0); + expect(usage.reclaimableVolumes).toBe(0); + }); +}); + +// ── pruneSystem ──────────────────────────────────────────────────────── + +describe('DockerController - pruneSystem', () => { + it('prunes containers and returns reclaimed bytes', async () => { + mockDocker.pruneContainers.mockResolvedValue({ SpaceReclaimed: 1024 }); + + const dc = DockerController.getInstance(1); + const result = await dc.pruneSystem('containers'); + + expect(result).toEqual({ success: true, reclaimedBytes: 1024 }); + }); + + it('prunes images with dangling false filter', async () => { + mockDocker.pruneImages.mockResolvedValue({ SpaceReclaimed: 2048 }); + + const dc = DockerController.getInstance(1); + await dc.pruneSystem('images'); + + expect(mockDocker.pruneImages).toHaveBeenCalledWith({ + filters: expect.objectContaining({ dangling: { 'false': true } }), + }); + }); + + it('includes label filter when provided', async () => { + mockDocker.pruneContainers.mockResolvedValue({ SpaceReclaimed: 0 }); + + const dc = DockerController.getInstance(1); + await dc.pruneSystem('containers', 'com.example=true'); + + expect(mockDocker.pruneContainers).toHaveBeenCalledWith({ + filters: { label: ['com.example=true'] }, + }); + }); + + it('prunes volumes with all true filter', async () => { + mockDocker.pruneVolumes.mockResolvedValue({ SpaceReclaimed: 4096 }); + + const dc = DockerController.getInstance(1); + await dc.pruneSystem('volumes'); + + expect(mockDocker.pruneVolumes).toHaveBeenCalledWith({ + filters: { all: ['true'] }, + }); + }); +}); + +// ── getClassifiedResources ───────────────────────────────────────────── + +describe('DockerController - getClassifiedResources', () => { + it('classifies managed and unmanaged images', async () => { + mockDocker.listImages.mockResolvedValue([ + { Id: 'img1', RepoTags: ['nginx:latest'], Size: 100, Containers: 1 }, + { Id: 'img2', RepoTags: ['redis:latest'], Size: 200, Containers: 1 }, + { Id: 'img3', RepoTags: ['old:v1'], Size: 50, Containers: 0 }, + ]); + mockDocker.listContainers.mockResolvedValue([ + { ImageID: 'img1', Labels: { 'com.docker.compose.project': 'my-stack' } }, + { ImageID: 'img2', Labels: { 'com.docker.compose.project': 'unknown-stack' } }, + ]); + mockDocker.listVolumes.mockResolvedValue({ Volumes: [] }); + mockDocker.listNetworks.mockResolvedValue([]); + + const dc = DockerController.getInstance(1); + const result = await dc.getClassifiedResources(['my-stack']); + + const managed = result.images.find(i => i.Id === 'img1'); + expect(managed!.managedStatus).toBe('managed'); + expect(managed!.managedBy).toBe('my-stack'); + + const unmanaged = result.images.find(i => i.Id === 'img2'); + expect(unmanaged!.managedStatus).toBe('unmanaged'); + + const unused = result.images.find(i => i.Id === 'img3'); + expect(unused!.managedStatus).toBe('unused'); + }); + + it('classifies system networks', async () => { + mockDocker.listImages.mockResolvedValue([]); + mockDocker.listContainers.mockResolvedValue([]); + mockDocker.listVolumes.mockResolvedValue({ Volumes: [] }); + mockDocker.listNetworks.mockResolvedValue([ + { Id: 'n1', Name: 'bridge', Driver: 'bridge', Scope: 'local' }, + { Id: 'n2', Name: 'host', Driver: 'host', Scope: 'local' }, + { Id: 'n3', Name: 'none', Driver: 'null', Scope: 'local' }, + { Id: 'n4', Name: 'my-stack_default', Driver: 'bridge', Scope: 'local', Labels: { 'com.docker.compose.project': 'my-stack' } }, + ]); + + const dc = DockerController.getInstance(1); + const result = await dc.getClassifiedResources(['my-stack']); + + expect(result.networks.filter(n => n.managedStatus === 'system')).toHaveLength(3); + expect(result.networks.find(n => n.Name === 'my-stack_default')!.managedStatus).toBe('managed'); + }); + + it('classifies managed and unmanaged volumes', async () => { + mockDocker.listImages.mockResolvedValue([]); + mockDocker.listContainers.mockResolvedValue([]); + mockDocker.listNetworks.mockResolvedValue([]); + mockDocker.listVolumes.mockResolvedValue({ + Volumes: [ + { Name: 'my-stack_data', Driver: 'local', Mountpoint: '/var/lib/docker/volumes/my-stack_data', Labels: { 'com.docker.compose.project': 'my-stack' } }, + { Name: 'random_vol', Driver: 'local', Mountpoint: '/var/lib/docker/volumes/random', Labels: {} }, + ], + }); + + const dc = DockerController.getInstance(1); + const result = await dc.getClassifiedResources(['my-stack']); + + expect(result.volumes.find(v => v.Name === 'my-stack_data')!.managedStatus).toBe('managed'); + expect(result.volumes.find(v => v.Name === 'random_vol')!.managedStatus).toBe('unmanaged'); + }); +}); + +// ── getOrphanContainers ──────────────────────────────────────────────── + +describe('DockerController - getOrphanContainers', () => { + it('returns containers whose project label is not in known stacks', async () => { + mockDocker.listContainers.mockResolvedValue([ + { Id: 'c1', Names: ['/c1'], State: 'running', Status: 'Up', Image: 'nginx', Labels: { 'com.docker.compose.project': 'orphan-stack' } }, + { Id: 'c2', Names: ['/c2'], State: 'running', Status: 'Up', Image: 'redis', Labels: { 'com.docker.compose.project': 'known-stack' } }, + ]); + + const dc = DockerController.getInstance(1); + const result = await dc.getOrphanContainers(['known-stack']); + + expect(result['orphan-stack']).toHaveLength(1); + expect(result['orphan-stack'][0].Id).toBe('c1'); + expect(result['known-stack']).toBeUndefined(); + }); + + it('returns empty when all containers belong to known stacks', async () => { + mockDocker.listContainers.mockResolvedValue([ + { Id: 'c1', Names: ['/c1'], State: 'running', Status: 'Up', Image: 'nginx', Labels: { 'com.docker.compose.project': 'my-stack' } }, + ]); + + const dc = DockerController.getInstance(1); + const result = await dc.getOrphanContainers(['my-stack']); + + expect(Object.keys(result)).toHaveLength(0); + }); +}); + +// ── Docker daemon unreachable ────────────────────────────────────────── + +describe('DockerController - error paths', () => { + it('propagates connection errors from Docker daemon', async () => { + mockDocker.listContainers.mockRejectedValue( + new Error('connect ECONNREFUSED /var/run/docker.sock') + ); + + const dc = DockerController.getInstance(1); + await expect(dc.getOrphanContainers(['x'])).rejects.toThrow('ECONNREFUSED'); + }); + + it('propagates errors from df() call', async () => { + mockDocker.df.mockRejectedValue(new Error('daemon not running')); + + const dc = DockerController.getInstance(1); + await expect(dc.getDiskUsage()).rejects.toThrow('daemon not running'); + }); +}); diff --git a/backend/src/__tests__/monitor-service.test.ts b/backend/src/__tests__/monitor-service.test.ts new file mode 100644 index 00000000..fd7bdab7 --- /dev/null +++ b/backend/src/__tests__/monitor-service.test.ts @@ -0,0 +1,521 @@ +/** + * Unit tests for MonitorService — alert state machine, metric calculations, + * cleanup delegation, global settings evaluation, and concurrency guards. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// ── Hoisted mocks ────────────────────────────────────────────────────── + +const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContainerMetric, + mockCleanupOldMetrics, mockCleanupOldNotifications, mockCleanupOldAuditLogs, + mockUpdateStackAlertLastFired, mockGetSystemState, mockSetSystemState, + mockGetRunningContainers, mockGetAllContainers, mockGetContainerStatsStream, + mockDispatchAlert, + mockCurrentLoad, mockMem, mockFsSize, + mockExecAsync, +} = vi.hoisted(() => ({ + mockGetGlobalSettings: vi.fn().mockReturnValue({}), + mockGetNodes: vi.fn().mockReturnValue([]), + mockGetStackAlerts: vi.fn().mockReturnValue([]), + mockAddContainerMetric: vi.fn(), + mockCleanupOldMetrics: vi.fn(), + mockCleanupOldNotifications: vi.fn(), + mockCleanupOldAuditLogs: vi.fn(), + mockUpdateStackAlertLastFired: vi.fn(), + mockGetSystemState: vi.fn().mockReturnValue(null), + mockSetSystemState: vi.fn(), + mockGetRunningContainers: vi.fn().mockResolvedValue([]), + mockGetAllContainers: vi.fn().mockResolvedValue([]), + mockGetContainerStatsStream: vi.fn().mockResolvedValue('{}'), + mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockCurrentLoad: vi.fn().mockResolvedValue({ currentLoad: 10 }), + mockMem: vi.fn().mockResolvedValue({ used: 4e9, total: 16e9 }), + mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]), + mockExecAsync: vi.fn().mockResolvedValue({ stdout: '' }), +})); + +vi.mock('../services/DatabaseService', () => ({ + DatabaseService: { + getInstance: () => ({ + getGlobalSettings: mockGetGlobalSettings, + getNodes: mockGetNodes, + getStackAlerts: mockGetStackAlerts, + addContainerMetric: mockAddContainerMetric, + cleanupOldMetrics: mockCleanupOldMetrics, + cleanupOldNotifications: mockCleanupOldNotifications, + cleanupOldAuditLogs: mockCleanupOldAuditLogs, + updateStackAlertLastFired: mockUpdateStackAlertLastFired, + getSystemState: mockGetSystemState, + setSystemState: mockSetSystemState, + }), + }, +})); + +vi.mock('../services/DockerController', () => ({ + default: { + getInstance: () => ({ + getRunningContainers: mockGetRunningContainers, + getAllContainers: mockGetAllContainers, + getContainerStatsStream: mockGetContainerStatsStream, + }), + }, +})); + +vi.mock('../services/NotificationService', () => ({ + NotificationService: { + getInstance: () => ({ + dispatchAlert: mockDispatchAlert, + }), + }, +})); + +vi.mock('systeminformation', () => ({ + default: { + currentLoad: (...args: unknown[]) => mockCurrentLoad(...args), + mem: (...args: unknown[]) => mockMem(...args), + fsSize: (...args: unknown[]) => mockFsSize(...args), + }, +})); + +vi.mock('child_process', () => ({ + exec: vi.fn(), +})); + +vi.mock('util', () => ({ + promisify: () => mockExecAsync, +})); + +import { MonitorService } from '../services/MonitorService'; + +beforeEach(() => { + vi.clearAllMocks(); + (MonitorService as any).instance = undefined; +}); + +// ── Pure calculation helpers (accessed via private method reflection) ─── + +describe('MonitorService - calculateCpuPercent', () => { + function calcCpu(stats: any): number { + const svc = MonitorService.getInstance(); + return (svc as any).calculateCpuPercent(stats); + } + + it('returns correct percentage for normal stats', () => { + const stats = { + cpu_stats: { cpu_usage: { total_usage: 2000 }, system_cpu_usage: 10000, online_cpus: 4 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + }; + // (1000 / 5000) * 4 * 100 = 80% + expect(calcCpu(stats)).toBeCloseTo(80, 1); + }); + + it('returns 0 when cpu_stats is missing', () => { + expect(calcCpu({})).toBe(0); + expect(calcCpu(null)).toBe(0); + expect(calcCpu({ cpu_stats: {} })).toBe(0); + }); + + it('returns 0 when systemDelta is zero', () => { + const stats = { + cpu_stats: { cpu_usage: { total_usage: 2000 }, system_cpu_usage: 5000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + }; + expect(calcCpu(stats)).toBe(0); + }); + + it('accounts for online_cpus count', () => { + const stats = { + cpu_stats: { cpu_usage: { total_usage: 2000 }, system_cpu_usage: 10000, online_cpus: 8 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + }; + // (1000/5000) * 8 * 100 = 160% + expect(calcCpu(stats)).toBeCloseTo(160, 1); + }); + + it('falls back to percpu_usage length when online_cpus missing', () => { + const stats = { + cpu_stats: { cpu_usage: { total_usage: 2000, percpu_usage: [0, 0] }, system_cpu_usage: 10000 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + }; + // (1000/5000) * 2 * 100 = 40% + expect(calcCpu(stats)).toBeCloseTo(40, 1); + }); +}); + +describe('MonitorService - calculateMemoryPercent', () => { + function calcMem(stats: any): number { + const svc = MonitorService.getInstance(); + return (svc as any).calculateMemoryPercent(stats); + } + + it('returns correct percentage subtracting cache', () => { + const stats = { + memory_stats: { usage: 500e6, limit: 1e9, stats: { cache: 100e6 } }, + }; + // (400e6 / 1e9) * 100 = 40% + expect(calcMem(stats)).toBeCloseTo(40, 1); + }); + + it('returns 0 when memory_stats is missing', () => { + expect(calcMem({})).toBe(0); + expect(calcMem({ memory_stats: {} })).toBe(0); + }); + + it('returns 0 when limit is zero', () => { + const stats = { memory_stats: { usage: 100, limit: 0 } }; + expect(calcMem(stats)).toBe(0); + }); + + it('handles missing cache field', () => { + const stats = { memory_stats: { usage: 500e6, limit: 1e9 } }; + // No cache → (500e6 / 1e9) * 100 = 50% + expect(calcMem(stats)).toBeCloseTo(50, 1); + }); +}); + +describe('MonitorService - calculateNetwork', () => { + function calcNet(stats: any, dir: 'rx' | 'tx'): number { + const svc = MonitorService.getInstance(); + return (svc as any).calculateNetwork(stats, dir); + } + + it('sums rx_bytes across all interfaces', () => { + const stats = { + networks: { + eth0: { rx_bytes: 1024 * 1024, tx_bytes: 0 }, + eth1: { rx_bytes: 2 * 1024 * 1024, tx_bytes: 0 }, + }, + }; + expect(calcNet(stats, 'rx')).toBeCloseTo(3, 0); // 3 MB + }); + + it('sums tx_bytes across all interfaces', () => { + const stats = { + networks: { + eth0: { rx_bytes: 0, tx_bytes: 512 * 1024 }, + }, + }; + expect(calcNet(stats, 'tx')).toBeCloseTo(0.5, 1); // 0.5 MB + }); + + it('returns 0 when no networks present', () => { + expect(calcNet({}, 'rx')).toBe(0); + expect(calcNet({ networks: null }, 'tx')).toBe(0); + }); +}); + +describe('MonitorService - evaluateCondition', () => { + function evalCond(actual: number, operator: string, threshold: number): boolean { + const svc = MonitorService.getInstance(); + return (svc as any).evaluateCondition(actual, operator, threshold); + } + + it('handles > operator', () => { + expect(evalCond(81, '>', 80)).toBe(true); + expect(evalCond(80, '>', 80)).toBe(false); + }); + + it('handles < operator', () => { + expect(evalCond(79, '<', 80)).toBe(true); + expect(evalCond(80, '<', 80)).toBe(false); + }); + + it('handles >= operator at boundary', () => { + expect(evalCond(80, '>=', 80)).toBe(true); + expect(evalCond(79, '>=', 80)).toBe(false); + }); + + it('handles <= operator at boundary', () => { + expect(evalCond(80, '<=', 80)).toBe(true); + expect(evalCond(81, '<=', 80)).toBe(false); + }); + + it('handles == operator', () => { + expect(evalCond(80, '==', 80)).toBe(true); + expect(evalCond(81, '==', 80)).toBe(false); + }); + + it('returns false for unknown operator', () => { + expect(evalCond(80, '!=', 80)).toBe(false); + expect(evalCond(80, 'foo', 80)).toBe(false); + }); +}); + +// ── Integration-level: evaluateGlobalSettings ────────────────────────── + +describe('MonitorService - evaluateGlobalSettings', () => { + it('dispatches CPU warning when over threshold', async () => { + mockGetGlobalSettings.mockReturnValue({ host_cpu_limit: '50' }); + mockCurrentLoad.mockResolvedValue({ currentLoad: 75 }); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' }); + + expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('CPU')); + }); + + it('does not dispatch when CPU below threshold', async () => { + mockCurrentLoad.mockResolvedValue({ currentLoad: 25 }); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '50' }); + + expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU')); + }); + + it('dispatches RAM warning when over threshold', async () => { + mockMem.mockResolvedValue({ used: 15e9, total: 16e9 }); // ~94% + + const svc = MonitorService.getInstance(); + await (svc as any).evaluateGlobalSettings({ host_ram_limit: '80' }); + + expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Memory')); + }); + + it('dispatches disk warning when over threshold', async () => { + mockFsSize.mockResolvedValue([{ mount: '/', use: 92 }]); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluateGlobalSettings({ host_disk_limit: '90' }); + + expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Disk')); + }); + + it('skips host limits when threshold is 0 or NaN', async () => { + mockCurrentLoad.mockResolvedValue({ currentLoad: 99 }); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluateGlobalSettings({ host_cpu_limit: '0' }); + expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU')); + + await (svc as any).evaluateGlobalSettings({ host_cpu_limit: 'abc' }); + expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU')); + }); +}); + +// ── Global crash detection ───────────────────────────────────────────── + +describe('MonitorService - global crash detection', () => { + it('detects exited containers with non-intentional exit codes', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetAllContainers.mockResolvedValue([{ + State: 'exited', + Status: 'Exited (1) 5 seconds ago', + Names: ['/my-container'], + }]); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluateGlobalSettings({ global_crash: '1' }); + + expect(mockDispatchAlert).toHaveBeenCalledWith('error', expect.stringContaining('Crash')); + }); + + it('ignores exit codes 0, 137, 143, 255', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + const intentionalExits = [0, 137, 143, 255]; + + for (const code of intentionalExits) { + mockDispatchAlert.mockClear(); + mockGetAllContainers.mockResolvedValue([{ + State: 'exited', + Status: `Exited (${code}) 5 seconds ago`, + Names: ['/safe-container'], + }]); + + const svc = MonitorService.getInstance(); + (MonitorService as any).instance = undefined; + await (svc as any).evaluateGlobalSettings({ global_crash: '1' }); + expect(mockDispatchAlert).not.toHaveBeenCalledWith('error', expect.stringContaining('Crash')); + } + }); + + it('detects unhealthy containers', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetAllContainers.mockResolvedValue([{ + State: 'running', + Status: 'Up 2 hours (unhealthy)', + Names: ['/sick-container'], + }]); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluateGlobalSettings({ global_crash: '1' }); + + expect(mockDispatchAlert).toHaveBeenCalledWith('error', expect.stringContaining('unhealthy')); + }); + + it('skips remote nodes', async () => { + mockGetNodes.mockReturnValue([{ id: 2, name: 'remote-node', type: 'remote' }]); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluateGlobalSettings({ global_crash: '1' }); + + expect(mockGetAllContainers).not.toHaveBeenCalled(); + }); +}); + +// ── Alert breach state machine ───────────────────────────────────────── + +describe('MonitorService - breach state machine', () => { + function setupAlertScenario(cpuPercent: number) { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetRunningContainers.mockResolvedValue([{ + Id: 'c1', + Labels: { 'com.docker.compose.project': 'my-stack' }, + }]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 1000 + cpuPercent * 50 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + mockGetStackAlerts.mockReturnValue([{ + id: 1, + stack_name: 'my-stack', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, // Fire immediately on breach + cooldown_mins: 60, + last_fired_at: 0, + }]); + mockGetGlobalSettings.mockReturnValue({}); + } + + it('fires alert when condition met and duration is 0', async () => { + setupAlertScenario(90); // Will produce CPU > 80% + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('CPU')); + expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number)); + }); + + it('does not fire when condition not met', async () => { + setupAlertScenario(10); // Will produce CPU < 80% + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU')); + }); + + it('respects cooldown after firing', async () => { + setupAlertScenario(90); + // Simulate that alert was fired 30 minutes ago (within 60-min cooldown) + mockGetStackAlerts.mockReturnValue([{ + id: 1, + stack_name: 'my-stack', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 60, + last_fired_at: Date.now() - 30 * 60 * 1000, + }]); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockUpdateStackAlertLastFired).not.toHaveBeenCalled(); + }); + + it('resets breach state when condition clears', async () => { + const svc = MonitorService.getInstance(); + + // First: breach starts + setupAlertScenario(90); + mockGetStackAlerts.mockReturnValue([{ + id: 42, + stack_name: 'my-stack', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 999, // Won't fire due to long duration + cooldown_mins: 0, + last_fired_at: 0, + }]); + await (svc as any).evaluate(); + expect((svc as any).activeBreaches.has(42)).toBe(true); + + // Second: condition clears + setupAlertScenario(10); + mockGetStackAlerts.mockReturnValue([{ + id: 42, + stack_name: 'my-stack', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 999, + cooldown_mins: 0, + last_fired_at: 0, + }]); + await (svc as any).evaluate(); + expect((svc as any).activeBreaches.has(42)).toBe(false); + }); +}); + +// ── Cleanup triggers ─────────────────────────────────────────────────── + +describe('MonitorService - cleanup triggers', () => { + it('calls cleanup methods with configured retention', async () => { + mockGetNodes.mockReturnValue([]); + mockGetStackAlerts.mockReturnValue([]); + mockGetGlobalSettings.mockReturnValue({ + metrics_retention_hours: '48', + log_retention_days: '7', + audit_retention_days: '30', + }); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockCleanupOldMetrics).toHaveBeenCalledWith(48); + expect(mockCleanupOldNotifications).toHaveBeenCalledWith(7); + expect(mockCleanupOldAuditLogs).toHaveBeenCalledWith(30); + }); + + it('uses defaults when settings are NaN', async () => { + mockGetNodes.mockReturnValue([]); + mockGetStackAlerts.mockReturnValue([]); + mockGetGlobalSettings.mockReturnValue({ + metrics_retention_hours: 'bad', + log_retention_days: 'bad', + audit_retention_days: 'bad', + }); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockCleanupOldMetrics).toHaveBeenCalledWith(24); + expect(mockCleanupOldNotifications).toHaveBeenCalledWith(30); + expect(mockCleanupOldAuditLogs).toHaveBeenCalledWith(90); + }); +}); + +// ── isProcessing guard ───────────────────────────────────────────────── + +describe('MonitorService - isProcessing guard', () => { + it('skips evaluation if already processing', async () => { + mockGetGlobalSettings.mockReturnValue({}); + mockGetNodes.mockReturnValue([]); + mockGetStackAlerts.mockReturnValue([]); + + const svc = MonitorService.getInstance(); + (svc as any).isProcessing = true; + + await (svc as any).evaluate(); + + // Should have been skipped — no DB calls + expect(mockGetGlobalSettings).not.toHaveBeenCalled(); + }); + + it('resets isProcessing after evaluate completes (even on error)', async () => { + mockGetGlobalSettings.mockImplementationOnce(() => { throw new Error('boom'); }); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + // isProcessing should be reset in finally block + expect((svc as any).isProcessing).toBe(false); + }); +}); diff --git a/backend/src/__tests__/scheduler-service.test.ts b/backend/src/__tests__/scheduler-service.test.ts new file mode 100644 index 00000000..e98aeea3 --- /dev/null +++ b/backend/src/__tests__/scheduler-service.test.ts @@ -0,0 +1,645 @@ +/** + * Unit tests for SchedulerService — task execution, concurrent prevention, + * license gating, cron parsing, and error handling. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// ── Hoisted mocks ────────────────────────────────────────────────────── + +const { + mockGetDueScheduledTasks, mockCreateScheduledTaskRun, mockUpdateScheduledTaskRun, + mockUpdateScheduledTask, mockCleanupOldTaskRuns, mockGetScheduledTask, mockGetNodes, + mockCreateSnapshot, mockInsertSnapshotFiles, mockClearStackUpdateStatus, + mockGetTier, mockGetVariant, + mockGetContainersByStack, mockRestartContainer, mockPruneSystem, + mockUpdateStack, + mockGetStacks, mockGetStackContent, mockGetEnvContent, + mockCheckImage, + mockDispatchAlert, +} = vi.hoisted(() => ({ + mockGetDueScheduledTasks: vi.fn().mockReturnValue([]), + mockCreateScheduledTaskRun: vi.fn().mockReturnValue(1), + mockUpdateScheduledTaskRun: vi.fn(), + mockUpdateScheduledTask: vi.fn(), + mockCleanupOldTaskRuns: vi.fn(), + mockGetScheduledTask: vi.fn(), + mockGetNodes: vi.fn().mockReturnValue([]), + mockCreateSnapshot: vi.fn().mockReturnValue(1), + mockInsertSnapshotFiles: vi.fn(), + mockClearStackUpdateStatus: vi.fn(), + mockGetTier: vi.fn().mockReturnValue('pro'), + mockGetVariant: vi.fn().mockReturnValue('team'), + mockGetContainersByStack: vi.fn().mockResolvedValue([]), + mockRestartContainer: vi.fn().mockResolvedValue(undefined), + mockPruneSystem: vi.fn().mockResolvedValue({ success: true, reclaimedBytes: 0 }), + mockUpdateStack: vi.fn().mockResolvedValue(undefined), + mockGetStacks: vi.fn().mockResolvedValue([]), + mockGetStackContent: vi.fn().mockResolvedValue(''), + mockGetEnvContent: vi.fn().mockResolvedValue(''), + mockCheckImage: vi.fn().mockResolvedValue(false), + mockDispatchAlert: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock('../services/DatabaseService', () => ({ + DatabaseService: { + getInstance: () => ({ + getDueScheduledTasks: mockGetDueScheduledTasks, + createScheduledTaskRun: mockCreateScheduledTaskRun, + updateScheduledTaskRun: mockUpdateScheduledTaskRun, + updateScheduledTask: mockUpdateScheduledTask, + cleanupOldTaskRuns: mockCleanupOldTaskRuns, + getScheduledTask: mockGetScheduledTask, + getNodes: mockGetNodes, + createSnapshot: mockCreateSnapshot, + insertSnapshotFiles: mockInsertSnapshotFiles, + clearStackUpdateStatus: mockClearStackUpdateStatus, + }), + }, +})); + +vi.mock('../services/LicenseService', () => ({ + LicenseService: { + getInstance: () => ({ + getTier: mockGetTier, + getVariant: mockGetVariant, + }), + }, +})); + +vi.mock('../services/DockerController', () => ({ + default: { + getInstance: () => ({ + getContainersByStack: mockGetContainersByStack, + restartContainer: mockRestartContainer, + pruneSystem: mockPruneSystem, + }), + }, +})); + +vi.mock('../services/ComposeService', () => ({ + ComposeService: { + getInstance: () => ({ + updateStack: mockUpdateStack, + }), + }, +})); + +vi.mock('../services/FileSystemService', () => ({ + FileSystemService: { + getInstance: () => ({ + getStacks: mockGetStacks, + getStackContent: mockGetStackContent, + getEnvContent: mockGetEnvContent, + }), + }, +})); + +vi.mock('../services/ImageUpdateService', () => ({ + ImageUpdateService: { + getInstance: () => ({ + checkImage: mockCheckImage, + }), + }, +})); + +vi.mock('../services/NotificationService', () => ({ + NotificationService: { + getInstance: () => ({ + dispatchAlert: mockDispatchAlert, + }), + }, +})); + +vi.mock('../services/NodeRegistry', () => ({ + NodeRegistry: { + getInstance: () => ({ + getDefaultNodeId: () => 1, + }), + }, +})); + +import { SchedulerService } from '../services/SchedulerService'; + +beforeEach(() => { + vi.clearAllMocks(); + (SchedulerService as any).instance = undefined; +}); + +// ── calculateNextRun ─────────────────────────────────────────────────── + +describe('SchedulerService - calculateNextRun', () => { + it('returns a future timestamp for valid cron expression', () => { + const svc = SchedulerService.getInstance(); + const next = svc.calculateNextRun('*/5 * * * *'); // Every 5 minutes + expect(next).toBeGreaterThan(Date.now()); + }); + + it('throws on invalid cron expression', () => { + const svc = SchedulerService.getInstance(); + expect(() => svc.calculateNextRun('not a cron')).toThrow(); + }); +}); + +// ── License gating ───────────────────────────────────────────────────── + +describe('SchedulerService - license gating', () => { + function makeTask(overrides: Partial = {}) { + return { + id: 1, + name: 'test-task', + action: 'restart', + cron_expression: '*/5 * * * *', + enabled: true, + target_id: 'my-stack', + node_id: 1, + created_by: 'admin', + last_status: null, + ...overrides, + }; + } + + it('skips all tasks when tier is not pro', async () => { + mockGetTier.mockReturnValue('community'); + mockGetDueScheduledTasks.mockReturnValue([makeTask()]); + + const svc = SchedulerService.getInstance(); + await (svc as any).tick(); + + expect(mockCreateScheduledTaskRun).not.toHaveBeenCalled(); + }); + + it('allows update tasks for non-admiral pro', async () => { + mockGetTier.mockReturnValue('pro'); + mockGetVariant.mockReturnValue('individual'); + mockGetDueScheduledTasks.mockReturnValue([makeTask({ action: 'update' })]); + mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }]); + mockCheckImage.mockResolvedValue(false); + + const svc = SchedulerService.getInstance(); + await (svc as any).tick(); + + // Wait for the async task to settle + await new Promise(r => setTimeout(r, 50)); + expect(mockCreateScheduledTaskRun).toHaveBeenCalled(); + }); + + it('skips non-update tasks for non-admiral pro', async () => { + mockGetTier.mockReturnValue('pro'); + mockGetVariant.mockReturnValue('individual'); + mockGetDueScheduledTasks.mockReturnValue([makeTask({ action: 'restart' })]); + + const svc = SchedulerService.getInstance(); + await (svc as any).tick(); + + expect(mockCreateScheduledTaskRun).not.toHaveBeenCalled(); + }); + + it('allows all actions for admiral (pro + team)', async () => { + mockGetTier.mockReturnValue('pro'); + mockGetVariant.mockReturnValue('team'); + mockGetDueScheduledTasks.mockReturnValue([makeTask({ action: 'restart' })]); + mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]); + + const svc = SchedulerService.getInstance(); + await (svc as any).tick(); + + await new Promise(r => setTimeout(r, 50)); + expect(mockCreateScheduledTaskRun).toHaveBeenCalled(); + }); +}); + +// ── Concurrent task prevention ───────────────────────────────────────── + +describe('SchedulerService - concurrent task prevention', () => { + it('does not execute a task that is already in runningTasks', async () => { + mockGetTier.mockReturnValue('pro'); + mockGetVariant.mockReturnValue('team'); + mockGetDueScheduledTasks.mockReturnValue([{ + id: 42, + name: 'running-task', + action: 'restart', + cron_expression: '*/5 * * * *', + enabled: true, + target_id: 'my-stack', + node_id: 1, + created_by: 'admin', + last_status: null, + }]); + + const svc = SchedulerService.getInstance(); + // Pre-add the task to runningTasks + (svc as any).runningTasks.add(42); + + await (svc as any).tick(); + await new Promise(r => setTimeout(r, 50)); + + expect(mockCreateScheduledTaskRun).not.toHaveBeenCalled(); + }); + + it('removes task from runningTasks after completion', async () => { + mockGetTier.mockReturnValue('pro'); + mockGetVariant.mockReturnValue('team'); + mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]); + + const svc = SchedulerService.getInstance(); + mockGetScheduledTask.mockReturnValue({ + id: 99, + name: 'trigger-test', + action: 'restart', + cron_expression: '*/5 * * * *', + enabled: true, + target_id: 'my-stack', + node_id: 1, + created_by: 'admin', + last_status: null, + }); + + await svc.triggerTask(99); + + expect((svc as any).runningTasks.has(99)).toBe(false); + }); + + it('removes task from runningTasks even on failure', async () => { + const svc = SchedulerService.getInstance(); + mockGetScheduledTask.mockReturnValue({ + id: 100, + name: 'fail-test', + action: 'restart', + cron_expression: '*/5 * * * *', + enabled: true, + target_id: null, // Will cause error: "requires target_id" + node_id: null, + created_by: 'admin', + last_status: null, + }); + + await svc.triggerTask(100); + + expect((svc as any).runningTasks.has(100)).toBe(false); + // Error should have been recorded + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( + expect.any(Number), + expect.objectContaining({ status: 'failure' }) + ); + }); +}); + +// ── triggerTask ──────────────────────────────────────────────────────── + +describe('SchedulerService - triggerTask', () => { + it('throws when task not found', async () => { + mockGetScheduledTask.mockReturnValue(undefined); + + const svc = SchedulerService.getInstance(); + await expect(svc.triggerTask(999)).rejects.toThrow('Task not found'); + }); + + it('throws when task is already running', async () => { + mockGetScheduledTask.mockReturnValue({ id: 50, name: 'busy' }); + + const svc = SchedulerService.getInstance(); + (svc as any).runningTasks.add(50); + + await expect(svc.triggerTask(50)).rejects.toThrow('already running'); + }); + + it('sets triggered_by to manual', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 55, + name: 'manual-test', + action: 'restart', + cron_expression: '*/5 * * * *', + enabled: false, // Disabled — but triggerTask should still work + target_id: 'my-stack', + node_id: 1, + created_by: 'admin', + last_status: null, + }); + mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]); + + const svc = SchedulerService.getInstance(); + await svc.triggerTask(55); + + expect(mockCreateScheduledTaskRun).toHaveBeenCalledWith( + expect.objectContaining({ triggered_by: 'manual' }) + ); + }); +}); + +// ── executeRestart ───────────────────────────────────────────────────── + +describe('SchedulerService - executeRestart', () => { + it('restarts all containers in a stack', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 60, + name: 'restart-all', + action: 'restart', + cron_expression: '*/5 * * * *', + enabled: true, + target_id: 'my-stack', + node_id: 1, + created_by: 'admin', + last_status: null, + }); + mockGetContainersByStack.mockResolvedValue([ + { Id: 'c1', Service: 'web' }, + { Id: 'c2', Service: 'db' }, + ]); + + const svc = SchedulerService.getInstance(); + await svc.triggerTask(60); + + expect(mockRestartContainer).toHaveBeenCalledTimes(2); + }); + + it('restarts only specified services when target_services set', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 61, + name: 'restart-filtered', + action: 'restart', + cron_expression: '*/5 * * * *', + enabled: true, + target_id: 'my-stack', + node_id: 1, + target_services: JSON.stringify(['web']), + created_by: 'admin', + last_status: null, + }); + mockGetContainersByStack.mockResolvedValue([ + { Id: 'c1', Service: 'web' }, + { Id: 'c2', Service: 'db' }, + ]); + + const svc = SchedulerService.getInstance(); + await svc.triggerTask(61); + + expect(mockRestartContainer).toHaveBeenCalledTimes(1); + expect(mockRestartContainer).toHaveBeenCalledWith('c1'); + }); + + it('records failure when no containers found', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 62, + name: 'restart-empty', + action: 'restart', + cron_expression: '*/5 * * * *', + enabled: true, + target_id: 'empty-stack', + node_id: 1, + created_by: 'admin', + last_status: null, + }); + mockGetContainersByStack.mockResolvedValue([]); + + const svc = SchedulerService.getInstance(); + await svc.triggerTask(62); + + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( + expect.any(Number), + expect.objectContaining({ status: 'failure', error: expect.stringContaining('No containers') }) + ); + }); +}); + +// ── executePrune ─────────────────────────────────────────────────────── + +describe('SchedulerService - executePrune', () => { + it('prunes all targets by default', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 70, + name: 'prune-all', + action: 'prune', + cron_expression: '0 3 * * *', + enabled: true, + node_id: 1, + created_by: 'admin', + last_status: null, + }); + + const svc = SchedulerService.getInstance(); + await svc.triggerTask(70); + + // Should prune all 4 targets + expect(mockPruneSystem).toHaveBeenCalledTimes(4); + }); + + it('prunes only specified targets', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 71, + name: 'prune-some', + action: 'prune', + cron_expression: '0 3 * * *', + enabled: true, + node_id: 1, + prune_targets: JSON.stringify(['images', 'volumes']), + created_by: 'admin', + last_status: null, + }); + + const svc = SchedulerService.getInstance(); + await svc.triggerTask(71); + + expect(mockPruneSystem).toHaveBeenCalledTimes(2); + expect(mockPruneSystem).toHaveBeenCalledWith('images', undefined); + expect(mockPruneSystem).toHaveBeenCalledWith('volumes', undefined); + }); + + it('includes label filter when configured', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 72, + name: 'prune-labeled', + action: 'prune', + cron_expression: '0 3 * * *', + enabled: true, + node_id: 1, + prune_targets: JSON.stringify(['containers']), + prune_label_filter: 'env=staging', + created_by: 'admin', + last_status: null, + }); + + const svc = SchedulerService.getInstance(); + await svc.triggerTask(72); + + expect(mockPruneSystem).toHaveBeenCalledWith('containers', 'env=staging'); + }); +}); + +// ── executeUpdate ────────────────────────────────────────────────────── + +describe('SchedulerService - executeUpdate', () => { + it('updates stack when image update available', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 80, + name: 'update-stack', + action: 'update', + cron_expression: '0 4 * * *', + enabled: true, + target_id: 'web-app', + node_id: 1, + created_by: 'admin', + last_status: null, + }); + mockGetContainersByStack.mockResolvedValue([ + { Id: 'c1', Image: 'nginx:latest' }, + ]); + mockCheckImage.mockResolvedValue(true); // Update available + + const svc = SchedulerService.getInstance(); + await svc.triggerTask(80); + + expect(mockUpdateStack).toHaveBeenCalledWith('web-app', undefined, true); + expect(mockClearStackUpdateStatus).toHaveBeenCalledWith('web-app'); + }); + + it('skips when all images up to date', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 81, + name: 'update-no-change', + action: 'update', + cron_expression: '0 4 * * *', + enabled: true, + target_id: 'web-app', + node_id: 1, + created_by: 'admin', + last_status: null, + }); + mockGetContainersByStack.mockResolvedValue([ + { Id: 'c1', Image: 'nginx:latest' }, + ]); + mockCheckImage.mockResolvedValue(false); // No update + + const svc = SchedulerService.getInstance(); + await svc.triggerTask(81); + + expect(mockUpdateStack).not.toHaveBeenCalled(); + }); + + it('handles wildcard target (*) by updating all stacks', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 82, + name: 'update-all', + action: 'update', + cron_expression: '0 4 * * *', + enabled: true, + target_id: '*', + node_id: 1, + created_by: 'admin', + last_status: null, + }); + mockGetStacks.mockResolvedValue(['app1', 'app2']); + mockGetContainersByStack.mockResolvedValue([ + { Id: 'c1', Image: 'nginx:latest' }, + ]); + mockCheckImage.mockResolvedValue(true); + + const svc = SchedulerService.getInstance(); + await svc.triggerTask(82); + + expect(mockUpdateStack).toHaveBeenCalledTimes(2); + }); +}); + +// ── Error handling & notifications ───────────────────────────────────── + +describe('SchedulerService - error handling', () => { + it('records failure status in DB on error', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 90, + name: 'error-task', + action: 'restart', + cron_expression: '*/5 * * * *', + enabled: true, + target_id: null, + node_id: null, + created_by: 'admin', + last_status: null, + }); + + const svc = SchedulerService.getInstance(); + await svc.triggerTask(90); + + expect(mockUpdateScheduledTask).toHaveBeenCalledWith( + 90, + expect.objectContaining({ last_status: 'failure' }) + ); + }); + + it('dispatches error notification on failure', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 91, + name: 'notify-fail', + action: 'restart', + cron_expression: '*/5 * * * *', + enabled: true, + target_id: null, + node_id: null, + created_by: 'admin', + last_status: null, + }); + + const svc = SchedulerService.getInstance(); + await svc.triggerTask(91); + + expect(mockDispatchAlert).toHaveBeenCalledWith('error', expect.stringContaining('failed')); + }); + + it('dispatches recovery notification when previous status was failure', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 92, + name: 'recovery-task', + action: 'restart', + cron_expression: '*/5 * * * *', + enabled: true, + target_id: 'my-stack', + node_id: 1, + created_by: 'admin', + last_status: 'failure', // Previous run failed + }); + mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]); + + const svc = SchedulerService.getInstance(); + await svc.triggerTask(92); + + expect(mockDispatchAlert).toHaveBeenCalledWith('info', expect.stringContaining('recovered')); + }); +}); + +// ── Cleanup ──────────────────────────────────────────────────────────── + +describe('SchedulerService - cleanup', () => { + it('calls cleanupOldTaskRuns(30) on every tick', async () => { + mockGetTier.mockReturnValue('pro'); + mockGetVariant.mockReturnValue('team'); + mockGetDueScheduledTasks.mockReturnValue([]); + + const svc = SchedulerService.getInstance(); + await (svc as any).tick(); + + expect(mockCleanupOldTaskRuns).toHaveBeenCalledWith(30); + }); +}); + +// ── isProcessing guard ───────────────────────────────────────────────── + +describe('SchedulerService - isProcessing guard', () => { + it('skips tick if already processing', async () => { + mockGetTier.mockReturnValue('pro'); + + const svc = SchedulerService.getInstance(); + (svc as any).isProcessing = true; + + await (svc as any).tick(); + + expect(mockGetTier).not.toHaveBeenCalled(); + }); + + it('resets isProcessing after tick completes (even on error)', async () => { + mockGetTier.mockImplementationOnce(() => { throw new Error('boom'); }); + + const svc = SchedulerService.getInstance(); + await (svc as any).tick(); + + expect((svc as any).isProcessing).toBe(false); + }); +});