/** * Unit tests for SchedulerService — task execution, concurrent prevention, * license gating, cron parsing, and error handling. */ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import type { ScheduledTask } from '../services/DatabaseService'; // ── Hoisted mocks ────────────────────────────────────────────────────── const { mockGetDueScheduledTasks, mockCreateScheduledTaskRun, mockUpdateScheduledTaskRun, mockUpdateScheduledTask, mockCleanupOldTaskRuns, mockGetScheduledTask, mockGetNodes, mockGetNode, mockGetGlobalSettings, mockGetStackDossier, mockCreateSnapshot, mockInsertSnapshotFiles, mockClearStackUpdateStatus, mockMarkStaleRunsAsFailed, mockDeleteOldScans, mockGetTier, mockGetProxyHeaders, mockGetContainersByStack, mockRestartContainer, mockPruneSystem, mockUpdateStack, mockGetStacks, mockGetStackContent, mockGetEnvContent, mockCheckImage, mockDispatchAlert, mockGetProxyTarget, mockIsTrivyAvailable, mockScanAllNodeImages, mockDeleteScheduledTask, mockGetMatchingPolicy, mockRunCommand, mockDeployStack, mockBackupStackFiles, mockEnforcePolicyPreDeploy, } = vi.hoisted(() => ({ mockGetDueScheduledTasks: vi.fn().mockReturnValue([]), mockCreateScheduledTaskRun: vi.fn().mockReturnValue(1), mockUpdateScheduledTaskRun: vi.fn(), mockUpdateScheduledTask: vi.fn(), mockCleanupOldTaskRuns: vi.fn(), mockGetScheduledTask: vi.fn(), mockGetNodes: vi.fn().mockReturnValue([]), mockGetNode: vi.fn().mockReturnValue({ id: 1, name: 'local', type: 'local', status: 'online' }), mockGetGlobalSettings: vi.fn().mockReturnValue({}), mockGetStackDossier: vi.fn().mockReturnValue(undefined), mockCreateSnapshot: vi.fn().mockReturnValue(1), mockInsertSnapshotFiles: vi.fn(), mockClearStackUpdateStatus: vi.fn(), mockMarkStaleRunsAsFailed: vi.fn().mockReturnValue(0), mockDeleteOldScans: vi.fn().mockReturnValue(0), mockGetTier: vi.fn().mockReturnValue('paid'), mockGetProxyHeaders: vi.fn().mockReturnValue({ tier: 'paid' }), mockGetContainersByStack: vi.fn().mockResolvedValue([]), mockRestartContainer: vi.fn().mockResolvedValue(undefined), mockPruneSystem: vi.fn().mockResolvedValue({ success: true, reclaimedBytes: 0 }), mockUpdateStack: vi.fn().mockResolvedValue(undefined), mockGetStacks: vi.fn().mockResolvedValue([]), mockGetStackContent: vi.fn().mockResolvedValue(''), mockGetEnvContent: vi.fn().mockResolvedValue(''), mockCheckImage: vi.fn().mockResolvedValue({ hasUpdate: false }), mockDispatchAlert: vi.fn().mockResolvedValue(undefined), mockGetProxyTarget: vi.fn().mockReturnValue(null), mockIsTrivyAvailable: vi.fn().mockReturnValue(true), mockScanAllNodeImages: vi.fn().mockResolvedValue({ scanned: 0, skipped: 0, failed: 0, severity: { critical: 0, high: 0, medium: 0, low: 0, unknown: 0 }, violations: [], }), mockDeleteScheduledTask: vi.fn(), mockGetMatchingPolicy: vi.fn().mockReturnValue(null), mockRunCommand: vi.fn().mockResolvedValue(undefined), mockDeployStack: vi.fn().mockResolvedValue(undefined), mockBackupStackFiles: vi.fn().mockResolvedValue(undefined), mockEnforcePolicyPreDeploy: vi.fn(), })); vi.mock('../services/DatabaseService', () => ({ DatabaseService: { getInstance: () => ({ getDueScheduledTasks: mockGetDueScheduledTasks, createScheduledTaskRun: mockCreateScheduledTaskRun, updateScheduledTaskRun: mockUpdateScheduledTaskRun, updateScheduledTask: mockUpdateScheduledTask, cleanupOldTaskRuns: mockCleanupOldTaskRuns, getScheduledTask: mockGetScheduledTask, getNodes: mockGetNodes, getNode: mockGetNode, getGlobalSettings: mockGetGlobalSettings, getStackDossier: mockGetStackDossier, createSnapshot: mockCreateSnapshot, insertSnapshotFiles: mockInsertSnapshotFiles, clearStackUpdateStatus: mockClearStackUpdateStatus, markStaleRunsAsFailed: mockMarkStaleRunsAsFailed, deleteOldScans: mockDeleteOldScans, deleteScheduledTask: mockDeleteScheduledTask, getMatchingPolicy: mockGetMatchingPolicy, }), }, })); vi.mock('../services/FleetSyncService', () => ({ FleetSyncService: { getSelfIdentity: () => 'self-node', }, })); vi.mock('../services/LicenseService', () => ({ LicenseService: { getInstance: () => ({ getTier: mockGetTier, getProxyHeaders: mockGetProxyHeaders, }), }, })); vi.mock('../services/DockerController', () => ({ default: { getInstance: () => ({ getContainersByStack: mockGetContainersByStack, restartContainer: mockRestartContainer, pruneSystem: mockPruneSystem, }), }, })); vi.mock('../services/ComposeService', () => ({ ComposeService: { getInstance: () => ({ updateStack: mockUpdateStack, runCommand: mockRunCommand, deployStack: mockDeployStack, }), }, })); vi.mock('../services/FileSystemService', () => ({ FileSystemService: { getInstance: () => ({ getStacks: mockGetStacks, getStackContent: mockGetStackContent, getEnvContent: mockGetEnvContent, backupStackFiles: mockBackupStackFiles, }), }, })); vi.mock('../services/ImageUpdateService', () => ({ ImageUpdateService: { getInstance: () => ({ checkImage: mockCheckImage, }), }, })); vi.mock('../services/NotificationService', () => ({ NotificationService: { getInstance: () => ({ dispatchAlert: mockDispatchAlert, }), }, })); vi.mock('../services/CloudBackupService', () => ({ CloudBackupService: { getInstance: () => ({ isEnabled: () => false, isAutoUploadOn: () => false, uploadSnapshot: vi.fn().mockResolvedValue(undefined), }), }, })); vi.mock('../services/NodeRegistry', () => ({ NodeRegistry: { getInstance: () => ({ getDefaultNodeId: () => 1, getNode: mockGetNode, getProxyTarget: mockGetProxyTarget, }), }, })); vi.mock('../services/TrivyService', () => ({ default: { getInstance: () => ({ isTrivyAvailable: mockIsTrivyAvailable, scanAllNodeImages: mockScanAllNodeImages, getSource: () => 'managed', detectTrivy: vi.fn().mockResolvedValue(undefined), }), }, })); vi.mock('../services/PolicyEnforcement', () => ({ enforcePolicyPreDeploy: mockEnforcePolicyPreDeploy, })); import { SchedulerService } from '../services/SchedulerService'; import { StackOpLockService } from '../services/StackOpLockService'; beforeEach(() => { vi.clearAllMocks(); // Lifecycle/update handlers run through the real StackOpLockService; reset it // so a lock left held by one test cannot make a later test skip its op. StackOpLockService.resetForTests(); // clearAllMocks only clears call history, not implementations, so restore the // mocks that individual tests mutate (tier, node lookup, proxy target) to // their documented defaults. Without this a test that points getNode at a // remote node or drops the tier leaks that state into every later test in the // file. mockGetTier.mockReturnValue('paid'); mockGetNode.mockReturnValue({ id: 1, name: 'local', type: 'local', status: 'online' }); mockGetProxyTarget.mockReturnValue(null); // Documentation capture is opt-in and off by default; reset so a test that // enables it does not leak into later snapshot tests. mockGetGlobalSettings.mockReturnValue({}); mockGetStackDossier.mockReturnValue(undefined); // Default: the scan-policy gate allows. Individual tests override to a block. mockEnforcePolicyPreDeploy.mockResolvedValue({ ok: true, bypassed: false, violations: [] }); (SchedulerService as any).instance = undefined; }); // ── calculateNextRun ─────────────────────────────────────────────────── describe('SchedulerService - calculateNextRun', () => { it('returns a future timestamp for valid cron expression', () => { const svc = SchedulerService.getInstance(); const next = svc.calculateNextRun('*/5 * * * *'); // Every 5 minutes expect(next).toBeGreaterThan(Date.now()); }); it('throws on invalid cron expression', () => { const svc = SchedulerService.getInstance(); expect(() => svc.calculateNextRun('not a cron')).toThrow(); }); }); describe('SchedulerService - calculateRunsWithin', () => { it('expands hourly cron into every firing within a 24h window when limit allows', () => { const svc = SchedulerService.getInstance(); const from = Date.now(); const to = from + 24 * 60 * 60 * 1000; const runs = svc.calculateRunsWithin('0 * * * *', from, to, 32); expect(runs.length).toBeGreaterThanOrEqual(23); expect(runs.length).toBeLessThanOrEqual(24); for (const run of runs) { expect(run).toBeGreaterThanOrEqual(from); expect(run).toBeLessThanOrEqual(to); } }); it('returns a single firing for a daily cron', () => { const svc = SchedulerService.getInstance(); const from = Date.now(); const to = from + 24 * 60 * 60 * 1000; const runs = svc.calculateRunsWithin('0 3 * * *', from, to); expect(runs.length).toBeLessThanOrEqual(2); expect(runs.length).toBeGreaterThanOrEqual(1); }); it('honours the limit parameter to avoid runaway expansions', () => { const svc = SchedulerService.getInstance(); const from = Date.now(); const to = from + 60 * 60 * 1000; const runs = svc.calculateRunsWithin('* * * * *', from, to, 5); expect(runs.length).toBe(5); }); it('returns empty array for invalid cron instead of throwing', () => { const svc = SchedulerService.getInstance(); const from = Date.now(); const to = from + 60 * 60 * 1000; expect(svc.calculateRunsWithin('not a cron', from, to)).toEqual([]); }); }); // ── License gating ───────────────────────────────────────────────────── describe('SchedulerService - scheduled tasks run on every tier', () => { function makeTask(overrides: Partial = {}) { return { id: 1, name: 'test-task', action: 'restart', cron_expression: '*/5 * * * *', enabled: true, target_id: 'my-stack', node_id: 1, created_by: 'admin', last_status: null, ...overrides, }; } it('runs tasks on the Community tier (no paid gate)', async () => { mockGetTier.mockReturnValue('community'); mockGetDueScheduledTasks.mockReturnValue([makeTask()]); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]); const svc = SchedulerService.getInstance(); await (svc as any).tick(); await new Promise(r => setTimeout(r, 50)); expect(mockCreateScheduledTaskRun).toHaveBeenCalled(); }); it('runs update tasks on the Community tier', async () => { mockGetTier.mockReturnValue('community'); mockGetDueScheduledTasks.mockReturnValue([makeTask({ action: 'update' })]); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }]); mockCheckImage.mockResolvedValue({ hasUpdate: false }); const svc = SchedulerService.getInstance(); await (svc as any).tick(); // Wait for the async task to settle await new Promise(r => setTimeout(r, 50)); expect(mockCreateScheduledTaskRun).toHaveBeenCalled(); }); it('runs snapshot tasks on the Community tier', async () => { mockGetTier.mockReturnValue('community'); mockGetDueScheduledTasks.mockReturnValue([makeTask({ action: 'snapshot', target_type: 'fleet' })]); const svc = SchedulerService.getInstance(); await (svc as any).tick(); await new Promise(r => setTimeout(r, 50)); expect(mockCreateScheduledTaskRun).toHaveBeenCalled(); }); it('runs tasks on the paid tier', async () => { mockGetTier.mockReturnValue('paid'); mockGetDueScheduledTasks.mockReturnValue([makeTask({ action: 'restart' })]); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]); const svc = SchedulerService.getInstance(); await (svc as any).tick(); await new Promise(r => setTimeout(r, 50)); expect(mockCreateScheduledTaskRun).toHaveBeenCalled(); }); }); // ── Concurrent task prevention ───────────────────────────────────────── describe('SchedulerService - concurrent task prevention', () => { it('does not execute a task that is already in runningTasks', async () => { mockGetTier.mockReturnValue('paid'); mockGetDueScheduledTasks.mockReturnValue([{ id: 42, name: 'running-task', action: 'restart', cron_expression: '*/5 * * * *', enabled: true, target_id: 'my-stack', node_id: 1, created_by: 'admin', last_status: null, }]); const svc = SchedulerService.getInstance(); // Pre-add the task to runningTasks (svc as any).runningTasks.add(42); await (svc as any).tick(); await new Promise(r => setTimeout(r, 50)); expect(mockCreateScheduledTaskRun).not.toHaveBeenCalled(); }); it('removes task from runningTasks after completion', async () => { mockGetTier.mockReturnValue('paid'); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]); const svc = SchedulerService.getInstance(); mockGetScheduledTask.mockReturnValue({ id: 99, name: 'trigger-test', action: 'restart', cron_expression: '*/5 * * * *', enabled: true, target_id: 'my-stack', node_id: 1, created_by: 'admin', last_status: null, }); await svc.triggerTask(99); expect((svc as any).runningTasks.has(99)).toBe(false); }); it('removes task from runningTasks even on failure', async () => { const svc = SchedulerService.getInstance(); mockGetScheduledTask.mockReturnValue({ id: 100, name: 'fail-test', action: 'restart', cron_expression: '*/5 * * * *', enabled: true, target_id: null, // Will cause error: "requires target_id" node_id: null, created_by: 'admin', last_status: null, }); await svc.triggerTask(100); expect((svc as any).runningTasks.has(100)).toBe(false); // Error should have been recorded expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( expect.any(Number), expect.objectContaining({ status: 'failure' }) ); }); }); // ── triggerTask ──────────────────────────────────────────────────────── describe('SchedulerService - triggerTask', () => { it('throws when task not found', async () => { mockGetScheduledTask.mockReturnValue(undefined); const svc = SchedulerService.getInstance(); await expect(svc.triggerTask(999)).rejects.toThrow('Task not found'); }); it('throws when task is already running', async () => { mockGetScheduledTask.mockReturnValue({ id: 50, name: 'busy' }); const svc = SchedulerService.getInstance(); (svc as any).runningTasks.add(50); await expect(svc.triggerTask(50)).rejects.toThrow('already running'); }); it('sets triggered_by to manual', async () => { mockGetScheduledTask.mockReturnValue({ id: 55, name: 'manual-test', action: 'restart', cron_expression: '*/5 * * * *', enabled: false, // Disabled — but triggerTask should still work target_id: 'my-stack', node_id: 1, created_by: 'admin', last_status: null, }); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]); const svc = SchedulerService.getInstance(); await svc.triggerTask(55); expect(mockCreateScheduledTaskRun).toHaveBeenCalledWith( expect.objectContaining({ triggered_by: 'manual' }) ); }); }); // ── executeRestart ───────────────────────────────────────────────────── describe('SchedulerService - executeRestart', () => { it('restarts all containers in a stack', async () => { mockGetScheduledTask.mockReturnValue({ id: 60, name: 'restart-all', action: 'restart', cron_expression: '*/5 * * * *', enabled: true, target_id: 'my-stack', node_id: 1, created_by: 'admin', last_status: null, }); mockGetContainersByStack.mockResolvedValue([ { Id: 'c1', Service: 'web' }, { Id: 'c2', Service: 'db' }, ]); const svc = SchedulerService.getInstance(); await svc.triggerTask(60); expect(mockRestartContainer).toHaveBeenCalledTimes(2); }); it('restarts only specified services when target_services set', async () => { mockGetScheduledTask.mockReturnValue({ id: 61, name: 'restart-filtered', action: 'restart', cron_expression: '*/5 * * * *', enabled: true, target_id: 'my-stack', node_id: 1, target_services: JSON.stringify(['web']), created_by: 'admin', last_status: null, }); mockGetContainersByStack.mockResolvedValue([ { Id: 'c1', Service: 'web' }, { Id: 'c2', Service: 'db' }, ]); const svc = SchedulerService.getInstance(); await svc.triggerTask(61); expect(mockRestartContainer).toHaveBeenCalledTimes(1); expect(mockRestartContainer).toHaveBeenCalledWith('c1'); }); it('records failure when no containers found', async () => { mockGetScheduledTask.mockReturnValue({ id: 62, name: 'restart-empty', action: 'restart', cron_expression: '*/5 * * * *', enabled: true, target_id: 'empty-stack', node_id: 1, created_by: 'admin', last_status: null, }); mockGetContainersByStack.mockResolvedValue([]); const svc = SchedulerService.getInstance(); await svc.triggerTask(62); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( expect.any(Number), expect.objectContaining({ status: 'failure', error: expect.stringContaining('No containers') }) ); }); }); // ── executePrune ─────────────────────────────────────────────────────── describe('SchedulerService - executePrune', () => { it('prunes all targets by default', async () => { mockGetScheduledTask.mockReturnValue({ id: 70, name: 'prune-all', action: 'prune', cron_expression: '0 3 * * *', enabled: true, node_id: 1, created_by: 'admin', last_status: null, }); const svc = SchedulerService.getInstance(); await svc.triggerTask(70); // Should prune all 4 targets expect(mockPruneSystem).toHaveBeenCalledTimes(4); }); it('prunes only specified targets', async () => { mockGetScheduledTask.mockReturnValue({ id: 71, name: 'prune-some', action: 'prune', cron_expression: '0 3 * * *', enabled: true, node_id: 1, prune_targets: JSON.stringify(['images', 'volumes']), created_by: 'admin', last_status: null, }); const svc = SchedulerService.getInstance(); await svc.triggerTask(71); expect(mockPruneSystem).toHaveBeenCalledTimes(2); expect(mockPruneSystem).toHaveBeenCalledWith('images', undefined); expect(mockPruneSystem).toHaveBeenCalledWith('volumes', undefined); }); it('includes label filter when configured', async () => { mockGetScheduledTask.mockReturnValue({ id: 72, name: 'prune-labeled', action: 'prune', cron_expression: '0 3 * * *', enabled: true, node_id: 1, prune_targets: JSON.stringify(['containers']), prune_label_filter: 'env=staging', created_by: 'admin', last_status: null, }); const svc = SchedulerService.getInstance(); await svc.triggerTask(72); expect(mockPruneSystem).toHaveBeenCalledWith('containers', 'env=staging'); }); it('marks scheduled prune runs as failed when a target prune fails', async () => { mockGetScheduledTask.mockReturnValue({ id: 74, name: 'prune-fails', action: 'prune', cron_expression: '0 3 * * *', enabled: true, node_id: 1, prune_targets: JSON.stringify(['images']), created_by: 'admin', last_status: null, }); mockPruneSystem.mockRejectedValueOnce(new Error('docker prune failed')); const svc = SchedulerService.getInstance(); await svc.triggerTask(74); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( expect.any(Number), expect.objectContaining({ status: 'failure', error: expect.stringContaining('images: failed (docker prune failed)'), }), ); expect(mockUpdateScheduledTask).toHaveBeenCalledWith( 74, expect.objectContaining({ last_status: 'failure' }), ); }); it('fails scheduled prune tasks that target remote nodes before pruning', async () => { mockGetScheduledTask.mockReturnValue({ id: 73, name: 'remote-prune', action: 'prune', cron_expression: '0 3 * * *', enabled: true, node_id: 2, created_by: 'admin', last_status: null, }); mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); const svc = SchedulerService.getInstance(); await svc.triggerTask(73); expect(mockPruneSystem).not.toHaveBeenCalled(); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( expect.any(Number), expect.objectContaining({ status: 'failure', error: expect.stringMatching(/local node/i), }), ); }); }); // ── executeUpdate ────────────────────────────────────────────────────── describe('SchedulerService - executeUpdate', () => { it('updates stack when image update available', async () => { mockGetScheduledTask.mockReturnValue({ id: 80, name: 'update-stack', action: 'update', cron_expression: '0 4 * * *', enabled: true, target_id: 'web-app', node_id: 1, created_by: 'admin', last_status: null, }); mockGetContainersByStack.mockResolvedValue([ { Id: 'c1', Image: 'nginx:latest' }, ]); mockCheckImage.mockResolvedValue({ hasUpdate: true }); // Update available const svc = SchedulerService.getInstance(); await svc.triggerTask(80); expect(mockUpdateStack).toHaveBeenCalledWith('web-app', undefined, true); expect(mockClearStackUpdateStatus).toHaveBeenCalledWith(1, 'web-app'); }); it('runs a scheduled update on the community tier (no paid gate)', async () => { // Scheduled tasks are free, so a community licence runs the update like any // other tier. mockGetTier.mockReturnValue('community'); mockGetScheduledTask.mockReturnValue({ id: 82, name: 'update-community', action: 'update', cron_expression: '0 4 * * *', enabled: true, target_id: 'web-app', node_id: 1, created_by: 'admin', last_status: null, }); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }]); mockCheckImage.mockResolvedValue({ hasUpdate: true }); const svc = SchedulerService.getInstance(); await svc.triggerTask(82); expect(mockUpdateStack).toHaveBeenCalledWith('web-app', undefined, true); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'success' }), ); }); it('begins a health gate after a scheduled update succeeds', async () => { const { HealthGateService } = await import('../services/HealthGateService'); const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-1'); try { mockGetScheduledTask.mockReturnValue({ id: 83, name: 'update-gated', action: 'update', cron_expression: '0 4 * * *', enabled: true, target_id: 'web-app', node_id: 1, created_by: 'admin', last_status: null, }); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }]); mockCheckImage.mockResolvedValue({ hasUpdate: true }); await SchedulerService.getInstance().triggerTask(83); expect(beginSpy).toHaveBeenCalledWith(1, 'web-app', 'update', 'system:scheduler'); } finally { beginSpy.mockRestore(); } }); it('skips when all images up to date', async () => { mockGetScheduledTask.mockReturnValue({ id: 81, name: 'update-no-change', action: 'update', cron_expression: '0 4 * * *', enabled: true, target_id: 'web-app', node_id: 1, created_by: 'admin', last_status: null, }); mockGetContainersByStack.mockResolvedValue([ { Id: 'c1', Image: 'nginx:latest' }, ]); mockCheckImage.mockResolvedValue({ hasUpdate: false }); // No update const svc = SchedulerService.getInstance(); await svc.triggerTask(81); expect(mockUpdateStack).not.toHaveBeenCalled(); }); it('handles wildcard target (*) by updating all stacks', async () => { mockGetScheduledTask.mockReturnValue({ id: 82, name: 'update-all', action: 'update', cron_expression: '0 4 * * *', enabled: true, target_id: '*', node_id: 1, created_by: 'admin', last_status: null, }); mockGetStacks.mockResolvedValue(['app1', 'app2']); mockGetContainersByStack.mockResolvedValue([ { Id: 'c1', Image: 'nginx:latest' }, ]); mockCheckImage.mockResolvedValue({ hasUpdate: true }); const svc = SchedulerService.getInstance(); await svc.triggerTask(82); expect(mockUpdateStack).toHaveBeenCalledTimes(2); }); it('reports warning when all image checks fail (B3 fix)', async () => { mockGetScheduledTask.mockReturnValue({ id: 83, name: 'update-check-fail', action: 'update', cron_expression: '0 4 * * *', enabled: true, target_id: 'web-app', node_id: 1, created_by: 'admin', last_status: null, }); mockGetContainersByStack.mockResolvedValue([ { Id: 'c1', Image: 'nginx:latest' }, ]); mockCheckImage.mockResolvedValue({ hasUpdate: false, error: 'Registry unreachable for registry-1.docker.io/library/nginx:latest' }); const svc = SchedulerService.getInstance(); await svc.triggerTask(83); // Should succeed (not throw) but output should contain warning expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'success', output: expect.stringContaining('WARNING'), }) ); expect(mockUpdateStack).not.toHaveBeenCalled(); }); it('reports partial check failures with success count (B3 fix)', async () => { mockGetScheduledTask.mockReturnValue({ id: 84, name: 'update-partial-fail', action: 'update', cron_expression: '0 4 * * *', enabled: true, target_id: 'web-app', node_id: 1, created_by: 'admin', last_status: null, }); mockGetContainersByStack.mockResolvedValue([ { Id: 'c1', Image: 'nginx:latest' }, { Id: 'c2', Image: 'redis:7' }, ]); // First image check succeeds (no update), second fails mockCheckImage .mockResolvedValueOnce({ hasUpdate: false }) .mockResolvedValueOnce({ hasUpdate: false, error: 'Registry unreachable' }); const svc = SchedulerService.getInstance(); await svc.triggerTask(84); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'success', output: expect.stringContaining('check(s) failed'), }) ); }); it('warns when targeted stack has 0 containers (E1 fix)', async () => { mockGetScheduledTask.mockReturnValue({ id: 85, name: 'update-missing-stack', action: 'update', cron_expression: '0 4 * * *', enabled: true, target_id: 'deleted-stack', node_id: 1, created_by: 'admin', last_status: null, }); mockGetContainersByStack.mockResolvedValue([]); const svc = SchedulerService.getInstance(); await svc.triggerTask(85); // Targeted (non-wildcard) stack with 0 containers should produce a WARNING expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'success', output: expect.stringContaining('WARNING'), }) ); }); it('silently skips empty stacks in wildcard mode', async () => { mockGetScheduledTask.mockReturnValue({ id: 86, name: 'update-wildcard-empty', action: 'update', cron_expression: '0 4 * * *', enabled: true, target_id: '*', node_id: 1, created_by: 'admin', last_status: null, }); mockGetStacks.mockResolvedValue(['active-stack', 'empty-stack']); // First stack has containers, second has none mockGetContainersByStack .mockResolvedValueOnce([{ Id: 'c1', Image: 'nginx:latest' }]) .mockResolvedValueOnce([]); mockCheckImage.mockResolvedValue({ hasUpdate: false }); const svc = SchedulerService.getInstance(); await svc.triggerTask(86); // Empty stack in wildcard mode should say "skipped", not "WARNING" expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'success', output: expect.not.stringContaining('WARNING'), }) ); }); it('dispatches a scan_finding warning and skips the stack when a policy blocks the update', async () => { mockGetScheduledTask.mockReturnValue({ id: 88, name: 'blocked-update', action: 'update', cron_expression: '0 4 * * *', enabled: true, target_id: 'web-app', node_id: 1, created_by: 'admin', last_status: null, }); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:1.14' }]); mockCheckImage.mockResolvedValue({ hasUpdate: true }); mockEnforcePolicyPreDeploy.mockResolvedValue({ ok: false, bypassed: false, policy: { id: 1, name: 'block-high', max_severity: 'HIGH' }, violations: [{ imageRef: 'nginx:1.14', severity: 'CRITICAL', criticalCount: 2, highCount: 5, scanId: 7 }], }); const svc = SchedulerService.getInstance(); await svc.triggerTask(88); // Gate blocked it: the stack must not be updated. expect(mockUpdateStack).not.toHaveBeenCalled(); // A scan_finding warning naming the policy and the offending image fired. const warn = mockDispatchAlert.mock.calls.find((c) => c[0] === 'warning' && c[1] === 'scan_finding'); expect(warn).toBeDefined(); expect(warn![2]).toContain('block-high'); expect(warn![2]).toContain('nginx:1.14'); // The run completes (skip-and-continue), not a hard task failure. expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' })); }); it('reports a failed run and an Auto-start warning when a policy blocks a scheduled auto-start', async () => { mockGetScheduledTask.mockReturnValue({ id: 89, name: 'blocked-start', action: 'auto_start', cron_expression: '0 4 * * *', enabled: true, target_id: 'web-app', node_id: 1, created_by: 'admin', last_status: null, }); mockEnforcePolicyPreDeploy.mockResolvedValue({ ok: false, bypassed: false, policy: { id: 1, name: 'block-high', max_severity: 'HIGH' }, violations: [{ imageRef: 'nginx:1.14', severity: 'CRITICAL', criticalCount: 1, highCount: 0, scanId: 3 }], }); const svc = SchedulerService.getInstance(); await svc.triggerTask(89); // Gate blocked it: the stack must not start. expect(mockDeployStack).not.toHaveBeenCalled(); const warn = mockDispatchAlert.mock.calls.find((c) => c[0] === 'warning' && c[1] === 'scan_finding'); expect(warn).toBeDefined(); expect(warn![2]).toContain('Auto-start'); expect(warn![2]).toContain('block-high'); // Auto-start does not skip-and-continue; the run is recorded as a failure. expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'failure' })); }); it('exposes isTaskRunning status', async () => { const svc = SchedulerService.getInstance(); expect(svc.isTaskRunning(999)).toBe(false); }); it('fleet target updates every stack discovered on the node', async () => { mockGetScheduledTask.mockReturnValue({ id: 87, name: 'fleet-update', action: 'update', target_type: 'fleet', cron_expression: '0 4 * * *', enabled: true, target_id: null, node_id: 1, created_by: 'admin', last_status: null, }); mockGetStacks.mockResolvedValue(['app1', 'app2', 'app3']); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }]); mockCheckImage.mockResolvedValue({ hasUpdate: true }); const svc = SchedulerService.getInstance(); await svc.triggerTask(87); expect(mockUpdateStack).toHaveBeenCalledTimes(3); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'success' }) ); }); it('fleet target on empty node returns early with skipped message', async () => { mockGetScheduledTask.mockReturnValue({ id: 89, name: 'fleet-update-empty-node', action: 'update', target_type: 'fleet', cron_expression: '0 4 * * *', enabled: true, target_id: null, node_id: 1, created_by: 'admin', last_status: null, }); mockGetStacks.mockResolvedValue([]); const svc = SchedulerService.getInstance(); await svc.triggerTask(89); expect(mockUpdateStack).not.toHaveBeenCalled(); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'success', output: expect.stringContaining('No stacks found'), }) ); }); }); // ── Error handling & notifications ───────────────────────────────────── describe('SchedulerService - error handling', () => { it('records failure status in DB on error', async () => { mockGetScheduledTask.mockReturnValue({ id: 90, name: 'error-task', action: 'restart', cron_expression: '*/5 * * * *', enabled: true, target_id: null, node_id: null, created_by: 'admin', last_status: null, }); const svc = SchedulerService.getInstance(); await svc.triggerTask(90); expect(mockUpdateScheduledTask).toHaveBeenCalledWith( 90, expect.objectContaining({ last_status: 'failure' }) ); }); it('dispatches error notification on failure', async () => { mockGetScheduledTask.mockReturnValue({ id: 91, name: 'notify-fail', action: 'restart', cron_expression: '*/5 * * * *', enabled: true, target_id: null, node_id: null, created_by: 'admin', last_status: null, }); const svc = SchedulerService.getInstance(); await svc.triggerTask(91); expect(mockDispatchAlert).toHaveBeenCalledWith('error', 'system', expect.stringContaining('failed'), { stackName: undefined, actor: 'system:scheduler' }); }); it('dispatches recovery notification when previous status was failure', async () => { mockGetScheduledTask.mockReturnValue({ id: 92, name: 'recovery-task', action: 'restart', cron_expression: '*/5 * * * *', enabled: true, target_id: 'my-stack', node_id: 1, created_by: 'admin', last_status: 'failure', // Previous run failed }); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]); const svc = SchedulerService.getInstance(); await svc.triggerTask(92); expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'system', expect.stringContaining('recovered'), { stackName: 'my-stack', actor: 'system:scheduler' }); }); }); // ── Scheduled scan completion notifications ─────────────────────────── describe('SchedulerService - scheduled scan notifications', () => { function makeScanTask(overrides: Partial = {}) { return { id: 200, name: 'nightly-scan', action: 'scan', cron_expression: '0 2 * * *', enabled: true, target_id: null, node_id: 1, created_by: 'admin', last_status: null, ...overrides, }; } function scanResult(opts: { scanned?: number; skipped?: number; failed?: number; critical?: number; high?: number; medium?: number; low?: number; unknown?: number; totalImages?: number; processedImages?: number; truncated?: boolean; limitReason?: string; } = {}) { return { scanned: opts.scanned ?? 0, skipped: opts.skipped ?? 0, failed: opts.failed ?? 0, totalImages: opts.totalImages, processedImages: opts.processedImages, truncated: opts.truncated, limitReason: opts.limitReason, severity: { critical: opts.critical ?? 0, high: opts.high ?? 0, medium: opts.medium ?? 0, low: opts.low ?? 0, unknown: opts.unknown ?? 0, }, violations: [], }; } it('dispatches info-level notification when scan completes cleanly', async () => { mockGetScheduledTask.mockReturnValue(makeScanTask()); mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 3, skipped: 1 })); const svc = SchedulerService.getInstance(); await svc.triggerTask(200); expect(mockDispatchAlert).toHaveBeenCalledWith( 'info', 'scan_finding', expect.stringContaining('nightly-scan'), { stackName: undefined, actor: 'system:scheduler' }, ); expect(mockDispatchAlert).toHaveBeenCalledWith( 'info', 'scan_finding', expect.stringContaining('Scanned 3 image(s)'), { stackName: undefined, actor: 'system:scheduler' }, ); }); it('dispatches warning-level notification when scan has failures', async () => { mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 201, name: 'flaky-scan' })); mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 5, failed: 2 })); const svc = SchedulerService.getInstance(); await svc.triggerTask(201); expect(mockDispatchAlert).toHaveBeenCalledWith( 'warning', 'scan_finding', expect.stringContaining('2 failed'), { stackName: undefined, actor: 'system:scheduler' }, ); }); it('passes target_id to dispatchAlert when set', async () => { mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 202, target_id: 'web-stack' })); mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 1 })); const svc = SchedulerService.getInstance(); await svc.triggerTask(202); expect(mockDispatchAlert).toHaveBeenCalledWith( 'info', 'scan_finding', expect.stringContaining('completed'), { stackName: 'web-stack', actor: 'system:scheduler' }, ); }); it('fires only the scan notification when previous run was failure (no duplicate recovery alert)', async () => { mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 204, name: 'recovered-scan', last_status: 'failure', })); mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 2 })); const svc = SchedulerService.getInstance(); await svc.triggerTask(204); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); expect(mockDispatchAlert).toHaveBeenCalledWith( 'info', 'scan_finding', expect.stringContaining('recovered-scan'), { stackName: undefined, actor: 'system:scheduler' }, ); }); it('does not dispatch a scan notification for non-scan actions', async () => { mockGetScheduledTask.mockReturnValue({ id: 203, name: 'restart-task', action: 'restart', cron_expression: '*/5 * * * *', enabled: true, target_id: 'my-stack', node_id: 1, created_by: 'admin', last_status: null, }); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]); const svc = SchedulerService.getInstance(); await svc.triggerTask(203); expect(mockDispatchAlert).not.toHaveBeenCalled(); }); it('dispatches error-level notification when Trivy is unavailable', async () => { mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 205, target_id: 'payment-stack' })); mockIsTrivyAvailable.mockReturnValueOnce(false); const svc = SchedulerService.getInstance(); await svc.triggerTask(205); expect(mockDispatchAlert).toHaveBeenCalledWith( 'error', 'system', expect.stringMatching(/failed.*Trivy/i), { stackName: 'payment-stack', actor: 'system:scheduler' }, ); }); it('fails scheduled scan tasks that target remote nodes before scanning', async () => { mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 210, node_id: 2 })); mockGetNode .mockReturnValueOnce({ id: 2, name: 'remote', type: 'remote', status: 'online' }) .mockReturnValueOnce({ id: 2, name: 'remote', type: 'remote', status: 'online' }); const svc = SchedulerService.getInstance(); await svc.triggerTask(210); expect(mockScanAllNodeImages).not.toHaveBeenCalled(); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( expect.any(Number), expect.objectContaining({ status: 'failure', error: expect.stringMatching(/local node/i), }), ); }); it('includes severity counts in the notification message', async () => { mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 206 })); mockScanAllNodeImages.mockResolvedValue( scanResult({ scanned: 3, skipped: 1, critical: 2, high: 5, medium: 10 }), ); const svc = SchedulerService.getInstance(); await svc.triggerTask(206); const message = mockDispatchAlert.mock.calls[0][2] as string; expect(message).toContain('2 critical'); expect(message).toContain('5 high'); expect(message).toContain('10 medium'); }); it('reports "No images to scan" when the node has nothing to scan', async () => { mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 207 })); mockScanAllNodeImages.mockResolvedValue(scanResult()); const svc = SchedulerService.getInstance(); await svc.triggerTask(207); expect(mockDispatchAlert).toHaveBeenCalledWith( 'info', 'scan_finding', expect.stringContaining('No images to scan'), { stackName: undefined, actor: 'system:scheduler' }, ); }); it('reports "All N image(s) already scanned recently" when every image was cached', async () => { mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 208 })); mockScanAllNodeImages.mockResolvedValue(scanResult({ skipped: 12 })); const svc = SchedulerService.getInstance(); await svc.triggerTask(208); expect(mockDispatchAlert).toHaveBeenCalledWith( 'info', 'scan_finding', expect.stringContaining('All 12 image(s) already scanned recently'), { stackName: undefined, actor: 'system:scheduler' }, ); }); it('reports when scan-all stops at a configured bound', async () => { mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 211 })); mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 100, totalImages: 250, processedImages: 100, truncated: true, limitReason: 'image limit 100 reached', })); const svc = SchedulerService.getInstance(); await svc.triggerTask(211); const message = mockDispatchAlert.mock.calls[0][2] as string; expect(message).toContain('Scan limited after 100 of 250 image(s)'); expect(message).toContain('image limit 100 reached'); }); it('persists the run as success even when notification dispatch throws', async () => { mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 209 })); mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 1 })); mockDispatchAlert.mockRejectedValueOnce(new Error('webhook down')); const svc = SchedulerService.getInstance(); await expect(svc.triggerTask(209)).resolves.not.toThrow(); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( expect.any(Number), expect.objectContaining({ status: 'success' }), ); }); }); // ── Cleanup ──────────────────────────────────────────────────────────── describe('SchedulerService - cleanup', () => { it('calls cleanupOldTaskRuns(30) on every tick', async () => { mockGetTier.mockReturnValue('paid'); mockGetDueScheduledTasks.mockReturnValue([]); const svc = SchedulerService.getInstance(); await (svc as any).tick(); expect(mockCleanupOldTaskRuns).toHaveBeenCalledWith(30); }); }); // ── isProcessing guard ───────────────────────────────────────────────── describe('SchedulerService - isProcessing guard', () => { it('skips tick if already processing', async () => { const svc = SchedulerService.getInstance(); (svc as any).isProcessing = true; await (svc as any).tick(); // Short-circuits before fetching due tasks (the first DB call inside tick). expect(mockGetDueScheduledTasks).not.toHaveBeenCalled(); }); it('resets isProcessing after tick completes (even on error)', async () => { mockGetDueScheduledTasks.mockImplementationOnce(() => { throw new Error('boom'); }); const svc = SchedulerService.getInstance(); await (svc as any).tick(); expect((svc as any).isProcessing).toBe(false); }); }); // ── Stale run cleanup (T1) ─────────────────────────────────────────── describe('SchedulerService - stale run cleanup', () => { it('calls markStaleRunsAsFailed on start and logs when records exist', () => { mockMarkStaleRunsAsFailed.mockReturnValue(2); const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); const svc = SchedulerService.getInstance(); svc.start(); expect(mockMarkStaleRunsAsFailed).toHaveBeenCalledTimes(1); expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('Cleaned up 2 stale run record(s)')); logSpy.mockRestore(); svc.stop(); }); it('does not log when no stale runs exist', () => { mockMarkStaleRunsAsFailed.mockReturnValue(0); const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); const svc = SchedulerService.getInstance(); svc.start(); expect(mockMarkStaleRunsAsFailed).toHaveBeenCalledTimes(1); expect(logSpy).not.toHaveBeenCalledWith(expect.stringContaining('stale')); logSpy.mockRestore(); svc.stop(); }); }); // ── Invalid cron at execution time (T2) ────────────────────────────── describe('SchedulerService - invalid cron at execution time', () => { it('disables task and records error when cron becomes invalid', async () => { mockGetScheduledTask.mockReturnValue({ id: 95, name: 'bad-cron-task', action: 'restart', cron_expression: 'INVALID CRON', enabled: true, target_id: null, node_id: null, created_by: 'admin', last_status: null, }); const svc = SchedulerService.getInstance(); await svc.triggerTask(95); expect(mockUpdateScheduledTask).toHaveBeenCalledWith( 95, expect.objectContaining({ enabled: 0, last_status: 'failure', last_error: expect.stringContaining('no longer valid'), }) ); expect(mockDispatchAlert).toHaveBeenCalledWith( 'error', 'system', expect.stringContaining('failed'), { stackName: undefined, actor: 'system:scheduler' }, ); }); }); // ── executeSnapshot (T3) ───────────────────────────────────────────── describe('SchedulerService - executeSnapshot', () => { it('creates a fleet snapshot capturing all local nodes', async () => { mockGetScheduledTask.mockReturnValue({ id: 75, name: 'nightly-snapshot', action: 'snapshot', target_type: 'fleet', cron_expression: '0 3 * * *', enabled: true, created_by: 'admin', last_status: null, }); mockGetNodes.mockReturnValue([ { id: 1, name: 'local', type: 'local' }, ]); mockGetStacks.mockResolvedValue(['app1']); mockGetStackContent.mockResolvedValue('version: "3"\nservices:\n web:\n image: nginx'); const svc = SchedulerService.getInstance(); await svc.triggerTask(75); expect(mockCreateSnapshot).toHaveBeenCalledWith( expect.stringContaining('nightly-snapshot'), 'admin', 1, 1, expect.any(String), expect.any(String), '', ); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'success' }) ); }); it('handles nodes with no stacks gracefully', async () => { mockGetScheduledTask.mockReturnValue({ id: 76, name: 'empty-snapshot', action: 'snapshot', target_type: 'fleet', cron_expression: '0 3 * * *', enabled: true, created_by: 'admin', last_status: null, }); mockGetNodes.mockReturnValue([ { id: 1, name: 'local', type: 'local' }, ]); mockGetStacks.mockResolvedValue([]); const svc = SchedulerService.getInstance(); await svc.triggerTask(76); expect(mockCreateSnapshot).toHaveBeenCalled(); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'success' }) ); }); it('captures dossier documentation when the snapshot_documentation setting is on', async () => { mockGetScheduledTask.mockReturnValue({ id: 77, name: 'documented-snapshot', action: 'snapshot', target_type: 'fleet', cron_expression: '0 3 * * *', enabled: true, created_by: 'admin', last_status: null, }); mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); mockGetStacks.mockResolvedValue(['app1']); mockGetStackContent.mockResolvedValue('services: {}\n'); mockGetGlobalSettings.mockReturnValue({ snapshot_documentation: '1' }); mockGetStackDossier.mockReturnValue({ purpose: 'documented', owner: '', access_urls: '', static_ip: '', vlan: '', firewall_notes: '', reverse_proxy_notes: '', backup_notes: '', upgrade_notes: '', recovery_notes: '', custom_notes: '' }); await SchedulerService.getInstance().triggerTask(77); const docArg = mockCreateSnapshot.mock.calls.at(-1)?.[6] as string; expect(docArg).toBeTruthy(); expect(JSON.parse(docArg).stacks[0]).toMatchObject({ stackName: 'app1', dossier: { purpose: 'documented' } }); }); }); // ── executeUpdateRemote (T4) ───────────────────────────────────────── describe('SchedulerService - executeUpdateRemote', () => { afterEach(() => { vi.unstubAllGlobals(); }); it('proxies update execution to remote node', async () => { mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'test-token', }); const mockFetch = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ result: 'Stack "web": updated (nginx:latest).' }), }); vi.stubGlobal('fetch', mockFetch); mockGetScheduledTask.mockReturnValue({ id: 88, name: 'remote-update', action: 'update', cron_expression: '0 4 * * *', enabled: true, target_id: 'web-app', node_id: 2, created_by: 'admin', last_status: null, }); const svc = SchedulerService.getInstance(); await svc.triggerTask(88); expect(mockFetch).toHaveBeenCalledWith( 'http://remote:1852/api/auto-update/execute', expect.objectContaining({ method: 'POST', headers: expect.objectContaining({ 'Authorization': 'Bearer test-token', 'x-sencho-tier': 'paid', }), body: JSON.stringify({ target: 'web-app' }), }) ); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'success' }) ); }); it('records failure when remote node returns error', async () => { mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'test-token', }); const mockFetch = vi.fn().mockResolvedValue({ ok: false, status: 500, json: async () => ({ error: 'Internal error' }), }); vi.stubGlobal('fetch', mockFetch); mockGetScheduledTask.mockReturnValue({ id: 89, name: 'remote-update-fail', action: 'update', cron_expression: '0 4 * * *', enabled: true, target_id: 'web-app', node_id: 2, created_by: 'admin', last_status: null, }); const svc = SchedulerService.getInstance(); await svc.triggerTask(89); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'failure', error: expect.stringContaining('Internal error') }) ); }); }); // ── Lifecycle actions (auto_backup, auto_stop, auto_down, auto_start) ─── function makeLifecycleTask(action: ScheduledTask['action'], overrides: Partial = {}): ScheduledTask { return { id: 300, name: `lifecycle-${action}`, action, target_type: 'stack', target_id: 'my-stack', node_id: 1, cron_expression: '0 2 * * *', enabled: 1, created_by: 'admin', created_at: 0, updated_at: 0, last_run_at: null, next_run_at: null, last_status: null, last_error: null, prune_targets: null, target_services: null, prune_label_filter: null, delete_after_run: 0, ...overrides, }; } describe('SchedulerService - lifecycle actions', () => { it('auto_stop calls runCommand with "stop"', async () => { mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop')); await SchedulerService.getInstance().triggerTask(300); expect(mockRunCommand).toHaveBeenCalledWith('my-stack', 'stop'); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' })); }); it('auto_down calls runCommand with "down"', async () => { mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_down')); await SchedulerService.getInstance().triggerTask(300); expect(mockRunCommand).toHaveBeenCalledWith('my-stack', 'down'); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' })); }); it('auto_start calls deployStack', async () => { mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_start')); await SchedulerService.getInstance().triggerTask(300); expect(mockDeployStack).toHaveBeenCalledWith('my-stack'); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' })); }); it('auto_backup calls backupStackFiles', async () => { mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup')); await SchedulerService.getInstance().triggerTask(300); expect(mockBackupStackFiles).toHaveBeenCalledWith('my-stack'); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' })); }); it('auto_stop records failure when target_id is missing', async () => { mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { target_id: null })); await SchedulerService.getInstance().triggerTask(300); expect(mockRunCommand).not.toHaveBeenCalled(); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'failure' })); }); it('auto_backup records failure when node_id is missing', async () => { mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup', { node_id: null })); await SchedulerService.getInstance().triggerTask(300); expect(mockBackupStackFiles).not.toHaveBeenCalled(); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'failure' })); }); it('records failure (not success) and skips the op when the stack lock is held', async () => { // A manual operation holds the lock; the scheduled lifecycle op must skip // rather than race, and surface as a failed run instead of a silent success. StackOpLockService.getInstance().tryAcquire(1, 'my-stack', 'deploy', 'admin'); mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop')); await SchedulerService.getInstance().triggerTask(300); expect(mockRunCommand).not.toHaveBeenCalled(); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'failure' })); expect(mockUpdateScheduledTask).toHaveBeenCalledWith(300, expect.objectContaining({ last_status: 'failure' })); }); it('paid tier executes lifecycle actions', async () => { mockGetTier.mockReturnValue('paid'); mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop')); mockGetDueScheduledTasks.mockReturnValue([makeLifecycleTask('auto_stop')]); const svc = SchedulerService.getInstance(); await (svc as any).tick(); await new Promise(r => setTimeout(r, 50)); expect(mockRunCommand).toHaveBeenCalled(); }); }); // ── Lifecycle remote proxy ────────────────────────────────────────────── describe('SchedulerService - lifecycle remote proxy', () => { afterEach(() => { vi.unstubAllGlobals(); }); const remoteHeaders = expect.objectContaining({ 'Authorization': 'Bearer tkn', 'x-sencho-tier': 'paid', }); function stubRemote(okBody: unknown = { success: true }) { mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' }); const mockFetch = vi.fn().mockResolvedValue({ ok: true, json: async () => okBody }); vi.stubGlobal('fetch', mockFetch); return mockFetch; } it('auto_stop proxies to the remote stop endpoint instead of running locally', async () => { const fetchMock = stubRemote(); mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 })); await SchedulerService.getInstance().triggerTask(300); expect(fetchMock).toHaveBeenCalledWith( 'http://remote:1852/api/stacks/my-stack/stop', expect.objectContaining({ method: 'POST', headers: remoteHeaders }), ); expect(mockRunCommand).not.toHaveBeenCalled(); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' })); }); it('auto_down proxies to the remote down endpoint', async () => { const fetchMock = stubRemote(); mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_down', { node_id: 2 })); await SchedulerService.getInstance().triggerTask(300); expect(fetchMock).toHaveBeenCalledWith( 'http://remote:1852/api/stacks/my-stack/down', expect.objectContaining({ method: 'POST' }), ); expect(mockRunCommand).not.toHaveBeenCalled(); }); it('auto_start proxies to the remote deploy endpoint and skips the hub policy gate', async () => { const fetchMock = stubRemote(); mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_start', { node_id: 2 })); await SchedulerService.getInstance().triggerTask(300); expect(fetchMock).toHaveBeenCalledWith( 'http://remote:1852/api/stacks/my-stack/deploy', expect.objectContaining({ method: 'POST' }), ); expect(mockDeployStack).not.toHaveBeenCalled(); expect(mockEnforcePolicyPreDeploy).not.toHaveBeenCalled(); }); it('auto_backup proxies to the remote backup endpoint', async () => { const fetchMock = stubRemote(); mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup', { node_id: 2 })); await SchedulerService.getInstance().triggerTask(300); expect(fetchMock).toHaveBeenCalledWith( 'http://remote:1852/api/stacks/my-stack/backup', expect.objectContaining({ method: 'POST' }), ); expect(mockBackupStackFiles).not.toHaveBeenCalled(); }); it('restart (all services) proxies to the remote restart endpoint', async () => { const fetchMock = stubRemote(); mockGetScheduledTask.mockReturnValue(makeLifecycleTask('restart', { node_id: 2 })); await SchedulerService.getInstance().triggerTask(300); expect(fetchMock).toHaveBeenCalledWith( 'http://remote:1852/api/stacks/my-stack/restart', expect.objectContaining({ method: 'POST' }), ); expect(mockGetContainersByStack).not.toHaveBeenCalled(); }); it('restart with target_services fans out to per-service restart endpoints', async () => { const fetchMock = stubRemote(); mockGetScheduledTask.mockReturnValue( makeLifecycleTask('restart', { node_id: 2, target_services: JSON.stringify(['api', 'worker']) }), ); await SchedulerService.getInstance().triggerTask(300); expect(fetchMock).toHaveBeenCalledWith( 'http://remote:1852/api/stacks/my-stack/services/api/restart', expect.objectContaining({ method: 'POST' }), ); expect(fetchMock).toHaveBeenCalledWith( 'http://remote:1852/api/stacks/my-stack/services/worker/restart', expect.objectContaining({ method: 'POST' }), ); expect(fetchMock).toHaveBeenCalledTimes(2); }); it('records failure when the remote node returns an error', async () => { mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' }); vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, status: 503, json: async () => ({ error: 'Docker daemon is unreachable' }), })); mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 })); await SchedulerService.getInstance().triggerTask(300); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'failure', error: expect.stringContaining('Docker daemon is unreachable') }), ); }); it('falls back to the HTTP status when the remote error body is not JSON', async () => { mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' }); vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, status: 502, json: async () => { throw new Error('not json'); }, })); mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_down', { node_id: 2 })); await SchedulerService.getInstance().triggerTask(300); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'failure', error: expect.stringContaining('HTTP 502') }), ); }); it('records failure when the remote node has no proxy credentials', async () => { mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); // getProxyTarget defaults to null (no credentials configured). const fetchMock = vi.fn(); vi.stubGlobal('fetch', fetchMock); mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 })); await SchedulerService.getInstance().triggerTask(300); expect(fetchMock).not.toHaveBeenCalled(); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'failure', error: expect.stringContaining('not configured or missing API credentials') }), ); }); it('restart fan-out fails fast and names already-restarted services', async () => { mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' }); const fetchMock = vi.fn() .mockResolvedValueOnce({ ok: true, json: async () => ({ success: true }) }) .mockResolvedValueOnce({ ok: false, status: 500, json: async () => ({ error: 'boom' }) }); vi.stubGlobal('fetch', fetchMock); mockGetScheduledTask.mockReturnValue( makeLifecycleTask('restart', { node_id: 2, target_services: JSON.stringify(['api', 'worker', 'cache']) }), ); await SchedulerService.getInstance().triggerTask(300); // Third service is never reached after the second fails. expect(fetchMock).toHaveBeenCalledTimes(2); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'failure', error: expect.stringContaining('already restarted: api') }), ); }); }); // ── Unpaid-tier guard in executeTask ──────────────────────────────────── describe('SchedulerService - community tier runs lifecycle actions', () => { it('executes a lifecycle action and records success on the community tier', async () => { mockGetTier.mockReturnValue('community'); mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop')); await SchedulerService.getInstance().triggerTask(300); expect(mockRunCommand).toHaveBeenCalled(); expect(mockCreateScheduledTaskRun).toHaveBeenCalled(); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, expect.objectContaining({ status: 'success' }), ); }); }); // ── delete_after_run ──────────────────────────────────────────────────── describe('SchedulerService - delete_after_run', () => { it('deletes task after successful run when delete_after_run is 1', async () => { mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup', { delete_after_run: 1 })); await SchedulerService.getInstance().triggerTask(300); expect(mockDeleteScheduledTask).toHaveBeenCalledWith(300); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' })); }); it('does not delete task when run fails even if delete_after_run is 1', async () => { mockBackupStackFiles.mockRejectedValueOnce(new Error('disk full')); mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup', { delete_after_run: 1 })); await SchedulerService.getInstance().triggerTask(300); expect(mockDeleteScheduledTask).not.toHaveBeenCalled(); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'failure' })); }); it('does not delete task when delete_after_run is 0', async () => { mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup', { delete_after_run: 0 })); await SchedulerService.getInstance().triggerTask(300); expect(mockDeleteScheduledTask).not.toHaveBeenCalled(); expect(mockUpdateScheduledTask).toHaveBeenCalledWith(300, expect.objectContaining({ last_status: 'success' })); }); });