mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-13 04:06:59 +00:00
Anso
db8bb70b7d
* fix(scheduler): reject 6-field cron in Scheduled Operations Create and edit validation parsed cron with cron-parser, which accepts both 5- and 6-field expressions, while the form, presets, and docs all describe a 5-field cron. Because the scheduler ticks once per minute, a leading seconds field can never improve precision, so a 6-field expression was silently accepted but never honored on its stated schedule. Add a field-count guard on both sides: the API rejects 6-field input at create and edit with a clear message, and the form surfaces the same error inline and disables save. Cron nicknames such as @daily still pass. Document the five-field requirement in the cron reference. * chore: merge main into scheduled cron validation * fix: avoid logging policy bypass actor in debug output
1931 lines
67 KiB
TypeScript
1931 lines
67 KiB
TypeScript
/**
|
|
* Unit tests for SchedulerService — task execution, concurrent prevention,
|
|
* license gating, cron parsing, and error handling.
|
|
*/
|
|
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
import type { ScheduledTask } from '../services/DatabaseService';
|
|
|
|
// ── Hoisted mocks ──────────────────────────────────────────────────────
|
|
|
|
const {
|
|
mockGetDueScheduledTasks, mockCreateScheduledTaskRun, mockUpdateScheduledTaskRun,
|
|
mockUpdateScheduledTask, mockCleanupOldTaskRuns, mockGetScheduledTask, mockGetNodes, mockGetNode,
|
|
mockGetGlobalSettings, mockGetStackDossier,
|
|
mockCreateSnapshot, mockInsertSnapshotFiles, mockClearStackUpdateStatus,
|
|
mockMarkStaleRunsAsFailed, mockDeleteOldScans,
|
|
mockGetTier, mockGetProxyHeaders,
|
|
mockGetContainersByStack, mockRestartContainer, mockPruneSystem,
|
|
mockUpdateStack,
|
|
mockGetStacks, mockGetStackContent, mockGetEnvContent,
|
|
mockCheckImage,
|
|
mockDispatchAlert,
|
|
mockGetProxyTarget,
|
|
mockIsTrivyAvailable,
|
|
mockScanAllNodeImages,
|
|
mockDeleteScheduledTask,
|
|
mockGetMatchingPolicy,
|
|
mockRunCommand,
|
|
mockDeployStack,
|
|
mockBackupStackFiles,
|
|
mockEnforcePolicyPreDeploy,
|
|
} = vi.hoisted(() => ({
|
|
mockGetDueScheduledTasks: vi.fn().mockReturnValue([]),
|
|
mockCreateScheduledTaskRun: vi.fn().mockReturnValue(1),
|
|
mockUpdateScheduledTaskRun: vi.fn(),
|
|
mockUpdateScheduledTask: vi.fn(),
|
|
mockCleanupOldTaskRuns: vi.fn(),
|
|
mockGetScheduledTask: vi.fn(),
|
|
mockGetNodes: vi.fn().mockReturnValue([]),
|
|
mockGetNode: vi.fn().mockReturnValue({ id: 1, name: 'local', type: 'local', status: 'online' }),
|
|
mockGetGlobalSettings: vi.fn().mockReturnValue({}),
|
|
mockGetStackDossier: vi.fn().mockReturnValue(undefined),
|
|
mockCreateSnapshot: vi.fn().mockReturnValue(1),
|
|
mockInsertSnapshotFiles: vi.fn(),
|
|
mockClearStackUpdateStatus: vi.fn(),
|
|
mockMarkStaleRunsAsFailed: vi.fn().mockReturnValue(0),
|
|
mockDeleteOldScans: vi.fn().mockReturnValue(0),
|
|
mockGetTier: vi.fn().mockReturnValue('paid'),
|
|
mockGetProxyHeaders: vi.fn().mockReturnValue({ tier: 'paid' }),
|
|
mockGetContainersByStack: vi.fn().mockResolvedValue([]),
|
|
mockRestartContainer: vi.fn().mockResolvedValue(undefined),
|
|
mockPruneSystem: vi.fn().mockResolvedValue({ success: true, reclaimedBytes: 0 }),
|
|
mockUpdateStack: vi.fn().mockResolvedValue(undefined),
|
|
mockGetStacks: vi.fn().mockResolvedValue([]),
|
|
mockGetStackContent: vi.fn().mockResolvedValue(''),
|
|
mockGetEnvContent: vi.fn().mockResolvedValue(''),
|
|
mockCheckImage: vi.fn().mockResolvedValue({ hasUpdate: false }),
|
|
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
|
|
mockGetProxyTarget: vi.fn().mockReturnValue(null),
|
|
mockIsTrivyAvailable: vi.fn().mockReturnValue(true),
|
|
mockScanAllNodeImages: vi.fn().mockResolvedValue({
|
|
scanned: 0,
|
|
skipped: 0,
|
|
failed: 0,
|
|
severity: { critical: 0, high: 0, medium: 0, low: 0, unknown: 0 },
|
|
violations: [],
|
|
}),
|
|
mockDeleteScheduledTask: vi.fn(),
|
|
mockGetMatchingPolicy: vi.fn().mockReturnValue(null),
|
|
mockRunCommand: vi.fn().mockResolvedValue(undefined),
|
|
mockDeployStack: vi.fn().mockResolvedValue(undefined),
|
|
mockBackupStackFiles: vi.fn().mockResolvedValue(undefined),
|
|
mockEnforcePolicyPreDeploy: vi.fn(),
|
|
}));
|
|
|
|
vi.mock('../services/DatabaseService', () => ({
|
|
DatabaseService: {
|
|
getInstance: () => ({
|
|
getDueScheduledTasks: mockGetDueScheduledTasks,
|
|
createScheduledTaskRun: mockCreateScheduledTaskRun,
|
|
updateScheduledTaskRun: mockUpdateScheduledTaskRun,
|
|
updateScheduledTask: mockUpdateScheduledTask,
|
|
cleanupOldTaskRuns: mockCleanupOldTaskRuns,
|
|
getScheduledTask: mockGetScheduledTask,
|
|
getNodes: mockGetNodes,
|
|
getNode: mockGetNode,
|
|
getGlobalSettings: mockGetGlobalSettings,
|
|
getStackDossier: mockGetStackDossier,
|
|
createSnapshot: mockCreateSnapshot,
|
|
insertSnapshotFiles: mockInsertSnapshotFiles,
|
|
clearStackUpdateStatus: mockClearStackUpdateStatus,
|
|
markStaleRunsAsFailed: mockMarkStaleRunsAsFailed,
|
|
deleteOldScans: mockDeleteOldScans,
|
|
deleteScheduledTask: mockDeleteScheduledTask,
|
|
getMatchingPolicy: mockGetMatchingPolicy,
|
|
}),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/FleetSyncService', () => ({
|
|
FleetSyncService: {
|
|
getSelfIdentity: () => 'self-node',
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/LicenseService', () => ({
|
|
LicenseService: {
|
|
getInstance: () => ({
|
|
getTier: mockGetTier,
|
|
getProxyHeaders: mockGetProxyHeaders,
|
|
}),
|
|
},
|
|
}));
|
|
|
|
|
|
vi.mock('../services/DockerController', () => ({
|
|
default: {
|
|
getInstance: () => ({
|
|
getContainersByStack: mockGetContainersByStack,
|
|
restartContainer: mockRestartContainer,
|
|
pruneSystem: mockPruneSystem,
|
|
}),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/ComposeService', () => ({
|
|
ComposeService: {
|
|
getInstance: () => ({
|
|
updateStack: mockUpdateStack,
|
|
runCommand: mockRunCommand,
|
|
deployStack: mockDeployStack,
|
|
}),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/FileSystemService', () => ({
|
|
FileSystemService: {
|
|
getInstance: () => ({
|
|
getStacks: mockGetStacks,
|
|
getStackContent: mockGetStackContent,
|
|
getEnvContent: mockGetEnvContent,
|
|
backupStackFiles: mockBackupStackFiles,
|
|
}),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/ImageUpdateService', () => ({
|
|
ImageUpdateService: {
|
|
getInstance: () => ({
|
|
checkImage: mockCheckImage,
|
|
}),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/NotificationService', () => ({
|
|
NotificationService: {
|
|
getInstance: () => ({
|
|
dispatchAlert: mockDispatchAlert,
|
|
}),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/CloudBackupService', () => ({
|
|
CloudBackupService: {
|
|
getInstance: () => ({
|
|
isEnabled: () => false,
|
|
isAutoUploadOn: () => false,
|
|
uploadSnapshot: vi.fn().mockResolvedValue(undefined),
|
|
}),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/NodeRegistry', () => ({
|
|
NodeRegistry: {
|
|
getInstance: () => ({
|
|
getDefaultNodeId: () => 1,
|
|
getNode: mockGetNode,
|
|
getProxyTarget: mockGetProxyTarget,
|
|
}),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/TrivyService', () => ({
|
|
default: {
|
|
getInstance: () => ({
|
|
isTrivyAvailable: mockIsTrivyAvailable,
|
|
scanAllNodeImages: mockScanAllNodeImages,
|
|
getSource: () => 'managed',
|
|
detectTrivy: vi.fn().mockResolvedValue(undefined),
|
|
}),
|
|
},
|
|
}));
|
|
|
|
vi.mock('../services/PolicyEnforcement', () => ({
|
|
enforcePolicyPreDeploy: mockEnforcePolicyPreDeploy,
|
|
}));
|
|
|
|
import { SchedulerService } from '../services/SchedulerService';
|
|
import { StackOpLockService } from '../services/StackOpLockService';
|
|
|
|
beforeEach(() => {
|
|
vi.clearAllMocks();
|
|
// Lifecycle/update handlers run through the real StackOpLockService; reset it
|
|
// so a lock left held by one test cannot make a later test skip its op.
|
|
StackOpLockService.resetForTests();
|
|
// clearAllMocks only clears call history, not implementations, so restore the
|
|
// mocks that individual tests mutate (tier, node lookup, proxy target) to
|
|
// their documented defaults. Without this a test that points getNode at a
|
|
// remote node or drops the tier leaks that state into every later test in the
|
|
// file.
|
|
mockGetTier.mockReturnValue('paid');
|
|
mockGetNode.mockReturnValue({ id: 1, name: 'local', type: 'local', status: 'online' });
|
|
mockGetProxyTarget.mockReturnValue(null);
|
|
// Documentation capture is opt-in and off by default; reset so a test that
|
|
// enables it does not leak into later snapshot tests.
|
|
mockGetGlobalSettings.mockReturnValue({});
|
|
mockGetStackDossier.mockReturnValue(undefined);
|
|
// Default: the scan-policy gate allows. Individual tests override to a block.
|
|
mockEnforcePolicyPreDeploy.mockResolvedValue({ ok: true, bypassed: false, violations: [] });
|
|
(SchedulerService as any).instance = undefined;
|
|
});
|
|
|
|
// ── calculateNextRun ───────────────────────────────────────────────────
|
|
|
|
describe('SchedulerService - calculateNextRun', () => {
|
|
it('returns a future timestamp for valid cron expression', () => {
|
|
const svc = SchedulerService.getInstance();
|
|
const next = svc.calculateNextRun('*/5 * * * *'); // Every 5 minutes
|
|
expect(next).toBeGreaterThan(Date.now());
|
|
});
|
|
|
|
it('throws on invalid cron expression', () => {
|
|
const svc = SchedulerService.getInstance();
|
|
expect(() => svc.calculateNextRun('not a cron')).toThrow();
|
|
});
|
|
});
|
|
|
|
describe('SchedulerService - calculateRunsWithin', () => {
|
|
it('expands hourly cron into every firing within a 24h window when limit allows', () => {
|
|
const svc = SchedulerService.getInstance();
|
|
const from = Date.now();
|
|
const to = from + 24 * 60 * 60 * 1000;
|
|
const runs = svc.calculateRunsWithin('0 * * * *', from, to, 32);
|
|
expect(runs.length).toBeGreaterThanOrEqual(23);
|
|
expect(runs.length).toBeLessThanOrEqual(24);
|
|
for (const run of runs) {
|
|
expect(run).toBeGreaterThanOrEqual(from);
|
|
expect(run).toBeLessThanOrEqual(to);
|
|
}
|
|
});
|
|
|
|
it('returns a single firing for a daily cron', () => {
|
|
const svc = SchedulerService.getInstance();
|
|
const from = Date.now();
|
|
const to = from + 24 * 60 * 60 * 1000;
|
|
const runs = svc.calculateRunsWithin('0 3 * * *', from, to);
|
|
expect(runs.length).toBeLessThanOrEqual(2);
|
|
expect(runs.length).toBeGreaterThanOrEqual(1);
|
|
});
|
|
|
|
it('honours the limit parameter to avoid runaway expansions', () => {
|
|
const svc = SchedulerService.getInstance();
|
|
const from = Date.now();
|
|
const to = from + 60 * 60 * 1000;
|
|
const runs = svc.calculateRunsWithin('* * * * *', from, to, 5);
|
|
expect(runs.length).toBe(5);
|
|
});
|
|
|
|
it('returns empty array for invalid cron instead of throwing', () => {
|
|
const svc = SchedulerService.getInstance();
|
|
const from = Date.now();
|
|
const to = from + 60 * 60 * 1000;
|
|
expect(svc.calculateRunsWithin('not a cron', from, to)).toEqual([]);
|
|
});
|
|
});
|
|
|
|
// ── License gating ─────────────────────────────────────────────────────
|
|
|
|
describe('SchedulerService - scheduled tasks run on every tier', () => {
|
|
function makeTask(overrides: Partial<any> = {}) {
|
|
return {
|
|
id: 1,
|
|
name: 'test-task',
|
|
action: 'restart',
|
|
cron_expression: '*/5 * * * *',
|
|
enabled: true,
|
|
target_id: 'my-stack',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
it('runs tasks on the Community tier (no paid gate)', async () => {
|
|
mockGetTier.mockReturnValue('community');
|
|
mockGetDueScheduledTasks.mockReturnValue([makeTask()]);
|
|
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await (svc as any).tick();
|
|
|
|
await new Promise(r => setTimeout(r, 50));
|
|
expect(mockCreateScheduledTaskRun).toHaveBeenCalled();
|
|
});
|
|
|
|
it('runs update tasks on the Community tier', async () => {
|
|
mockGetTier.mockReturnValue('community');
|
|
mockGetDueScheduledTasks.mockReturnValue([makeTask({ action: 'update' })]);
|
|
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }]);
|
|
mockCheckImage.mockResolvedValue({ hasUpdate: false });
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await (svc as any).tick();
|
|
|
|
// Wait for the async task to settle
|
|
await new Promise(r => setTimeout(r, 50));
|
|
expect(mockCreateScheduledTaskRun).toHaveBeenCalled();
|
|
});
|
|
|
|
it('runs snapshot tasks on the Community tier', async () => {
|
|
mockGetTier.mockReturnValue('community');
|
|
mockGetDueScheduledTasks.mockReturnValue([makeTask({ action: 'snapshot', target_type: 'fleet' })]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await (svc as any).tick();
|
|
|
|
await new Promise(r => setTimeout(r, 50));
|
|
expect(mockCreateScheduledTaskRun).toHaveBeenCalled();
|
|
});
|
|
|
|
it('runs tasks on the paid tier', async () => {
|
|
mockGetTier.mockReturnValue('paid');
|
|
mockGetDueScheduledTasks.mockReturnValue([makeTask({ action: 'restart' })]);
|
|
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await (svc as any).tick();
|
|
|
|
await new Promise(r => setTimeout(r, 50));
|
|
expect(mockCreateScheduledTaskRun).toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
// ── Concurrent task prevention ─────────────────────────────────────────
|
|
|
|
describe('SchedulerService - concurrent task prevention', () => {
|
|
it('does not execute a task that is already in runningTasks', async () => {
|
|
mockGetTier.mockReturnValue('paid');
|
|
mockGetDueScheduledTasks.mockReturnValue([{
|
|
id: 42,
|
|
name: 'running-task',
|
|
action: 'restart',
|
|
cron_expression: '*/5 * * * *',
|
|
enabled: true,
|
|
target_id: 'my-stack',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
}]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
// Pre-add the task to runningTasks
|
|
(svc as any).runningTasks.add(42);
|
|
|
|
await (svc as any).tick();
|
|
await new Promise(r => setTimeout(r, 50));
|
|
|
|
expect(mockCreateScheduledTaskRun).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('removes task from runningTasks after completion', async () => {
|
|
mockGetTier.mockReturnValue('paid');
|
|
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 99,
|
|
name: 'trigger-test',
|
|
action: 'restart',
|
|
cron_expression: '*/5 * * * *',
|
|
enabled: true,
|
|
target_id: 'my-stack',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
|
|
await svc.triggerTask(99);
|
|
|
|
expect((svc as any).runningTasks.has(99)).toBe(false);
|
|
});
|
|
|
|
it('removes task from runningTasks even on failure', async () => {
|
|
const svc = SchedulerService.getInstance();
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 100,
|
|
name: 'fail-test',
|
|
action: 'restart',
|
|
cron_expression: '*/5 * * * *',
|
|
enabled: true,
|
|
target_id: null, // Will cause error: "requires target_id"
|
|
node_id: null,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
|
|
await svc.triggerTask(100);
|
|
|
|
expect((svc as any).runningTasks.has(100)).toBe(false);
|
|
// Error should have been recorded
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
expect.any(Number),
|
|
expect.objectContaining({ status: 'failure' })
|
|
);
|
|
});
|
|
});
|
|
|
|
// ── triggerTask ────────────────────────────────────────────────────────
|
|
|
|
describe('SchedulerService - triggerTask', () => {
|
|
it('throws when task not found', async () => {
|
|
mockGetScheduledTask.mockReturnValue(undefined);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await expect(svc.triggerTask(999)).rejects.toThrow('Task not found');
|
|
});
|
|
|
|
it('throws when task is already running', async () => {
|
|
mockGetScheduledTask.mockReturnValue({ id: 50, name: 'busy' });
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
(svc as any).runningTasks.add(50);
|
|
|
|
await expect(svc.triggerTask(50)).rejects.toThrow('already running');
|
|
});
|
|
|
|
it('sets triggered_by to manual', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 55,
|
|
name: 'manual-test',
|
|
action: 'restart',
|
|
cron_expression: '*/5 * * * *',
|
|
enabled: false, // Disabled — but triggerTask should still work
|
|
target_id: 'my-stack',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(55);
|
|
|
|
expect(mockCreateScheduledTaskRun).toHaveBeenCalledWith(
|
|
expect.objectContaining({ triggered_by: 'manual' })
|
|
);
|
|
});
|
|
});
|
|
|
|
// ── executeRestart ─────────────────────────────────────────────────────
|
|
|
|
describe('SchedulerService - executeRestart', () => {
|
|
it('restarts all containers in a stack', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 60,
|
|
name: 'restart-all',
|
|
action: 'restart',
|
|
cron_expression: '*/5 * * * *',
|
|
enabled: true,
|
|
target_id: 'my-stack',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetContainersByStack.mockResolvedValue([
|
|
{ Id: 'c1', Service: 'web' },
|
|
{ Id: 'c2', Service: 'db' },
|
|
]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(60);
|
|
|
|
expect(mockRestartContainer).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('restarts only specified services when target_services set', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 61,
|
|
name: 'restart-filtered',
|
|
action: 'restart',
|
|
cron_expression: '*/5 * * * *',
|
|
enabled: true,
|
|
target_id: 'my-stack',
|
|
node_id: 1,
|
|
target_services: JSON.stringify(['web']),
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetContainersByStack.mockResolvedValue([
|
|
{ Id: 'c1', Service: 'web' },
|
|
{ Id: 'c2', Service: 'db' },
|
|
]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(61);
|
|
|
|
expect(mockRestartContainer).toHaveBeenCalledTimes(1);
|
|
expect(mockRestartContainer).toHaveBeenCalledWith('c1');
|
|
});
|
|
|
|
it('records failure when no containers found', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 62,
|
|
name: 'restart-empty',
|
|
action: 'restart',
|
|
cron_expression: '*/5 * * * *',
|
|
enabled: true,
|
|
target_id: 'empty-stack',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetContainersByStack.mockResolvedValue([]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(62);
|
|
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
expect.any(Number),
|
|
expect.objectContaining({ status: 'failure', error: expect.stringContaining('No containers') })
|
|
);
|
|
});
|
|
});
|
|
|
|
// ── executePrune ───────────────────────────────────────────────────────
|
|
|
|
describe('SchedulerService - executePrune', () => {
|
|
it('prunes all targets by default', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 70,
|
|
name: 'prune-all',
|
|
action: 'prune',
|
|
cron_expression: '0 3 * * *',
|
|
enabled: true,
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(70);
|
|
|
|
// Should prune all 4 targets
|
|
expect(mockPruneSystem).toHaveBeenCalledTimes(4);
|
|
});
|
|
|
|
it('prunes only specified targets', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 71,
|
|
name: 'prune-some',
|
|
action: 'prune',
|
|
cron_expression: '0 3 * * *',
|
|
enabled: true,
|
|
node_id: 1,
|
|
prune_targets: JSON.stringify(['images', 'volumes']),
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(71);
|
|
|
|
expect(mockPruneSystem).toHaveBeenCalledTimes(2);
|
|
expect(mockPruneSystem).toHaveBeenCalledWith('images', undefined);
|
|
expect(mockPruneSystem).toHaveBeenCalledWith('volumes', undefined);
|
|
});
|
|
|
|
it('includes label filter when configured', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 72,
|
|
name: 'prune-labeled',
|
|
action: 'prune',
|
|
cron_expression: '0 3 * * *',
|
|
enabled: true,
|
|
node_id: 1,
|
|
prune_targets: JSON.stringify(['containers']),
|
|
prune_label_filter: 'env=staging',
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(72);
|
|
|
|
expect(mockPruneSystem).toHaveBeenCalledWith('containers', 'env=staging');
|
|
});
|
|
|
|
it('marks scheduled prune runs as failed when a target prune fails', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 74,
|
|
name: 'prune-fails',
|
|
action: 'prune',
|
|
cron_expression: '0 3 * * *',
|
|
enabled: true,
|
|
node_id: 1,
|
|
prune_targets: JSON.stringify(['images']),
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockPruneSystem.mockRejectedValueOnce(new Error('docker prune failed'));
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(74);
|
|
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
expect.any(Number),
|
|
expect.objectContaining({
|
|
status: 'failure',
|
|
error: expect.stringContaining('images: failed (docker prune failed)'),
|
|
}),
|
|
);
|
|
expect(mockUpdateScheduledTask).toHaveBeenCalledWith(
|
|
74,
|
|
expect.objectContaining({ last_status: 'failure' }),
|
|
);
|
|
});
|
|
|
|
it('fails scheduled prune tasks that target remote nodes before pruning', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 73,
|
|
name: 'remote-prune',
|
|
action: 'prune',
|
|
cron_expression: '0 3 * * *',
|
|
enabled: true,
|
|
node_id: 2,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(73);
|
|
|
|
expect(mockPruneSystem).not.toHaveBeenCalled();
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
expect.any(Number),
|
|
expect.objectContaining({
|
|
status: 'failure',
|
|
error: expect.stringMatching(/local node/i),
|
|
}),
|
|
);
|
|
});
|
|
});
|
|
|
|
// ── executeUpdate ──────────────────────────────────────────────────────
|
|
|
|
describe('SchedulerService - executeUpdate', () => {
|
|
it('updates stack when image update available', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 80,
|
|
name: 'update-stack',
|
|
action: 'update',
|
|
cron_expression: '0 4 * * *',
|
|
enabled: true,
|
|
target_id: 'web-app',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetContainersByStack.mockResolvedValue([
|
|
{ Id: 'c1', Image: 'nginx:latest' },
|
|
]);
|
|
mockCheckImage.mockResolvedValue({ hasUpdate: true }); // Update available
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(80);
|
|
|
|
expect(mockUpdateStack).toHaveBeenCalledWith('web-app', undefined, true);
|
|
expect(mockClearStackUpdateStatus).toHaveBeenCalledWith(1, 'web-app');
|
|
});
|
|
|
|
it('runs a scheduled update on the community tier (no paid gate)', async () => {
|
|
// Scheduled tasks are free, so a community licence runs the update like any
|
|
// other tier.
|
|
mockGetTier.mockReturnValue('community');
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 82,
|
|
name: 'update-community',
|
|
action: 'update',
|
|
cron_expression: '0 4 * * *',
|
|
enabled: true,
|
|
target_id: 'web-app',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }]);
|
|
mockCheckImage.mockResolvedValue({ hasUpdate: true });
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(82);
|
|
|
|
expect(mockUpdateStack).toHaveBeenCalledWith('web-app', undefined, true);
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({ status: 'success' }),
|
|
);
|
|
});
|
|
|
|
it('begins a health gate after a scheduled update succeeds', async () => {
|
|
const { HealthGateService } = await import('../services/HealthGateService');
|
|
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-1');
|
|
try {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 83,
|
|
name: 'update-gated',
|
|
action: 'update',
|
|
cron_expression: '0 4 * * *',
|
|
enabled: true,
|
|
target_id: 'web-app',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }]);
|
|
mockCheckImage.mockResolvedValue({ hasUpdate: true });
|
|
|
|
await SchedulerService.getInstance().triggerTask(83);
|
|
|
|
expect(beginSpy).toHaveBeenCalledWith(1, 'web-app', 'update', 'system:scheduler');
|
|
} finally {
|
|
beginSpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it('skips when all images up to date', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 81,
|
|
name: 'update-no-change',
|
|
action: 'update',
|
|
cron_expression: '0 4 * * *',
|
|
enabled: true,
|
|
target_id: 'web-app',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetContainersByStack.mockResolvedValue([
|
|
{ Id: 'c1', Image: 'nginx:latest' },
|
|
]);
|
|
mockCheckImage.mockResolvedValue({ hasUpdate: false }); // No update
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(81);
|
|
|
|
expect(mockUpdateStack).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('handles wildcard target (*) by updating all stacks', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 82,
|
|
name: 'update-all',
|
|
action: 'update',
|
|
cron_expression: '0 4 * * *',
|
|
enabled: true,
|
|
target_id: '*',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetStacks.mockResolvedValue(['app1', 'app2']);
|
|
mockGetContainersByStack.mockResolvedValue([
|
|
{ Id: 'c1', Image: 'nginx:latest' },
|
|
]);
|
|
mockCheckImage.mockResolvedValue({ hasUpdate: true });
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(82);
|
|
|
|
expect(mockUpdateStack).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('reports warning when all image checks fail (B3 fix)', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 83,
|
|
name: 'update-check-fail',
|
|
action: 'update',
|
|
cron_expression: '0 4 * * *',
|
|
enabled: true,
|
|
target_id: 'web-app',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetContainersByStack.mockResolvedValue([
|
|
{ Id: 'c1', Image: 'nginx:latest' },
|
|
]);
|
|
mockCheckImage.mockResolvedValue({ hasUpdate: false, error: 'Registry unreachable for registry-1.docker.io/library/nginx:latest' });
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(83);
|
|
|
|
// Should succeed (not throw) but output should contain warning
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({
|
|
status: 'success',
|
|
output: expect.stringContaining('WARNING'),
|
|
})
|
|
);
|
|
expect(mockUpdateStack).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('reports partial check failures with success count (B3 fix)', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 84,
|
|
name: 'update-partial-fail',
|
|
action: 'update',
|
|
cron_expression: '0 4 * * *',
|
|
enabled: true,
|
|
target_id: 'web-app',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetContainersByStack.mockResolvedValue([
|
|
{ Id: 'c1', Image: 'nginx:latest' },
|
|
{ Id: 'c2', Image: 'redis:7' },
|
|
]);
|
|
// First image check succeeds (no update), second fails
|
|
mockCheckImage
|
|
.mockResolvedValueOnce({ hasUpdate: false })
|
|
.mockResolvedValueOnce({ hasUpdate: false, error: 'Registry unreachable' });
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(84);
|
|
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({
|
|
status: 'success',
|
|
output: expect.stringContaining('check(s) failed'),
|
|
})
|
|
);
|
|
});
|
|
|
|
it('warns when targeted stack has 0 containers (E1 fix)', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 85,
|
|
name: 'update-missing-stack',
|
|
action: 'update',
|
|
cron_expression: '0 4 * * *',
|
|
enabled: true,
|
|
target_id: 'deleted-stack',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetContainersByStack.mockResolvedValue([]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(85);
|
|
|
|
// Targeted (non-wildcard) stack with 0 containers should produce a WARNING
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({
|
|
status: 'success',
|
|
output: expect.stringContaining('WARNING'),
|
|
})
|
|
);
|
|
});
|
|
|
|
it('silently skips empty stacks in wildcard mode', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 86,
|
|
name: 'update-wildcard-empty',
|
|
action: 'update',
|
|
cron_expression: '0 4 * * *',
|
|
enabled: true,
|
|
target_id: '*',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetStacks.mockResolvedValue(['active-stack', 'empty-stack']);
|
|
// First stack has containers, second has none
|
|
mockGetContainersByStack
|
|
.mockResolvedValueOnce([{ Id: 'c1', Image: 'nginx:latest' }])
|
|
.mockResolvedValueOnce([]);
|
|
mockCheckImage.mockResolvedValue({ hasUpdate: false });
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(86);
|
|
|
|
// Empty stack in wildcard mode should say "skipped", not "WARNING"
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({
|
|
status: 'success',
|
|
output: expect.not.stringContaining('WARNING'),
|
|
})
|
|
);
|
|
});
|
|
|
|
it('dispatches a scan_finding warning and skips the stack when a policy blocks the update', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 88,
|
|
name: 'blocked-update',
|
|
action: 'update',
|
|
cron_expression: '0 4 * * *',
|
|
enabled: true,
|
|
target_id: 'web-app',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:1.14' }]);
|
|
mockCheckImage.mockResolvedValue({ hasUpdate: true });
|
|
mockEnforcePolicyPreDeploy.mockResolvedValue({
|
|
ok: false,
|
|
bypassed: false,
|
|
policy: { id: 1, name: 'block-high', max_severity: 'HIGH' },
|
|
violations: [{ imageRef: 'nginx:1.14', severity: 'CRITICAL', criticalCount: 2, highCount: 5, scanId: 7 }],
|
|
});
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(88);
|
|
|
|
// Gate blocked it: the stack must not be updated.
|
|
expect(mockUpdateStack).not.toHaveBeenCalled();
|
|
// A scan_finding warning naming the policy and the offending image fired.
|
|
const warn = mockDispatchAlert.mock.calls.find((c) => c[0] === 'warning' && c[1] === 'scan_finding');
|
|
expect(warn).toBeDefined();
|
|
expect(warn![2]).toContain('block-high');
|
|
expect(warn![2]).toContain('nginx:1.14');
|
|
// The run completes (skip-and-continue), not a hard task failure.
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' }));
|
|
});
|
|
|
|
it('reports a failed run and an Auto-start warning when a policy blocks a scheduled auto-start', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 89,
|
|
name: 'blocked-start',
|
|
action: 'auto_start',
|
|
cron_expression: '0 4 * * *',
|
|
enabled: true,
|
|
target_id: 'web-app',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockEnforcePolicyPreDeploy.mockResolvedValue({
|
|
ok: false,
|
|
bypassed: false,
|
|
policy: { id: 1, name: 'block-high', max_severity: 'HIGH' },
|
|
violations: [{ imageRef: 'nginx:1.14', severity: 'CRITICAL', criticalCount: 1, highCount: 0, scanId: 3 }],
|
|
});
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(89);
|
|
|
|
// Gate blocked it: the stack must not start.
|
|
expect(mockDeployStack).not.toHaveBeenCalled();
|
|
const warn = mockDispatchAlert.mock.calls.find((c) => c[0] === 'warning' && c[1] === 'scan_finding');
|
|
expect(warn).toBeDefined();
|
|
expect(warn![2]).toContain('Auto-start');
|
|
expect(warn![2]).toContain('block-high');
|
|
// Auto-start does not skip-and-continue; the run is recorded as a failure.
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'failure' }));
|
|
});
|
|
|
|
it('exposes isTaskRunning status', async () => {
|
|
const svc = SchedulerService.getInstance();
|
|
expect(svc.isTaskRunning(999)).toBe(false);
|
|
});
|
|
|
|
it('fleet target updates every stack discovered on the node', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 87,
|
|
name: 'fleet-update',
|
|
action: 'update',
|
|
target_type: 'fleet',
|
|
cron_expression: '0 4 * * *',
|
|
enabled: true,
|
|
target_id: null,
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetStacks.mockResolvedValue(['app1', 'app2', 'app3']);
|
|
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }]);
|
|
mockCheckImage.mockResolvedValue({ hasUpdate: true });
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(87);
|
|
|
|
expect(mockUpdateStack).toHaveBeenCalledTimes(3);
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({ status: 'success' })
|
|
);
|
|
});
|
|
|
|
it('fleet target on empty node returns early with skipped message', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 89,
|
|
name: 'fleet-update-empty-node',
|
|
action: 'update',
|
|
target_type: 'fleet',
|
|
cron_expression: '0 4 * * *',
|
|
enabled: true,
|
|
target_id: null,
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetStacks.mockResolvedValue([]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(89);
|
|
|
|
expect(mockUpdateStack).not.toHaveBeenCalled();
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({
|
|
status: 'success',
|
|
output: expect.stringContaining('No stacks found'),
|
|
})
|
|
);
|
|
});
|
|
});
|
|
|
|
// ── Error handling & notifications ─────────────────────────────────────
|
|
|
|
describe('SchedulerService - error handling', () => {
|
|
it('records failure status in DB on error', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 90,
|
|
name: 'error-task',
|
|
action: 'restart',
|
|
cron_expression: '*/5 * * * *',
|
|
enabled: true,
|
|
target_id: null,
|
|
node_id: null,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(90);
|
|
|
|
expect(mockUpdateScheduledTask).toHaveBeenCalledWith(
|
|
90,
|
|
expect.objectContaining({ last_status: 'failure' })
|
|
);
|
|
});
|
|
|
|
it('dispatches error notification on failure', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 91,
|
|
name: 'notify-fail',
|
|
action: 'restart',
|
|
cron_expression: '*/5 * * * *',
|
|
enabled: true,
|
|
target_id: null,
|
|
node_id: null,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(91);
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith('error', 'system', expect.stringContaining('failed'), { stackName: undefined, actor: 'system:scheduler' });
|
|
});
|
|
|
|
it('dispatches recovery notification when previous status was failure', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 92,
|
|
name: 'recovery-task',
|
|
action: 'restart',
|
|
cron_expression: '*/5 * * * *',
|
|
enabled: true,
|
|
target_id: 'my-stack',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: 'failure', // Previous run failed
|
|
});
|
|
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(92);
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith('info', 'system', expect.stringContaining('recovered'), { stackName: 'my-stack', actor: 'system:scheduler' });
|
|
});
|
|
});
|
|
|
|
// ── Scheduled scan completion notifications ───────────────────────────
|
|
|
|
describe('SchedulerService - scheduled scan notifications', () => {
|
|
function makeScanTask(overrides: Partial<any> = {}) {
|
|
return {
|
|
id: 200,
|
|
name: 'nightly-scan',
|
|
action: 'scan',
|
|
cron_expression: '0 2 * * *',
|
|
enabled: true,
|
|
target_id: null,
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function scanResult(opts: {
|
|
scanned?: number;
|
|
skipped?: number;
|
|
failed?: number;
|
|
critical?: number;
|
|
high?: number;
|
|
medium?: number;
|
|
low?: number;
|
|
unknown?: number;
|
|
totalImages?: number;
|
|
processedImages?: number;
|
|
truncated?: boolean;
|
|
limitReason?: string;
|
|
} = {}) {
|
|
return {
|
|
scanned: opts.scanned ?? 0,
|
|
skipped: opts.skipped ?? 0,
|
|
failed: opts.failed ?? 0,
|
|
totalImages: opts.totalImages,
|
|
processedImages: opts.processedImages,
|
|
truncated: opts.truncated,
|
|
limitReason: opts.limitReason,
|
|
severity: {
|
|
critical: opts.critical ?? 0,
|
|
high: opts.high ?? 0,
|
|
medium: opts.medium ?? 0,
|
|
low: opts.low ?? 0,
|
|
unknown: opts.unknown ?? 0,
|
|
},
|
|
violations: [],
|
|
};
|
|
}
|
|
|
|
it('dispatches info-level notification when scan completes cleanly', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeScanTask());
|
|
mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 3, skipped: 1 }));
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(200);
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'info',
|
|
'scan_finding',
|
|
expect.stringContaining('nightly-scan'),
|
|
{ stackName: undefined, actor: 'system:scheduler' },
|
|
);
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'info',
|
|
'scan_finding',
|
|
expect.stringContaining('Scanned 3 image(s)'),
|
|
{ stackName: undefined, actor: 'system:scheduler' },
|
|
);
|
|
});
|
|
|
|
it('dispatches warning-level notification when scan has failures', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 201, name: 'flaky-scan' }));
|
|
mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 5, failed: 2 }));
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(201);
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'warning',
|
|
'scan_finding',
|
|
expect.stringContaining('2 failed'),
|
|
{ stackName: undefined, actor: 'system:scheduler' },
|
|
);
|
|
});
|
|
|
|
it('passes target_id to dispatchAlert when set', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 202, target_id: 'web-stack' }));
|
|
mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 1 }));
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(202);
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'info',
|
|
'scan_finding',
|
|
expect.stringContaining('completed'),
|
|
{ stackName: 'web-stack', actor: 'system:scheduler' },
|
|
);
|
|
});
|
|
|
|
it('fires only the scan notification when previous run was failure (no duplicate recovery alert)', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeScanTask({
|
|
id: 204,
|
|
name: 'recovered-scan',
|
|
last_status: 'failure',
|
|
}));
|
|
mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 2 }));
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(204);
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'info',
|
|
'scan_finding',
|
|
expect.stringContaining('recovered-scan'),
|
|
{ stackName: undefined, actor: 'system:scheduler' },
|
|
);
|
|
});
|
|
|
|
it('does not dispatch a scan notification for non-scan actions', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 203,
|
|
name: 'restart-task',
|
|
action: 'restart',
|
|
cron_expression: '*/5 * * * *',
|
|
enabled: true,
|
|
target_id: 'my-stack',
|
|
node_id: 1,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(203);
|
|
|
|
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('dispatches error-level notification when Trivy is unavailable', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 205, target_id: 'payment-stack' }));
|
|
mockIsTrivyAvailable.mockReturnValueOnce(false);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(205);
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'error',
|
|
'system',
|
|
expect.stringMatching(/failed.*Trivy/i),
|
|
{ stackName: 'payment-stack', actor: 'system:scheduler' },
|
|
);
|
|
});
|
|
|
|
it('fails scheduled scan tasks that target remote nodes before scanning', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 210, node_id: 2 }));
|
|
mockGetNode
|
|
.mockReturnValueOnce({ id: 2, name: 'remote', type: 'remote', status: 'online' })
|
|
.mockReturnValueOnce({ id: 2, name: 'remote', type: 'remote', status: 'online' });
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(210);
|
|
|
|
expect(mockScanAllNodeImages).not.toHaveBeenCalled();
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
expect.any(Number),
|
|
expect.objectContaining({
|
|
status: 'failure',
|
|
error: expect.stringMatching(/local node/i),
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('includes severity counts in the notification message', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 206 }));
|
|
mockScanAllNodeImages.mockResolvedValue(
|
|
scanResult({ scanned: 3, skipped: 1, critical: 2, high: 5, medium: 10 }),
|
|
);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(206);
|
|
|
|
const message = mockDispatchAlert.mock.calls[0][2] as string;
|
|
expect(message).toContain('2 critical');
|
|
expect(message).toContain('5 high');
|
|
expect(message).toContain('10 medium');
|
|
});
|
|
|
|
it('reports "No images to scan" when the node has nothing to scan', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 207 }));
|
|
mockScanAllNodeImages.mockResolvedValue(scanResult());
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(207);
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'info',
|
|
'scan_finding',
|
|
expect.stringContaining('No images to scan'),
|
|
{ stackName: undefined, actor: 'system:scheduler' },
|
|
);
|
|
});
|
|
|
|
it('reports "All N image(s) already scanned recently" when every image was cached', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 208 }));
|
|
mockScanAllNodeImages.mockResolvedValue(scanResult({ skipped: 12 }));
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(208);
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'info',
|
|
'scan_finding',
|
|
expect.stringContaining('All 12 image(s) already scanned recently'),
|
|
{ stackName: undefined, actor: 'system:scheduler' },
|
|
);
|
|
});
|
|
|
|
it('reports when scan-all stops at a configured bound', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 211 }));
|
|
mockScanAllNodeImages.mockResolvedValue(scanResult({
|
|
scanned: 100,
|
|
totalImages: 250,
|
|
processedImages: 100,
|
|
truncated: true,
|
|
limitReason: 'image limit 100 reached',
|
|
}));
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(211);
|
|
|
|
const message = mockDispatchAlert.mock.calls[0][2] as string;
|
|
expect(message).toContain('Scan limited after 100 of 250 image(s)');
|
|
expect(message).toContain('image limit 100 reached');
|
|
});
|
|
|
|
it('persists the run as success even when notification dispatch throws', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 209 }));
|
|
mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 1 }));
|
|
mockDispatchAlert.mockRejectedValueOnce(new Error('webhook down'));
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await expect(svc.triggerTask(209)).resolves.not.toThrow();
|
|
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
expect.any(Number),
|
|
expect.objectContaining({ status: 'success' }),
|
|
);
|
|
});
|
|
});
|
|
|
|
// ── Cleanup ────────────────────────────────────────────────────────────
|
|
|
|
describe('SchedulerService - cleanup', () => {
|
|
it('calls cleanupOldTaskRuns(30) on every tick', async () => {
|
|
mockGetTier.mockReturnValue('paid');
|
|
mockGetDueScheduledTasks.mockReturnValue([]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await (svc as any).tick();
|
|
|
|
expect(mockCleanupOldTaskRuns).toHaveBeenCalledWith(30);
|
|
});
|
|
});
|
|
|
|
// ── isProcessing guard ─────────────────────────────────────────────────
|
|
|
|
describe('SchedulerService - isProcessing guard', () => {
|
|
it('skips tick if already processing', async () => {
|
|
const svc = SchedulerService.getInstance();
|
|
(svc as any).isProcessing = true;
|
|
|
|
await (svc as any).tick();
|
|
|
|
// Short-circuits before fetching due tasks (the first DB call inside tick).
|
|
expect(mockGetDueScheduledTasks).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('resets isProcessing after tick completes (even on error)', async () => {
|
|
mockGetDueScheduledTasks.mockImplementationOnce(() => { throw new Error('boom'); });
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await (svc as any).tick();
|
|
|
|
expect((svc as any).isProcessing).toBe(false);
|
|
});
|
|
});
|
|
|
|
// ── Stale run cleanup (T1) ───────────────────────────────────────────
|
|
|
|
describe('SchedulerService - stale run cleanup', () => {
|
|
it('calls markStaleRunsAsFailed on start and logs when records exist', () => {
|
|
mockMarkStaleRunsAsFailed.mockReturnValue(2);
|
|
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
svc.start();
|
|
|
|
expect(mockMarkStaleRunsAsFailed).toHaveBeenCalledTimes(1);
|
|
expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('Cleaned up 2 stale run record(s)'));
|
|
|
|
logSpy.mockRestore();
|
|
svc.stop();
|
|
});
|
|
|
|
it('does not log when no stale runs exist', () => {
|
|
mockMarkStaleRunsAsFailed.mockReturnValue(0);
|
|
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
svc.start();
|
|
|
|
expect(mockMarkStaleRunsAsFailed).toHaveBeenCalledTimes(1);
|
|
expect(logSpy).not.toHaveBeenCalledWith(expect.stringContaining('stale'));
|
|
|
|
logSpy.mockRestore();
|
|
svc.stop();
|
|
});
|
|
});
|
|
|
|
// ── Invalid cron at execution time (T2) ──────────────────────────────
|
|
|
|
describe('SchedulerService - invalid cron at execution time', () => {
|
|
it('disables task and records error when cron becomes invalid', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 95,
|
|
name: 'bad-cron-task',
|
|
action: 'restart',
|
|
cron_expression: 'INVALID CRON',
|
|
enabled: true,
|
|
target_id: null,
|
|
node_id: null,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(95);
|
|
|
|
expect(mockUpdateScheduledTask).toHaveBeenCalledWith(
|
|
95,
|
|
expect.objectContaining({
|
|
enabled: 0,
|
|
last_status: 'failure',
|
|
last_error: expect.stringContaining('no longer valid'),
|
|
})
|
|
);
|
|
|
|
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
|
'error',
|
|
'system',
|
|
expect.stringContaining('failed'),
|
|
{ stackName: undefined, actor: 'system:scheduler' },
|
|
);
|
|
});
|
|
});
|
|
|
|
// ── executeSnapshot (T3) ─────────────────────────────────────────────
|
|
|
|
describe('SchedulerService - executeSnapshot', () => {
|
|
it('creates a fleet snapshot capturing all local nodes', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 75,
|
|
name: 'nightly-snapshot',
|
|
action: 'snapshot',
|
|
target_type: 'fleet',
|
|
cron_expression: '0 3 * * *',
|
|
enabled: true,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetNodes.mockReturnValue([
|
|
{ id: 1, name: 'local', type: 'local' },
|
|
]);
|
|
mockGetStacks.mockResolvedValue(['app1']);
|
|
mockGetStackContent.mockResolvedValue('version: "3"\nservices:\n web:\n image: nginx');
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(75);
|
|
|
|
expect(mockCreateSnapshot).toHaveBeenCalledWith(
|
|
expect.stringContaining('nightly-snapshot'),
|
|
'admin',
|
|
1,
|
|
1,
|
|
expect.any(String),
|
|
expect.any(String),
|
|
'',
|
|
);
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({ status: 'success' })
|
|
);
|
|
});
|
|
|
|
it('handles nodes with no stacks gracefully', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 76,
|
|
name: 'empty-snapshot',
|
|
action: 'snapshot',
|
|
target_type: 'fleet',
|
|
cron_expression: '0 3 * * *',
|
|
enabled: true,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetNodes.mockReturnValue([
|
|
{ id: 1, name: 'local', type: 'local' },
|
|
]);
|
|
mockGetStacks.mockResolvedValue([]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(76);
|
|
|
|
expect(mockCreateSnapshot).toHaveBeenCalled();
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({ status: 'success' })
|
|
);
|
|
});
|
|
|
|
it('captures dossier documentation when the snapshot_documentation setting is on', async () => {
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 77,
|
|
name: 'documented-snapshot',
|
|
action: 'snapshot',
|
|
target_type: 'fleet',
|
|
cron_expression: '0 3 * * *',
|
|
enabled: true,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
|
mockGetStacks.mockResolvedValue(['app1']);
|
|
mockGetStackContent.mockResolvedValue('services: {}\n');
|
|
mockGetGlobalSettings.mockReturnValue({ snapshot_documentation: '1' });
|
|
mockGetStackDossier.mockReturnValue({ purpose: 'documented', owner: '', access_urls: '', static_ip: '', vlan: '', firewall_notes: '', reverse_proxy_notes: '', backup_notes: '', upgrade_notes: '', recovery_notes: '', custom_notes: '' });
|
|
|
|
await SchedulerService.getInstance().triggerTask(77);
|
|
|
|
const docArg = mockCreateSnapshot.mock.calls.at(-1)?.[6] as string;
|
|
expect(docArg).toBeTruthy();
|
|
expect(JSON.parse(docArg).stacks[0]).toMatchObject({ stackName: 'app1', dossier: { purpose: 'documented' } });
|
|
});
|
|
});
|
|
|
|
// ── executeUpdateRemote (T4) ─────────────────────────────────────────
|
|
|
|
describe('SchedulerService - executeUpdateRemote', () => {
|
|
afterEach(() => {
|
|
vi.unstubAllGlobals();
|
|
});
|
|
|
|
it('proxies update execution to remote node', async () => {
|
|
mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
|
|
mockGetProxyTarget.mockReturnValue({
|
|
apiUrl: 'http://remote:1852',
|
|
apiToken: 'test-token',
|
|
});
|
|
|
|
const mockFetch = vi.fn().mockResolvedValue({
|
|
ok: true,
|
|
json: async () => ({ result: 'Stack "web": updated (nginx:latest).' }),
|
|
});
|
|
vi.stubGlobal('fetch', mockFetch);
|
|
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 88,
|
|
name: 'remote-update',
|
|
action: 'update',
|
|
cron_expression: '0 4 * * *',
|
|
enabled: true,
|
|
target_id: 'web-app',
|
|
node_id: 2,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(88);
|
|
|
|
expect(mockFetch).toHaveBeenCalledWith(
|
|
'http://remote:1852/api/auto-update/execute',
|
|
expect.objectContaining({
|
|
method: 'POST',
|
|
headers: expect.objectContaining({
|
|
'Authorization': 'Bearer test-token',
|
|
'x-sencho-tier': 'paid',
|
|
}),
|
|
body: JSON.stringify({ target: 'web-app' }),
|
|
})
|
|
);
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({ status: 'success' })
|
|
);
|
|
});
|
|
|
|
it('records failure when remote node returns error', async () => {
|
|
mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
|
|
mockGetProxyTarget.mockReturnValue({
|
|
apiUrl: 'http://remote:1852',
|
|
apiToken: 'test-token',
|
|
});
|
|
|
|
const mockFetch = vi.fn().mockResolvedValue({
|
|
ok: false,
|
|
status: 500,
|
|
json: async () => ({ error: 'Internal error' }),
|
|
});
|
|
vi.stubGlobal('fetch', mockFetch);
|
|
|
|
mockGetScheduledTask.mockReturnValue({
|
|
id: 89,
|
|
name: 'remote-update-fail',
|
|
action: 'update',
|
|
cron_expression: '0 4 * * *',
|
|
enabled: true,
|
|
target_id: 'web-app',
|
|
node_id: 2,
|
|
created_by: 'admin',
|
|
last_status: null,
|
|
});
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await svc.triggerTask(89);
|
|
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({ status: 'failure', error: expect.stringContaining('Internal error') })
|
|
);
|
|
});
|
|
});
|
|
|
|
// ── Lifecycle actions (auto_backup, auto_stop, auto_down, auto_start) ───
|
|
|
|
function makeLifecycleTask(action: ScheduledTask['action'], overrides: Partial<ScheduledTask> = {}): ScheduledTask {
|
|
return {
|
|
id: 300,
|
|
name: `lifecycle-${action}`,
|
|
action,
|
|
target_type: 'stack',
|
|
target_id: 'my-stack',
|
|
node_id: 1,
|
|
cron_expression: '0 2 * * *',
|
|
enabled: 1,
|
|
created_by: 'admin',
|
|
created_at: 0,
|
|
updated_at: 0,
|
|
last_run_at: null,
|
|
next_run_at: null,
|
|
last_status: null,
|
|
last_error: null,
|
|
prune_targets: null,
|
|
target_services: null,
|
|
prune_label_filter: null,
|
|
delete_after_run: 0,
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
describe('SchedulerService - lifecycle actions', () => {
|
|
it('auto_stop calls runCommand with "stop"', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop'));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(mockRunCommand).toHaveBeenCalledWith('my-stack', 'stop');
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' }));
|
|
});
|
|
|
|
it('auto_down calls runCommand with "down"', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_down'));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(mockRunCommand).toHaveBeenCalledWith('my-stack', 'down');
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' }));
|
|
});
|
|
|
|
it('auto_start calls deployStack', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_start'));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(mockDeployStack).toHaveBeenCalledWith('my-stack');
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' }));
|
|
});
|
|
|
|
it('auto_backup calls backupStackFiles', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup'));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(mockBackupStackFiles).toHaveBeenCalledWith('my-stack');
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' }));
|
|
});
|
|
|
|
it('auto_stop records failure when target_id is missing', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { target_id: null }));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(mockRunCommand).not.toHaveBeenCalled();
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'failure' }));
|
|
});
|
|
|
|
it('auto_backup records failure when node_id is missing', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup', { node_id: null }));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(mockBackupStackFiles).not.toHaveBeenCalled();
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'failure' }));
|
|
});
|
|
|
|
it('records failure (not success) and skips the op when the stack lock is held', async () => {
|
|
// A manual operation holds the lock; the scheduled lifecycle op must skip
|
|
// rather than race, and surface as a failed run instead of a silent success.
|
|
StackOpLockService.getInstance().tryAcquire(1, 'my-stack', 'deploy', 'admin');
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop'));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(mockRunCommand).not.toHaveBeenCalled();
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'failure' }));
|
|
expect(mockUpdateScheduledTask).toHaveBeenCalledWith(300, expect.objectContaining({ last_status: 'failure' }));
|
|
});
|
|
|
|
it('paid tier executes lifecycle actions', async () => {
|
|
mockGetTier.mockReturnValue('paid');
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop'));
|
|
mockGetDueScheduledTasks.mockReturnValue([makeLifecycleTask('auto_stop')]);
|
|
|
|
const svc = SchedulerService.getInstance();
|
|
await (svc as any).tick();
|
|
|
|
await new Promise(r => setTimeout(r, 50));
|
|
expect(mockRunCommand).toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
// ── Lifecycle remote proxy ──────────────────────────────────────────────
|
|
|
|
describe('SchedulerService - lifecycle remote proxy', () => {
|
|
afterEach(() => {
|
|
vi.unstubAllGlobals();
|
|
});
|
|
|
|
const remoteHeaders = expect.objectContaining({
|
|
'Authorization': 'Bearer tkn',
|
|
'x-sencho-tier': 'paid',
|
|
});
|
|
|
|
function stubRemote(okBody: unknown = { success: true }) {
|
|
mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
|
|
mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' });
|
|
const mockFetch = vi.fn().mockResolvedValue({ ok: true, json: async () => okBody });
|
|
vi.stubGlobal('fetch', mockFetch);
|
|
return mockFetch;
|
|
}
|
|
|
|
it('auto_stop proxies to the remote stop endpoint instead of running locally', async () => {
|
|
const fetchMock = stubRemote();
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 }));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(fetchMock).toHaveBeenCalledWith(
|
|
'http://remote:1852/api/stacks/my-stack/stop',
|
|
expect.objectContaining({ method: 'POST', headers: remoteHeaders }),
|
|
);
|
|
expect(mockRunCommand).not.toHaveBeenCalled();
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' }));
|
|
});
|
|
|
|
it('auto_down proxies to the remote down endpoint', async () => {
|
|
const fetchMock = stubRemote();
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_down', { node_id: 2 }));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(fetchMock).toHaveBeenCalledWith(
|
|
'http://remote:1852/api/stacks/my-stack/down',
|
|
expect.objectContaining({ method: 'POST' }),
|
|
);
|
|
expect(mockRunCommand).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('auto_start proxies to the remote deploy endpoint and skips the hub policy gate', async () => {
|
|
const fetchMock = stubRemote();
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_start', { node_id: 2 }));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(fetchMock).toHaveBeenCalledWith(
|
|
'http://remote:1852/api/stacks/my-stack/deploy',
|
|
expect.objectContaining({ method: 'POST' }),
|
|
);
|
|
expect(mockDeployStack).not.toHaveBeenCalled();
|
|
expect(mockEnforcePolicyPreDeploy).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('auto_backup proxies to the remote backup endpoint', async () => {
|
|
const fetchMock = stubRemote();
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup', { node_id: 2 }));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(fetchMock).toHaveBeenCalledWith(
|
|
'http://remote:1852/api/stacks/my-stack/backup',
|
|
expect.objectContaining({ method: 'POST' }),
|
|
);
|
|
expect(mockBackupStackFiles).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('restart (all services) proxies to the remote restart endpoint', async () => {
|
|
const fetchMock = stubRemote();
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('restart', { node_id: 2 }));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(fetchMock).toHaveBeenCalledWith(
|
|
'http://remote:1852/api/stacks/my-stack/restart',
|
|
expect.objectContaining({ method: 'POST' }),
|
|
);
|
|
expect(mockGetContainersByStack).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('restart with target_services fans out to per-service restart endpoints', async () => {
|
|
const fetchMock = stubRemote();
|
|
mockGetScheduledTask.mockReturnValue(
|
|
makeLifecycleTask('restart', { node_id: 2, target_services: JSON.stringify(['api', 'worker']) }),
|
|
);
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(fetchMock).toHaveBeenCalledWith(
|
|
'http://remote:1852/api/stacks/my-stack/services/api/restart',
|
|
expect.objectContaining({ method: 'POST' }),
|
|
);
|
|
expect(fetchMock).toHaveBeenCalledWith(
|
|
'http://remote:1852/api/stacks/my-stack/services/worker/restart',
|
|
expect.objectContaining({ method: 'POST' }),
|
|
);
|
|
expect(fetchMock).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('records failure when the remote node returns an error', async () => {
|
|
mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
|
|
mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' });
|
|
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
|
|
ok: false,
|
|
status: 503,
|
|
json: async () => ({ error: 'Docker daemon is unreachable' }),
|
|
}));
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 }));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({ status: 'failure', error: expect.stringContaining('Docker daemon is unreachable') }),
|
|
);
|
|
});
|
|
|
|
it('falls back to the HTTP status when the remote error body is not JSON', async () => {
|
|
mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
|
|
mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' });
|
|
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
|
|
ok: false,
|
|
status: 502,
|
|
json: async () => { throw new Error('not json'); },
|
|
}));
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_down', { node_id: 2 }));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({ status: 'failure', error: expect.stringContaining('HTTP 502') }),
|
|
);
|
|
});
|
|
|
|
it('records failure when the remote node has no proxy credentials', async () => {
|
|
mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
|
|
// getProxyTarget defaults to null (no credentials configured).
|
|
const fetchMock = vi.fn();
|
|
vi.stubGlobal('fetch', fetchMock);
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 }));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(fetchMock).not.toHaveBeenCalled();
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({ status: 'failure', error: expect.stringContaining('not configured or missing API credentials') }),
|
|
);
|
|
});
|
|
|
|
it('restart fan-out fails fast and names already-restarted services', async () => {
|
|
mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
|
|
mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' });
|
|
const fetchMock = vi.fn()
|
|
.mockResolvedValueOnce({ ok: true, json: async () => ({ success: true }) })
|
|
.mockResolvedValueOnce({ ok: false, status: 500, json: async () => ({ error: 'boom' }) });
|
|
vi.stubGlobal('fetch', fetchMock);
|
|
mockGetScheduledTask.mockReturnValue(
|
|
makeLifecycleTask('restart', { node_id: 2, target_services: JSON.stringify(['api', 'worker', 'cache']) }),
|
|
);
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
// Third service is never reached after the second fails.
|
|
expect(fetchMock).toHaveBeenCalledTimes(2);
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({ status: 'failure', error: expect.stringContaining('already restarted: api') }),
|
|
);
|
|
});
|
|
});
|
|
|
|
// ── Unpaid-tier guard in executeTask ────────────────────────────────────
|
|
|
|
describe('SchedulerService - community tier runs lifecycle actions', () => {
|
|
it('executes a lifecycle action and records success on the community tier', async () => {
|
|
mockGetTier.mockReturnValue('community');
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop'));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(mockRunCommand).toHaveBeenCalled();
|
|
expect(mockCreateScheduledTaskRun).toHaveBeenCalled();
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
|
1,
|
|
expect.objectContaining({ status: 'success' }),
|
|
);
|
|
});
|
|
});
|
|
|
|
// ── delete_after_run ────────────────────────────────────────────────────
|
|
|
|
describe('SchedulerService - delete_after_run', () => {
|
|
it('deletes task after successful run when delete_after_run is 1', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup', { delete_after_run: 1 }));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(mockDeleteScheduledTask).toHaveBeenCalledWith(300);
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' }));
|
|
});
|
|
|
|
it('does not delete task when run fails even if delete_after_run is 1', async () => {
|
|
mockBackupStackFiles.mockRejectedValueOnce(new Error('disk full'));
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup', { delete_after_run: 1 }));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(mockDeleteScheduledTask).not.toHaveBeenCalled();
|
|
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'failure' }));
|
|
});
|
|
|
|
it('does not delete task when delete_after_run is 0', async () => {
|
|
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup', { delete_after_run: 0 }));
|
|
await SchedulerService.getInstance().triggerTask(300);
|
|
expect(mockDeleteScheduledTask).not.toHaveBeenCalled();
|
|
expect(mockUpdateScheduledTask).toHaveBeenCalledWith(300, expect.objectContaining({ last_status: 'success' }));
|
|
});
|
|
});
|