mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-08 01:43:55 +00:00
Anso
c491d309c1
Collapse the Admiral carveout that restricted restart, prune, auto_backup, auto_stop, auto_down, and auto_start schedules to the Admiral variant. Scheduled Operations stays at Skipper+ (paid). The action picker now lists every supported operation for any paid admin, and the scheduler runner executes every action on either variant.
830 lines
35 KiB
TypeScript
830 lines
35 KiB
TypeScript
import { CronExpressionParser } from 'cron-parser';
|
|
import { DatabaseService } from './DatabaseService';
|
|
import type { ScheduledTask } from './DatabaseService';
|
|
import { LicenseService } from './LicenseService';
|
|
import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from './license-headers';
|
|
import DockerController from './DockerController';
|
|
import { ComposeService } from './ComposeService';
|
|
import { FileSystemService } from './FileSystemService';
|
|
import { ImageUpdateService } from './ImageUpdateService';
|
|
import type { ImageCheckResult } from './ImageUpdateService';
|
|
import { isDebugEnabled } from '../utils/debug';
|
|
import { getErrorMessage } from '../utils/errors';
|
|
import { sanitizeForLog } from '../utils/safeLog';
|
|
import { captureLocalNodeFiles, captureRemoteNodeFiles } from '../utils/snapshot-capture';
|
|
import { NodeRegistry } from './NodeRegistry';
|
|
import { NotificationService } from './NotificationService';
|
|
import TrivyService from './TrivyService';
|
|
import type { ScanAllNodeImagesResult } from './TrivyService';
|
|
import TrivyInstaller from './TrivyInstaller';
|
|
import { CloudBackupService } from './CloudBackupService';
|
|
import { assertPolicyGateAllows, buildSystemPolicyGateOptions } from '../helpers/policyGate';
|
|
|
|
const TRIVY_UPDATE_CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
const TRIVY_UPDATE_CHECK_STARTUP_DELAY_MS = 5 * 60 * 1000;
|
|
|
|
const TRIVY_REDETECT_INTERVAL_MS = 10 * 60 * 1000;
|
|
const STALE_SCAN_THRESHOLD_MS = 15 * 60 * 1000;
|
|
|
|
export class SchedulerService {
|
|
private static instance: SchedulerService;
|
|
private intervalId: ReturnType<typeof setInterval> | null = null;
|
|
private trivyUpdateIntervalId: ReturnType<typeof setInterval> | null = null;
|
|
private trivyUpdateStartupTimer: ReturnType<typeof setTimeout> | null = null;
|
|
private isProcessing = false;
|
|
private isCheckingTrivyUpdate = false;
|
|
private runningTasks = new Set<number>();
|
|
private lastTrivyRedetect = 0;
|
|
|
|
private constructor() {}
|
|
|
|
public static getInstance(): SchedulerService {
|
|
if (!SchedulerService.instance) {
|
|
SchedulerService.instance = new SchedulerService();
|
|
}
|
|
return SchedulerService.instance;
|
|
}
|
|
|
|
public start(): void {
|
|
if (this.intervalId) return;
|
|
this.cleanupStaleRuns();
|
|
this.intervalId = setInterval(() => this.tick(), 60_000);
|
|
setTimeout(() => this.tick(), 10_000);
|
|
this.trivyUpdateStartupTimer = setTimeout(() => this.runTrivyUpdateCheck(), TRIVY_UPDATE_CHECK_STARTUP_DELAY_MS);
|
|
this.trivyUpdateIntervalId = setInterval(() => this.runTrivyUpdateCheck(), TRIVY_UPDATE_CHECK_INTERVAL_MS);
|
|
console.log('[SchedulerService] Started');
|
|
}
|
|
|
|
public stop(): void {
|
|
if (this.intervalId) {
|
|
clearInterval(this.intervalId);
|
|
this.intervalId = null;
|
|
}
|
|
if (this.trivyUpdateIntervalId) {
|
|
clearInterval(this.trivyUpdateIntervalId);
|
|
this.trivyUpdateIntervalId = null;
|
|
}
|
|
if (this.trivyUpdateStartupTimer) {
|
|
clearTimeout(this.trivyUpdateStartupTimer);
|
|
this.trivyUpdateStartupTimer = null;
|
|
}
|
|
console.log('[SchedulerService] Stopped');
|
|
}
|
|
|
|
private async runTrivyUpdateCheck(): Promise<void> {
|
|
if (this.isCheckingTrivyUpdate) return;
|
|
this.isCheckingTrivyUpdate = true;
|
|
try {
|
|
const trivy = TrivyService.getInstance();
|
|
if (!trivy.isTrivyAvailable() || trivy.getSource() !== 'managed') return;
|
|
const db = DatabaseService.getInstance();
|
|
const settings = db.getGlobalSettings();
|
|
const autoUpdate = settings.trivy_auto_update === '1';
|
|
const installer = TrivyInstaller.getInstance();
|
|
if (installer.isBusy()) return;
|
|
const check = await installer.checkForUpdate(trivy.getVersion(), 'managed');
|
|
if (!check.updateAvailable) return;
|
|
|
|
if (autoUpdate) {
|
|
const previous = trivy.getVersion() ?? 'unknown';
|
|
console.log(`[SchedulerService] Auto-updating Trivy from ${previous} to ${check.latest}`);
|
|
try {
|
|
await installer.update();
|
|
await trivy.detectTrivy();
|
|
this.safeDispatch(
|
|
'info',
|
|
'system',
|
|
`Trivy updated from v${previous} to v${check.latest}`,
|
|
);
|
|
db.updateGlobalSetting('trivy_last_notified_version', check.latest);
|
|
} catch (err) {
|
|
console.error('[SchedulerService] Trivy auto-update failed:', getErrorMessage(err, 'unknown error'));
|
|
}
|
|
} else {
|
|
const lastNotified = settings.trivy_last_notified_version || '';
|
|
if (lastNotified === check.latest) return;
|
|
this.safeDispatch(
|
|
'info',
|
|
'system',
|
|
`Trivy update available: v${check.latest} (currently v${check.current ?? 'unknown'})`,
|
|
);
|
|
db.updateGlobalSetting('trivy_last_notified_version', check.latest);
|
|
}
|
|
} catch (err) {
|
|
console.warn('[SchedulerService] Trivy update check failed:', getErrorMessage(err, 'unknown error'));
|
|
} finally {
|
|
this.isCheckingTrivyUpdate = false;
|
|
}
|
|
}
|
|
|
|
private cleanupStaleRuns(): void {
|
|
try {
|
|
const count = DatabaseService.getInstance().markStaleRunsAsFailed();
|
|
if (count > 0) {
|
|
console.log(`[SchedulerService] Cleaned up ${count} stale run record(s)`);
|
|
}
|
|
} catch (error) {
|
|
console.error('[SchedulerService] Failed to clean up stale runs:', error);
|
|
}
|
|
}
|
|
|
|
private async maybeRedetectTrivy(): Promise<void> {
|
|
const now = Date.now();
|
|
if (now - this.lastTrivyRedetect < TRIVY_REDETECT_INTERVAL_MS) return;
|
|
this.lastTrivyRedetect = now;
|
|
try {
|
|
await TrivyService.getInstance().detectTrivy();
|
|
} catch (error) {
|
|
if (isDebugEnabled()) {
|
|
console.warn('[SchedulerService:debug] Trivy re-detect failed:', error);
|
|
}
|
|
}
|
|
}
|
|
|
|
public calculateNextRun(cronExpression: string): number {
|
|
const expr = CronExpressionParser.parse(cronExpression);
|
|
return expr.next().toDate().getTime();
|
|
}
|
|
|
|
public calculateRunsWithin(cronExpression: string, fromMs: number, toMs: number, limit = 16): number[] {
|
|
try {
|
|
const expr = CronExpressionParser.parse(cronExpression, { currentDate: new Date(fromMs) });
|
|
const runs: number[] = [];
|
|
while (runs.length < limit) {
|
|
const next = expr.next().toDate().getTime();
|
|
if (next > toMs) break;
|
|
runs.push(next);
|
|
}
|
|
return runs;
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Fire a notification without awaiting completion, catching any promise
|
|
* rejection so the scheduler never crashes on a failed dispatch.
|
|
*/
|
|
private safeDispatch(level: 'info' | 'warning' | 'error', category: import('./NotificationService').NotificationCategory, message: string, stackName?: string): void {
|
|
NotificationService.getInstance()
|
|
.dispatchAlert(level, category, message, { stackName })
|
|
.catch(err => console.error('[SchedulerService] Notification dispatch failed:', getErrorMessage(err, 'unknown error')));
|
|
}
|
|
|
|
private async tick(): Promise<void> {
|
|
if (this.isProcessing) {
|
|
console.warn('[SchedulerService] Tick skipped: previous tick still processing');
|
|
return;
|
|
}
|
|
this.isProcessing = true;
|
|
try {
|
|
const db = DatabaseService.getInstance();
|
|
|
|
// Vulnerability scanning is available on every tier, so the stale-scan sweep
|
|
// and Trivy re-detect run before the paid-tier gate below.
|
|
try {
|
|
const staleScans = db.markStaleScansAsFailed(STALE_SCAN_THRESHOLD_MS);
|
|
if (staleScans > 0) {
|
|
console.log(
|
|
`[SchedulerService] Marked ${staleScans} stale vulnerability scan(s) as failed`,
|
|
);
|
|
}
|
|
} catch (error) {
|
|
console.error('[SchedulerService] Stale scan sweep failed:', error);
|
|
}
|
|
await this.maybeRedetectTrivy();
|
|
|
|
const ls = LicenseService.getInstance();
|
|
if (ls.getTier() !== 'paid') return;
|
|
|
|
const now = Date.now();
|
|
const dueTasks = db.getDueScheduledTasks(now);
|
|
|
|
if (dueTasks.length > 0) {
|
|
console.log(`[SchedulerService] Found ${dueTasks.length} due task(s)`);
|
|
}
|
|
|
|
// Clean up old runs periodically (piggyback on tick)
|
|
db.cleanupOldTaskRuns(30);
|
|
db.deleteOldScans(90 * 24 * 60 * 60 * 1000);
|
|
|
|
for (const task of dueTasks) {
|
|
if (this.runningTasks.has(task.id)) {
|
|
if (isDebugEnabled()) console.log(`[SchedulerService] Task ${task.id} skipped: already running`);
|
|
continue;
|
|
}
|
|
this.runningTasks.add(task.id);
|
|
if (isDebugEnabled()) console.log(`[SchedulerService] Executing task ${task.id} ("${task.name}")`);
|
|
this.executeTask(task).finally(() => this.runningTasks.delete(task.id));
|
|
}
|
|
} catch (error) {
|
|
console.error('[SchedulerService] Tick error:', error);
|
|
} finally {
|
|
this.isProcessing = false;
|
|
}
|
|
}
|
|
|
|
public isTaskRunning(taskId: number): boolean {
|
|
return this.runningTasks.has(taskId);
|
|
}
|
|
|
|
// Intentionally allows triggering disabled tasks, useful for testing before enabling a schedule.
|
|
// Manual triggers are attributed as 'manual' in the run record (see triggered_by column).
|
|
public async triggerTask(taskId: number): Promise<void> {
|
|
const db = DatabaseService.getInstance();
|
|
const task = db.getScheduledTask(taskId);
|
|
if (!task) throw new Error('Task not found');
|
|
if (this.runningTasks.has(task.id)) throw new Error('Task is already running');
|
|
console.log(`[SchedulerService] Manual trigger: task "${task.name}" (id=${task.id})`);
|
|
this.runningTasks.add(task.id);
|
|
try {
|
|
await this.executeTask(task, 'manual');
|
|
} finally {
|
|
this.runningTasks.delete(task.id);
|
|
}
|
|
}
|
|
|
|
private async executeTask(task: ScheduledTask, triggeredBy: 'scheduler' | 'manual' = 'scheduler'): Promise<void> {
|
|
const db = DatabaseService.getInstance();
|
|
const runId = db.createScheduledTaskRun({
|
|
task_id: task.id,
|
|
started_at: Date.now(),
|
|
completed_at: null,
|
|
status: 'running',
|
|
output: null,
|
|
error: null,
|
|
triggered_by: triggeredBy,
|
|
});
|
|
|
|
try {
|
|
// Pre-check: ensure target node exists and is reachable
|
|
if (task.node_id != null && task.action !== 'snapshot') {
|
|
const node = db.getNode(task.node_id);
|
|
if (!node) throw new Error(`Target node (id=${task.node_id}) no longer exists`);
|
|
if (node.status === 'offline') throw new Error(`Target node "${node.name}" is offline`);
|
|
}
|
|
|
|
if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} pre-checks passed, executing ${task.action}`);
|
|
const actionStart = Date.now();
|
|
let output = '';
|
|
let scanFailedCount = 0;
|
|
switch (task.action) {
|
|
case 'restart':
|
|
output = await this.executeRestart(task);
|
|
break;
|
|
case 'snapshot':
|
|
output = await this.executeSnapshot(task);
|
|
break;
|
|
case 'prune':
|
|
output = await this.executePrune(task);
|
|
break;
|
|
case 'update':
|
|
output = await this.executeUpdate(task);
|
|
break;
|
|
case 'scan': {
|
|
const result = await this.executeScan(task);
|
|
output = result.output;
|
|
scanFailedCount = result.failed;
|
|
break;
|
|
}
|
|
case 'auto_backup':
|
|
output = await this.executeAutoBackup(task);
|
|
break;
|
|
case 'auto_stop':
|
|
output = await this.executeAutoStop(task);
|
|
break;
|
|
case 'auto_down':
|
|
output = await this.executeAutoDown(task);
|
|
break;
|
|
case 'auto_start':
|
|
output = await this.executeAutoStart(task);
|
|
break;
|
|
}
|
|
|
|
if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} action completed in ${Date.now() - actionStart}ms`);
|
|
|
|
db.updateScheduledTaskRun(runId, {
|
|
completed_at: Date.now(),
|
|
status: 'success',
|
|
output,
|
|
});
|
|
console.log(`[SchedulerService] Task "${task.name}" (id=${task.id}) completed successfully`);
|
|
|
|
if (task.delete_after_run === 1) {
|
|
console.log(`[SchedulerService] Task "${task.name}" (id=${task.id}) self-deleting after successful one-shot run`);
|
|
db.deleteScheduledTask(task.id);
|
|
return;
|
|
}
|
|
|
|
const nextRun = this.calculateNextRun(task.cron_expression);
|
|
db.updateScheduledTask(task.id, {
|
|
last_run_at: Date.now(),
|
|
next_run_at: nextRun,
|
|
last_status: 'success',
|
|
last_error: null,
|
|
updated_at: Date.now(),
|
|
});
|
|
|
|
if (task.action === 'scan') {
|
|
const scanLevel: 'info' | 'warning' = scanFailedCount > 0 ? 'warning' : 'info';
|
|
if (isDebugEnabled()) {
|
|
console.log(
|
|
`[SchedulerService:debug] Dispatching scan completion notification (level=${scanLevel}, stackContext=${task.target_id ?? 'none'})`,
|
|
);
|
|
}
|
|
this.safeDispatch(
|
|
scanLevel,
|
|
'scan_finding',
|
|
`Scheduled scan "${task.name}" completed: ${output}`,
|
|
task.target_id ?? undefined
|
|
);
|
|
} else if (task.last_status === 'failure') {
|
|
this.safeDispatch(
|
|
'info',
|
|
'system',
|
|
`Scheduled task "${task.name}" (${task.action}) recovered successfully`,
|
|
task.target_id ?? undefined
|
|
);
|
|
}
|
|
} catch (error: unknown) {
|
|
const errMsg = error instanceof Error ? error.message : String(error);
|
|
let nextRun: number | null = null;
|
|
let cronInvalid = false;
|
|
try {
|
|
nextRun = this.calculateNextRun(task.cron_expression);
|
|
} catch {
|
|
cronInvalid = true;
|
|
}
|
|
const updates: Partial<Omit<ScheduledTask, 'id'>> = {
|
|
last_run_at: Date.now(),
|
|
next_run_at: nextRun,
|
|
last_status: 'failure',
|
|
last_error: cronInvalid
|
|
? `${errMsg}. Cron expression "${task.cron_expression}" is no longer valid; task has been disabled.`
|
|
: errMsg,
|
|
updated_at: Date.now(),
|
|
};
|
|
if (cronInvalid) {
|
|
updates.enabled = 0;
|
|
console.warn(`[SchedulerService] Task "${task.name}" (id=${task.id}) auto-disabled: cron expression invalid`);
|
|
}
|
|
db.updateScheduledTask(task.id, updates);
|
|
db.updateScheduledTaskRun(runId, {
|
|
completed_at: Date.now(),
|
|
status: 'failure',
|
|
error: errMsg,
|
|
});
|
|
console.error(`[SchedulerService] Task "${task.name}" (id=${task.id}) failed:`, errMsg);
|
|
this.safeDispatch(
|
|
'error',
|
|
'system',
|
|
`Scheduled task "${task.name}" (${task.action}) failed: ${errMsg}`,
|
|
task.target_id ?? undefined
|
|
);
|
|
}
|
|
}
|
|
|
|
private async executeRestart(task: ScheduledTask): Promise<string> {
|
|
if (!task.target_id || task.node_id == null) {
|
|
throw new Error('Stack restart requires target_id and node_id');
|
|
}
|
|
const docker = DockerController.getInstance(task.node_id);
|
|
const containers = await docker.getContainersByStack(task.target_id);
|
|
if (!containers || containers.length === 0) {
|
|
throw new Error(`No containers found for stack "${task.target_id}"`);
|
|
}
|
|
|
|
let filtered = containers;
|
|
if (task.target_services) {
|
|
const serviceNames: string[] = JSON.parse(task.target_services);
|
|
filtered = containers.filter(c => c.Service && serviceNames.includes(c.Service));
|
|
if (filtered.length === 0) {
|
|
throw new Error(`No containers found matching services [${serviceNames.join(', ')}] in stack "${task.target_id}"`);
|
|
}
|
|
}
|
|
|
|
await Promise.all(filtered.map(c => docker.restartContainer(c.Id)));
|
|
const servicesSuffix = task.target_services
|
|
? ` (services: ${(JSON.parse(task.target_services) as string[]).join(', ')})`
|
|
: '';
|
|
return `Restarted ${filtered.length} container(s) in stack "${task.target_id}"${servicesSuffix}`;
|
|
}
|
|
|
|
private assertStackTarget(task: ScheduledTask, label: string): asserts task is ScheduledTask & { target_id: string; node_id: number } {
|
|
if (!task.target_id || task.node_id == null) {
|
|
throw new Error(`${label} requires target_id and node_id`);
|
|
}
|
|
}
|
|
|
|
private async executeAutoBackup(task: ScheduledTask): Promise<string> {
|
|
this.assertStackTarget(task, 'Auto-backup');
|
|
await FileSystemService.getInstance(task.node_id).backupStackFiles(task.target_id);
|
|
return `Backed up stack "${task.target_id}" files`;
|
|
}
|
|
|
|
private async executeAutoStop(task: ScheduledTask): Promise<string> {
|
|
this.assertStackTarget(task, 'Auto-stop');
|
|
await ComposeService.getInstance(task.node_id).runCommand(task.target_id, 'stop');
|
|
return `Stopped stack "${task.target_id}" (containers preserved)`;
|
|
}
|
|
|
|
private async executeAutoDown(task: ScheduledTask): Promise<string> {
|
|
this.assertStackTarget(task, 'Auto-down');
|
|
await ComposeService.getInstance(task.node_id).runCommand(task.target_id, 'down');
|
|
return `Took down stack "${task.target_id}" (containers removed)`;
|
|
}
|
|
|
|
private async executeAutoStart(task: ScheduledTask): Promise<string> {
|
|
this.assertStackTarget(task, 'Auto-start');
|
|
await assertPolicyGateAllows(
|
|
task.target_id,
|
|
task.node_id,
|
|
buildSystemPolicyGateOptions('scheduler:auto-start', {
|
|
auditPath: `/api/scheduled-tasks/${task.id}/run`,
|
|
}),
|
|
);
|
|
await ComposeService.getInstance(task.node_id).deployStack(task.target_id);
|
|
return `Started stack "${task.target_id}"`;
|
|
}
|
|
|
|
private async executeSnapshot(task: ScheduledTask): Promise<string> {
|
|
const db = DatabaseService.getInstance();
|
|
const nodes = db.getNodes();
|
|
|
|
const results = await Promise.allSettled(
|
|
nodes.map(async (node) => {
|
|
if (node.type === 'remote') {
|
|
return captureRemoteNodeFiles(node);
|
|
}
|
|
return captureLocalNodeFiles(node);
|
|
})
|
|
);
|
|
|
|
const capturedNodes: Array<{ nodeId: number; nodeName: string; stacks: Array<{ stackName: string; files: Array<{ filename: string; content: string }> }> }> = [];
|
|
const skippedNodes: Array<{ nodeId: number; nodeName: string; reason: string }> = [];
|
|
|
|
results.forEach((result, i) => {
|
|
if (result.status === 'fulfilled') {
|
|
capturedNodes.push(result.value);
|
|
} else {
|
|
skippedNodes.push({
|
|
nodeId: nodes[i].id,
|
|
nodeName: nodes[i].name,
|
|
reason: result.reason instanceof Error ? result.reason.message : 'Unknown error',
|
|
});
|
|
}
|
|
});
|
|
|
|
let totalStacks = 0;
|
|
const allFiles: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }> = [];
|
|
|
|
for (const nodeData of capturedNodes) {
|
|
totalStacks += nodeData.stacks.length;
|
|
for (const stack of nodeData.stacks) {
|
|
for (const file of stack.files) {
|
|
allFiles.push({
|
|
nodeId: nodeData.nodeId,
|
|
nodeName: nodeData.nodeName,
|
|
stackName: stack.stackName,
|
|
filename: file.filename,
|
|
content: file.content,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
const description = `Scheduled snapshot: ${task.name}`;
|
|
const snapshotId = db.createSnapshot(
|
|
description,
|
|
task.created_by,
|
|
capturedNodes.length,
|
|
totalStacks,
|
|
JSON.stringify(skippedNodes),
|
|
);
|
|
|
|
if (allFiles.length > 0) {
|
|
db.insertSnapshotFiles(snapshotId, allFiles);
|
|
}
|
|
|
|
let cloudUploadNote = '';
|
|
const cloudSvc = CloudBackupService.getInstance();
|
|
if (cloudSvc.isEnabled() && cloudSvc.isAutoUploadOn()) {
|
|
try {
|
|
await cloudSvc.uploadSnapshot(snapshotId);
|
|
cloudUploadNote = ', cloud upload OK';
|
|
} catch (err) {
|
|
const message = getErrorMessage(err, 'Cloud upload failed');
|
|
console.error('[SchedulerService] Cloud upload failed:', message);
|
|
this.safeDispatch('warning', 'system', `Cloud backup failed for scheduled snapshot ${snapshotId}: ${message}`);
|
|
cloudUploadNote = ', cloud upload FAILED';
|
|
}
|
|
}
|
|
|
|
if (isDebugEnabled()) {
|
|
console.debug(`[SchedulerService:debug] Snapshot task ${task.id}: captured ${capturedNodes.length} node(s), ${totalStacks} stack(s), ${allFiles.length} file(s), skipped ${skippedNodes.length}${cloudUploadNote}`);
|
|
}
|
|
|
|
return `Fleet snapshot created (id=${snapshotId}, ${capturedNodes.length} node(s), ${totalStacks} stack(s)${skippedNodes.length > 0 ? `, ${skippedNodes.length} skipped` : ''}${cloudUploadNote})`;
|
|
}
|
|
|
|
private async executePrune(task: ScheduledTask): Promise<string> {
|
|
const nodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
|
|
if (task.node_id == null && isDebugEnabled()) {
|
|
console.log(`[SchedulerService:debug] Prune task ${task.id}: no node_id specified, using default node ${nodeId}`);
|
|
}
|
|
const docker = DockerController.getInstance(nodeId);
|
|
const allTargets = ['containers', 'images', 'networks', 'volumes'] as const;
|
|
type PruneTarget = typeof allTargets[number];
|
|
const targets: PruneTarget[] = task.prune_targets
|
|
? (JSON.parse(task.prune_targets) as string[]).filter((t): t is PruneTarget => allTargets.includes(t as PruneTarget))
|
|
: [...allTargets];
|
|
const labelFilter = task.prune_label_filter || undefined;
|
|
const results: string[] = [];
|
|
|
|
for (const target of targets) {
|
|
try {
|
|
const result = await docker.pruneSystem(target, labelFilter);
|
|
results.push(`${target}: ${result.reclaimedBytes ?? 0} bytes reclaimed`);
|
|
} catch (error: unknown) {
|
|
const msg = error instanceof Error ? error.message : String(error);
|
|
results.push(`${target}: failed (${msg})`);
|
|
}
|
|
}
|
|
|
|
const filterSuffix = labelFilter ? ` (label: ${labelFilter})` : '';
|
|
return `System prune completed${filterSuffix}: ${results.join('; ')}`;
|
|
}
|
|
|
|
private async executeUpdate(task: ScheduledTask): Promise<string> {
|
|
if (task.node_id == null) {
|
|
throw new Error('Auto-update requires node_id');
|
|
}
|
|
|
|
const isFleet = task.target_type === 'fleet';
|
|
|
|
if (!isFleet && !task.target_id) {
|
|
throw new Error('Auto-update requires target_id (stack name or "*")');
|
|
}
|
|
|
|
// For remote nodes, proxy the entire execution to the remote Sencho instance.
|
|
// The remote /api/auto-update/execute endpoint already handles per-stack
|
|
// auto-update policy, so passing '*' for fleet is sufficient.
|
|
const node = NodeRegistry.getInstance().getNode(task.node_id);
|
|
if (node?.type === 'remote') {
|
|
return this.executeUpdateRemote(task.node_id, isFleet ? '*' : task.target_id!);
|
|
}
|
|
|
|
// Local node: execute directly
|
|
const isWildcard = task.target_id === '*';
|
|
let stackNames: string[];
|
|
if (isFleet || isWildcard) {
|
|
stackNames = await FileSystemService.getInstance(task.node_id).getStacks();
|
|
if (stackNames.length === 0) {
|
|
return 'No stacks found on node; skipped.';
|
|
}
|
|
} else {
|
|
stackNames = [task.target_id!];
|
|
}
|
|
|
|
if (isDebugEnabled()) {
|
|
console.log(`[SchedulerService] executeUpdate: ${stackNames.length} stack(s) to check, fleet=${isFleet}, wildcard=${isWildcard}`);
|
|
}
|
|
|
|
const db = DatabaseService.getInstance();
|
|
const docker = DockerController.getInstance(task.node_id);
|
|
const imageUpdateService = ImageUpdateService.getInstance();
|
|
const compose = ComposeService.getInstance(task.node_id);
|
|
const results: string[] = [];
|
|
|
|
// Single batch query for fleet mode; per-stack default is enabled (true) when no explicit row exists.
|
|
const policyMap = isFleet ? db.getStackAutoUpdateSettingsForNode(task.node_id) : null;
|
|
|
|
for (const stackName of stackNames) {
|
|
try {
|
|
if (isFleet && (policyMap![stackName] ?? true) === false) {
|
|
results.push(`Stack "${stackName}": auto-updates disabled; skipped.`);
|
|
continue;
|
|
}
|
|
const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, compose, db, isFleet || isWildcard);
|
|
results.push(output);
|
|
} catch (e) {
|
|
const msg = getErrorMessage(e, String(e));
|
|
results.push(`Stack "${stackName}" failed: ${msg}`);
|
|
console.error(`[SchedulerService] Auto-update failed for stack "${stackName}":`, e);
|
|
}
|
|
}
|
|
|
|
return results.join('\n');
|
|
}
|
|
|
|
/**
|
|
* Proxy auto-update execution to a remote Sencho instance.
|
|
* The remote node runs the image checks and compose update locally.
|
|
*/
|
|
private async executeUpdateRemote(nodeId: number, target: string): Promise<string> {
|
|
const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId);
|
|
if (!proxyTarget) {
|
|
throw new Error('Remote node is not configured or missing API credentials');
|
|
}
|
|
|
|
const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
|
|
const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
|
|
if (isDebugEnabled()) {
|
|
console.log(`[SchedulerService] executeUpdateRemote: node=${nodeId} target=${target}`);
|
|
}
|
|
const startTime = Date.now();
|
|
const response = await fetch(`${baseUrl}/api/auto-update/execute`, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
'Authorization': `Bearer ${proxyTarget.apiToken}`,
|
|
[PROXY_TIER_HEADER]: proxyHeaders.tier,
|
|
[PROXY_VARIANT_HEADER]: proxyHeaders.variant ?? '',
|
|
},
|
|
body: JSON.stringify({ target }),
|
|
signal: AbortSignal.timeout(300_000), // 5 minute timeout for long updates
|
|
});
|
|
|
|
if (!response.ok) {
|
|
const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` }));
|
|
throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`);
|
|
}
|
|
|
|
const body = await response.json() as { result?: string };
|
|
if (isDebugEnabled()) {
|
|
console.log(`[SchedulerService] executeUpdateRemote: completed in ${Date.now() - startTime}ms`);
|
|
}
|
|
return body.result || 'Remote auto-update completed (no details returned).';
|
|
}
|
|
|
|
private async executeUpdateForStack(
|
|
stackName: string,
|
|
nodeId: number,
|
|
docker: DockerController,
|
|
imageUpdateService: ImageUpdateService,
|
|
compose: ComposeService,
|
|
db: DatabaseService,
|
|
isWildcard = false
|
|
): Promise<string> {
|
|
const containers = await docker.getContainersByStack(stackName);
|
|
if (!containers || containers.length === 0) {
|
|
if (!isWildcard) {
|
|
console.warn(`[SchedulerService] Stack "${stackName}": no containers found. The stack may have been removed or renamed.`);
|
|
return `Stack "${stackName}": WARNING - no containers found. The stack may have been removed or renamed.`;
|
|
}
|
|
return `Stack "${stackName}": no containers found; skipped.`;
|
|
}
|
|
|
|
const imageRefs = [...new Set(
|
|
containers
|
|
.map((c: { Image?: string }) => c.Image)
|
|
.filter((img): img is string => !!img && !img.startsWith('sha256:'))
|
|
)];
|
|
|
|
if (imageRefs.length === 0) {
|
|
return `Stack "${stackName}": no pullable images; skipped.`;
|
|
}
|
|
|
|
if (isDebugEnabled()) {
|
|
console.log(`[SchedulerService] Stack "${stackName}": checking ${imageRefs.length} image(s): ${imageRefs.join(', ')}`);
|
|
}
|
|
|
|
let hasUpdate = false;
|
|
const updatedImages: string[] = [];
|
|
const checkErrors: string[] = [];
|
|
|
|
for (const imageRef of imageRefs) {
|
|
try {
|
|
const result: ImageCheckResult = await imageUpdateService.checkImage(docker, imageRef);
|
|
if (result.error) {
|
|
checkErrors.push(result.error);
|
|
} else if (result.hasUpdate) {
|
|
hasUpdate = true;
|
|
updatedImages.push(imageRef);
|
|
}
|
|
} catch (e) {
|
|
const msg = getErrorMessage(e, String(e));
|
|
checkErrors.push(msg);
|
|
console.warn(`[SchedulerService] Failed to check image ${sanitizeForLog(imageRef)}:`, sanitizeForLog((e as Error)?.message ?? String(e)));
|
|
}
|
|
}
|
|
|
|
if (!hasUpdate) {
|
|
if (checkErrors.length > 0 && checkErrors.length === imageRefs.length) {
|
|
return `Stack "${stackName}": WARNING - all image checks failed (${checkErrors.join('; ')}). Unable to determine update status.`;
|
|
}
|
|
if (checkErrors.length > 0) {
|
|
return `Stack "${stackName}": all reachable images up to date (${checkErrors.length} check(s) failed).`;
|
|
}
|
|
return `Stack "${stackName}": all images up to date.`;
|
|
}
|
|
|
|
await assertPolicyGateAllows(
|
|
stackName,
|
|
nodeId,
|
|
buildSystemPolicyGateOptions('scheduler:auto-update', {
|
|
auditPath: `/api/scheduled-tasks/auto-update/${stackName}`,
|
|
}),
|
|
);
|
|
await compose.updateStack(stackName, undefined, true);
|
|
db.clearStackUpdateStatus(nodeId, stackName);
|
|
|
|
this.safeDispatch(
|
|
'info',
|
|
'image_update_applied',
|
|
`Auto-update: stack "${stackName}" updated with new images`,
|
|
stackName
|
|
);
|
|
|
|
return `Stack "${stackName}": updated (${updatedImages.join(', ')}).`;
|
|
}
|
|
|
|
private async executeScan(task: ScheduledTask): Promise<{ output: string; failed: number }> {
|
|
const trivy = TrivyService.getInstance();
|
|
if (!trivy.isTrivyAvailable()) {
|
|
throw new Error('Trivy binary is not available on this node');
|
|
}
|
|
|
|
const nodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
|
|
if (task.node_id == null && isDebugEnabled()) {
|
|
console.log(`[SchedulerService:debug] Scan task ${task.id}: no node_id specified, using default node ${nodeId}`);
|
|
}
|
|
const node = NodeRegistry.getInstance().getNode(nodeId);
|
|
if (!node) {
|
|
throw new Error('Scheduled vulnerability scans require an existing local node.');
|
|
}
|
|
if (node?.type === 'remote') {
|
|
throw new Error('Scheduled vulnerability scans currently require a local node.');
|
|
}
|
|
|
|
const scanStart = Date.now();
|
|
if (isDebugEnabled()) console.log(`[SchedulerService:debug] executeScan start: task=${task.id} node=${nodeId}`);
|
|
|
|
const summary = await trivy.scanAllNodeImages(nodeId, 'scheduled');
|
|
|
|
if (isDebugEnabled()) {
|
|
console.log(
|
|
`[SchedulerService:debug] executeScan summary: scanned=${summary.scanned} skipped=${summary.skipped} failed=${summary.failed} ` +
|
|
`critical=${summary.severity.critical} high=${summary.severity.high} medium=${summary.severity.medium} ` +
|
|
`low=${summary.severity.low} unknown=${summary.severity.unknown} violations=${summary.violations.length} durationMs=${Date.now() - scanStart}`,
|
|
);
|
|
}
|
|
|
|
// Scheduled scans never auto-quarantine; violations surface as alerts
|
|
// so an operator can review and remediate. One alert per violation so
|
|
// the notification panel keeps per-image granularity.
|
|
for (const v of summary.violations ?? []) {
|
|
NotificationService.getInstance().dispatchAlert(
|
|
'warning',
|
|
'scan_finding',
|
|
`Policy "${v.policyName}" violated by ${v.imageRef}: ${v.severity} exceeds ${v.maxSeverity}`,
|
|
);
|
|
}
|
|
|
|
const output = formatScanOutput(summary);
|
|
return { output, failed: summary.failed };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Build the human-readable completion message from a bulk scan summary.
|
|
* Exported for unit tests.
|
|
*/
|
|
export function formatScanOutput(summary: ScanAllNodeImagesResult): string {
|
|
const { scanned, skipped, failed, severity } = summary;
|
|
|
|
let header: string;
|
|
if (scanned === 0 && skipped === 0 && failed === 0) {
|
|
header = 'No images to scan';
|
|
} else if (scanned === 0 && skipped > 0 && failed === 0) {
|
|
header = `All ${skipped} image(s) already scanned recently (cache hit)`;
|
|
} else {
|
|
const parts: string[] = [`Scanned ${scanned} image(s)`];
|
|
if (skipped > 0) parts.push(`${skipped} skipped (cached)`);
|
|
if (failed > 0) parts.push(`${failed} failed`);
|
|
header = parts.join('; ');
|
|
}
|
|
|
|
if (summary.truncated) {
|
|
const total = summary.totalImages ?? scanned + skipped + failed;
|
|
const processed = summary.processedImages ?? scanned + skipped + failed;
|
|
header += `. Scan limited after ${processed} of ${total} image(s)`;
|
|
if (summary.limitReason) header += ` (${summary.limitReason})`;
|
|
}
|
|
|
|
const severityTiers: Array<[string, number]> = [
|
|
['critical', severity.critical],
|
|
['high', severity.high],
|
|
['medium', severity.medium],
|
|
];
|
|
const nonZero = severityTiers.filter(([, n]) => n > 0);
|
|
if (nonZero.length === 0) {
|
|
if (scanned === 0 && skipped === 0 && failed === 0) {
|
|
return header + '.';
|
|
}
|
|
return `${header}. No critical, high, or medium findings.`;
|
|
}
|
|
const findings = nonZero.map(([label, n]) => `${n} ${label}`).join(', ');
|
|
return `${header}. Found ${findings}.`;
|
|
}
|