mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-05 16:37:46 +00:00
Anso
836 lines
36 KiB
TypeScript
836 lines
36 KiB
TypeScript
import { CronExpressionParser } from 'cron-parser';
|
|
import { DatabaseService } from './DatabaseService';
|
|
import type { ScheduledTask } from './DatabaseService';
|
|
import { LicenseService } from './LicenseService';
|
|
import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from './license-headers';
|
|
import DockerController from './DockerController';
|
|
import { ComposeService } from './ComposeService';
|
|
import { FileSystemService } from './FileSystemService';
|
|
import { ImageUpdateService } from './ImageUpdateService';
|
|
import type { ImageCheckResult } from './ImageUpdateService';
|
|
import { isDebugEnabled } from '../utils/debug';
|
|
import { getErrorMessage } from '../utils/errors';
|
|
import { sanitizeForLog } from '../utils/safeLog';
|
|
import { captureLocalNodeFiles, captureRemoteNodeFiles } from '../utils/snapshot-capture';
|
|
import { NodeRegistry } from './NodeRegistry';
|
|
import { NotificationService } from './NotificationService';
|
|
import TrivyService from './TrivyService';
|
|
import type { ScanAllNodeImagesResult } from './TrivyService';
|
|
import TrivyInstaller from './TrivyInstaller';
|
|
import { CloudBackupService } from './CloudBackupService';
|
|
import { assertPolicyGateAllows, buildSystemPolicyGateOptions } from '../helpers/policyGate';
|
|
|
|
const TRIVY_UPDATE_CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
const TRIVY_UPDATE_CHECK_STARTUP_DELAY_MS = 5 * 60 * 1000;
|
|
|
|
const TRIVY_REDETECT_INTERVAL_MS = 10 * 60 * 1000;
|
|
const STALE_SCAN_THRESHOLD_MS = 15 * 60 * 1000;
|
|
|
|
export class SchedulerService {
|
|
private static instance: SchedulerService;
|
|
private intervalId: ReturnType<typeof setInterval> | null = null;
|
|
private trivyUpdateIntervalId: ReturnType<typeof setInterval> | null = null;
|
|
private trivyUpdateStartupTimer: ReturnType<typeof setTimeout> | null = null;
|
|
private isProcessing = false;
|
|
private isCheckingTrivyUpdate = false;
|
|
private runningTasks = new Set<number>();
|
|
private lastTrivyRedetect = 0;
|
|
|
|
private constructor() {}
|
|
|
|
public static getInstance(): SchedulerService {
|
|
if (!SchedulerService.instance) {
|
|
SchedulerService.instance = new SchedulerService();
|
|
}
|
|
return SchedulerService.instance;
|
|
}
|
|
|
|
public start(): void {
|
|
if (this.intervalId) return;
|
|
this.cleanupStaleRuns();
|
|
this.intervalId = setInterval(() => this.tick(), 60_000);
|
|
setTimeout(() => this.tick(), 10_000);
|
|
this.trivyUpdateStartupTimer = setTimeout(() => this.runTrivyUpdateCheck(), TRIVY_UPDATE_CHECK_STARTUP_DELAY_MS);
|
|
this.trivyUpdateIntervalId = setInterval(() => this.runTrivyUpdateCheck(), TRIVY_UPDATE_CHECK_INTERVAL_MS);
|
|
console.log('[SchedulerService] Started');
|
|
}
|
|
|
|
public stop(): void {
|
|
if (this.intervalId) {
|
|
clearInterval(this.intervalId);
|
|
this.intervalId = null;
|
|
}
|
|
if (this.trivyUpdateIntervalId) {
|
|
clearInterval(this.trivyUpdateIntervalId);
|
|
this.trivyUpdateIntervalId = null;
|
|
}
|
|
if (this.trivyUpdateStartupTimer) {
|
|
clearTimeout(this.trivyUpdateStartupTimer);
|
|
this.trivyUpdateStartupTimer = null;
|
|
}
|
|
console.log('[SchedulerService] Stopped');
|
|
}
|
|
|
|
private async runTrivyUpdateCheck(): Promise<void> {
|
|
if (this.isCheckingTrivyUpdate) return;
|
|
this.isCheckingTrivyUpdate = true;
|
|
try {
|
|
const trivy = TrivyService.getInstance();
|
|
if (!trivy.isTrivyAvailable() || trivy.getSource() !== 'managed') return;
|
|
const db = DatabaseService.getInstance();
|
|
const settings = db.getGlobalSettings();
|
|
const autoUpdate = settings.trivy_auto_update === '1';
|
|
const installer = TrivyInstaller.getInstance();
|
|
if (installer.isBusy()) return;
|
|
const check = await installer.checkForUpdate(trivy.getVersion(), 'managed');
|
|
if (!check.updateAvailable) return;
|
|
|
|
if (autoUpdate) {
|
|
const previous = trivy.getVersion() ?? 'unknown';
|
|
console.log(`[SchedulerService] Auto-updating Trivy from ${previous} to ${check.latest}`);
|
|
try {
|
|
await installer.update();
|
|
await trivy.detectTrivy();
|
|
this.safeDispatch(
|
|
'info',
|
|
'system',
|
|
`Trivy updated from v${previous} to v${check.latest}`,
|
|
);
|
|
db.updateGlobalSetting('trivy_last_notified_version', check.latest);
|
|
} catch (err) {
|
|
console.error('[SchedulerService] Trivy auto-update failed:', getErrorMessage(err, 'unknown error'));
|
|
}
|
|
} else {
|
|
const lastNotified = settings.trivy_last_notified_version || '';
|
|
if (lastNotified === check.latest) return;
|
|
this.safeDispatch(
|
|
'info',
|
|
'system',
|
|
`Trivy update available: v${check.latest} (currently v${check.current ?? 'unknown'})`,
|
|
);
|
|
db.updateGlobalSetting('trivy_last_notified_version', check.latest);
|
|
}
|
|
} catch (err) {
|
|
console.warn('[SchedulerService] Trivy update check failed:', getErrorMessage(err, 'unknown error'));
|
|
} finally {
|
|
this.isCheckingTrivyUpdate = false;
|
|
}
|
|
}
|
|
|
|
private cleanupStaleRuns(): void {
|
|
try {
|
|
const count = DatabaseService.getInstance().markStaleRunsAsFailed();
|
|
if (count > 0) {
|
|
console.log(`[SchedulerService] Cleaned up ${count} stale run record(s)`);
|
|
}
|
|
} catch (error) {
|
|
console.error('[SchedulerService] Failed to clean up stale runs:', error);
|
|
}
|
|
}
|
|
|
|
private async maybeRedetectTrivy(): Promise<void> {
|
|
const now = Date.now();
|
|
if (now - this.lastTrivyRedetect < TRIVY_REDETECT_INTERVAL_MS) return;
|
|
this.lastTrivyRedetect = now;
|
|
try {
|
|
await TrivyService.getInstance().detectTrivy();
|
|
} catch (error) {
|
|
if (isDebugEnabled()) {
|
|
console.warn('[SchedulerService:debug] Trivy re-detect failed:', error);
|
|
}
|
|
}
|
|
}
|
|
|
|
public calculateNextRun(cronExpression: string): number {
|
|
const expr = CronExpressionParser.parse(cronExpression);
|
|
return expr.next().toDate().getTime();
|
|
}
|
|
|
|
public calculateRunsWithin(cronExpression: string, fromMs: number, toMs: number, limit = 16): number[] {
|
|
try {
|
|
const expr = CronExpressionParser.parse(cronExpression, { currentDate: new Date(fromMs) });
|
|
const runs: number[] = [];
|
|
while (runs.length < limit) {
|
|
const next = expr.next().toDate().getTime();
|
|
if (next > toMs) break;
|
|
runs.push(next);
|
|
}
|
|
return runs;
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Fire a notification without awaiting completion, catching any promise
|
|
* rejection so the scheduler never crashes on a failed dispatch.
|
|
*/
|
|
private safeDispatch(level: 'info' | 'warning' | 'error', category: import('./NotificationService').NotificationCategory, message: string, stackName?: string): void {
|
|
NotificationService.getInstance()
|
|
.dispatchAlert(level, category, message, { stackName })
|
|
.catch(err => console.error('[SchedulerService] Notification dispatch failed:', getErrorMessage(err, 'unknown error')));
|
|
}
|
|
|
|
private async tick(): Promise<void> {
|
|
if (this.isProcessing) {
|
|
console.warn('[SchedulerService] Tick skipped: previous tick still processing');
|
|
return;
|
|
}
|
|
this.isProcessing = true;
|
|
try {
|
|
const db = DatabaseService.getInstance();
|
|
|
|
// Vulnerability scanning is available on every tier, so the stale-scan sweep
|
|
// and Trivy re-detect run before the paid-tier gate below.
|
|
try {
|
|
const staleScans = db.markStaleScansAsFailed(STALE_SCAN_THRESHOLD_MS);
|
|
if (staleScans > 0) {
|
|
console.log(
|
|
`[SchedulerService] Marked ${staleScans} stale vulnerability scan(s) as failed`,
|
|
);
|
|
}
|
|
} catch (error) {
|
|
console.error('[SchedulerService] Stale scan sweep failed:', error);
|
|
}
|
|
await this.maybeRedetectTrivy();
|
|
|
|
const ls = LicenseService.getInstance();
|
|
const isPaid = ls.getTier() === 'paid';
|
|
const isAdmiral = isPaid && ls.getVariant() === 'admiral';
|
|
if (!isPaid) return;
|
|
|
|
const now = Date.now();
|
|
const dueTasks = db.getDueScheduledTasks(now);
|
|
|
|
if (dueTasks.length > 0) {
|
|
console.log(`[SchedulerService] Found ${dueTasks.length} due task(s)`);
|
|
}
|
|
|
|
// Clean up old runs periodically (piggyback on tick)
|
|
db.cleanupOldTaskRuns(30);
|
|
db.deleteOldScans(90 * 24 * 60 * 60 * 1000);
|
|
|
|
for (const task of dueTasks) {
|
|
if (!isAdmiral && task.action !== 'update' && task.action !== 'scan' && task.action !== 'snapshot') {
|
|
if (isDebugEnabled()) console.log(`[SchedulerService] Task ${task.id} skipped: action "${task.action}" requires Admiral tier`);
|
|
continue;
|
|
}
|
|
if (this.runningTasks.has(task.id)) {
|
|
if (isDebugEnabled()) console.log(`[SchedulerService] Task ${task.id} skipped: already running`);
|
|
continue;
|
|
}
|
|
this.runningTasks.add(task.id);
|
|
if (isDebugEnabled()) console.log(`[SchedulerService] Executing task ${task.id} ("${task.name}")`);
|
|
this.executeTask(task).finally(() => this.runningTasks.delete(task.id));
|
|
}
|
|
} catch (error) {
|
|
console.error('[SchedulerService] Tick error:', error);
|
|
} finally {
|
|
this.isProcessing = false;
|
|
}
|
|
}
|
|
|
|
public isTaskRunning(taskId: number): boolean {
|
|
return this.runningTasks.has(taskId);
|
|
}
|
|
|
|
// Intentionally allows triggering disabled tasks, useful for testing before enabling a schedule.
|
|
// Manual triggers are attributed as 'manual' in the run record (see triggered_by column).
|
|
public async triggerTask(taskId: number): Promise<void> {
|
|
const db = DatabaseService.getInstance();
|
|
const task = db.getScheduledTask(taskId);
|
|
if (!task) throw new Error('Task not found');
|
|
if (this.runningTasks.has(task.id)) throw new Error('Task is already running');
|
|
console.log(`[SchedulerService] Manual trigger: task "${task.name}" (id=${task.id})`);
|
|
this.runningTasks.add(task.id);
|
|
try {
|
|
await this.executeTask(task, 'manual');
|
|
} finally {
|
|
this.runningTasks.delete(task.id);
|
|
}
|
|
}
|
|
|
|
private async executeTask(task: ScheduledTask, triggeredBy: 'scheduler' | 'manual' = 'scheduler'): Promise<void> {
|
|
const db = DatabaseService.getInstance();
|
|
const runId = db.createScheduledTaskRun({
|
|
task_id: task.id,
|
|
started_at: Date.now(),
|
|
completed_at: null,
|
|
status: 'running',
|
|
output: null,
|
|
error: null,
|
|
triggered_by: triggeredBy,
|
|
});
|
|
|
|
try {
|
|
// Pre-check: ensure target node exists and is reachable
|
|
if (task.node_id != null && task.action !== 'snapshot') {
|
|
const node = db.getNode(task.node_id);
|
|
if (!node) throw new Error(`Target node (id=${task.node_id}) no longer exists`);
|
|
if (node.status === 'offline') throw new Error(`Target node "${node.name}" is offline`);
|
|
}
|
|
|
|
if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} pre-checks passed, executing ${task.action}`);
|
|
const actionStart = Date.now();
|
|
let output = '';
|
|
let scanFailedCount = 0;
|
|
switch (task.action) {
|
|
case 'restart':
|
|
output = await this.executeRestart(task);
|
|
break;
|
|
case 'snapshot':
|
|
output = await this.executeSnapshot(task);
|
|
break;
|
|
case 'prune':
|
|
output = await this.executePrune(task);
|
|
break;
|
|
case 'update':
|
|
output = await this.executeUpdate(task);
|
|
break;
|
|
case 'scan': {
|
|
const result = await this.executeScan(task);
|
|
output = result.output;
|
|
scanFailedCount = result.failed;
|
|
break;
|
|
}
|
|
case 'auto_backup':
|
|
output = await this.executeAutoBackup(task);
|
|
break;
|
|
case 'auto_stop':
|
|
output = await this.executeAutoStop(task);
|
|
break;
|
|
case 'auto_down':
|
|
output = await this.executeAutoDown(task);
|
|
break;
|
|
case 'auto_start':
|
|
output = await this.executeAutoStart(task);
|
|
break;
|
|
}
|
|
|
|
if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} action completed in ${Date.now() - actionStart}ms`);
|
|
|
|
db.updateScheduledTaskRun(runId, {
|
|
completed_at: Date.now(),
|
|
status: 'success',
|
|
output,
|
|
});
|
|
console.log(`[SchedulerService] Task "${task.name}" (id=${task.id}) completed successfully`);
|
|
|
|
if (task.delete_after_run === 1) {
|
|
console.log(`[SchedulerService] Task "${task.name}" (id=${task.id}) self-deleting after successful one-shot run`);
|
|
db.deleteScheduledTask(task.id);
|
|
return;
|
|
}
|
|
|
|
const nextRun = this.calculateNextRun(task.cron_expression);
|
|
db.updateScheduledTask(task.id, {
|
|
last_run_at: Date.now(),
|
|
next_run_at: nextRun,
|
|
last_status: 'success',
|
|
last_error: null,
|
|
updated_at: Date.now(),
|
|
});
|
|
|
|
if (task.action === 'scan') {
|
|
const scanLevel: 'info' | 'warning' = scanFailedCount > 0 ? 'warning' : 'info';
|
|
if (isDebugEnabled()) {
|
|
console.log(
|
|
`[SchedulerService:debug] Dispatching scan completion notification (level=${scanLevel}, stackContext=${task.target_id ?? 'none'})`,
|
|
);
|
|
}
|
|
this.safeDispatch(
|
|
scanLevel,
|
|
'scan_finding',
|
|
`Scheduled scan "${task.name}" completed: ${output}`,
|
|
task.target_id ?? undefined
|
|
);
|
|
} else if (task.last_status === 'failure') {
|
|
this.safeDispatch(
|
|
'info',
|
|
'system',
|
|
`Scheduled task "${task.name}" (${task.action}) recovered successfully`,
|
|
task.target_id ?? undefined
|
|
);
|
|
}
|
|
} catch (error: unknown) {
|
|
const errMsg = error instanceof Error ? error.message : String(error);
|
|
let nextRun: number | null = null;
|
|
let cronInvalid = false;
|
|
try {
|
|
nextRun = this.calculateNextRun(task.cron_expression);
|
|
} catch {
|
|
cronInvalid = true;
|
|
}
|
|
const updates: Partial<Omit<ScheduledTask, 'id'>> = {
|
|
last_run_at: Date.now(),
|
|
next_run_at: nextRun,
|
|
last_status: 'failure',
|
|
last_error: cronInvalid
|
|
? `${errMsg}. Cron expression "${task.cron_expression}" is no longer valid; task has been disabled.`
|
|
: errMsg,
|
|
updated_at: Date.now(),
|
|
};
|
|
if (cronInvalid) {
|
|
updates.enabled = 0;
|
|
console.warn(`[SchedulerService] Task "${task.name}" (id=${task.id}) auto-disabled: cron expression invalid`);
|
|
}
|
|
db.updateScheduledTask(task.id, updates);
|
|
db.updateScheduledTaskRun(runId, {
|
|
completed_at: Date.now(),
|
|
status: 'failure',
|
|
error: errMsg,
|
|
});
|
|
console.error(`[SchedulerService] Task "${task.name}" (id=${task.id}) failed:`, errMsg);
|
|
this.safeDispatch(
|
|
'error',
|
|
'system',
|
|
`Scheduled task "${task.name}" (${task.action}) failed: ${errMsg}`,
|
|
task.target_id ?? undefined
|
|
);
|
|
}
|
|
}
|
|
|
|
private async executeRestart(task: ScheduledTask): Promise<string> {
|
|
if (!task.target_id || task.node_id == null) {
|
|
throw new Error('Stack restart requires target_id and node_id');
|
|
}
|
|
const docker = DockerController.getInstance(task.node_id);
|
|
const containers = await docker.getContainersByStack(task.target_id);
|
|
if (!containers || containers.length === 0) {
|
|
throw new Error(`No containers found for stack "${task.target_id}"`);
|
|
}
|
|
|
|
let filtered = containers;
|
|
if (task.target_services) {
|
|
const serviceNames: string[] = JSON.parse(task.target_services);
|
|
filtered = containers.filter(c => c.Service && serviceNames.includes(c.Service));
|
|
if (filtered.length === 0) {
|
|
throw new Error(`No containers found matching services [${serviceNames.join(', ')}] in stack "${task.target_id}"`);
|
|
}
|
|
}
|
|
|
|
await Promise.all(filtered.map(c => docker.restartContainer(c.Id)));
|
|
const servicesSuffix = task.target_services
|
|
? ` (services: ${(JSON.parse(task.target_services) as string[]).join(', ')})`
|
|
: '';
|
|
return `Restarted ${filtered.length} container(s) in stack "${task.target_id}"${servicesSuffix}`;
|
|
}
|
|
|
|
private assertStackTarget(task: ScheduledTask, label: string): asserts task is ScheduledTask & { target_id: string; node_id: number } {
|
|
if (!task.target_id || task.node_id == null) {
|
|
throw new Error(`${label} requires target_id and node_id`);
|
|
}
|
|
}
|
|
|
|
private async executeAutoBackup(task: ScheduledTask): Promise<string> {
|
|
this.assertStackTarget(task, 'Auto-backup');
|
|
await FileSystemService.getInstance(task.node_id).backupStackFiles(task.target_id);
|
|
return `Backed up stack "${task.target_id}" files`;
|
|
}
|
|
|
|
private async executeAutoStop(task: ScheduledTask): Promise<string> {
|
|
this.assertStackTarget(task, 'Auto-stop');
|
|
await ComposeService.getInstance(task.node_id).runCommand(task.target_id, 'stop');
|
|
return `Stopped stack "${task.target_id}" (containers preserved)`;
|
|
}
|
|
|
|
private async executeAutoDown(task: ScheduledTask): Promise<string> {
|
|
this.assertStackTarget(task, 'Auto-down');
|
|
await ComposeService.getInstance(task.node_id).runCommand(task.target_id, 'down');
|
|
return `Took down stack "${task.target_id}" (containers removed)`;
|
|
}
|
|
|
|
private async executeAutoStart(task: ScheduledTask): Promise<string> {
|
|
this.assertStackTarget(task, 'Auto-start');
|
|
await assertPolicyGateAllows(
|
|
task.target_id,
|
|
task.node_id,
|
|
buildSystemPolicyGateOptions('scheduler:auto-start', {
|
|
auditPath: `/api/scheduled-tasks/${task.id}/run`,
|
|
}),
|
|
);
|
|
await ComposeService.getInstance(task.node_id).deployStack(task.target_id);
|
|
return `Started stack "${task.target_id}"`;
|
|
}
|
|
|
|
private async executeSnapshot(task: ScheduledTask): Promise<string> {
|
|
const db = DatabaseService.getInstance();
|
|
const nodes = db.getNodes();
|
|
|
|
const results = await Promise.allSettled(
|
|
nodes.map(async (node) => {
|
|
if (node.type === 'remote') {
|
|
return captureRemoteNodeFiles(node);
|
|
}
|
|
return captureLocalNodeFiles(node);
|
|
})
|
|
);
|
|
|
|
const capturedNodes: Array<{ nodeId: number; nodeName: string; stacks: Array<{ stackName: string; files: Array<{ filename: string; content: string }> }> }> = [];
|
|
const skippedNodes: Array<{ nodeId: number; nodeName: string; reason: string }> = [];
|
|
|
|
results.forEach((result, i) => {
|
|
if (result.status === 'fulfilled') {
|
|
capturedNodes.push(result.value);
|
|
} else {
|
|
skippedNodes.push({
|
|
nodeId: nodes[i].id,
|
|
nodeName: nodes[i].name,
|
|
reason: result.reason instanceof Error ? result.reason.message : 'Unknown error',
|
|
});
|
|
}
|
|
});
|
|
|
|
let totalStacks = 0;
|
|
const allFiles: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }> = [];
|
|
|
|
for (const nodeData of capturedNodes) {
|
|
totalStacks += nodeData.stacks.length;
|
|
for (const stack of nodeData.stacks) {
|
|
for (const file of stack.files) {
|
|
allFiles.push({
|
|
nodeId: nodeData.nodeId,
|
|
nodeName: nodeData.nodeName,
|
|
stackName: stack.stackName,
|
|
filename: file.filename,
|
|
content: file.content,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
const description = `Scheduled snapshot: ${task.name}`;
|
|
const snapshotId = db.createSnapshot(
|
|
description,
|
|
task.created_by,
|
|
capturedNodes.length,
|
|
totalStacks,
|
|
JSON.stringify(skippedNodes),
|
|
);
|
|
|
|
if (allFiles.length > 0) {
|
|
db.insertSnapshotFiles(snapshotId, allFiles);
|
|
}
|
|
|
|
let cloudUploadNote = '';
|
|
const cloudSvc = CloudBackupService.getInstance();
|
|
if (cloudSvc.isEnabled() && cloudSvc.isAutoUploadOn()) {
|
|
try {
|
|
await cloudSvc.uploadSnapshot(snapshotId);
|
|
cloudUploadNote = ', cloud upload OK';
|
|
} catch (err) {
|
|
const message = getErrorMessage(err, 'Cloud upload failed');
|
|
console.error('[SchedulerService] Cloud upload failed:', message);
|
|
this.safeDispatch('warning', 'system', `Cloud backup failed for scheduled snapshot ${snapshotId}: ${message}`);
|
|
cloudUploadNote = ', cloud upload FAILED';
|
|
}
|
|
}
|
|
|
|
if (isDebugEnabled()) {
|
|
console.debug(`[SchedulerService:debug] Snapshot task ${task.id}: captured ${capturedNodes.length} node(s), ${totalStacks} stack(s), ${allFiles.length} file(s), skipped ${skippedNodes.length}${cloudUploadNote}`);
|
|
}
|
|
|
|
return `Fleet snapshot created (id=${snapshotId}, ${capturedNodes.length} node(s), ${totalStacks} stack(s)${skippedNodes.length > 0 ? `, ${skippedNodes.length} skipped` : ''}${cloudUploadNote})`;
|
|
}
|
|
|
|
private async executePrune(task: ScheduledTask): Promise<string> {
|
|
const nodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
|
|
if (task.node_id == null && isDebugEnabled()) {
|
|
console.log(`[SchedulerService:debug] Prune task ${task.id}: no node_id specified, using default node ${nodeId}`);
|
|
}
|
|
const docker = DockerController.getInstance(nodeId);
|
|
const allTargets = ['containers', 'images', 'networks', 'volumes'] as const;
|
|
type PruneTarget = typeof allTargets[number];
|
|
const targets: PruneTarget[] = task.prune_targets
|
|
? (JSON.parse(task.prune_targets) as string[]).filter((t): t is PruneTarget => allTargets.includes(t as PruneTarget))
|
|
: [...allTargets];
|
|
const labelFilter = task.prune_label_filter || undefined;
|
|
const results: string[] = [];
|
|
|
|
for (const target of targets) {
|
|
try {
|
|
const result = await docker.pruneSystem(target, labelFilter);
|
|
results.push(`${target}: ${result.reclaimedBytes ?? 0} bytes reclaimed`);
|
|
} catch (error: unknown) {
|
|
const msg = error instanceof Error ? error.message : String(error);
|
|
results.push(`${target}: failed (${msg})`);
|
|
}
|
|
}
|
|
|
|
const filterSuffix = labelFilter ? ` (label: ${labelFilter})` : '';
|
|
return `System prune completed${filterSuffix}: ${results.join('; ')}`;
|
|
}
|
|
|
|
private async executeUpdate(task: ScheduledTask): Promise<string> {
|
|
if (task.node_id == null) {
|
|
throw new Error('Auto-update requires node_id');
|
|
}
|
|
|
|
const isFleet = task.target_type === 'fleet';
|
|
|
|
if (!isFleet && !task.target_id) {
|
|
throw new Error('Auto-update requires target_id (stack name or "*")');
|
|
}
|
|
|
|
// For remote nodes, proxy the entire execution to the remote Sencho instance.
|
|
// The remote /api/auto-update/execute endpoint already handles per-stack
|
|
// auto-update policy, so passing '*' for fleet is sufficient.
|
|
const node = NodeRegistry.getInstance().getNode(task.node_id);
|
|
if (node?.type === 'remote') {
|
|
return this.executeUpdateRemote(task.node_id, isFleet ? '*' : task.target_id!);
|
|
}
|
|
|
|
// Local node: execute directly
|
|
const isWildcard = task.target_id === '*';
|
|
let stackNames: string[];
|
|
if (isFleet || isWildcard) {
|
|
stackNames = await FileSystemService.getInstance(task.node_id).getStacks();
|
|
if (stackNames.length === 0) {
|
|
return 'No stacks found on node; skipped.';
|
|
}
|
|
} else {
|
|
stackNames = [task.target_id!];
|
|
}
|
|
|
|
if (isDebugEnabled()) {
|
|
console.log(`[SchedulerService] executeUpdate: ${stackNames.length} stack(s) to check, fleet=${isFleet}, wildcard=${isWildcard}`);
|
|
}
|
|
|
|
const db = DatabaseService.getInstance();
|
|
const docker = DockerController.getInstance(task.node_id);
|
|
const imageUpdateService = ImageUpdateService.getInstance();
|
|
const compose = ComposeService.getInstance(task.node_id);
|
|
const results: string[] = [];
|
|
|
|
// Single batch query for fleet mode; per-stack default is enabled (true) when no explicit row exists.
|
|
const policyMap = isFleet ? db.getStackAutoUpdateSettingsForNode(task.node_id) : null;
|
|
|
|
for (const stackName of stackNames) {
|
|
try {
|
|
if (isFleet && (policyMap![stackName] ?? true) === false) {
|
|
results.push(`Stack "${stackName}": auto-updates disabled; skipped.`);
|
|
continue;
|
|
}
|
|
const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, compose, db, isFleet || isWildcard);
|
|
results.push(output);
|
|
} catch (e) {
|
|
const msg = getErrorMessage(e, String(e));
|
|
results.push(`Stack "${stackName}" failed: ${msg}`);
|
|
console.error(`[SchedulerService] Auto-update failed for stack "${stackName}":`, e);
|
|
}
|
|
}
|
|
|
|
return results.join('\n');
|
|
}
|
|
|
|
/**
|
|
* Proxy auto-update execution to a remote Sencho instance.
|
|
* The remote node runs the image checks and compose update locally.
|
|
*/
|
|
private async executeUpdateRemote(nodeId: number, target: string): Promise<string> {
|
|
const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId);
|
|
if (!proxyTarget) {
|
|
throw new Error('Remote node is not configured or missing API credentials');
|
|
}
|
|
|
|
const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
|
|
const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
|
|
if (isDebugEnabled()) {
|
|
console.log(`[SchedulerService] executeUpdateRemote: node=${nodeId} target=${target}`);
|
|
}
|
|
const startTime = Date.now();
|
|
const response = await fetch(`${baseUrl}/api/auto-update/execute`, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
'Authorization': `Bearer ${proxyTarget.apiToken}`,
|
|
[PROXY_TIER_HEADER]: proxyHeaders.tier,
|
|
[PROXY_VARIANT_HEADER]: proxyHeaders.variant ?? '',
|
|
},
|
|
body: JSON.stringify({ target }),
|
|
signal: AbortSignal.timeout(300_000), // 5 minute timeout for long updates
|
|
});
|
|
|
|
if (!response.ok) {
|
|
const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` }));
|
|
throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`);
|
|
}
|
|
|
|
const body = await response.json() as { result?: string };
|
|
if (isDebugEnabled()) {
|
|
console.log(`[SchedulerService] executeUpdateRemote: completed in ${Date.now() - startTime}ms`);
|
|
}
|
|
return body.result || 'Remote auto-update completed (no details returned).';
|
|
}
|
|
|
|
private async executeUpdateForStack(
|
|
stackName: string,
|
|
nodeId: number,
|
|
docker: DockerController,
|
|
imageUpdateService: ImageUpdateService,
|
|
compose: ComposeService,
|
|
db: DatabaseService,
|
|
isWildcard = false
|
|
): Promise<string> {
|
|
const containers = await docker.getContainersByStack(stackName);
|
|
if (!containers || containers.length === 0) {
|
|
if (!isWildcard) {
|
|
console.warn(`[SchedulerService] Stack "${stackName}": no containers found. The stack may have been removed or renamed.`);
|
|
return `Stack "${stackName}": WARNING - no containers found. The stack may have been removed or renamed.`;
|
|
}
|
|
return `Stack "${stackName}": no containers found; skipped.`;
|
|
}
|
|
|
|
const imageRefs = [...new Set(
|
|
containers
|
|
.map((c: { Image?: string }) => c.Image)
|
|
.filter((img): img is string => !!img && !img.startsWith('sha256:'))
|
|
)];
|
|
|
|
if (imageRefs.length === 0) {
|
|
return `Stack "${stackName}": no pullable images; skipped.`;
|
|
}
|
|
|
|
if (isDebugEnabled()) {
|
|
console.log(`[SchedulerService] Stack "${stackName}": checking ${imageRefs.length} image(s): ${imageRefs.join(', ')}`);
|
|
}
|
|
|
|
let hasUpdate = false;
|
|
const updatedImages: string[] = [];
|
|
const checkErrors: string[] = [];
|
|
|
|
for (const imageRef of imageRefs) {
|
|
try {
|
|
const result: ImageCheckResult = await imageUpdateService.checkImage(docker, imageRef);
|
|
if (result.error) {
|
|
checkErrors.push(result.error);
|
|
} else if (result.hasUpdate) {
|
|
hasUpdate = true;
|
|
updatedImages.push(imageRef);
|
|
}
|
|
} catch (e) {
|
|
const msg = getErrorMessage(e, String(e));
|
|
checkErrors.push(msg);
|
|
console.warn(`[SchedulerService] Failed to check image ${sanitizeForLog(imageRef)}:`, sanitizeForLog((e as Error)?.message ?? String(e)));
|
|
}
|
|
}
|
|
|
|
if (!hasUpdate) {
|
|
if (checkErrors.length > 0 && checkErrors.length === imageRefs.length) {
|
|
return `Stack "${stackName}": WARNING - all image checks failed (${checkErrors.join('; ')}). Unable to determine update status.`;
|
|
}
|
|
if (checkErrors.length > 0) {
|
|
return `Stack "${stackName}": all reachable images up to date (${checkErrors.length} check(s) failed).`;
|
|
}
|
|
return `Stack "${stackName}": all images up to date.`;
|
|
}
|
|
|
|
await assertPolicyGateAllows(
|
|
stackName,
|
|
nodeId,
|
|
buildSystemPolicyGateOptions('scheduler:auto-update', {
|
|
auditPath: `/api/scheduled-tasks/auto-update/${stackName}`,
|
|
}),
|
|
);
|
|
await compose.updateStack(stackName, undefined, true);
|
|
db.clearStackUpdateStatus(nodeId, stackName);
|
|
|
|
this.safeDispatch(
|
|
'info',
|
|
'image_update_applied',
|
|
`Auto-update: stack "${stackName}" updated with new images`,
|
|
stackName
|
|
);
|
|
|
|
return `Stack "${stackName}": updated (${updatedImages.join(', ')}).`;
|
|
}
|
|
|
|
private async executeScan(task: ScheduledTask): Promise<{ output: string; failed: number }> {
|
|
const trivy = TrivyService.getInstance();
|
|
if (!trivy.isTrivyAvailable()) {
|
|
throw new Error('Trivy binary is not available on this node');
|
|
}
|
|
|
|
const nodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
|
|
if (task.node_id == null && isDebugEnabled()) {
|
|
console.log(`[SchedulerService:debug] Scan task ${task.id}: no node_id specified, using default node ${nodeId}`);
|
|
}
|
|
const node = NodeRegistry.getInstance().getNode(nodeId);
|
|
if (!node) {
|
|
throw new Error('Scheduled vulnerability scans require an existing local node.');
|
|
}
|
|
if (node?.type === 'remote') {
|
|
throw new Error('Scheduled vulnerability scans currently require a local node.');
|
|
}
|
|
|
|
const scanStart = Date.now();
|
|
if (isDebugEnabled()) console.log(`[SchedulerService:debug] executeScan start: task=${task.id} node=${nodeId}`);
|
|
|
|
const summary = await trivy.scanAllNodeImages(nodeId, 'scheduled');
|
|
|
|
if (isDebugEnabled()) {
|
|
console.log(
|
|
`[SchedulerService:debug] executeScan summary: scanned=${summary.scanned} skipped=${summary.skipped} failed=${summary.failed} ` +
|
|
`critical=${summary.severity.critical} high=${summary.severity.high} medium=${summary.severity.medium} ` +
|
|
`low=${summary.severity.low} unknown=${summary.severity.unknown} violations=${summary.violations.length} durationMs=${Date.now() - scanStart}`,
|
|
);
|
|
}
|
|
|
|
// Scheduled scans never auto-quarantine; violations surface as alerts
|
|
// so an operator can review and remediate. One alert per violation so
|
|
// the notification panel keeps per-image granularity.
|
|
for (const v of summary.violations ?? []) {
|
|
NotificationService.getInstance().dispatchAlert(
|
|
'warning',
|
|
'scan_finding',
|
|
`Policy "${v.policyName}" violated by ${v.imageRef}: ${v.severity} exceeds ${v.maxSeverity}`,
|
|
);
|
|
}
|
|
|
|
const output = formatScanOutput(summary);
|
|
return { output, failed: summary.failed };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Build the human-readable completion message from a bulk scan summary.
|
|
* Exported for unit tests.
|
|
*/
|
|
export function formatScanOutput(summary: ScanAllNodeImagesResult): string {
|
|
const { scanned, skipped, failed, severity } = summary;
|
|
|
|
let header: string;
|
|
if (scanned === 0 && skipped === 0 && failed === 0) {
|
|
header = 'No images to scan';
|
|
} else if (scanned === 0 && skipped > 0 && failed === 0) {
|
|
header = `All ${skipped} image(s) already scanned recently (cache hit)`;
|
|
} else {
|
|
const parts: string[] = [`Scanned ${scanned} image(s)`];
|
|
if (skipped > 0) parts.push(`${skipped} skipped (cached)`);
|
|
if (failed > 0) parts.push(`${failed} failed`);
|
|
header = parts.join('; ');
|
|
}
|
|
|
|
if (summary.truncated) {
|
|
const total = summary.totalImages ?? scanned + skipped + failed;
|
|
const processed = summary.processedImages ?? scanned + skipped + failed;
|
|
header += `. Scan limited after ${processed} of ${total} image(s)`;
|
|
if (summary.limitReason) header += ` (${summary.limitReason})`;
|
|
}
|
|
|
|
const severityTiers: Array<[string, number]> = [
|
|
['critical', severity.critical],
|
|
['high', severity.high],
|
|
['medium', severity.medium],
|
|
];
|
|
const nonZero = severityTiers.filter(([, n]) => n > 0);
|
|
if (nonZero.length === 0) {
|
|
if (scanned === 0 && skipped === 0 && failed === 0) {
|
|
return header + '.';
|
|
}
|
|
return `${header}. No critical, high, or medium findings.`;
|
|
}
|
|
const findings = nonZero.map(([label, n]) => `${n} ${label}`).join(', ');
|
|
return `${header}. Found ${findings}.`;
|
|
}
|