mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-06 17:08:10 +00:00
Anso
29ed0524c1
Enrich scheduled vulnerability scan completion notifications with per-severity CVE counts so recipients can triage from the message body alone. Expose the scan action in the schedule creation UI, require an explicit node_id, and harden fire-and-forget alert dispatches so a failing webhook cannot crash the scheduler. Notification body now reports scanned/skipped/failed counts plus critical/high/medium totals aggregated across fresh and cached scans, reflecting the current node posture rather than only what was newly scanned on this run.
690 lines
29 KiB
TypeScript
690 lines
29 KiB
TypeScript
import { CronExpressionParser } from 'cron-parser';
|
|
import { DatabaseService } from './DatabaseService';
|
|
import type { ScheduledTask } from './DatabaseService';
|
|
import { LicenseService } from './LicenseService';
|
|
import DockerController from './DockerController';
|
|
import { ComposeService } from './ComposeService';
|
|
import { FileSystemService } from './FileSystemService';
|
|
import { ImageUpdateService } from './ImageUpdateService';
|
|
import type { ImageCheckResult } from './ImageUpdateService';
|
|
import { isDebugEnabled } from '../utils/debug';
|
|
import { getErrorMessage } from '../utils/errors';
|
|
import { captureLocalNodeFiles, captureRemoteNodeFiles } from '../utils/snapshot-capture';
|
|
import { NodeRegistry } from './NodeRegistry';
|
|
import { NotificationService } from './NotificationService';
|
|
import TrivyService from './TrivyService';
|
|
import type { ScanAllNodeImagesResult } from './TrivyService';
|
|
import TrivyInstaller from './TrivyInstaller';
|
|
|
|
const TRIVY_UPDATE_CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
const TRIVY_UPDATE_CHECK_STARTUP_DELAY_MS = 5 * 60 * 1000;
|
|
|
|
const TRIVY_REDETECT_INTERVAL_MS = 10 * 60 * 1000;
|
|
const STALE_SCAN_THRESHOLD_MS = 15 * 60 * 1000;
|
|
|
|
export class SchedulerService {
|
|
private static instance: SchedulerService;
|
|
private intervalId: ReturnType<typeof setInterval> | null = null;
|
|
private trivyUpdateIntervalId: ReturnType<typeof setInterval> | null = null;
|
|
private trivyUpdateStartupTimer: ReturnType<typeof setTimeout> | null = null;
|
|
private isProcessing = false;
|
|
private isCheckingTrivyUpdate = false;
|
|
private runningTasks = new Set<number>();
|
|
private lastTrivyRedetect = 0;
|
|
|
|
private constructor() {}
|
|
|
|
public static getInstance(): SchedulerService {
|
|
if (!SchedulerService.instance) {
|
|
SchedulerService.instance = new SchedulerService();
|
|
}
|
|
return SchedulerService.instance;
|
|
}
|
|
|
|
public start(): void {
|
|
if (this.intervalId) return;
|
|
this.cleanupStaleRuns();
|
|
this.intervalId = setInterval(() => this.tick(), 60_000);
|
|
setTimeout(() => this.tick(), 10_000);
|
|
this.trivyUpdateStartupTimer = setTimeout(() => this.runTrivyUpdateCheck(), TRIVY_UPDATE_CHECK_STARTUP_DELAY_MS);
|
|
this.trivyUpdateIntervalId = setInterval(() => this.runTrivyUpdateCheck(), TRIVY_UPDATE_CHECK_INTERVAL_MS);
|
|
console.log('[SchedulerService] Started');
|
|
}
|
|
|
|
public stop(): void {
|
|
if (this.intervalId) {
|
|
clearInterval(this.intervalId);
|
|
this.intervalId = null;
|
|
}
|
|
if (this.trivyUpdateIntervalId) {
|
|
clearInterval(this.trivyUpdateIntervalId);
|
|
this.trivyUpdateIntervalId = null;
|
|
}
|
|
if (this.trivyUpdateStartupTimer) {
|
|
clearTimeout(this.trivyUpdateStartupTimer);
|
|
this.trivyUpdateStartupTimer = null;
|
|
}
|
|
console.log('[SchedulerService] Stopped');
|
|
}
|
|
|
|
private async runTrivyUpdateCheck(): Promise<void> {
|
|
if (this.isCheckingTrivyUpdate) return;
|
|
this.isCheckingTrivyUpdate = true;
|
|
try {
|
|
const trivy = TrivyService.getInstance();
|
|
if (!trivy.isTrivyAvailable() || trivy.getSource() !== 'managed') return;
|
|
const db = DatabaseService.getInstance();
|
|
const settings = db.getGlobalSettings();
|
|
const autoUpdate = settings.trivy_auto_update === '1';
|
|
const installer = TrivyInstaller.getInstance();
|
|
if (installer.isBusy()) return;
|
|
const check = await installer.checkForUpdate(trivy.getVersion(), 'managed');
|
|
if (!check.updateAvailable) return;
|
|
|
|
if (autoUpdate) {
|
|
const previous = trivy.getVersion() ?? 'unknown';
|
|
console.log(`[SchedulerService] Auto-updating Trivy from ${previous} to ${check.latest}`);
|
|
try {
|
|
await installer.update();
|
|
await trivy.detectTrivy();
|
|
this.safeDispatch(
|
|
'info',
|
|
`Trivy updated from v${previous} to v${check.latest}`,
|
|
);
|
|
db.updateGlobalSetting('trivy_last_notified_version', check.latest);
|
|
} catch (err) {
|
|
console.error('[SchedulerService] Trivy auto-update failed:', getErrorMessage(err, 'unknown error'));
|
|
}
|
|
} else {
|
|
const lastNotified = settings.trivy_last_notified_version || '';
|
|
if (lastNotified === check.latest) return;
|
|
this.safeDispatch(
|
|
'info',
|
|
`Trivy update available: v${check.latest} (currently v${check.current ?? 'unknown'})`,
|
|
);
|
|
db.updateGlobalSetting('trivy_last_notified_version', check.latest);
|
|
}
|
|
} catch (err) {
|
|
console.warn('[SchedulerService] Trivy update check failed:', getErrorMessage(err, 'unknown error'));
|
|
} finally {
|
|
this.isCheckingTrivyUpdate = false;
|
|
}
|
|
}
|
|
|
|
private cleanupStaleRuns(): void {
|
|
try {
|
|
const count = DatabaseService.getInstance().markStaleRunsAsFailed();
|
|
if (count > 0) {
|
|
console.log(`[SchedulerService] Cleaned up ${count} stale run record(s)`);
|
|
}
|
|
} catch (error) {
|
|
console.error('[SchedulerService] Failed to clean up stale runs:', error);
|
|
}
|
|
}
|
|
|
|
private async maybeRedetectTrivy(): Promise<void> {
|
|
const now = Date.now();
|
|
if (now - this.lastTrivyRedetect < TRIVY_REDETECT_INTERVAL_MS) return;
|
|
this.lastTrivyRedetect = now;
|
|
try {
|
|
await TrivyService.getInstance().detectTrivy();
|
|
} catch (error) {
|
|
if (isDebugEnabled()) {
|
|
console.warn('[SchedulerService:debug] Trivy re-detect failed:', error);
|
|
}
|
|
}
|
|
}
|
|
|
|
public calculateNextRun(cronExpression: string): number {
|
|
const expr = CronExpressionParser.parse(cronExpression);
|
|
return expr.next().toDate().getTime();
|
|
}
|
|
|
|
/**
|
|
* Fire a notification without awaiting completion, catching any promise
|
|
* rejection so the scheduler never crashes on a failed dispatch.
|
|
*/
|
|
private safeDispatch(level: 'info' | 'warning' | 'error', message: string, stackName?: string): void {
|
|
NotificationService.getInstance()
|
|
.dispatchAlert(level, message, stackName)
|
|
.catch(err => console.error('[SchedulerService] Notification dispatch failed:', getErrorMessage(err, 'unknown error')));
|
|
}
|
|
|
|
private async tick(): Promise<void> {
|
|
if (this.isProcessing) {
|
|
console.warn('[SchedulerService] Tick skipped: previous tick still processing');
|
|
return;
|
|
}
|
|
this.isProcessing = true;
|
|
try {
|
|
const db = DatabaseService.getInstance();
|
|
|
|
// Vulnerability scanning is available on every tier, so the stale-scan sweep
|
|
// and Trivy re-detect run before the paid-tier gate below.
|
|
try {
|
|
const staleScans = db.markStaleScansAsFailed(STALE_SCAN_THRESHOLD_MS);
|
|
if (staleScans > 0) {
|
|
console.log(
|
|
`[SchedulerService] Marked ${staleScans} stale vulnerability scan(s) as failed`,
|
|
);
|
|
}
|
|
} catch (error) {
|
|
console.error('[SchedulerService] Stale scan sweep failed:', error);
|
|
}
|
|
await this.maybeRedetectTrivy();
|
|
|
|
const ls = LicenseService.getInstance();
|
|
const isPaid = ls.getTier() === 'paid';
|
|
const isAdmiral = isPaid && ls.getVariant() === 'admiral';
|
|
if (!isPaid) return;
|
|
|
|
const now = Date.now();
|
|
const dueTasks = db.getDueScheduledTasks(now);
|
|
|
|
if (dueTasks.length > 0) {
|
|
console.log(`[SchedulerService] Found ${dueTasks.length} due task(s)`);
|
|
}
|
|
|
|
// Clean up old runs periodically (piggyback on tick)
|
|
db.cleanupOldTaskRuns(30);
|
|
db.deleteOldScans(90 * 24 * 60 * 60 * 1000);
|
|
|
|
for (const task of dueTasks) {
|
|
if (!isAdmiral && task.action !== 'update' && task.action !== 'scan') {
|
|
if (isDebugEnabled()) console.log(`[SchedulerService] Task ${task.id} skipped: action "${task.action}" requires Admiral tier`);
|
|
continue;
|
|
}
|
|
if (this.runningTasks.has(task.id)) {
|
|
if (isDebugEnabled()) console.log(`[SchedulerService] Task ${task.id} skipped: already running`);
|
|
continue;
|
|
}
|
|
this.runningTasks.add(task.id);
|
|
if (isDebugEnabled()) console.log(`[SchedulerService] Executing task ${task.id} ("${task.name}")`);
|
|
this.executeTask(task).finally(() => this.runningTasks.delete(task.id));
|
|
}
|
|
} catch (error) {
|
|
console.error('[SchedulerService] Tick error:', error);
|
|
} finally {
|
|
this.isProcessing = false;
|
|
}
|
|
}
|
|
|
|
public isTaskRunning(taskId: number): boolean {
|
|
return this.runningTasks.has(taskId);
|
|
}
|
|
|
|
// Intentionally allows triggering disabled tasks, useful for testing before enabling a schedule.
|
|
// Manual triggers are attributed as 'manual' in the run record (see triggered_by column).
|
|
public async triggerTask(taskId: number): Promise<void> {
|
|
const db = DatabaseService.getInstance();
|
|
const task = db.getScheduledTask(taskId);
|
|
if (!task) throw new Error('Task not found');
|
|
if (this.runningTasks.has(task.id)) throw new Error('Task is already running');
|
|
console.log(`[SchedulerService] Manual trigger: task "${task.name}" (id=${task.id})`);
|
|
this.runningTasks.add(task.id);
|
|
try {
|
|
await this.executeTask(task, 'manual');
|
|
} finally {
|
|
this.runningTasks.delete(task.id);
|
|
}
|
|
}
|
|
|
|
private async executeTask(task: ScheduledTask, triggeredBy: 'scheduler' | 'manual' = 'scheduler'): Promise<void> {
|
|
const db = DatabaseService.getInstance();
|
|
const runId = db.createScheduledTaskRun({
|
|
task_id: task.id,
|
|
started_at: Date.now(),
|
|
completed_at: null,
|
|
status: 'running',
|
|
output: null,
|
|
error: null,
|
|
triggered_by: triggeredBy,
|
|
});
|
|
|
|
try {
|
|
// Pre-check: ensure target node exists and is reachable
|
|
if (task.node_id != null && task.action !== 'snapshot') {
|
|
const node = db.getNode(task.node_id);
|
|
if (!node) throw new Error(`Target node (id=${task.node_id}) no longer exists`);
|
|
if (node.status === 'offline') throw new Error(`Target node "${node.name}" is offline`);
|
|
}
|
|
|
|
if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} pre-checks passed, executing ${task.action}`);
|
|
const actionStart = Date.now();
|
|
let output = '';
|
|
let scanFailedCount = 0;
|
|
switch (task.action) {
|
|
case 'restart':
|
|
output = await this.executeRestart(task);
|
|
break;
|
|
case 'snapshot':
|
|
output = await this.executeSnapshot(task);
|
|
break;
|
|
case 'prune':
|
|
output = await this.executePrune(task);
|
|
break;
|
|
case 'update':
|
|
output = await this.executeUpdate(task);
|
|
break;
|
|
case 'scan': {
|
|
const result = await this.executeScan(task);
|
|
output = result.output;
|
|
scanFailedCount = result.failed;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} action completed in ${Date.now() - actionStart}ms`);
|
|
|
|
const nextRun = this.calculateNextRun(task.cron_expression);
|
|
db.updateScheduledTask(task.id, {
|
|
last_run_at: Date.now(),
|
|
next_run_at: nextRun,
|
|
last_status: 'success',
|
|
last_error: null,
|
|
updated_at: Date.now(),
|
|
});
|
|
db.updateScheduledTaskRun(runId, {
|
|
completed_at: Date.now(),
|
|
status: 'success',
|
|
output,
|
|
});
|
|
console.log(`[SchedulerService] Task "${task.name}" (id=${task.id}) completed successfully`);
|
|
if (task.action === 'scan') {
|
|
const scanLevel: 'info' | 'warning' = scanFailedCount > 0 ? 'warning' : 'info';
|
|
if (isDebugEnabled()) {
|
|
console.log(
|
|
`[SchedulerService:debug] Dispatching scan completion notification (level=${scanLevel}, stackContext=${task.target_id ?? 'none'})`,
|
|
);
|
|
}
|
|
this.safeDispatch(
|
|
scanLevel,
|
|
`Scheduled scan "${task.name}" completed: ${output}`,
|
|
task.target_id ?? undefined
|
|
);
|
|
} else if (task.last_status === 'failure') {
|
|
this.safeDispatch(
|
|
'info',
|
|
`Scheduled task "${task.name}" (${task.action}) recovered successfully`,
|
|
task.target_id ?? undefined
|
|
);
|
|
}
|
|
} catch (error: unknown) {
|
|
const errMsg = error instanceof Error ? error.message : String(error);
|
|
let nextRun: number | null = null;
|
|
let cronInvalid = false;
|
|
try {
|
|
nextRun = this.calculateNextRun(task.cron_expression);
|
|
} catch {
|
|
cronInvalid = true;
|
|
}
|
|
const updates: Partial<Omit<ScheduledTask, 'id'>> = {
|
|
last_run_at: Date.now(),
|
|
next_run_at: nextRun,
|
|
last_status: 'failure',
|
|
last_error: cronInvalid
|
|
? `${errMsg}. Cron expression "${task.cron_expression}" is no longer valid; task has been disabled.`
|
|
: errMsg,
|
|
updated_at: Date.now(),
|
|
};
|
|
if (cronInvalid) {
|
|
updates.enabled = 0;
|
|
console.warn(`[SchedulerService] Task "${task.name}" (id=${task.id}) auto-disabled: cron expression invalid`);
|
|
}
|
|
db.updateScheduledTask(task.id, updates);
|
|
db.updateScheduledTaskRun(runId, {
|
|
completed_at: Date.now(),
|
|
status: 'failure',
|
|
error: errMsg,
|
|
});
|
|
console.error(`[SchedulerService] Task "${task.name}" (id=${task.id}) failed:`, errMsg);
|
|
this.safeDispatch(
|
|
'error',
|
|
`Scheduled task "${task.name}" (${task.action}) failed: ${errMsg}`,
|
|
task.target_id ?? undefined
|
|
);
|
|
}
|
|
}
|
|
|
|
private async executeRestart(task: ScheduledTask): Promise<string> {
|
|
if (!task.target_id || task.node_id == null) {
|
|
throw new Error('Stack restart requires target_id and node_id');
|
|
}
|
|
const docker = DockerController.getInstance(task.node_id);
|
|
const containers = await docker.getContainersByStack(task.target_id);
|
|
if (!containers || containers.length === 0) {
|
|
throw new Error(`No containers found for stack "${task.target_id}"`);
|
|
}
|
|
|
|
let filtered = containers;
|
|
if (task.target_services) {
|
|
const serviceNames: string[] = JSON.parse(task.target_services);
|
|
filtered = containers.filter(c => c.Service && serviceNames.includes(c.Service));
|
|
if (filtered.length === 0) {
|
|
throw new Error(`No containers found matching services [${serviceNames.join(', ')}] in stack "${task.target_id}"`);
|
|
}
|
|
}
|
|
|
|
await Promise.all(filtered.map(c => docker.restartContainer(c.Id)));
|
|
const servicesSuffix = task.target_services
|
|
? ` (services: ${(JSON.parse(task.target_services) as string[]).join(', ')})`
|
|
: '';
|
|
return `Restarted ${filtered.length} container(s) in stack "${task.target_id}"${servicesSuffix}`;
|
|
}
|
|
|
|
private async executeSnapshot(task: ScheduledTask): Promise<string> {
|
|
const db = DatabaseService.getInstance();
|
|
const nodes = db.getNodes();
|
|
|
|
const results = await Promise.allSettled(
|
|
nodes.map(async (node) => {
|
|
if (node.type === 'remote') {
|
|
return captureRemoteNodeFiles(node);
|
|
}
|
|
return captureLocalNodeFiles(node);
|
|
})
|
|
);
|
|
|
|
const capturedNodes: Array<{ nodeId: number; nodeName: string; stacks: Array<{ stackName: string; files: Array<{ filename: string; content: string }> }> }> = [];
|
|
const skippedNodes: Array<{ nodeId: number; nodeName: string; reason: string }> = [];
|
|
|
|
results.forEach((result, i) => {
|
|
if (result.status === 'fulfilled') {
|
|
capturedNodes.push(result.value);
|
|
} else {
|
|
skippedNodes.push({
|
|
nodeId: nodes[i].id,
|
|
nodeName: nodes[i].name,
|
|
reason: result.reason instanceof Error ? result.reason.message : 'Unknown error',
|
|
});
|
|
}
|
|
});
|
|
|
|
let totalStacks = 0;
|
|
const allFiles: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }> = [];
|
|
|
|
for (const nodeData of capturedNodes) {
|
|
totalStacks += nodeData.stacks.length;
|
|
for (const stack of nodeData.stacks) {
|
|
for (const file of stack.files) {
|
|
allFiles.push({
|
|
nodeId: nodeData.nodeId,
|
|
nodeName: nodeData.nodeName,
|
|
stackName: stack.stackName,
|
|
filename: file.filename,
|
|
content: file.content,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
const description = `Scheduled snapshot: ${task.name}`;
|
|
const snapshotId = db.createSnapshot(
|
|
description,
|
|
task.created_by,
|
|
capturedNodes.length,
|
|
totalStacks,
|
|
JSON.stringify(skippedNodes),
|
|
);
|
|
|
|
if (allFiles.length > 0) {
|
|
db.insertSnapshotFiles(snapshotId, allFiles);
|
|
}
|
|
|
|
if (isDebugEnabled()) {
|
|
console.debug(`[SchedulerService:debug] Snapshot task ${task.id}: captured ${capturedNodes.length} node(s), ${totalStacks} stack(s), ${allFiles.length} file(s), skipped ${skippedNodes.length}`);
|
|
}
|
|
|
|
return `Fleet snapshot created (id=${snapshotId}, ${capturedNodes.length} node(s), ${totalStacks} stack(s)${skippedNodes.length > 0 ? `, ${skippedNodes.length} skipped` : ''})`;
|
|
}
|
|
|
|
private async executePrune(task: ScheduledTask): Promise<string> {
|
|
const nodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
|
|
if (task.node_id == null && isDebugEnabled()) {
|
|
console.log(`[SchedulerService:debug] Prune task ${task.id}: no node_id specified, using default node ${nodeId}`);
|
|
}
|
|
const docker = DockerController.getInstance(nodeId);
|
|
const allTargets = ['containers', 'images', 'networks', 'volumes'] as const;
|
|
type PruneTarget = typeof allTargets[number];
|
|
const targets: PruneTarget[] = task.prune_targets
|
|
? (JSON.parse(task.prune_targets) as string[]).filter((t): t is PruneTarget => allTargets.includes(t as PruneTarget))
|
|
: [...allTargets];
|
|
const labelFilter = task.prune_label_filter || undefined;
|
|
const results: string[] = [];
|
|
|
|
for (const target of targets) {
|
|
try {
|
|
const result = await docker.pruneSystem(target, labelFilter);
|
|
results.push(`${target}: ${result.reclaimedBytes ?? 0} bytes reclaimed`);
|
|
} catch (error: unknown) {
|
|
const msg = error instanceof Error ? error.message : String(error);
|
|
results.push(`${target}: failed (${msg})`);
|
|
}
|
|
}
|
|
|
|
const filterSuffix = labelFilter ? ` (label: ${labelFilter})` : '';
|
|
return `System prune completed${filterSuffix}: ${results.join('; ')}`;
|
|
}
|
|
|
|
private async executeUpdate(task: ScheduledTask): Promise<string> {
|
|
if (!task.target_id || task.node_id == null) {
|
|
throw new Error('Auto-update requires target_id (stack name or "*") and node_id');
|
|
}
|
|
|
|
// For remote nodes, proxy the entire execution to the remote Sencho instance
|
|
const node = NodeRegistry.getInstance().getNode(task.node_id);
|
|
if (node?.type === 'remote') {
|
|
return this.executeUpdateRemote(task.node_id, task.target_id);
|
|
}
|
|
|
|
// Local node: execute directly
|
|
const isWildcard = task.target_id === '*';
|
|
let stackNames: string[];
|
|
if (isWildcard) {
|
|
stackNames = await FileSystemService.getInstance(task.node_id).getStacks();
|
|
if (stackNames.length === 0) {
|
|
return 'No stacks found on node; skipped.';
|
|
}
|
|
} else {
|
|
stackNames = [task.target_id];
|
|
}
|
|
|
|
if (isDebugEnabled()) {
|
|
console.log(`[SchedulerService] executeUpdate: ${stackNames.length} stack(s) to check, wildcard=${isWildcard}`);
|
|
}
|
|
|
|
const docker = DockerController.getInstance(task.node_id);
|
|
const imageUpdateService = ImageUpdateService.getInstance();
|
|
const compose = ComposeService.getInstance(task.node_id);
|
|
const db = DatabaseService.getInstance();
|
|
const results: string[] = [];
|
|
|
|
for (const stackName of stackNames) {
|
|
try {
|
|
const output = await this.executeUpdateForStack(stackName, task.node_id ?? 0, docker, imageUpdateService, compose, db, isWildcard);
|
|
results.push(output);
|
|
} catch (e) {
|
|
const msg = getErrorMessage(e, String(e));
|
|
results.push(`Stack "${stackName}" failed: ${msg}`);
|
|
console.error(`[SchedulerService] Auto-update failed for stack "${stackName}":`, e);
|
|
}
|
|
}
|
|
|
|
return results.join('\n');
|
|
}
|
|
|
|
/**
|
|
* Proxy auto-update execution to a remote Sencho instance.
|
|
* The remote node runs the image checks and compose update locally.
|
|
*/
|
|
private async executeUpdateRemote(nodeId: number, target: string): Promise<string> {
|
|
const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId);
|
|
if (!proxyTarget) {
|
|
throw new Error('Remote node is not configured or missing API credentials');
|
|
}
|
|
|
|
const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
|
|
if (isDebugEnabled()) {
|
|
console.log(`[SchedulerService] executeUpdateRemote: node=${nodeId} target=${target}`);
|
|
}
|
|
const startTime = Date.now();
|
|
const response = await fetch(`${baseUrl}/api/auto-update/execute`, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
'Authorization': `Bearer ${proxyTarget.apiToken}`,
|
|
},
|
|
body: JSON.stringify({ target }),
|
|
signal: AbortSignal.timeout(300_000), // 5 minute timeout for long updates
|
|
});
|
|
|
|
if (!response.ok) {
|
|
const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` }));
|
|
throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`);
|
|
}
|
|
|
|
const body = await response.json() as { result?: string };
|
|
if (isDebugEnabled()) {
|
|
console.log(`[SchedulerService] executeUpdateRemote: completed in ${Date.now() - startTime}ms`);
|
|
}
|
|
return body.result || 'Remote auto-update completed (no details returned).';
|
|
}
|
|
|
|
private async executeUpdateForStack(
|
|
stackName: string,
|
|
nodeId: number,
|
|
docker: DockerController,
|
|
imageUpdateService: ImageUpdateService,
|
|
compose: ComposeService,
|
|
db: DatabaseService,
|
|
isWildcard = false
|
|
): Promise<string> {
|
|
const containers = await docker.getContainersByStack(stackName);
|
|
if (!containers || containers.length === 0) {
|
|
if (!isWildcard) {
|
|
console.warn(`[SchedulerService] Stack "${stackName}": no containers found. The stack may have been removed or renamed.`);
|
|
return `Stack "${stackName}": WARNING - no containers found. The stack may have been removed or renamed.`;
|
|
}
|
|
return `Stack "${stackName}": no containers found; skipped.`;
|
|
}
|
|
|
|
const imageRefs = [...new Set(
|
|
containers
|
|
.map((c: { Image?: string }) => c.Image)
|
|
.filter((img): img is string => !!img && !img.startsWith('sha256:'))
|
|
)];
|
|
|
|
if (imageRefs.length === 0) {
|
|
return `Stack "${stackName}": no pullable images; skipped.`;
|
|
}
|
|
|
|
if (isDebugEnabled()) {
|
|
console.log(`[SchedulerService] Stack "${stackName}": checking ${imageRefs.length} image(s): ${imageRefs.join(', ')}`);
|
|
}
|
|
|
|
let hasUpdate = false;
|
|
const updatedImages: string[] = [];
|
|
const checkErrors: string[] = [];
|
|
|
|
for (const imageRef of imageRefs) {
|
|
try {
|
|
const result: ImageCheckResult = await imageUpdateService.checkImage(docker, imageRef);
|
|
if (result.error) {
|
|
checkErrors.push(result.error);
|
|
} else if (result.hasUpdate) {
|
|
hasUpdate = true;
|
|
updatedImages.push(imageRef);
|
|
}
|
|
} catch (e) {
|
|
const msg = getErrorMessage(e, String(e));
|
|
checkErrors.push(msg);
|
|
console.warn(`[SchedulerService] Failed to check image ${imageRef}:`, e);
|
|
}
|
|
}
|
|
|
|
if (!hasUpdate) {
|
|
if (checkErrors.length > 0 && checkErrors.length === imageRefs.length) {
|
|
return `Stack "${stackName}": WARNING - all image checks failed (${checkErrors.join('; ')}). Unable to determine update status.`;
|
|
}
|
|
if (checkErrors.length > 0) {
|
|
return `Stack "${stackName}": all reachable images up to date (${checkErrors.length} check(s) failed).`;
|
|
}
|
|
return `Stack "${stackName}": all images up to date.`;
|
|
}
|
|
|
|
await compose.updateStack(stackName, undefined, true);
|
|
db.clearStackUpdateStatus(nodeId, stackName);
|
|
|
|
this.safeDispatch(
|
|
'info',
|
|
`Auto-update: stack "${stackName}" updated with new images`,
|
|
stackName
|
|
);
|
|
|
|
return `Stack "${stackName}": updated (${updatedImages.join(', ')}).`;
|
|
}
|
|
|
|
private async executeScan(task: ScheduledTask): Promise<{ output: string; failed: number }> {
|
|
const trivy = TrivyService.getInstance();
|
|
if (!trivy.isTrivyAvailable()) {
|
|
throw new Error('Trivy binary is not available on this node');
|
|
}
|
|
|
|
const nodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
|
|
if (task.node_id == null && isDebugEnabled()) {
|
|
console.log(`[SchedulerService:debug] Scan task ${task.id}: no node_id specified, using default node ${nodeId}`);
|
|
}
|
|
|
|
const scanStart = Date.now();
|
|
if (isDebugEnabled()) console.log(`[SchedulerService:debug] executeScan start: task=${task.id} node=${nodeId}`);
|
|
|
|
const summary = await trivy.scanAllNodeImages(nodeId, 'scheduled');
|
|
|
|
if (isDebugEnabled()) {
|
|
console.log(
|
|
`[SchedulerService:debug] executeScan summary: scanned=${summary.scanned} skipped=${summary.skipped} failed=${summary.failed} ` +
|
|
`critical=${summary.severity.critical} high=${summary.severity.high} medium=${summary.severity.medium} ` +
|
|
`low=${summary.severity.low} unknown=${summary.severity.unknown} durationMs=${Date.now() - scanStart}`,
|
|
);
|
|
}
|
|
|
|
const output = formatScanOutput(summary);
|
|
return { output, failed: summary.failed };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Build the human-readable completion message from a bulk scan summary.
|
|
* Exported for unit tests.
|
|
*/
|
|
export function formatScanOutput(summary: ScanAllNodeImagesResult): string {
|
|
const { scanned, skipped, failed, severity } = summary;
|
|
|
|
let header: string;
|
|
if (scanned === 0 && skipped === 0 && failed === 0) {
|
|
header = 'No images to scan';
|
|
} else if (scanned === 0 && skipped > 0 && failed === 0) {
|
|
header = `All ${skipped} image(s) already scanned recently (cache hit)`;
|
|
} else {
|
|
const parts: string[] = [`Scanned ${scanned} image(s)`];
|
|
if (skipped > 0) parts.push(`${skipped} skipped (cached)`);
|
|
if (failed > 0) parts.push(`${failed} failed`);
|
|
header = parts.join('; ');
|
|
}
|
|
|
|
const severityTiers: Array<[string, number]> = [
|
|
['critical', severity.critical],
|
|
['high', severity.high],
|
|
['medium', severity.medium],
|
|
];
|
|
const nonZero = severityTiers.filter(([, n]) => n > 0);
|
|
if (nonZero.length === 0) {
|
|
if (scanned === 0 && skipped === 0 && failed === 0) {
|
|
return header + '.';
|
|
}
|
|
return `${header}. No critical, high, or medium findings.`;
|
|
}
|
|
const findings = nonZero.map(([label, n]) => `${n} ${label}`).join(', ');
|
|
return `${header}. Found ${findings}.`;
|
|
}
|