feat(scheduled-ops): add scheduled operations for Team Pro users (#231)

Adds the ability to schedule recurring Docker operations (stack restarts,
fleet snapshots, system prunes) via cron expressions with full execution
history logging. Includes Run Now for on-demand execution.
This commit is contained in:
Anso
2026-03-28 23:56:37 -04:00
committed by GitHub
parent 89c4a9cb77
commit 31e1795af0
31 changed files with 1552 additions and 160 deletions
+1 -1
View File
@@ -242,7 +242,7 @@ describe('SSO Config CRUD (DB layer)', () => {
});
});
describe('Database migration SSO columns', () => {
describe('Database migration - SSO columns', () => {
it('users table has auth_provider, provider_id, email columns', async () => {
const { DatabaseService } = await import('../services/DatabaseService');
const db = DatabaseService.getInstance();
+258 -3
View File
@@ -18,7 +18,7 @@ import httpProxy from 'http-proxy';
import { createProxyMiddleware } from 'http-proxy-middleware';
import path from 'path';
import { HostTerminalService } from './services/HostTerminalService';
import { DatabaseService, Node, AuthProvider } from './services/DatabaseService';
import { DatabaseService, Node, AuthProvider, ScheduledTask } from './services/DatabaseService';
import { NotificationService } from './services/NotificationService';
import { MonitorService } from './services/MonitorService';
import { ImageUpdateService } from './services/ImageUpdateService';
@@ -28,6 +28,8 @@ import { NodeRegistry } from './services/NodeRegistry';
import { LicenseService } from './services/LicenseService';
import { WebhookService } from './services/WebhookService';
import { SSOService } from './services/SSOService';
import { SchedulerService } from './services/SchedulerService';
import { CronExpressionParser } from 'cron-parser';
import { isValidStackName, isValidRemoteUrl } from './utils/validation';
import YAML from 'yaml';
import fs, { promises as fsPromises } from 'fs';
@@ -652,7 +654,7 @@ app.use('/api', (req: Request, res: Response, next: NextFunction): void => {
authMiddleware(req, res, next);
});
// Audit logging middleware records all mutating API actions for Team Pro accountability.
// Audit logging middleware - records all mutating API actions for Team Pro accountability.
// Runs for POST/PUT/DELETE/PATCH on /api/* routes. Uses res.on('finish') to capture status code.
const AUDIT_ROUTE_SUMMARIES: Record<string, string> = {
'POST /stacks': 'Created stack',
@@ -684,6 +686,10 @@ const AUDIT_ROUTE_SUMMARIES: Record<string, string> = {
'DELETE /sso/config': 'Deleted SSO configuration',
'POST /api-tokens': 'Created API token',
'DELETE /api-tokens': 'Revoked API token',
'POST /scheduled-tasks': 'Created scheduled task',
'PUT /scheduled-tasks': 'Updated scheduled task',
'DELETE /scheduled-tasks': 'Deleted scheduled task',
'PATCH /scheduled-tasks': 'Toggled scheduled task',
};
function getAuditSummary(method: string, apiPath: string): string {
@@ -765,7 +771,7 @@ const requireAdmin = (req: Request, res: Response): boolean => {
return true;
};
// Scope enforcement for API tokens restricts which endpoints a token can reach.
// Scope enforcement for API tokens - restricts which endpoints a token can reach.
const DEPLOY_ALLOWED_PATTERNS: RegExp[] = [
/^\/api\/stacks\/[^/]+\/deploy$/,
/^\/api\/stacks\/[^/]+\/down$/,
@@ -3364,6 +3370,251 @@ app.delete('/api/api-tokens/:id', authMiddleware, async (req: Request, res: Resp
}
});
// --- Scheduled Operations Routes (Team Pro, admin-only, local-only) ---
app.get('/api/scheduled-tasks', (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
if (!requireTeamPro(req, res)) return;
try {
const tasks = DatabaseService.getInstance().getScheduledTasks();
res.json(tasks);
} catch (error) {
console.error('[ScheduledTasks] List error:', error);
res.status(500).json({ error: 'Failed to fetch scheduled tasks' });
}
});
app.post('/api/scheduled-tasks', (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
if (!requireTeamPro(req, res)) return;
try {
const { name, target_type, target_id, node_id, action, cron_expression, enabled } = req.body;
if (!name || typeof name !== 'string') {
res.status(400).json({ error: 'Name is required' }); return;
}
if (!['stack', 'fleet', 'system'].includes(target_type)) {
res.status(400).json({ error: 'Invalid target_type. Must be stack, fleet, or system.' }); return;
}
if (!['restart', 'snapshot', 'prune'].includes(action)) {
res.status(400).json({ error: 'Invalid action. Must be restart, snapshot, or prune.' }); return;
}
// Validate action-target combos
if (action === 'restart' && target_type !== 'stack') {
res.status(400).json({ error: 'Restart action requires target_type "stack".' }); return;
}
if (action === 'snapshot' && target_type !== 'fleet') {
res.status(400).json({ error: 'Snapshot action requires target_type "fleet".' }); return;
}
if (action === 'prune' && target_type !== 'system') {
res.status(400).json({ error: 'Prune action requires target_type "system".' }); return;
}
if (target_type === 'stack' && (!target_id || !node_id)) {
res.status(400).json({ error: 'Stack operations require target_id and node_id.' }); return;
}
// Validate cron expression
try { CronExpressionParser.parse(cron_expression); } catch {
res.status(400).json({ error: 'Invalid cron expression.' }); return;
}
const scheduler = SchedulerService.getInstance();
const now = Date.now();
const nextRun = (enabled !== false) ? scheduler.calculateNextRun(cron_expression) : null;
const id = DatabaseService.getInstance().createScheduledTask({
name,
target_type,
target_id: target_id || null,
node_id: node_id != null ? Number(node_id) : null,
action,
cron_expression,
enabled: enabled !== false ? 1 : 0,
created_by: req.user?.username || 'admin',
created_at: now,
updated_at: now,
last_run_at: null,
next_run_at: nextRun,
last_status: null,
last_error: null,
});
const task = DatabaseService.getInstance().getScheduledTask(id);
res.status(201).json(task);
} catch (error) {
console.error('[ScheduledTasks] Create error:', error);
res.status(500).json({ error: 'Failed to create scheduled task' });
}
});
app.get('/api/scheduled-tasks/:id', (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
if (!requireTeamPro(req, res)) return;
try {
const id = parseInt(req.params.id as string, 10);
if (isNaN(id)) { res.status(400).json({ error: 'Invalid task ID' }); return; }
const task = DatabaseService.getInstance().getScheduledTask(id);
if (!task) { res.status(404).json({ error: 'Scheduled task not found' }); return; }
res.json(task);
} catch (error) {
console.error('[ScheduledTasks] Get error:', error);
res.status(500).json({ error: 'Failed to fetch scheduled task' });
}
});
app.put('/api/scheduled-tasks/:id', (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
if (!requireTeamPro(req, res)) return;
try {
const id = parseInt(req.params.id as string, 10);
if (isNaN(id)) { res.status(400).json({ error: 'Invalid task ID' }); return; }
const db = DatabaseService.getInstance();
const existing = db.getScheduledTask(id);
if (!existing) { res.status(404).json({ error: 'Scheduled task not found' }); return; }
const { name, target_type, target_id, node_id, action, cron_expression, enabled } = req.body;
if (target_type && !['stack', 'fleet', 'system'].includes(target_type)) {
res.status(400).json({ error: 'Invalid target_type' }); return;
}
if (action && !['restart', 'snapshot', 'prune'].includes(action)) {
res.status(400).json({ error: 'Invalid action' }); return;
}
const finalAction = action || existing.action;
const finalTargetType = target_type || existing.target_type;
if (finalAction === 'restart' && finalTargetType !== 'stack') {
res.status(400).json({ error: 'Restart action requires target_type "stack".' }); return;
}
if (finalAction === 'snapshot' && finalTargetType !== 'fleet') {
res.status(400).json({ error: 'Snapshot action requires target_type "fleet".' }); return;
}
if (finalAction === 'prune' && finalTargetType !== 'system') {
res.status(400).json({ error: 'Prune action requires target_type "system".' }); return;
}
if (cron_expression) {
try { CronExpressionParser.parse(cron_expression); } catch {
res.status(400).json({ error: 'Invalid cron expression.' }); return;
}
}
const updates: Record<string, unknown> = { updated_at: Date.now() };
if (name !== undefined) updates.name = name;
if (target_type !== undefined) updates.target_type = target_type;
if (target_id !== undefined) updates.target_id = target_id || null;
if (node_id !== undefined) updates.node_id = node_id != null ? Number(node_id) : null;
if (action !== undefined) updates.action = action;
if (cron_expression !== undefined) updates.cron_expression = cron_expression;
if (enabled !== undefined) updates.enabled = enabled ? 1 : 0;
// Recalculate next_run if cron changed or if enabling
const finalCron = cron_expression || existing.cron_expression;
const finalEnabled = enabled !== undefined ? enabled : existing.enabled;
if (finalEnabled) {
updates.next_run_at = SchedulerService.getInstance().calculateNextRun(finalCron);
} else {
updates.next_run_at = null;
}
db.updateScheduledTask(id, updates as Partial<Omit<ScheduledTask, 'id'>>);
const task = db.getScheduledTask(id);
res.json(task);
} catch (error) {
console.error('[ScheduledTasks] Update error:', error);
res.status(500).json({ error: 'Failed to update scheduled task' });
}
});
app.delete('/api/scheduled-tasks/:id', (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
if (!requireTeamPro(req, res)) return;
try {
const id = parseInt(req.params.id as string, 10);
if (isNaN(id)) { res.status(400).json({ error: 'Invalid task ID' }); return; }
const db = DatabaseService.getInstance();
const existing = db.getScheduledTask(id);
if (!existing) { res.status(404).json({ error: 'Scheduled task not found' }); return; }
db.deleteScheduledTask(id);
res.json({ success: true });
} catch (error) {
console.error('[ScheduledTasks] Delete error:', error);
res.status(500).json({ error: 'Failed to delete scheduled task' });
}
});
app.patch('/api/scheduled-tasks/:id/toggle', (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
if (!requireTeamPro(req, res)) return;
try {
const id = parseInt(req.params.id as string, 10);
if (isNaN(id)) { res.status(400).json({ error: 'Invalid task ID' }); return; }
const db = DatabaseService.getInstance();
const existing = db.getScheduledTask(id);
if (!existing) { res.status(404).json({ error: 'Scheduled task not found' }); return; }
const newEnabled = existing.enabled ? 0 : 1;
const nextRun = newEnabled ? SchedulerService.getInstance().calculateNextRun(existing.cron_expression) : null;
db.updateScheduledTask(id, {
enabled: newEnabled,
next_run_at: nextRun,
updated_at: Date.now(),
});
const task = db.getScheduledTask(id);
res.json(task);
} catch (error) {
console.error('[ScheduledTasks] Toggle error:', error);
res.status(500).json({ error: 'Failed to toggle scheduled task' });
}
});
app.post('/api/scheduled-tasks/:id/run', async (req: Request, res: Response): Promise<void> => {
if (!requireAdmin(req, res)) return;
if (!requireTeamPro(req, res)) return;
try {
const id = parseInt(req.params.id as string, 10);
if (isNaN(id)) { res.status(400).json({ error: 'Invalid task ID' }); return; }
const db = DatabaseService.getInstance();
const existing = db.getScheduledTask(id);
if (!existing) { res.status(404).json({ error: 'Scheduled task not found' }); return; }
await SchedulerService.getInstance().triggerTask(id);
const task = db.getScheduledTask(id);
res.json(task);
} catch (error: unknown) {
const msg = error instanceof Error ? error.message : 'Failed to run task';
console.error('[ScheduledTasks] Run error:', msg);
res.status(500).json({ error: msg });
}
});
app.get('/api/scheduled-tasks/:id/runs', (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
if (!requireTeamPro(req, res)) return;
try {
const id = parseInt(req.params.id as string, 10);
if (isNaN(id)) { res.status(400).json({ error: 'Invalid task ID' }); return; }
const db = DatabaseService.getInstance();
const existing = db.getScheduledTask(id);
if (!existing) { res.status(404).json({ error: 'Scheduled task not found' }); return; }
const limit = Math.min(parseInt(req.query.limit as string, 10) || 50, 100);
const runs = db.getScheduledTaskRuns(id, limit);
res.json(runs);
} catch (error) {
console.error('[ScheduledTasks] Runs error:', error);
res.status(500).json({ error: 'Failed to fetch task runs' });
}
});
// --- System Maintenance Routes (The System Janitor) ---
app.get('/api/system/orphans', async (req: Request, res: Response) => {
@@ -3817,6 +4068,9 @@ async function startServer() {
// Start Background Image Update Checker
ImageUpdateService.getInstance().start();
// Start Scheduled Operations Service
SchedulerService.getInstance().start();
server.listen(PORT, () => {
console.log(`Server running on port ${PORT}`);
});
@@ -3841,6 +4095,7 @@ const gracefulShutdown = (signal: string) => {
try { LicenseService.getInstance().destroy(); } catch { /* already stopped */ }
try { MonitorService.getInstance().stop(); } catch { /* already stopped */ }
try { ImageUpdateService.getInstance().stop(); } catch { /* already stopped */ }
try { SchedulerService.getInstance().stop(); } catch { /* already stopped */ }
try { DatabaseService.getInstance().getDb().close(); } catch { /* already closed */ }
console.log('[Shutdown] Done - exiting');
process.exit(0);
+148
View File
@@ -137,6 +137,34 @@ export interface ApiToken {
revoked_at: number | null;
}
export interface ScheduledTask {
id: number;
name: string;
target_type: 'stack' | 'fleet' | 'system';
target_id: string | null;
node_id: number | null;
action: 'restart' | 'snapshot' | 'prune';
cron_expression: string;
enabled: number;
created_by: string;
created_at: number;
updated_at: number;
last_run_at: number | null;
next_run_at: number | null;
last_status: string | null;
last_error: string | null;
}
export interface ScheduledTaskRun {
id: number;
task_id: number;
started_at: number;
completed_at: number | null;
status: 'running' | 'success' | 'failure';
output: string | null;
error: string | null;
}
export class DatabaseService {
private static instance: DatabaseService;
private db: Database.Database;
@@ -324,6 +352,38 @@ export class DatabaseService {
CREATE INDEX IF NOT EXISTS idx_api_tokens_hash ON api_tokens(token_hash);
CREATE INDEX IF NOT EXISTS idx_api_tokens_user ON api_tokens(user_id);
CREATE TABLE IF NOT EXISTS scheduled_tasks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
target_type TEXT NOT NULL,
target_id TEXT,
node_id INTEGER,
action TEXT NOT NULL,
cron_expression TEXT NOT NULL,
enabled INTEGER DEFAULT 1,
created_by TEXT NOT NULL,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
last_run_at INTEGER,
next_run_at INTEGER,
last_status TEXT,
last_error TEXT
);
CREATE TABLE IF NOT EXISTS scheduled_task_runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
task_id INTEGER NOT NULL,
started_at INTEGER NOT NULL,
completed_at INTEGER,
status TEXT NOT NULL DEFAULT 'running',
output TEXT,
error TEXT,
FOREIGN KEY(task_id) REFERENCES scheduled_tasks(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_scheduled_task_runs_task ON scheduled_task_runs(task_id);
CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_next_run ON scheduled_tasks(next_run_at);
`);
// Apply migrations safely (ignore if columns already exist)
@@ -1016,4 +1076,92 @@ export class DatabaseService {
public updateApiTokenLastUsed(id: number): void {
this.db.prepare('UPDATE api_tokens SET last_used_at = ? WHERE id = ?').run(Date.now(), id);
}
// --- Scheduled Tasks ---
public getScheduledTasks(): ScheduledTask[] {
return this.db.prepare('SELECT * FROM scheduled_tasks ORDER BY created_at DESC').all() as ScheduledTask[];
}
public getScheduledTask(id: number): ScheduledTask | undefined {
return this.db.prepare('SELECT * FROM scheduled_tasks WHERE id = ?').get(id) as ScheduledTask | undefined;
}
public createScheduledTask(task: Omit<ScheduledTask, 'id'>): number {
const result = this.db.prepare(
'INSERT INTO scheduled_tasks (name, target_type, target_id, node_id, action, cron_expression, enabled, created_by, created_at, updated_at, last_run_at, next_run_at, last_status, last_error) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
).run(
task.name, task.target_type, task.target_id, task.node_id,
task.action, task.cron_expression, task.enabled, task.created_by,
task.created_at, task.updated_at, task.last_run_at, task.next_run_at,
task.last_status, task.last_error
);
return result.lastInsertRowid as number;
}
public updateScheduledTask(id: number, updates: Partial<Omit<ScheduledTask, 'id'>>): void {
const fields: string[] = [];
const values: unknown[] = [];
const map: Record<string, unknown> = {
name: updates.name, target_type: updates.target_type, target_id: updates.target_id,
node_id: updates.node_id, action: updates.action, cron_expression: updates.cron_expression,
enabled: updates.enabled, created_by: updates.created_by, updated_at: updates.updated_at,
last_run_at: updates.last_run_at, next_run_at: updates.next_run_at,
last_status: updates.last_status, last_error: updates.last_error,
};
for (const [col, val] of Object.entries(map)) {
if (val !== undefined) {
fields.push(`${col} = ?`);
values.push(val);
}
}
if (fields.length === 0) return;
values.push(id);
this.db.prepare(`UPDATE scheduled_tasks SET ${fields.join(', ')} WHERE id = ?`).run(...values);
}
public deleteScheduledTask(id: number): void {
this.db.prepare('DELETE FROM scheduled_tasks WHERE id = ?').run(id);
}
public getDueScheduledTasks(now: number): ScheduledTask[] {
return this.db.prepare(
'SELECT * FROM scheduled_tasks WHERE enabled = 1 AND next_run_at IS NOT NULL AND next_run_at <= ?'
).all(now) as ScheduledTask[];
}
public getScheduledTaskRuns(taskId: number, limit = 50): ScheduledTaskRun[] {
return this.db.prepare(
'SELECT * FROM scheduled_task_runs WHERE task_id = ? ORDER BY started_at DESC LIMIT ?'
).all(taskId, limit) as ScheduledTaskRun[];
}
public createScheduledTaskRun(run: Omit<ScheduledTaskRun, 'id'>): number {
const result = this.db.prepare(
'INSERT INTO scheduled_task_runs (task_id, started_at, completed_at, status, output, error) VALUES (?, ?, ?, ?, ?, ?)'
).run(run.task_id, run.started_at, run.completed_at, run.status, run.output, run.error);
return result.lastInsertRowid as number;
}
public updateScheduledTaskRun(id: number, updates: Partial<Omit<ScheduledTaskRun, 'id' | 'task_id'>>): void {
const fields: string[] = [];
const values: unknown[] = [];
if (updates.completed_at !== undefined) { fields.push('completed_at = ?'); values.push(updates.completed_at); }
if (updates.status !== undefined) { fields.push('status = ?'); values.push(updates.status); }
if (updates.output !== undefined) { fields.push('output = ?'); values.push(updates.output); }
if (updates.error !== undefined) { fields.push('error = ?'); values.push(updates.error); }
if (fields.length === 0) return;
values.push(id);
this.db.prepare(`UPDATE scheduled_task_runs SET ${fields.join(', ')} WHERE id = ?`).run(...values);
}
public cleanupOldTaskRuns(retentionDays = 30): void {
const cutoff = Date.now() - (retentionDays * 24 * 60 * 60 * 1000);
this.db.prepare('DELETE FROM scheduled_task_runs WHERE started_at < ?').run(cutoff);
}
}
+1 -1
View File
@@ -179,7 +179,7 @@ export class LicenseService {
/**
* Get the license variant (personal or team) from stored metadata.
* Trial licenses default to "personal" Team Pro features require a Team Pro license.
* Trial licenses default to "personal" - Team Pro features require a Team Pro license.
*/
public getVariant(): LicenseVariant {
const db = DatabaseService.getInstance();
+1 -1
View File
@@ -432,7 +432,7 @@ export class SSOService {
let issuer: InstanceType<typeof Issuer>;
if (provider === 'oidc_github') {
// GitHub is not a standard OIDC provider manually configure
// GitHub is not a standard OIDC provider - manually configure
issuer = new Issuer({
issuer: 'https://github.com',
authorization_endpoint: 'https://github.com/login/oauth/authorize',
+315
View File
@@ -0,0 +1,315 @@
import { CronExpressionParser } from 'cron-parser';
import { DatabaseService } from './DatabaseService';
import type { ScheduledTask } from './DatabaseService';
import { LicenseService } from './LicenseService';
import DockerController from './DockerController';
import { FileSystemService } from './FileSystemService';
import { NodeRegistry } from './NodeRegistry';
export class SchedulerService {
private static instance: SchedulerService;
private intervalId: ReturnType<typeof setInterval> | null = null;
private isProcessing = false;
private runningTasks = new Set<number>();
private constructor() {}
public static getInstance(): SchedulerService {
if (!SchedulerService.instance) {
SchedulerService.instance = new SchedulerService();
}
return SchedulerService.instance;
}
public start(): void {
if (this.intervalId) return;
this.intervalId = setInterval(() => this.tick(), 60_000);
setTimeout(() => this.tick(), 10_000);
console.log('[SchedulerService] Started');
}
public stop(): void {
if (this.intervalId) {
clearInterval(this.intervalId);
this.intervalId = null;
}
console.log('[SchedulerService] Stopped');
}
public calculateNextRun(cronExpression: string): number {
const expr = CronExpressionParser.parse(cronExpression);
return expr.next().toDate().getTime();
}
private async tick(): Promise<void> {
if (this.isProcessing) return;
this.isProcessing = true;
try {
const ls = LicenseService.getInstance();
if (ls.getTier() !== 'pro' || ls.getVariant() !== 'team') return;
const db = DatabaseService.getInstance();
const now = Date.now();
const dueTasks = db.getDueScheduledTasks(now);
// Clean up old runs periodically (piggyback on tick)
db.cleanupOldTaskRuns(30);
for (const task of dueTasks) {
if (this.runningTasks.has(task.id)) continue;
this.runningTasks.add(task.id);
this.executeTask(task).finally(() => this.runningTasks.delete(task.id));
}
} catch (error) {
console.error('[SchedulerService] Tick error:', error);
} finally {
this.isProcessing = false;
}
}
public async triggerTask(taskId: number): Promise<void> {
const db = DatabaseService.getInstance();
const task = db.getScheduledTask(taskId);
if (!task) throw new Error('Task not found');
if (this.runningTasks.has(task.id)) throw new Error('Task is already running');
this.runningTasks.add(task.id);
try {
await this.executeTask(task);
} finally {
this.runningTasks.delete(task.id);
}
}
private async executeTask(task: ScheduledTask): Promise<void> {
const db = DatabaseService.getInstance();
const runId = db.createScheduledTaskRun({
task_id: task.id,
started_at: Date.now(),
completed_at: null,
status: 'running',
output: null,
error: null,
});
try {
let output = '';
switch (task.action) {
case 'restart':
output = await this.executeRestart(task);
break;
case 'snapshot':
output = await this.executeSnapshot(task);
break;
case 'prune':
output = await this.executePrune(task);
break;
}
const nextRun = this.calculateNextRun(task.cron_expression);
db.updateScheduledTask(task.id, {
last_run_at: Date.now(),
next_run_at: nextRun,
last_status: 'success',
last_error: null,
updated_at: Date.now(),
});
db.updateScheduledTaskRun(runId, {
completed_at: Date.now(),
status: 'success',
output,
});
console.log(`[SchedulerService] Task "${task.name}" (id=${task.id}) completed successfully`);
} catch (error: unknown) {
const errMsg = error instanceof Error ? error.message : String(error);
let nextRun: number | null = null;
try {
nextRun = this.calculateNextRun(task.cron_expression);
} catch {
// If cron expression is somehow invalid, disable the task
}
db.updateScheduledTask(task.id, {
last_run_at: Date.now(),
next_run_at: nextRun,
last_status: 'failure',
last_error: errMsg,
updated_at: Date.now(),
});
db.updateScheduledTaskRun(runId, {
completed_at: Date.now(),
status: 'failure',
error: errMsg,
});
console.error(`[SchedulerService] Task "${task.name}" (id=${task.id}) failed:`, errMsg);
}
}
private async executeRestart(task: ScheduledTask): Promise<string> {
if (!task.target_id || task.node_id == null) {
throw new Error('Stack restart requires target_id and node_id');
}
const docker = DockerController.getInstance(task.node_id);
const containers = await docker.getContainersByStack(task.target_id);
if (!containers || containers.length === 0) {
throw new Error(`No containers found for stack "${task.target_id}"`);
}
await Promise.all(containers.map(c => docker.restartContainer(c.Id)));
return `Restarted ${containers.length} container(s) in stack "${task.target_id}"`;
}
private async executeSnapshot(task: ScheduledTask): Promise<string> {
const db = DatabaseService.getInstance();
const nodes = db.getNodes();
const results = await Promise.allSettled(
nodes.map(async (node) => {
if (node.type === 'remote') {
return this.captureRemoteNodeFiles(node);
}
return this.captureLocalNodeFiles(node);
})
);
const capturedNodes: Array<{ nodeId: number; nodeName: string; stacks: Array<{ stackName: string; files: Array<{ filename: string; content: string }> }> }> = [];
const skippedNodes: Array<{ nodeId: number; nodeName: string; reason: string }> = [];
results.forEach((result, i) => {
if (result.status === 'fulfilled') {
capturedNodes.push(result.value);
} else {
skippedNodes.push({
nodeId: nodes[i].id,
nodeName: nodes[i].name,
reason: result.reason instanceof Error ? result.reason.message : 'Unknown error',
});
}
});
let totalStacks = 0;
const allFiles: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }> = [];
for (const nodeData of capturedNodes) {
totalStacks += nodeData.stacks.length;
for (const stack of nodeData.stacks) {
for (const file of stack.files) {
allFiles.push({
nodeId: nodeData.nodeId,
nodeName: nodeData.nodeName,
stackName: stack.stackName,
filename: file.filename,
content: file.content,
});
}
}
}
const description = `Scheduled snapshot: ${task.name}`;
const snapshotId = db.createSnapshot(
description,
task.created_by,
capturedNodes.length,
totalStacks,
JSON.stringify(skippedNodes),
);
if (allFiles.length > 0) {
db.insertSnapshotFiles(snapshotId, allFiles);
}
return `Fleet snapshot created (id=${snapshotId}, ${capturedNodes.length} node(s), ${totalStacks} stack(s)${skippedNodes.length > 0 ? `, ${skippedNodes.length} skipped` : ''})`;
}
private async captureLocalNodeFiles(node: { id: number; name: string }) {
const fsService = FileSystemService.getInstance(node.id);
const stackNames = await fsService.getStacks();
const stacks: Array<{ stackName: string; files: Array<{ filename: string; content: string }> }> = [];
for (const stackName of stackNames) {
const files: Array<{ filename: string; content: string }> = [];
try {
const composeContent = await fsService.getStackContent(stackName);
files.push({ filename: 'compose.yaml', content: composeContent });
} catch {
continue;
}
try {
const envContent = await fsService.getEnvContent(stackName);
files.push({ filename: '.env', content: envContent });
} catch {
// No .env file
}
stacks.push({ stackName, files });
}
return { nodeId: node.id, nodeName: node.name, stacks };
}
private async captureRemoteNodeFiles(node: { id: number; name: string; api_url?: string; api_token?: string }) {
if (!node.api_url || !node.api_token) {
throw new Error('Remote node not configured');
}
const baseUrl = node.api_url.replace(/\/$/, '');
const headers = { Authorization: `Bearer ${node.api_token}` };
const stacksRes = await fetch(`${baseUrl}/api/stacks`, {
headers,
signal: AbortSignal.timeout(15000),
});
if (!stacksRes.ok) throw new Error('Failed to fetch stacks from remote node');
const stackNames = await stacksRes.json() as string[];
const stacks: Array<{ stackName: string; files: Array<{ filename: string; content: string }> }> = [];
for (const stackName of stackNames) {
const files: Array<{ filename: string; content: string }> = [];
try {
const composeRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, {
headers,
signal: AbortSignal.timeout(15000),
});
if (composeRes.ok) {
const content = await composeRes.text();
files.push({ filename: 'compose.yaml', content });
}
} catch {
continue;
}
try {
const envRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, {
headers,
signal: AbortSignal.timeout(15000),
});
if (envRes.ok) {
const content = await envRes.text();
files.push({ filename: '.env', content });
}
} catch {
// No .env
}
if (files.length > 0) {
stacks.push({ stackName, files });
}
}
return { nodeId: node.id, nodeName: node.name, stacks };
}
private async executePrune(task: ScheduledTask): Promise<string> {
const nodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
const docker = DockerController.getInstance(nodeId);
const targets = ['containers', 'images', 'networks', 'volumes'] as const;
const results: string[] = [];
for (const target of targets) {
try {
const result = await docker.pruneSystem(target);
results.push(`${target}: ${result.reclaimedBytes ?? 0} bytes reclaimed`);
} catch (error: unknown) {
const msg = error instanceof Error ? error.message : String(error);
results.push(`${target}: failed (${msg})`);
}
}
return `System prune completed: ${results.join('; ')}`;
}
}