feat(scheduled-ops): add scheduled operations for Team Pro users (#231)

Adds the ability to schedule recurring Docker operations (stack restarts, fleet snapshots, system prunes) via cron expressions with full execution history logging. Includes Run Now for on-demand execution.
2026-08-18 22:36:19 +00:00 · 2026-03-28 23:56:37 -04:00
parent 89c4a9cb77
commit 31e1795af0
31 changed files with 1552 additions and 160 deletions
@@ -137,6 +137,34 @@ export interface ApiToken {
    revoked_at: number | null;
 }

+export interface ScheduledTask {
+    id: number;
+    name: string;
+    target_type: 'stack' | 'fleet' | 'system';
+    target_id: string | null;
+    node_id: number | null;
+    action: 'restart' | 'snapshot' | 'prune';
+    cron_expression: string;
+    enabled: number;
+    created_by: string;
+    created_at: number;
+    updated_at: number;
+    last_run_at: number | null;
+    next_run_at: number | null;
+    last_status: string | null;
+    last_error: string | null;
+}
+
+export interface ScheduledTaskRun {
+    id: number;
+    task_id: number;
+    started_at: number;
+    completed_at: number | null;
+    status: 'running' | 'success' | 'failure';
+    output: string | null;
+    error: string | null;
+}
+
 export class DatabaseService {
    private static instance: DatabaseService;
    private db: Database.Database;
@@ -324,6 +352,38 @@ export class DatabaseService {

      CREATE INDEX IF NOT EXISTS idx_api_tokens_hash ON api_tokens(token_hash);
      CREATE INDEX IF NOT EXISTS idx_api_tokens_user ON api_tokens(user_id);
+
+      CREATE TABLE IF NOT EXISTS scheduled_tasks (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        name TEXT NOT NULL,
+        target_type TEXT NOT NULL,
+        target_id TEXT,
+        node_id INTEGER,
+        action TEXT NOT NULL,
+        cron_expression TEXT NOT NULL,
+        enabled INTEGER DEFAULT 1,
+        created_by TEXT NOT NULL,
+        created_at INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL,
+        last_run_at INTEGER,
+        next_run_at INTEGER,
+        last_status TEXT,
+        last_error TEXT
+      );
+
+      CREATE TABLE IF NOT EXISTS scheduled_task_runs (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        task_id INTEGER NOT NULL,
+        started_at INTEGER NOT NULL,
+        completed_at INTEGER,
+        status TEXT NOT NULL DEFAULT 'running',
+        output TEXT,
+        error TEXT,
+        FOREIGN KEY(task_id) REFERENCES scheduled_tasks(id) ON DELETE CASCADE
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_scheduled_task_runs_task ON scheduled_task_runs(task_id);
+      CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_next_run ON scheduled_tasks(next_run_at);
    `);

        // Apply migrations safely (ignore if columns already exist)
@@ -1016,4 +1076,92 @@ export class DatabaseService {
    public updateApiTokenLastUsed(id: number): void {
        this.db.prepare('UPDATE api_tokens SET last_used_at = ? WHERE id = ?').run(Date.now(), id);
    }
+
+    // --- Scheduled Tasks ---
+
+    public getScheduledTasks(): ScheduledTask[] {
+        return this.db.prepare('SELECT * FROM scheduled_tasks ORDER BY created_at DESC').all() as ScheduledTask[];
+    }
+
+    public getScheduledTask(id: number): ScheduledTask | undefined {
+        return this.db.prepare('SELECT * FROM scheduled_tasks WHERE id = ?').get(id) as ScheduledTask | undefined;
+    }
+
+    public createScheduledTask(task: Omit<ScheduledTask, 'id'>): number {
+        const result = this.db.prepare(
+            'INSERT INTO scheduled_tasks (name, target_type, target_id, node_id, action, cron_expression, enabled, created_by, created_at, updated_at, last_run_at, next_run_at, last_status, last_error) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
+        ).run(
+            task.name, task.target_type, task.target_id, task.node_id,
+            task.action, task.cron_expression, task.enabled, task.created_by,
+            task.created_at, task.updated_at, task.last_run_at, task.next_run_at,
+            task.last_status, task.last_error
+        );
+        return result.lastInsertRowid as number;
+    }
+
+    public updateScheduledTask(id: number, updates: Partial<Omit<ScheduledTask, 'id'>>): void {
+        const fields: string[] = [];
+        const values: unknown[] = [];
+
+        const map: Record<string, unknown> = {
+            name: updates.name, target_type: updates.target_type, target_id: updates.target_id,
+            node_id: updates.node_id, action: updates.action, cron_expression: updates.cron_expression,
+            enabled: updates.enabled, created_by: updates.created_by, updated_at: updates.updated_at,
+            last_run_at: updates.last_run_at, next_run_at: updates.next_run_at,
+            last_status: updates.last_status, last_error: updates.last_error,
+        };
+
+        for (const [col, val] of Object.entries(map)) {
+            if (val !== undefined) {
+                fields.push(`${col} = ?`);
+                values.push(val);
+            }
+        }
+
+        if (fields.length === 0) return;
+        values.push(id);
+        this.db.prepare(`UPDATE scheduled_tasks SET ${fields.join(', ')} WHERE id = ?`).run(...values);
+    }
+
+    public deleteScheduledTask(id: number): void {
+        this.db.prepare('DELETE FROM scheduled_tasks WHERE id = ?').run(id);
+    }
+
+    public getDueScheduledTasks(now: number): ScheduledTask[] {
+        return this.db.prepare(
+            'SELECT * FROM scheduled_tasks WHERE enabled = 1 AND next_run_at IS NOT NULL AND next_run_at <= ?'
+        ).all(now) as ScheduledTask[];
+    }
+
+    public getScheduledTaskRuns(taskId: number, limit = 50): ScheduledTaskRun[] {
+        return this.db.prepare(
+            'SELECT * FROM scheduled_task_runs WHERE task_id = ? ORDER BY started_at DESC LIMIT ?'
+        ).all(taskId, limit) as ScheduledTaskRun[];
+    }
+
+    public createScheduledTaskRun(run: Omit<ScheduledTaskRun, 'id'>): number {
+        const result = this.db.prepare(
+            'INSERT INTO scheduled_task_runs (task_id, started_at, completed_at, status, output, error) VALUES (?, ?, ?, ?, ?, ?)'
+        ).run(run.task_id, run.started_at, run.completed_at, run.status, run.output, run.error);
+        return result.lastInsertRowid as number;
+    }
+
+    public updateScheduledTaskRun(id: number, updates: Partial<Omit<ScheduledTaskRun, 'id' | 'task_id'>>): void {
+        const fields: string[] = [];
+        const values: unknown[] = [];
+
+        if (updates.completed_at !== undefined) { fields.push('completed_at = ?'); values.push(updates.completed_at); }
+        if (updates.status !== undefined) { fields.push('status = ?'); values.push(updates.status); }
+        if (updates.output !== undefined) { fields.push('output = ?'); values.push(updates.output); }
+        if (updates.error !== undefined) { fields.push('error = ?'); values.push(updates.error); }
+
+        if (fields.length === 0) return;
+        values.push(id);
+        this.db.prepare(`UPDATE scheduled_task_runs SET ${fields.join(', ')} WHERE id = ?`).run(...values);
+    }
+
+    public cleanupOldTaskRuns(retentionDays = 30): void {
+        const cutoff = Date.now() - (retentionDays * 24 * 60 * 60 * 1000);
+        this.db.prepare('DELETE FROM scheduled_task_runs WHERE started_at < ?').run(cutoff);
+    }
 }
@@ -179,7 +179,7 @@ export class LicenseService {

    /**
     * Get the license variant (personal or team) from stored metadata.
-     * Trial licenses default to "personal" — Team Pro features require a Team Pro license.
+     * Trial licenses default to "personal" - Team Pro features require a Team Pro license.
     */
    public getVariant(): LicenseVariant {
        const db = DatabaseService.getInstance();
@@ -432,7 +432,7 @@ export class SSOService {
        let issuer: InstanceType<typeof Issuer>;

        if (provider === 'oidc_github') {
-            // GitHub is not a standard OIDC provider — manually configure
+            // GitHub is not a standard OIDC provider - manually configure
            issuer = new Issuer({
                issuer: 'https://github.com',
                authorization_endpoint: 'https://github.com/login/oauth/authorize',
@@ -0,0 +1,315 @@
+import { CronExpressionParser } from 'cron-parser';
+import { DatabaseService } from './DatabaseService';
+import type { ScheduledTask } from './DatabaseService';
+import { LicenseService } from './LicenseService';
+import DockerController from './DockerController';
+import { FileSystemService } from './FileSystemService';
+import { NodeRegistry } from './NodeRegistry';
+
+export class SchedulerService {
+    private static instance: SchedulerService;
+    private intervalId: ReturnType<typeof setInterval> | null = null;
+    private isProcessing = false;
+    private runningTasks = new Set<number>();
+
+    private constructor() {}
+
+    public static getInstance(): SchedulerService {
+        if (!SchedulerService.instance) {
+            SchedulerService.instance = new SchedulerService();
+        }
+        return SchedulerService.instance;
+    }
+
+    public start(): void {
+        if (this.intervalId) return;
+        this.intervalId = setInterval(() => this.tick(), 60_000);
+        setTimeout(() => this.tick(), 10_000);
+        console.log('[SchedulerService] Started');
+    }
+
+    public stop(): void {
+        if (this.intervalId) {
+            clearInterval(this.intervalId);
+            this.intervalId = null;
+        }
+        console.log('[SchedulerService] Stopped');
+    }
+
+    public calculateNextRun(cronExpression: string): number {
+        const expr = CronExpressionParser.parse(cronExpression);
+        return expr.next().toDate().getTime();
+    }
+
+    private async tick(): Promise<void> {
+        if (this.isProcessing) return;
+        this.isProcessing = true;
+        try {
+            const ls = LicenseService.getInstance();
+            if (ls.getTier() !== 'pro' || ls.getVariant() !== 'team') return;
+
+            const db = DatabaseService.getInstance();
+            const now = Date.now();
+            const dueTasks = db.getDueScheduledTasks(now);
+
+            // Clean up old runs periodically (piggyback on tick)
+            db.cleanupOldTaskRuns(30);
+
+            for (const task of dueTasks) {
+                if (this.runningTasks.has(task.id)) continue;
+                this.runningTasks.add(task.id);
+                this.executeTask(task).finally(() => this.runningTasks.delete(task.id));
+            }
+        } catch (error) {
+            console.error('[SchedulerService] Tick error:', error);
+        } finally {
+            this.isProcessing = false;
+        }
+    }
+
+    public async triggerTask(taskId: number): Promise<void> {
+        const db = DatabaseService.getInstance();
+        const task = db.getScheduledTask(taskId);
+        if (!task) throw new Error('Task not found');
+        if (this.runningTasks.has(task.id)) throw new Error('Task is already running');
+        this.runningTasks.add(task.id);
+        try {
+            await this.executeTask(task);
+        } finally {
+            this.runningTasks.delete(task.id);
+        }
+    }
+
+    private async executeTask(task: ScheduledTask): Promise<void> {
+        const db = DatabaseService.getInstance();
+        const runId = db.createScheduledTaskRun({
+            task_id: task.id,
+            started_at: Date.now(),
+            completed_at: null,
+            status: 'running',
+            output: null,
+            error: null,
+        });
+
+        try {
+            let output = '';
+            switch (task.action) {
+                case 'restart':
+                    output = await this.executeRestart(task);
+                    break;
+                case 'snapshot':
+                    output = await this.executeSnapshot(task);
+                    break;
+                case 'prune':
+                    output = await this.executePrune(task);
+                    break;
+            }
+
+            const nextRun = this.calculateNextRun(task.cron_expression);
+            db.updateScheduledTask(task.id, {
+                last_run_at: Date.now(),
+                next_run_at: nextRun,
+                last_status: 'success',
+                last_error: null,
+                updated_at: Date.now(),
+            });
+            db.updateScheduledTaskRun(runId, {
+                completed_at: Date.now(),
+                status: 'success',
+                output,
+            });
+            console.log(`[SchedulerService] Task "${task.name}" (id=${task.id}) completed successfully`);
+        } catch (error: unknown) {
+            const errMsg = error instanceof Error ? error.message : String(error);
+            let nextRun: number | null = null;
+            try {
+                nextRun = this.calculateNextRun(task.cron_expression);
+            } catch {
+                // If cron expression is somehow invalid, disable the task
+            }
+            db.updateScheduledTask(task.id, {
+                last_run_at: Date.now(),
+                next_run_at: nextRun,
+                last_status: 'failure',
+                last_error: errMsg,
+                updated_at: Date.now(),
+            });
+            db.updateScheduledTaskRun(runId, {
+                completed_at: Date.now(),
+                status: 'failure',
+                error: errMsg,
+            });
+            console.error(`[SchedulerService] Task "${task.name}" (id=${task.id}) failed:`, errMsg);
+        }
+    }
+
+    private async executeRestart(task: ScheduledTask): Promise<string> {
+        if (!task.target_id || task.node_id == null) {
+            throw new Error('Stack restart requires target_id and node_id');
+        }
+        const docker = DockerController.getInstance(task.node_id);
+        const containers = await docker.getContainersByStack(task.target_id);
+        if (!containers || containers.length === 0) {
+            throw new Error(`No containers found for stack "${task.target_id}"`);
+        }
+        await Promise.all(containers.map(c => docker.restartContainer(c.Id)));
+        return `Restarted ${containers.length} container(s) in stack "${task.target_id}"`;
+    }
+
+    private async executeSnapshot(task: ScheduledTask): Promise<string> {
+        const db = DatabaseService.getInstance();
+        const nodes = db.getNodes();
+
+        const results = await Promise.allSettled(
+            nodes.map(async (node) => {
+                if (node.type === 'remote') {
+                    return this.captureRemoteNodeFiles(node);
+                }
+                return this.captureLocalNodeFiles(node);
+            })
+        );
+
+        const capturedNodes: Array<{ nodeId: number; nodeName: string; stacks: Array<{ stackName: string; files: Array<{ filename: string; content: string }> }> }> = [];
+        const skippedNodes: Array<{ nodeId: number; nodeName: string; reason: string }> = [];
+
+        results.forEach((result, i) => {
+            if (result.status === 'fulfilled') {
+                capturedNodes.push(result.value);
+            } else {
+                skippedNodes.push({
+                    nodeId: nodes[i].id,
+                    nodeName: nodes[i].name,
+                    reason: result.reason instanceof Error ? result.reason.message : 'Unknown error',
+                });
+            }
+        });
+
+        let totalStacks = 0;
+        const allFiles: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }> = [];
+
+        for (const nodeData of capturedNodes) {
+            totalStacks += nodeData.stacks.length;
+            for (const stack of nodeData.stacks) {
+                for (const file of stack.files) {
+                    allFiles.push({
+                        nodeId: nodeData.nodeId,
+                        nodeName: nodeData.nodeName,
+                        stackName: stack.stackName,
+                        filename: file.filename,
+                        content: file.content,
+                    });
+                }
+            }
+        }
+
+        const description = `Scheduled snapshot: ${task.name}`;
+        const snapshotId = db.createSnapshot(
+            description,
+            task.created_by,
+            capturedNodes.length,
+            totalStacks,
+            JSON.stringify(skippedNodes),
+        );
+
+        if (allFiles.length > 0) {
+            db.insertSnapshotFiles(snapshotId, allFiles);
+        }
+
+        return `Fleet snapshot created (id=${snapshotId}, ${capturedNodes.length} node(s), ${totalStacks} stack(s)${skippedNodes.length > 0 ? `, ${skippedNodes.length} skipped` : ''})`;
+    }
+
+    private async captureLocalNodeFiles(node: { id: number; name: string }) {
+        const fsService = FileSystemService.getInstance(node.id);
+        const stackNames = await fsService.getStacks();
+        const stacks: Array<{ stackName: string; files: Array<{ filename: string; content: string }> }> = [];
+
+        for (const stackName of stackNames) {
+            const files: Array<{ filename: string; content: string }> = [];
+            try {
+                const composeContent = await fsService.getStackContent(stackName);
+                files.push({ filename: 'compose.yaml', content: composeContent });
+            } catch {
+                continue;
+            }
+            try {
+                const envContent = await fsService.getEnvContent(stackName);
+                files.push({ filename: '.env', content: envContent });
+            } catch {
+                // No .env file
+            }
+            stacks.push({ stackName, files });
+        }
+
+        return { nodeId: node.id, nodeName: node.name, stacks };
+    }
+
+    private async captureRemoteNodeFiles(node: { id: number; name: string; api_url?: string; api_token?: string }) {
+        if (!node.api_url || !node.api_token) {
+            throw new Error('Remote node not configured');
+        }
+
+        const baseUrl = node.api_url.replace(/\/$/, '');
+        const headers = { Authorization: `Bearer ${node.api_token}` };
+
+        const stacksRes = await fetch(`${baseUrl}/api/stacks`, {
+            headers,
+            signal: AbortSignal.timeout(15000),
+        });
+        if (!stacksRes.ok) throw new Error('Failed to fetch stacks from remote node');
+        const stackNames = await stacksRes.json() as string[];
+
+        const stacks: Array<{ stackName: string; files: Array<{ filename: string; content: string }> }> = [];
+
+        for (const stackName of stackNames) {
+            const files: Array<{ filename: string; content: string }> = [];
+            try {
+                const composeRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, {
+                    headers,
+                    signal: AbortSignal.timeout(15000),
+                });
+                if (composeRes.ok) {
+                    const content = await composeRes.text();
+                    files.push({ filename: 'compose.yaml', content });
+                }
+            } catch {
+                continue;
+            }
+            try {
+                const envRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, {
+                    headers,
+                    signal: AbortSignal.timeout(15000),
+                });
+                if (envRes.ok) {
+                    const content = await envRes.text();
+                    files.push({ filename: '.env', content });
+                }
+            } catch {
+                // No .env
+            }
+            if (files.length > 0) {
+                stacks.push({ stackName, files });
+            }
+        }
+
+        return { nodeId: node.id, nodeName: node.name, stacks };
+    }
+
+    private async executePrune(task: ScheduledTask): Promise<string> {
+        const nodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
+        const docker = DockerController.getInstance(nodeId);
+        const targets = ['containers', 'images', 'networks', 'volumes'] as const;
+        const results: string[] = [];
+
+        for (const target of targets) {
+            try {
+                const result = await docker.pruneSystem(target);
+                results.push(`${target}: ${result.reclaimedBytes ?? 0} bytes reclaimed`);
+            } catch (error: unknown) {
+                const msg = error instanceof Error ? error.message : String(error);
+                results.push(`${target}: failed (${msg})`);
+            }
+        }
+
+        return `System prune completed: ${results.join('; ')}`;
+    }
+}