fix(scheduler): harden scheduled operations with stale cleanup, cron validation, and design fixes (#549)

* fix(scheduler): clean up stale runs on startup and auto-disable invalid cron tasks

- Add markStaleRunsAsFailed() bulk DB method with status index
- Clean up orphaned 'running' records on scheduler startup
- Auto-disable tasks when cron expression becomes invalid at execution time
- Promote CRUD debug logs to standard logs for scheduled task admin actions
- Add diagnostic logging for task pre-checks, action timing, and prune fallback

* refactor(scheduling): extract shared types and fix design system violations

- Extract ScheduledTask, TaskRun, NodeOption to shared types file
- Extract getCronDescription and formatTimestamp to shared utilities
- Fix formatTimestamp falsy-zero null check
- Tighten last_status type to 'success' | 'failure' | null
- Add strokeWidth={1.5} to all action icons per design system
- Add sr-only DialogDescription for accessibility
- Wrap Sheet run history in ScrollArea
- Fix delete button styling to match design system pattern
- Change manual trigger toast from "executed" to "triggered"

* test(scheduler): add tests for snapshot, remote update, stale cleanup, and cron invalidation

- Add stale run cleanup tests (bulk markStaleRunsAsFailed, logging)
- Add cron invalidation test (auto-disable, error message)
- Add executeSnapshot tests (fleet capture, empty stacks)
- Add executeUpdateRemote tests (proxy success, remote error)
- Document stale run cleanup and cron auto-disable in troubleshooting docs

backend/src/services/DatabaseService.ts

@@ -437,6 +437,7 @@ export class DatabaseService {
       );
 
       CREATE INDEX IF NOT EXISTS idx_scheduled_task_runs_task ON scheduled_task_runs(task_id);
+      CREATE INDEX IF NOT EXISTS idx_scheduled_task_runs_status ON scheduled_task_runs(status);
       CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_next_run ON scheduled_tasks(next_run_at);
 
       CREATE TABLE IF NOT EXISTS stack_labels (
@@ -1531,6 +1532,13 @@ export class DatabaseService {
         ).all(taskId) as ScheduledTaskRun[];
     }
 
+    public markStaleRunsAsFailed(): number {
+        const result = this.db.prepare(
+            'UPDATE scheduled_task_runs SET status = ?, completed_at = ?, error = ? WHERE status = ?'
+        ).run('failure', Date.now(), 'Server restarted during execution', 'running');
+        return result.changes;
+    }
+
     public cleanupOldTaskRuns(retentionDays = 30): void {
         const cutoff = Date.now() - (retentionDays * 24 * 60 * 60 * 1000);
         this.db.prepare('DELETE FROM scheduled_task_runs WHERE started_at < ?').run(cutoff);

backend/src/services/SchedulerService.ts

@@ -29,6 +29,7 @@ export class SchedulerService {
 
     public start(): void {
         if (this.intervalId) return;
+        this.cleanupStaleRuns();
         this.intervalId = setInterval(() => this.tick(), 60_000);
         setTimeout(() => this.tick(), 10_000);
         console.log('[SchedulerService] Started');
@@ -42,6 +43,17 @@ export class SchedulerService {
         console.log('[SchedulerService] Stopped');
     }
 
+    private cleanupStaleRuns(): void {
+        try {
+            const count = DatabaseService.getInstance().markStaleRunsAsFailed();
+            if (count > 0) {
+                console.log(`[SchedulerService] Cleaned up ${count} stale run record(s)`);
+            }
+        } catch (error) {
+            console.error('[SchedulerService] Failed to clean up stale runs:', error);
+        }
+    }
+
     public calculateNextRun(cronExpression: string): number {
         const expr = CronExpressionParser.parse(cronExpression);
         return expr.next().toDate().getTime();
@@ -101,6 +113,7 @@ export class SchedulerService {
         const task = db.getScheduledTask(taskId);
         if (!task) throw new Error('Task not found');
         if (this.runningTasks.has(task.id)) throw new Error('Task is already running');
+        console.log(`[SchedulerService] Manual trigger: task "${task.name}" (id=${task.id})`);
         this.runningTasks.add(task.id);
         try {
             await this.executeTask(task, 'manual');
@@ -129,6 +142,8 @@ export class SchedulerService {
                 if (node.status === 'offline') throw new Error(`Target node "${node.name}" is offline`);
             }
 
+            if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} pre-checks passed, executing ${task.action}`);
+            const actionStart = Date.now();
             let output = '';
             switch (task.action) {
                 case 'restart':
@@ -145,6 +160,8 @@ export class SchedulerService {
                     break;
             }
 
+            if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} action completed in ${Date.now() - actionStart}ms`);
+
             const nextRun = this.calculateNextRun(task.cron_expression);
             db.updateScheduledTask(task.id, {
                 last_run_at: Date.now(),
@@ -169,18 +186,26 @@ export class SchedulerService {
         } catch (error: unknown) {
             const errMsg = error instanceof Error ? error.message : String(error);
             let nextRun: number | null = null;
+            let cronInvalid = false;
             try {
                 nextRun = this.calculateNextRun(task.cron_expression);
             } catch {
-                // If cron expression is somehow invalid, disable the task
+                cronInvalid = true;
             }
-            db.updateScheduledTask(task.id, {
+            const updates: Partial<Omit<ScheduledTask, 'id'>> = {
                 last_run_at: Date.now(),
                 next_run_at: nextRun,
                 last_status: 'failure',
-                last_error: errMsg,
+                last_error: cronInvalid
+                    ? `${errMsg}. Cron expression "${task.cron_expression}" is no longer valid; task has been disabled.`
+                    : errMsg,
                 updated_at: Date.now(),
-            });
+            };
+            if (cronInvalid) {
+                updates.enabled = 0;
+                console.warn(`[SchedulerService] Task "${task.name}" (id=${task.id}) auto-disabled: cron expression invalid`);
+            }
+            db.updateScheduledTask(task.id, updates);
             db.updateScheduledTaskRun(runId, {
                 completed_at: Date.now(),
                 status: 'failure',
@@ -361,6 +386,9 @@ export class SchedulerService {
 
     private async executePrune(task: ScheduledTask): Promise<string> {
         const nodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
+        if (task.node_id == null && isDebugEnabled()) {
+            console.log(`[SchedulerService:debug] Prune task ${task.id}: no node_id specified, using default node ${nodeId}`);
+        }
         const docker = DockerController.getInstance(nodeId);
         const allTargets = ['containers', 'images', 'networks', 'volumes'] as const;
         type PruneTarget = typeof allTargets[number];