import { CronExpressionParser } from 'cron-parser';
import { DatabaseService } from './DatabaseService';
import type { ScheduledTask } from './DatabaseService';
import { LicenseService } from './LicenseService';
import { PROXY_TIER_HEADER, deployProvenanceHeaders } from './license-headers';
import DockerController from './DockerController';
import { ComposeService } from './ComposeService';
import { StackUpdateOrchestrator } from './StackUpdateOrchestrator';
import { StackOpLockService, stackOpSkipMessage as skipMessage } from './StackOpLockService';
import { FileSystemService } from './FileSystemService';
import { StackUpdateRecoveryService } from './StackUpdateRecoveryService';
import { HealthGateService } from './HealthGateService';
import { ServiceUpdateRecoveryService } from './ServiceUpdateRecoveryService';
import {
    createAutoUpdateDigestGateState,
    messageWhenDigestApplyBlockedByCheckErrors,
    messageWhenNoDigestUpdate,
    recordAutoUpdateImageCheck,
} from '../helpers/autoUpdateDigestGate';
import { ImageUpdateService, UPDATE_VERIFICATION_INCOMPLETE_WARNING } from './ImageUpdateService';
import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache';
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
import { isDebugEnabled } from '../utils/debug';
import { getErrorMessage } from '../utils/errors';
import { safeRemoteFetch } from '../utils/outboundTarget';
import { formatNoTargetError } from '../utils/remoteTarget';
import { sanitizeForLog } from '../utils/safeLog';
import { captureLocalNodeFiles, captureRemoteNodeFiles, buildSnapshotDocumentation, type SnapshotNodeData } from '../utils/snapshot-capture';
import { NodeRegistry, type ProxyTarget } from './NodeRegistry';
import { NotificationService } from './NotificationService';
import TrivyService from './TrivyService';
import type { ScanAllNodeImagesResult } from './TrivyService';
import TrivyInstaller from './TrivyInstaller';
import { CloudBackupService } from './CloudBackupService';
import { buildSystemPolicyGateOptions } from '../helpers/policyGate';
import { prepareOutboundRegistryDeliveryBody } from '../helpers/registryDeliveryOutbound';
import { filterContainersByComposeService } from '../helpers/composeServiceMatch';
import { excludeSelfContainers } from '../helpers/excludeSelfContainers';
import { enforcePolicyPreDeploy } from './PolicyEnforcement';
import { summarizeBlockReasons } from '../utils/policy-risk';
import { resolveTaskPermissionScope, type BackendScheduledAction, type TargetType } from './scheduledActionRegistry';
import { checkPermissionForSubject } from '../middleware/permissions';

const TRIVY_UPDATE_CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000;
const TRIVY_UPDATE_CHECK_STARTUP_DELAY_MS = 5 * 60 * 1000;

const TRIVY_REDETECT_INTERVAL_MS = 10 * 60 * 1000;
const STALE_SCAN_THRESHOLD_MS = 15 * 60 * 1000;

/** Thrown when a scheduled task's creator no longer has permission for the target action. */
export class TaskAuthorizationError extends Error {
    constructor(message: string) {
        super(message);
        this.name = 'TaskAuthorizationError';
    }
}

export class SchedulerService {
    private static instance: SchedulerService;
    private intervalId: ReturnType<typeof setInterval> | null = null;
    private trivyUpdateIntervalId: ReturnType<typeof setInterval> | null = null;
    private trivyUpdateStartupTimer: ReturnType<typeof setTimeout> | null = null;
    private isProcessing = false;
    private isCheckingTrivyUpdate = false;
    private runningTasks = new Set<number>();
    private lastTrivyRedetect = 0;

    private constructor() {}

    public static getInstance(): SchedulerService {
        if (!SchedulerService.instance) {
            SchedulerService.instance = new SchedulerService();
        }
        return SchedulerService.instance;
    }

    public start(): void {
        if (this.intervalId) return;
        this.cleanupStaleRuns();
        this.intervalId = setInterval(() => this.tick(), 60_000);
        setTimeout(() => this.tick(), 10_000);
        this.trivyUpdateStartupTimer = setTimeout(() => this.runTrivyUpdateCheck(), TRIVY_UPDATE_CHECK_STARTUP_DELAY_MS);
        this.trivyUpdateIntervalId = setInterval(() => this.runTrivyUpdateCheck(), TRIVY_UPDATE_CHECK_INTERVAL_MS);
        console.log('[SchedulerService] Started');
    }

    public stop(): void {
        if (this.intervalId) {
            clearInterval(this.intervalId);
            this.intervalId = null;
        }
        if (this.trivyUpdateIntervalId) {
            clearInterval(this.trivyUpdateIntervalId);
            this.trivyUpdateIntervalId = null;
        }
        if (this.trivyUpdateStartupTimer) {
            clearTimeout(this.trivyUpdateStartupTimer);
            this.trivyUpdateStartupTimer = null;
        }
        console.log('[SchedulerService] Stopped');
    }

    private async runTrivyUpdateCheck(): Promise<void> {
        if (this.isCheckingTrivyUpdate) return;
        this.isCheckingTrivyUpdate = true;
        try {
            const trivy = TrivyService.getInstance();
            if (!trivy.isTrivyAvailable() || trivy.getSource() !== 'managed') return;
            const db = DatabaseService.getInstance();
            const settings = db.getGlobalSettings();
            const autoUpdate = settings.trivy_auto_update === '1';
            const installer = TrivyInstaller.getInstance();
            if (installer.isBusy()) return;
            const check = await installer.checkForUpdate(trivy.getVersion(), 'managed');
            if (!check.updateAvailable) return;

            if (autoUpdate) {
                const previous = trivy.getVersion() ?? 'unknown';
                console.log(`[SchedulerService] Auto-updating Trivy from ${previous} to ${check.latest}`);
                try {
                    await installer.update();
                    await trivy.detectTrivy();
                    this.safeDispatch(
                        'info',
                        'system',
                        `Trivy updated from v${previous} to v${check.latest}`,
                    );
                    db.updateGlobalSetting('trivy_last_notified_version', check.latest);
                } catch (err) {
                    console.error('[SchedulerService] Trivy auto-update failed:', getErrorMessage(err, 'unknown error'));
                }
            } else {
                const lastNotified = settings.trivy_last_notified_version || '';
                if (lastNotified === check.latest) return;
                this.safeDispatch(
                    'info',
                    'system',
                    `Trivy update available: v${check.latest} (currently v${check.current ?? 'unknown'})`,
                );
                db.updateGlobalSetting('trivy_last_notified_version', check.latest);
            }
        } catch (err) {
            console.warn('[SchedulerService] Trivy update check failed:', getErrorMessage(err, 'unknown error'));
        } finally {
            this.isCheckingTrivyUpdate = false;
        }
    }

    private cleanupStaleRuns(): void {
        try {
            const count = DatabaseService.getInstance().markStaleRunsAsFailed();
            if (count > 0) {
                console.log(`[SchedulerService] Cleaned up ${count} stale run record(s)`);
            }
        } catch (error) {
            console.error('[SchedulerService] Failed to clean up stale runs:', error);
        }
    }

    private async maybeRedetectTrivy(): Promise<void> {
        const now = Date.now();
        if (now - this.lastTrivyRedetect < TRIVY_REDETECT_INTERVAL_MS) return;
        this.lastTrivyRedetect = now;
        try {
            await TrivyService.getInstance().detectTrivy();
        } catch (error) {
            if (isDebugEnabled()) {
                console.warn('[SchedulerService:debug] Trivy re-detect failed:', error);
            }
        }
    }

    public calculateNextRun(cronExpression: string): number {
        const expr = CronExpressionParser.parse(cronExpression);
        return expr.next().toDate().getTime();
    }

    public calculateRunsWithin(cronExpression: string, fromMs: number, toMs: number, limit = 16): number[] {
        try {
            const expr = CronExpressionParser.parse(cronExpression, { currentDate: new Date(fromMs) });
            const runs: number[] = [];
            while (runs.length < limit) {
                const next = expr.next().toDate().getTime();
                if (next > toMs) break;
                runs.push(next);
            }
            return runs;
        } catch {
            return [];
        }
    }

    /**
     * Fire a notification without awaiting completion, catching any promise
     * rejection so the scheduler never crashes on a failed dispatch.
     */
    private safeDispatch(level: 'info' | 'warning' | 'error', category: import('./NotificationService').NotificationCategory, message: string, stackName?: string): void {
        NotificationService.getInstance()
            .dispatchAlert(level, category, message, { stackName, actor: 'system:scheduler' })
            .catch(err => console.error('[SchedulerService] Notification dispatch failed:', getErrorMessage(err, 'unknown error')));
    }

    /**
     * Run the pre-deploy scan-policy gate for a scheduler-driven action. On a
     * block, dispatch the documented `scan_finding` warning naming the policy
     * and the offending images, then throw so the caller records the outcome:
     * the auto-update loop catches per stack and continues the rest of the run,
     * while a single-stack auto-start surfaces as a task failure. The gate
     * fails open when Trivy is missing.
     */
    private async enforceSchedulerPolicyGate(
        stackName: string,
        nodeId: number,
        action: 'Auto-start' | 'Auto-update',
        auditPath: string,
    ): Promise<void> {
        const actor = action === 'Auto-start' ? 'scheduler:auto-start' : 'scheduler:auto-update';
        const gate = await enforcePolicyPreDeploy(
            stackName,
            nodeId,
            buildSystemPolicyGateOptions(actor, { auditPath }),
        );
        if (gate.ok) return;
        const images = gate.violations.map((v) => v.imageRef).join(', ');
        const reasons = summarizeBlockReasons(gate.violations);
        this.safeDispatch(
            'warning',
            'scan_finding',
            `${action} blocked for "${stackName}" by policy "${gate.policy?.name}": ${gate.violations.length} image(s) matched ${reasons}${images ? ` (${images})` : ''}`,
            stackName,
        );
        throw new Error(
            `${action} blocked by policy "${gate.policy?.name}": ${gate.violations.length} image(s) matched ${reasons}`,
        );
    }

    private async tick(): Promise<void> {
        if (this.isProcessing) {
            console.warn('[SchedulerService] Tick skipped: previous tick still processing');
            return;
        }
        this.isProcessing = true;
        try {
            const db = DatabaseService.getInstance();

            // Sweep stale vulnerability scans and re-detect Trivy on every tick.
            try {
                const staleScans = db.markStaleScansAsFailed(STALE_SCAN_THRESHOLD_MS);
                if (staleScans > 0) {
                    console.log(
                        `[SchedulerService] Marked ${staleScans} stale vulnerability scan(s) as failed`,
                    );
                }
            } catch (error) {
                console.error('[SchedulerService] Stale scan sweep failed:', error);
            }
            await this.maybeRedetectTrivy();

            const now = Date.now();
            const dueTasks = db.getDueScheduledTasks(now);

            if (dueTasks.length > 0) {
                console.log(`[SchedulerService] Found ${dueTasks.length} due task(s)`);
            }

            // Clean up old runs periodically (piggyback on tick)
            db.cleanupOldTaskRuns(30);
            db.deleteOldScans(90 * 24 * 60 * 60 * 1000);

            for (const task of dueTasks) {
                if (this.runningTasks.has(task.id)) {
                    if (isDebugEnabled()) console.log(`[SchedulerService] Task ${task.id} skipped: already running`);
                    continue;
                }
                this.runningTasks.add(task.id);
                if (isDebugEnabled()) console.log(`[SchedulerService] Executing task ${task.id} ("${task.name}")`);
                this.executeTask(task).finally(() => this.runningTasks.delete(task.id));
            }
        } catch (error) {
            console.error('[SchedulerService] Tick error:', error);
        } finally {
            this.isProcessing = false;
        }
    }

    public isTaskRunning(taskId: number): boolean {
        return this.runningTasks.has(taskId);
    }

    // Intentionally allows triggering disabled tasks, useful for testing before enabling a schedule.
    // Manual triggers are attributed as 'manual' in the run record (see triggered_by column).
    public async triggerTask(taskId: number): Promise<void> {
        const db = DatabaseService.getInstance();
        const task = db.getScheduledTask(taskId);
        if (!task) throw new Error('Task not found');
        if (this.runningTasks.has(task.id)) throw new Error('Task is already running');
        console.log(`[SchedulerService] Manual trigger: task "${task.name}" (id=${task.id})`);
        this.runningTasks.add(task.id);
        try {
            await this.executeTask(task, 'manual');
        } finally {
            this.runningTasks.delete(task.id);
        }
    }

    private async executeTask(task: ScheduledTask, triggeredBy: 'scheduler' | 'manual' = 'scheduler'): Promise<void> {
        const db = DatabaseService.getInstance();
        const runId = db.createScheduledTaskRun({
            task_id: task.id,
            started_at: Date.now(),
            completed_at: null,
            status: 'running',
            output: null,
            error: null,
            triggered_by: triggeredBy,
        });

        try {
            // Permission revalidation: for automatic runs, verify the creator still holds
            // the required permission. Runs before the node-reachability check so that
            // a revoked authorization is surfaced even when the target node is offline
            // — a misleading "target node is offline" error must not hide the real
            // reason the task cannot execute. Manual runs skip this; the route's
            // acting-user check is the gate. Legacy tasks (creator_user_id NULL)
            // execute as before.
            if (triggeredBy === 'scheduler' && task.creator_user_id != null) {
                const creator = db.getUserById(task.creator_user_id);
                if (!creator) {
                    throw new TaskAuthorizationError('Scheduled task no longer authorized: creator account no longer exists.');
                }
                const scope = resolveTaskPermissionScope(
                    task.action as BackendScheduledAction,
                    task.target_type as TargetType,
                    task.target_id,
                    task.node_id,
                    task.selector_type,
                );
                if (!checkPermissionForSubject(
                    { username: creator.username, role: creator.role, userId: creator.id },
                    scope.action,
                    scope.resourceType,
                    scope.resourceId,
                    scope.resourceNodeId,
                )) {
                    throw new TaskAuthorizationError('Scheduled task no longer authorized: creator permission was revoked.');
                }
            }

            // Pre-check: ensure target node exists and is reachable
            if (task.node_id != null && task.action !== 'snapshot') {
                const node = db.getNode(task.node_id);
                if (!node) throw new Error(`Target node (id=${task.node_id}) no longer exists`);
                if (node.status === 'offline') {
                    // Local seed name ("Local") must not reach fleet-aggregated alerts;
                    // remote roster names stay for diagnosis.
                    throw new Error(
                        node.type === 'local'
                            ? 'Target node is offline'
                            : `Target node "${node.name}" is offline`,
                    );
                }
            }

            if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} pre-checks passed, executing ${task.action}`);
            const actionStart = Date.now();
            let output = '';
            let scanFailedCount = 0;
            switch (task.action) {
                case 'restart':
                    output = await this.executeRestart(task);
                    break;
                case 'snapshot':
                    output = await this.executeSnapshot(task);
                    break;
                case 'prune':
                    output = await this.executePrune(task);
                    break;
                case 'update':
                    output = await this.executeUpdate(task);
                    break;
                case 'scan': {
                    const result = await this.executeScan(task);
                    output = result.output;
                    scanFailedCount = result.failed;
                    break;
                }
                case 'auto_backup':
                    output = await this.executeAutoBackup(task);
                    break;
                case 'auto_stop':
                    output = await this.executeAutoStop(task);
                    break;
                case 'auto_down':
                    output = await this.executeAutoDown(task);
                    break;
                case 'auto_start':
                    output = await this.executeAutoStart(task);
                    break;
                default: {
                    const unhandledAction: never = task.action;
                    throw new Error(`Unsupported scheduled action: ${unhandledAction}`);
                }
            }

            if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} action completed in ${Date.now() - actionStart}ms`);

            db.updateScheduledTaskRun(runId, {
                completed_at: Date.now(),
                status: 'success',
                output,
            });
            console.log(`[SchedulerService] Task "${task.name}" (id=${task.id}) completed successfully`);

            if (task.delete_after_run === 1) {
                console.log(`[SchedulerService] Task "${task.name}" (id=${task.id}) self-deleting after successful one-shot run`);
                db.deleteScheduledTask(task.id);
                return;
            }

            const nextRun = this.calculateNextRun(task.cron_expression);
            db.updateScheduledTask(task.id, {
                last_run_at: Date.now(),
                next_run_at: nextRun,
                last_status: 'success',
                last_error: null,
                updated_at: Date.now(),
            });

            if (task.action === 'scan') {
                const scanLevel: 'info' | 'warning' = scanFailedCount > 0 ? 'warning' : 'info';
                if (isDebugEnabled()) {
                    console.log(
                        `[SchedulerService:debug] Dispatching scan completion notification (level=${scanLevel}, stackContext=${task.target_id ?? 'none'})`,
                    );
                }
                this.safeDispatch(
                    scanLevel,
                    'scan_finding',
                    `Scheduled scan "${task.name}" completed: ${output}`,
                    task.target_id ?? undefined
                );
            } else if (task.last_status === 'failure') {
                this.safeDispatch(
                    'info',
                    'system',
                    `Scheduled task "${task.name}" (${task.action}) recovered successfully`,
                    task.target_id ?? undefined
                );
            }
        } catch (error: unknown) {
            const errMsg = error instanceof Error ? error.message : String(error);
            let nextRun: number | null = null;
            let cronInvalid = false;
            try {
                nextRun = this.calculateNextRun(task.cron_expression);
            } catch {
                cronInvalid = true;
            }
            const updates: Partial<Omit<ScheduledTask, 'id'>> = {
                last_run_at: Date.now(),
                next_run_at: nextRun,
                last_status: 'failure',
                last_error: cronInvalid
                    ? `${errMsg}. Cron expression "${task.cron_expression}" is no longer valid; task has been disabled.`
                    : errMsg,
                updated_at: Date.now(),
            };
            if (cronInvalid) {
                updates.enabled = 0;
                console.warn(`[SchedulerService] Task "${task.name}" (id=${task.id}) auto-disabled: cron expression invalid`);
            }
            if (error instanceof TaskAuthorizationError) {
                updates.enabled = 0;
                console.warn(`[SchedulerService] Task "${task.name}" (id=${task.id}) auto-disabled: creator authorization revoked`);
            }
            db.updateScheduledTask(task.id, updates);
            db.updateScheduledTaskRun(runId, {
                completed_at: Date.now(),
                status: 'failure',
                error: errMsg,
            });
            console.error(`[SchedulerService] Task "${task.name}" (id=${task.id}) failed:`, errMsg);
            this.safeDispatch(
                'error',
                'system',
                `Scheduled task "${task.name}" (${task.action}) failed: ${errMsg}`,
                task.target_id ?? undefined
            );
        }
    }

    private async executeRestart(task: ScheduledTask): Promise<string> {
        if (task.target_type === 'container') {
            return this.executeContainerRestart(task);
        }
        if (!task.target_id || task.node_id == null) {
            throw new Error('Stack restart requires target_id and node_id');
        }
        if (this.isRemoteNode(task.node_id)) {
            return this.executeRestartRemote(task.node_id, task.target_id, task.target_services);
        }
        const docker = DockerController.getInstance(task.node_id);
        const containers = await docker.getContainersByStack(task.target_id);
        if (!containers || containers.length === 0) {
            throw new Error(`No containers found for stack "${task.target_id}"`);
        }

        let filtered = containers;
        if (task.target_services) {
            const serviceNames: string[] = JSON.parse(task.target_services);
            filtered = serviceNames.flatMap(svc => filterContainersByComposeService(containers, svc));
            if (filtered.length === 0) {
                throw new Error(`No containers found matching services [${serviceNames.join(', ')}] in stack "${task.target_id}"`);
            }
        }

        await Promise.all(filtered.map(c => docker.restartContainer(c.Id)));
        const servicesSuffix = task.target_services
            ? ` (services: ${(JSON.parse(task.target_services) as string[]).join(', ')})`
            : '';
        return `Restarted ${filtered.length} container(s) in stack "${task.target_id}"${servicesSuffix}`;
    }

    /**
     * Remote restart. The remote bulk-restart endpoint restarts every container
     * in the stack, so when the task targets specific services we fan out to the
     * per-service restart route to preserve the filter.
     */
    private async executeRestartRemote(nodeId: number, stackName: string, targetServices: string | null): Promise<string> {
        const stackSeg = encodeURIComponent(stackName);
        if (targetServices) {
            const serviceNames: string[] = JSON.parse(targetServices);
            // Fail fast, but name the services already restarted so a mid-loop failure
            // records the partial state of the remote stack in run history.
            const restarted: string[] = [];
            for (const svc of serviceNames) {
                try {
                    await this.postToRemoteStack(nodeId, `${stackSeg}/services/${encodeURIComponent(svc)}/restart`);
                    restarted.push(svc);
                } catch (e) {
                    const done = restarted.length ? ` (already restarted: ${restarted.join(', ')})` : '';
                    throw new Error(`Restart of service "${svc}" failed${done}: ${getErrorMessage(e, String(e))}`);
                }
            }
            return `Restarted services [${serviceNames.join(', ')}] in stack "${stackName}" on remote node`;
        }
        await this.postToRemoteStack(nodeId, `${stackSeg}/restart`);
        return `Restarted stack "${stackName}" on remote node`;
    }

    private assertStackTarget(task: ScheduledTask, label: string): asserts task is ScheduledTask & { target_id: string; node_id: number } {
        if (!task.target_id || task.node_id == null) {
            throw new Error(`${label} requires target_id and node_id`);
        }
    }

    private isRemoteNode(nodeId: number): boolean {
        return NodeRegistry.getInstance().getNode(nodeId)?.type === 'remote';
    }

    private async executeAutoBackup(task: ScheduledTask): Promise<string> {
        this.assertStackTarget(task, 'Auto-backup');
        if (this.isRemoteNode(task.node_id)) {
            await this.postToRemoteStack(task.node_id, `${encodeURIComponent(task.target_id)}/backup`);
            return `Captured a recovery generation for stack "${task.target_id}" on remote node`;
        }
        const localNodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
        const lock = await StackOpLockService.getInstance().runExclusive(
            localNodeId, task.target_id, 'backup', 'system',
            () => StackUpdateRecoveryService.getInstance().captureCurrentBackup({
                nodeId: localNodeId,
                stackName: task.target_id,
                createdBy: 'system:scheduler',
            }),
        );
        // Throw (not return) so the skip records as a failed run instead of a
        // silent success; the next scheduled tick retries once the lock frees.
        if (!lock.ran) throw new Error(skipMessage(task.target_id, lock.existing.action));
        return `Captured a recovery generation for stack "${task.target_id}"`;
    }

    private async executeAutoStop(task: ScheduledTask): Promise<string> {
        if (task.target_type === 'container') {
            return this.executeContainerStop(task);
        }
        this.assertStackTarget(task, 'Auto-stop');
        if (this.isRemoteNode(task.node_id)) {
            await this.postToRemoteStack(task.node_id, `${encodeURIComponent(task.target_id)}/stop`);
            return `Stopped stack "${task.target_id}" (containers preserved) on remote node`;
        }
        const localNodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
        const lock = await StackOpLockService.getInstance().runExclusive(
            localNodeId, task.target_id, 'stop', 'system',
            () => ComposeService.getInstance(localNodeId).runCommand(task.target_id, 'stop'),
        );
        if (!lock.ran) throw new Error(skipMessage(task.target_id, lock.existing.action));
        return `Stopped stack "${task.target_id}" (containers preserved)`;
    }

    private async executeAutoDown(task: ScheduledTask): Promise<string> {
        this.assertStackTarget(task, 'Auto-down');
        if (this.isRemoteNode(task.node_id)) {
            await this.postToRemoteStack(task.node_id, `${encodeURIComponent(task.target_id)}/down`);
            return `Took down stack "${task.target_id}" (containers removed) on remote node`;
        }
        const localNodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
        const lock = await StackOpLockService.getInstance().runExclusive(
            localNodeId, task.target_id, 'down', 'system',
            () => ComposeService.getInstance(localNodeId).runCommand(task.target_id, 'down'),
        );
        if (!lock.ran) throw new Error(skipMessage(task.target_id, lock.existing.action));
        return `Took down stack "${task.target_id}" (containers removed)`;
    }

    private async executeAutoStart(task: ScheduledTask): Promise<string> {
        if (task.target_type === 'container') {
            return this.executeContainerStart(task);
        }
        this.assertStackTarget(task, 'Auto-start');
        // Remote auto-start proxies to the remote's own deploy route, which runs
        // that node's scan-policy gate against the images it actually holds. The
        // hub-side enforceSchedulerPolicyGate below is for local nodes only.
        if (this.isRemoteNode(task.node_id)) {
            await this.postToRemoteStack(
                task.node_id,
                `${encodeURIComponent(task.target_id)}/deploy`,
                deployProvenanceHeaders('scheduler', 'system:scheduler'),
            );
            return `Started stack "${task.target_id}" on remote node`;
        }
        await this.enforceSchedulerPolicyGate(
            task.target_id,
            task.node_id,
            'Auto-start',
            `/api/scheduled-tasks/${task.id}/run`,
        );
        const localNodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
        const lock = await StackOpLockService.getInstance().runExclusive(
            localNodeId, task.target_id, 'deploy', 'system',
            () => ComposeService.getInstance(localNodeId).deployStack(
                task.target_id,
                undefined,
                undefined,
                { source: 'scheduler', actor: 'system:scheduler' },
            ),
        );
        if (!lock.ran) throw new Error(skipMessage(task.target_id, lock.existing.action));
        return `Started stack "${task.target_id}"`;
    }

    private async executeSnapshot(task: ScheduledTask): Promise<string> {
        const db = DatabaseService.getInstance();
        const nodes = db.getNodes();
        const captureDocs = db.getGlobalSettings().snapshot_documentation === '1';

        const results = await Promise.allSettled(
            nodes.map(async (node) => {
                if (node.type === 'remote') {
                    return captureRemoteNodeFiles(node, captureDocs);
                }
                return captureLocalNodeFiles(node, captureDocs);
            })
        );

        const capturedNodes: SnapshotNodeData[] = [];
        const skippedNodes: Array<{ nodeId: number; nodeName: string; reason: string }> = [];

        results.forEach((result, i) => {
            if (result.status === 'fulfilled') {
                capturedNodes.push(result.value);
            } else {
                skippedNodes.push({
                    nodeId: nodes[i].id,
                    nodeName: nodes[i].name,
                    reason: result.reason instanceof Error ? result.reason.message : 'Unknown error',
                });
            }
        });

        let totalStacks = 0;
        const allFiles: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }> = [];
        const skippedStacks: Array<{ nodeId: number; nodeName: string; stackName: string; reason: string }> = [];

        for (const nodeData of capturedNodes) {
            totalStacks += nodeData.stacks.length;
            for (const stack of nodeData.stacks) {
                for (const file of stack.files) {
                    allFiles.push({
                        nodeId: nodeData.nodeId,
                        nodeName: nodeData.nodeName,
                        stackName: stack.stackName,
                        filename: file.filename,
                        content: file.content,
                    });
                }
            }
            for (const warning of nodeData.warnings) {
                skippedStacks.push({
                    nodeId: nodeData.nodeId,
                    nodeName: nodeData.nodeName,
                    stackName: warning.stackName,
                    reason: warning.reason,
                });
            }
        }

        const documentation = captureDocs
            ? buildSnapshotDocumentation(capturedNodes, new Date().toISOString())
            : null;

        const description = `Scheduled snapshot: ${task.name}`;
        const snapshotId = db.createSnapshot(
            description,
            task.created_by,
            capturedNodes.length,
            totalStacks,
            JSON.stringify(skippedNodes),
            JSON.stringify(skippedStacks),
            documentation ? JSON.stringify(documentation) : '',
        );

        if (allFiles.length > 0) {
            db.insertSnapshotFiles(snapshotId, allFiles);
        }

        let cloudUploadNote = '';
        const cloudSvc = CloudBackupService.getInstance();
        if (cloudSvc.isEnabled() && cloudSvc.isAutoUploadOn()) {
            try {
                await cloudSvc.uploadSnapshot(snapshotId);
                cloudUploadNote = ', cloud upload OK';
            } catch (err) {
                const message = getErrorMessage(err, 'Cloud upload failed');
                console.error('[SchedulerService] Cloud upload failed:', message);
                this.safeDispatch('warning', 'system', `Cloud backup failed for scheduled snapshot ${snapshotId}: ${message}`);
                cloudUploadNote = ', cloud upload FAILED';
            }
        }

        if (skippedNodes.length > 0 || skippedStacks.length > 0) {
            console.warn(`[SchedulerService] Snapshot task ${task.id} partial: skipped ${skippedNodes.length} node(s), ${skippedStacks.length} stack(s)`);
        }
        if (isDebugEnabled()) {
            console.debug(`[SchedulerService:debug] Snapshot task ${task.id}: captured ${capturedNodes.length} node(s), ${totalStacks} stack(s), ${allFiles.length} file(s), skipped ${skippedNodes.length} node(s)/${skippedStacks.length} stack(s)${cloudUploadNote}`);
        }

        const skippedNote = [
            skippedNodes.length > 0 ? `${skippedNodes.length} node(s)` : '',
            skippedStacks.length > 0 ? `${skippedStacks.length} stack(s)` : '',
        ].filter(Boolean).join(', ');
        return `Fleet snapshot created (id=${snapshotId}, ${capturedNodes.length} node(s), ${totalStacks} stack(s)${skippedNote ? `, skipped ${skippedNote}` : ''}${cloudUploadNote})`;
    }

    private async executePrune(task: ScheduledTask): Promise<string> {
        const nodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
        if (task.node_id == null && isDebugEnabled()) {
            console.log(`[SchedulerService:debug] Prune task ${task.id}: no node_id specified, using default node ${nodeId}`);
        }
        if (this.isRemoteNode(nodeId)) {
            throw new Error('Scheduled prunes currently require a local node.');
        }
        const docker = DockerController.getInstance(nodeId);
        const allTargets = ['containers', 'images', 'networks', 'volumes'] as const;
        type PruneTarget = typeof allTargets[number];
        const targets: PruneTarget[] = task.prune_targets
            ? (JSON.parse(task.prune_targets) as string[]).filter((t): t is PruneTarget => allTargets.includes(t as PruneTarget))
            : [...allTargets];
        const labelFilter = task.prune_label_filter || undefined;
        const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(nodeId);
        const results: string[] = [];
        const failures: string[] = [];

        for (const target of targets) {
            try {
                const result = await docker.pruneSystem(target, labelFilter, isImageHeld);
                results.push(`${target}: ${result.reclaimedBytes ?? 0} bytes reclaimed`);
            } catch (error: unknown) {
                const msg = error instanceof Error ? error.message : String(error);
                const failure = `${target}: failed (${msg})`;
                results.push(failure);
                failures.push(failure);
            }
        }

        const filterSuffix = labelFilter ? ` (label: ${labelFilter})` : '';
        if (failures.length > 0) {
            throw new Error(`System prune failed${filterSuffix}: ${results.join('; ')}`);
        }
        return `System prune completed${filterSuffix}: ${results.join('; ')}`;
    }

    private async executeUpdate(task: ScheduledTask): Promise<string> {
        if (task.selector_type === 'stack-label') {
            return this.executeUpdateByStackLabel(task);
        }

        if (task.node_id == null) {
            throw new Error('Auto-update requires node_id');
        }

        const isFleet = task.target_type === 'fleet';

        if (!isFleet && !task.target_id) {
            throw new Error('Auto-update requires target_id (stack name or "*")');
        }

        // For remote nodes, proxy the entire execution to the remote Sencho instance.
        // The remote /api/auto-update/execute endpoint already handles per-stack
        // auto-update policy, so passing '*' for fleet is sufficient.
        const node = NodeRegistry.getInstance().getNode(task.node_id);
        if (node?.type === 'remote') {
            return this.executeUpdateRemote(task.node_id, isFleet ? '*' : task.target_id!);
        }

        // Local node: execute directly
        const isWildcard = task.target_id === '*';
        let stackNames: string[];
        if (isFleet || isWildcard) {
            stackNames = await FileSystemService.getInstance(task.node_id).getStacks();
            if (stackNames.length === 0) {
                return 'No stacks found on node; skipped.';
            }
        } else {
            stackNames = [task.target_id!];
        }

        if (isDebugEnabled()) {
            console.log(`[SchedulerService] executeUpdate: ${stackNames.length} stack(s) to check, fleet=${isFleet}, wildcard=${isWildcard}`);
        }

        const docker = DockerController.getInstance(task.node_id);
        const imageUpdateService = ImageUpdateService.getInstance();
        const results: string[] = [];

        for (const stackName of stackNames) {
            try {
                const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, isFleet || isWildcard);
                results.push(output);
            } catch (e) {
                const msg = getErrorMessage(e, String(e));
                results.push(`Stack "${stackName}" failed: ${msg}`);
                console.error(`[SchedulerService] Auto-update failed for stack "${stackName}":`, e);
            }
        }

        return results.join('\n');
    }

    /**
     * Resolve live stack-label membership (fleet-wide or one node) and run the
     * existing per-stack auto-update path. Remotes receive an explicit stack
     * list; they do not evaluate the selector themselves.
     */
    private async executeUpdateByStackLabel(task: ScheduledTask): Promise<string> {
        const labelName = (task.selector_value ?? '').trim();
        if (!labelName) {
            throw new Error('Label-targeted auto-update requires selector_value');
        }

        const { collectFleetLabelSummaries } = await import('../helpers/fleetLabelSummary');
        let summaries = await collectFleetLabelSummaries();
        if (task.node_id != null) {
            summaries = summaries.filter(s => s.nodeId === task.node_id);
            if (summaries.length === 0) {
                throw new Error(`Target node (id=${task.node_id}) no longer exists`);
            }
        }

        type NodePlan = {
            nodeId: number;
            nodeName: string;
            reachable: boolean;
            stacks: string[];
            error?: string;
        };

        const plans: NodePlan[] = [];
        const seen = new Set<string>();
        for (const summary of summaries) {
            if (!summary.reachable) {
                plans.push({
                    nodeId: summary.nodeId,
                    nodeName: summary.nodeName,
                    reachable: false,
                    stacks: [],
                    error: summary.error ?? 'unreachable',
                });
                continue;
            }
            const match = summary.labels.find(l => l.name === labelName);
            const stacks: string[] = [];
            if (match) {
                for (const stackName of match.stackNames) {
                    const key = `${summary.nodeId}\0${stackName}`;
                    if (seen.has(key)) continue;
                    seen.add(key);
                    stacks.push(stackName);
                }
            }
            plans.push({
                nodeId: summary.nodeId,
                nodeName: summary.nodeName,
                reachable: true,
                stacks,
            });
        }

        const lines: string[] = [
            `Selector: stack-label="${labelName}" · scope=${task.node_id == null ? 'entire fleet' : `node ${task.node_id}`}`,
        ];
        for (const plan of plans) {
            if (!plan.reachable) {
                lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}): unreachable (${plan.error})`);
            } else if (plan.stacks.length === 0) {
                lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}): no stacks with label "${labelName}"`);
            } else {
                lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}): ${plan.stacks.length} stack(s) → ${plan.stacks.join(', ')}`);
            }
        }

        const matchedStacks = plans.reduce((n, p) => n + p.stacks.length, 0);
        const unreachableCount = plans.filter(p => !p.reachable).length;
        if (matchedStacks === 0 && unreachableCount === 0) {
            lines.push(`No stacks currently match label "${labelName}"; skipped.`);
            return lines.join('\n');
        }

        let materialFailure = unreachableCount > 0;
        const work = plans.filter(p => p.reachable && p.stacks.length > 0);
        const NODE_CONCURRENCY = 3;
        // Label-targeted runs fail closed on material stack failures or
        // unreachable scoped nodes (unlike plain node fleet update, which
        // historically returns failure lines as a successful run output).

        const looksLikeStackFailure = (text: string): boolean =>
          /^Stack ".+" failed:/m.test(text);

        const runNode = async (plan: NodePlan): Promise<void> => {
            const node = NodeRegistry.getInstance().getNode(plan.nodeId);
            try {
                if (node?.type === 'remote') {
                    const remoteOut = await this.executeUpdateRemoteTargets(plan.nodeId, plan.stacks);
                    lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}) results:\n${remoteOut}`);
                    if (looksLikeStackFailure(remoteOut)) materialFailure = true;
                } else {
                    const docker = DockerController.getInstance(plan.nodeId);
                    const imageUpdateService = ImageUpdateService.getInstance();
                    const stackLines: string[] = [];
                    for (const stackName of plan.stacks) {
                        try {
                            const out = await this.executeUpdateForStack(
                                stackName, plan.nodeId, docker, imageUpdateService, true,
                            );
                            stackLines.push(out);
                        } catch (e) {
                            materialFailure = true;
                            const msg = getErrorMessage(e, String(e));
                            stackLines.push(`Stack "${stackName}" failed: ${msg}`);
                            console.error(`[SchedulerService] Label auto-update failed for stack "${stackName}" on node ${plan.nodeId}:`, e);
                        }
                    }
                    lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}) results:\n${stackLines.join('\n')}`);
                }
            } catch (e) {
                materialFailure = true;
                const msg = getErrorMessage(e, String(e));
                lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}): failed (${msg})`);
                console.error(`[SchedulerService] Label auto-update failed for node ${plan.nodeId}:`, e);
            }
        };

        for (let i = 0; i < work.length; i += NODE_CONCURRENCY) {
            const batch = work.slice(i, i + NODE_CONCURRENCY);
            await Promise.all(batch.map(runNode));
        }

        if (materialFailure) {
            throw new Error(lines.join('\n'));
        }
        return lines.join('\n');
    }

    /**
     * Proxy auto-update execution to a remote Sencho instance.
     * The remote node runs the image checks and compose update locally.
     */
    private async executeUpdateRemote(nodeId: number, target: string): Promise<string> {
        const proxyTarget = this.requireRemoteProxyTarget(nodeId);
        const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
        const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
        if (isDebugEnabled()) {
            console.log(`[SchedulerService] executeUpdateRemote: node=${nodeId} target=${target}`);
        }
        const startTime = Date.now();
        try {
            const response = await safeRemoteFetch(`${baseUrl}/api/auto-update/execute`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                    'Authorization': `Bearer ${proxyTarget.apiToken}`,
                    [PROXY_TIER_HEADER]: proxyHeaders.tier,
                },
                body: JSON.stringify({ target }),
                signal: AbortSignal.timeout(300_000), // 5 minute timeout for long updates
            }, proxyTarget.trustedLoopback);

            if (!response.ok) {
                throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
            }

            const body = await response.json() as { result?: string };
            if (isDebugEnabled()) {
                console.log(`[SchedulerService] executeUpdateRemote: completed in ${Date.now() - startTime}ms`);
            }
            return body.result || 'Remote auto-update completed (no details returned).';
        } catch (err) {
            this.rethrowRemoteProxyError(nodeId, err);
        }
    }

    /** Proxy auto-update for an explicit stack list on a remote node (label selector). */
    private async executeUpdateRemoteTargets(nodeId: number, targets: string[]): Promise<string> {
        const proxyTarget = this.requireRemoteProxyTarget(nodeId);
        const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
        const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
        if (isDebugEnabled()) {
            console.log(`[SchedulerService] executeUpdateRemoteTargets: node=${nodeId} count=${targets.length}`);
        }
        const startTime = Date.now();
        try {
            const response = await safeRemoteFetch(`${baseUrl}/api/auto-update/execute`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                    'Authorization': `Bearer ${proxyTarget.apiToken}`,
                    [PROXY_TIER_HEADER]: proxyHeaders.tier,
                },
                body: JSON.stringify({ targets }),
                signal: AbortSignal.timeout(300_000),
            }, proxyTarget.trustedLoopback);

            // Older remotes only accept { target }. Fall back to one call per
            // stack so mixed-version fleets still complete the label schedule.
            if (response.status === 400) {
                const detail = await this.remoteResponseDetail(response);
                if (/target/i.test(detail)) {
                    const parts: string[] = [];
                    for (const stackName of targets) {
                        parts.push(await this.executeUpdateRemote(nodeId, stackName));
                    }
                    return parts.join('\n');
                }
                throw new Error(this.remoteProxyFailureMessage(nodeId, detail));
            }

            if (!response.ok) {
                throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
            }

            const body = await response.json() as { result?: string };
            if (isDebugEnabled()) {
                console.log(`[SchedulerService] executeUpdateRemoteTargets: completed in ${Date.now() - startTime}ms`);
            }
            return body.result || 'Remote auto-update completed (no details returned).';
        } catch (err) {
            this.rethrowRemoteProxyError(nodeId, err);
        }
    }

    /**
     * Proxy a stack lifecycle action to a remote Sencho instance. ComposeService,
     * DockerController, and FileSystemService are local-only, so for a remote node
     * we POST to the remote's own stack-operation endpoint with the node Bearer
     * token and the licence proxy headers, exactly as executeUpdateRemote does.
     * `routeSuffix` is the path under `/api/stacks/`; the caller URL-encodes each
     * segment.
     */
    private containerNotFoundMessage(containerName: string, nodeId: number): string {
        const node = NodeRegistry.getInstance().getNode(nodeId);
        // Only an explicit local node is neutralized; missing lookup keeps the id fallback.
        const location = node?.type === 'local'
            ? 'on this node'
            : `on node "${node?.name ?? String(nodeId)}"`;
        return `Container "${containerName}" not found ${location}. It may have been renamed or removed.`;
    }

    /** Hub target tag used for startsWith rethrow guards (no trailing detail). */
    private remoteProxyNodeTag(nodeId: number): string {
        const nodeName = NodeRegistry.getInstance().getNode(nodeId)?.name ?? String(nodeId);
        return `Remote node "${nodeName}" (id=${nodeId})`;
    }

    /** Operator-facing remote proxy failure with stable node correlation. */
    private remoteProxyFailureMessage(nodeId: number, detail: string): string {
        return `${this.remoteProxyNodeTag(nodeId)}: ${detail}`;
    }

    private noProxyTargetDetail(nodeId: number): string {
        const node = NodeRegistry.getInstance().getNode(nodeId);
        if (!node) return 'Remote node is not configured or missing API credentials';
        return formatNoTargetError(node);
    }

    private async remoteResponseDetail(response: Response): Promise<string> {
        try {
            const body = await response.json() as { error?: string };
            return body.error || `Remote node returned ${response.status}`;
        } catch {
            return `HTTP ${response.status}`;
        }
    }

    private requireRemoteProxyTarget(nodeId: number): ProxyTarget {
        const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId);
        if (!proxyTarget) {
            throw new Error(this.remoteProxyFailureMessage(nodeId, this.noProxyTargetDetail(nodeId)));
        }
        return proxyTarget;
    }

    private rethrowRemoteProxyError(nodeId: number, err: unknown): never {
        const tag = this.remoteProxyNodeTag(nodeId);
        if (err instanceof Error && err.message.startsWith(tag)) throw err;
        throw new Error(this.remoteProxyFailureMessage(nodeId, getErrorMessage(err, 'Remote proxy request failed')));
    }

    private async resolveContainerId(task: ScheduledTask): Promise<{ id: string; name: string }> {
        if (!task.target_id || task.node_id == null) {
            throw new Error('Container operations require target_id and node_id');
        }
        const name = task.target_id;
        if (this.isRemoteNode(task.node_id)) {
            const containers = await this.getRemoteContainers(task.node_id);
            const match = containers.find(
                c => c.Names?.[0]?.replace(/^\//, '') === name,
            );
            if (!match) throw new Error(this.containerNotFoundMessage(name, task.node_id));
            return { id: match.Id, name };
        }
        const found = await DockerController.getInstance(task.node_id).findContainerByName(name);
        if (!found) throw new Error(this.containerNotFoundMessage(name, task.node_id));
        return { id: found.id, name: found.name };
    }

    private async executeContainerRestart(task: ScheduledTask): Promise<string> {
        const { id, name } = await this.resolveContainerId(task);
        const nodeId = task.node_id!;
        if (this.isRemoteNode(nodeId)) {
            await this.postToRemoteContainer(nodeId, id, 'restart');
        } else {
            await DockerController.getInstance(nodeId).restartContainer(id);
        }
        const nodeName = NodeRegistry.getInstance().getNode(nodeId)?.name ?? String(nodeId);
        return `Restarted container "${name}" (id ${id.slice(0, 12)}) on node "${nodeName}"`;
    }

    private async executeContainerStop(task: ScheduledTask): Promise<string> {
        const { id, name } = await this.resolveContainerId(task);
        const nodeId = task.node_id!;
        if (this.isRemoteNode(nodeId)) {
            await this.postToRemoteContainer(nodeId, id, 'stop');
        } else {
            await DockerController.getInstance(nodeId).stopContainer(id);
        }
        const nodeName = NodeRegistry.getInstance().getNode(nodeId)?.name ?? String(nodeId);
        return `Stopped container "${name}" (id ${id.slice(0, 12)}) on node "${nodeName}"`;
    }

    private async executeContainerStart(task: ScheduledTask): Promise<string> {
        const { id, name } = await this.resolveContainerId(task);
        const nodeId = task.node_id!;
        if (this.isRemoteNode(nodeId)) {
            await this.postToRemoteContainer(nodeId, id, 'start');
        } else {
            await DockerController.getInstance(nodeId).startContainer(id);
        }
        const nodeName = NodeRegistry.getInstance().getNode(nodeId)?.name ?? String(nodeId);
        return `Started container "${name}" (id ${id.slice(0, 12)}) on node "${nodeName}"`;
    }

    private async getRemoteContainers(nodeId: number): Promise<Array<{
        Id: string;
        Names?: string[];
        State?: string;
        Image?: string;
    }>> {
        const proxyTarget = this.requireRemoteProxyTarget(nodeId);
        const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
        const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
        try {
            const response = await safeRemoteFetch(`${baseUrl}/api/containers?all=true`, {
                headers: {
                    'Authorization': `Bearer ${proxyTarget.apiToken}`,
                    [PROXY_TIER_HEADER]: proxyHeaders.tier,
                },
                signal: AbortSignal.timeout(60_000),
            }, proxyTarget.trustedLoopback);
            if (!response.ok) {
                throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
            }
            return excludeSelfContainers(await response.json() as Array<{
                Id: string;
                Names?: string[];
                State?: string;
                Image?: string;
                ImageID?: string;
            }>);
        } catch (err) {
            this.rethrowRemoteProxyError(nodeId, err);
        }
    }

    private async postToRemoteContainer(
        nodeId: number,
        containerId: string,
        action: 'start' | 'stop' | 'restart',
    ): Promise<void> {
        const proxyTarget = this.requireRemoteProxyTarget(nodeId);
        const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
        const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
        try {
            const response = await safeRemoteFetch(
                `${baseUrl}/api/containers/${encodeURIComponent(containerId)}/${action}`,
                {
                    method: 'POST',
                    headers: {
                        'Content-Type': 'application/json',
                        'Authorization': `Bearer ${proxyTarget.apiToken}`,
                        [PROXY_TIER_HEADER]: proxyHeaders.tier,
                    },
                    signal: AbortSignal.timeout(300_000),
                },
                proxyTarget.trustedLoopback,
            );
            if (!response.ok) {
                throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
            }
        } catch (err) {
            this.rethrowRemoteProxyError(nodeId, err);
        }
    }

    private async postToRemoteStack(
        nodeId: number,
        routeSuffix: string,
        extraHeaders?: Record<string, string>,
    ): Promise<void> {
        const proxyTarget = this.requireRemoteProxyTarget(nodeId);
        const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
        const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
        const apiPath = `/api/stacks/${routeSuffix}`;
        if (isDebugEnabled()) {
            console.log(`[SchedulerService:debug] postToRemoteStack: node=${nodeId} route=${routeSuffix}`);
        }
        try {
            const augmented = await prepareOutboundRegistryDeliveryBody({
                method: 'POST',
                apiPath,
                nodeId,
                body: {},
            });
            if (!augmented.ok) {
                throw new Error(augmented.error);
            }
            const response = await safeRemoteFetch(`${baseUrl}/api/stacks/${routeSuffix}`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                    'Authorization': `Bearer ${proxyTarget.apiToken}`,
                    [PROXY_TIER_HEADER]: proxyHeaders.tier,
                    ...extraHeaders,
                },
                body: JSON.stringify(augmented.body),
                signal: AbortSignal.timeout(300_000),
            }, proxyTarget.trustedLoopback);
            if (!response.ok) {
                throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
            }
        } catch (err) {
            this.rethrowRemoteProxyError(nodeId, err);
        }
    }

    private async executeUpdateForStack(
        stackName: string,
        nodeId: number,
        docker: DockerController,
        imageUpdateService: ImageUpdateService,
        isWildcard = false
    ): Promise<string> {
        if (!ImageUpdateService.isChecksEnabled()) {
            console.log(`[SchedulerService] Stack "${stackName}": image update detection is disabled; skipped.`);
            return `Stack "${stackName}": image update detection is disabled; skipped.`;
        }
        const containers = await docker.getContainersByStack(stackName);
        if (!containers || containers.length === 0) {
            if (!isWildcard) {
                console.warn(`[SchedulerService] Stack "${stackName}": no containers found. The stack may have been removed or renamed.`);
                return `Stack "${stackName}": WARNING - no containers found. The stack may have been removed or renamed.`;
            }
            return `Stack "${stackName}": no containers found; skipped.`;
        }

        const imageRefs = [...new Set(
            containers
                .map((c: { Image?: string }) => c.Image)
                .filter((img): img is string => !!img && !img.startsWith('sha256:'))
        )];

        if (imageRefs.length === 0) {
            return `Stack "${stackName}": no pullable images; skipped.`;
        }

        if (isDebugEnabled()) {
            console.log(`[SchedulerService] Stack "${stackName}": checking ${imageRefs.length} image(s): ${imageRefs.join(', ')}`);
        }

        const gate = createAutoUpdateDigestGateState();

        for (const imageRef of imageRefs) {
            try {
                const result = await imageUpdateService.checkImage(docker, imageRef);
                recordAutoUpdateImageCheck(gate, imageRef, result);
            } catch (e) {
                const msg = getErrorMessage(e, String(e));
                gate.checkErrors.push(msg);
                console.warn(`[SchedulerService] Failed to check image ${sanitizeForLog(imageRef)}:`, sanitizeForLog((e as Error)?.message ?? String(e)));
            }
        }

        if (!gate.hasDigestUpdate) {
            return messageWhenNoDigestUpdate(stackName, gate, imageRefs.length);
        }
        const checkErrorBlock = messageWhenDigestApplyBlockedByCheckErrors(stackName, gate);
        if (checkErrorBlock) return checkErrorBlock;

        const { updatedImages } = gate;

        await this.enforceSchedulerPolicyGate(
            stackName,
            nodeId,
            'Auto-update',
            `/api/scheduled-tasks/auto-update/${stackName}`,
        );
        // Atomic backup/rollback is the default deploy mode: take a pre-op
        // backup and roll back on failure for every scheduled auto-update.
        const atomic = true;
        const lock = await StackOpLockService.getInstance().runExclusive(
            nodeId, stackName, 'update', 'system',
            () => StackUpdateOrchestrator.getInstance().execute(
                { nodeId, stackName, target: { scope: 'stack' }, trigger: 'scheduled', actor: 'system:scheduler' },
                { atomic, terminalWs: null },
            ),
        );
        if (!lock.ran) return skipMessage(stackName, lock.existing.action);

        // Health observation starts immediately after Compose; registry recheck is
        // isolated so a verification failure cannot turn Compose success into a failure.
        const orchResult = lock.result;
        const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler', { deployedGenerationId: orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.deployedGenerationId : null });
        const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
        if (recoveryId) {
            const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
            StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
        }

        // Recheck persists digest-cleared / tag-advisory state. Do not blind-clear.
        let recheckWarning: string | undefined;
        try {
            const recheck = await imageUpdateService.recheckStack(nodeId, stackName);
            if (recheck.warning) recheckWarning = recheck.warning;
        } catch (recheckErr) {
            console.warn(
                `[SchedulerService] Post-update recheck failed for ${sanitizeForLog(stackName)}:`,
                sanitizeForLog(getErrorMessage(recheckErr, 'unknown')),
            );
            recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING;
        }

        invalidateFleetUpdateCache();
        invalidateNodeCaches(nodeId);
        NotificationService.getInstance().broadcastEvent({
            type: 'state-invalidate',
            scope: 'image-updates',
            nodeId,
            stackName,
            action: 'stack-updated',
            ts: Date.now(),
        });

        this.safeDispatch(
            'info',
            'image_update_applied',
            `Auto-update: stack "${stackName}" updated with new images`,
            stackName
        );

        const base = `Stack "${stackName}": updated (${updatedImages.join(', ')}).`;
        return recheckWarning ? `${base} ${recheckWarning}` : base;
    }

    private async executeScan(task: ScheduledTask): Promise<{ output: string; failed: number }> {
        const trivy = TrivyService.getInstance();
        if (!trivy.isTrivyAvailable()) {
            throw new Error('Trivy binary is not available on this node');
        }

        const nodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
        if (task.node_id == null && isDebugEnabled()) {
            console.log(`[SchedulerService:debug] Scan task ${task.id}: no node_id specified, using default node ${nodeId}`);
        }
        const node = NodeRegistry.getInstance().getNode(nodeId);
        if (!node) {
            throw new Error('Scheduled vulnerability scans require an existing local node.');
        }
        if (node?.type === 'remote') {
            throw new Error('Scheduled vulnerability scans currently require a local node.');
        }

        const scanStart = Date.now();
        if (isDebugEnabled()) console.log(`[SchedulerService:debug] executeScan start: task=${task.id} node=${nodeId}`);

        const summary = await trivy.scanAllNodeImages(nodeId, 'scheduled');

        if (isDebugEnabled()) {
            console.log(
                `[SchedulerService:debug] executeScan summary: scanned=${summary.scanned} skipped=${summary.skipped} failed=${summary.failed} ` +
                `critical=${summary.severity.critical} high=${summary.severity.high} medium=${summary.severity.medium} ` +
                `low=${summary.severity.low} unknown=${summary.severity.unknown} violations=${summary.violations.length} durationMs=${Date.now() - scanStart}`,
            );
        }

        // Scheduled scans never auto-quarantine; violations surface as alerts
        // so an operator can review and remediate. One alert per violation so
        // the notification panel keeps per-image granularity.
        for (const v of summary.violations ?? []) {
            NotificationService.getInstance().dispatchAlert(
                'warning',
                'scan_finding',
                `Policy "${v.policyName}" violated by ${v.imageRef}: matched ${summarizeBlockReasons([v])}`,
                { actor: 'system:scheduler' },
            );
        }

        const output = formatScanOutput(summary);
        return { output, failed: summary.failed };
    }
}

/**
 * Build the human-readable completion message from a bulk scan summary.
 * Exported for unit tests.
 */
export function formatScanOutput(summary: ScanAllNodeImagesResult): string {
    const { scanned, skipped, failed, severity } = summary;

    let header: string;
    if (scanned === 0 && skipped === 0 && failed === 0) {
        header = 'No images to scan';
    } else if (scanned === 0 && skipped > 0 && failed === 0) {
        header = `All ${skipped} image(s) already scanned recently (cache hit)`;
    } else {
        const parts: string[] = [`Scanned ${scanned} image(s)`];
        if (skipped > 0) parts.push(`${skipped} skipped (cached)`);
        if (failed > 0) parts.push(`${failed} failed`);
        header = parts.join('; ');
    }

    if (summary.truncated) {
        const total = summary.totalImages ?? scanned + skipped + failed;
        const processed = summary.processedImages ?? scanned + skipped + failed;
        header += `. Scan limited after ${processed} of ${total} image(s)`;
        if (summary.limitReason) header += ` (${summary.limitReason})`;
    }

    const severityTiers: Array<[string, number]> = [
        ['critical', severity.critical],
        ['high', severity.high],
        ['medium', severity.medium],
    ];
    const nonZero = severityTiers.filter(([, n]) => n > 0);
    if (nonZero.length === 0) {
        if (scanned === 0 && skipped === 0 && failed === 0) {
            return header + '.';
        }
        return `${header}. No critical, high, or medium findings.`;
    }
    const findings = nonZero.map(([label, n]) => `${n} ${label}`).join(', ');
    return `${header}. Found ${findings}.`;
}
