sencho/backend/src/routes/fleet.ts

import { Router, type Request, type Response } from 'express';
import path from 'path';
import semver from 'semver';
import si from 'systeminformation';
import type Dockerode from 'dockerode';
import { DatabaseService, type Node } from '../services/DatabaseService';
import { ControlIdentityMismatchError, FleetSyncService, StaleSyncPushError } from '../services/FleetSyncService';
import { MAX_SYNC_ROWS, SYNC_ERROR_CODES } from '../services/fleetSyncConstants';
import { FleetUpdateTrackerService } from '../services/FleetUpdateTrackerService';
import { NodeRegistry } from '../services/NodeRegistry';
import DockerController from '../services/DockerController';
import { FileSystemService } from '../services/FileSystemService';
import { ComposeService } from '../services/ComposeService';
import SelfUpdateService from '../services/SelfUpdateService';
import { fetchRemoteMeta, getSenchoVersion, isValidVersion } from '../services/CapabilityRegistry';
import { authMiddleware } from '../middleware/auth';
import { requirePaid, requireAdmin, requireNodeProxy } from '../middleware/tierGates';
import { scheduleLocalUpdate } from './license';
import { captureLocalNodeFiles, captureRemoteNodeFiles, type SnapshotNodeData } from '../utils/snapshot-capture';
import { getLatestVersion } from '../utils/version-check';
import { isValidStackName } from '../utils/validation';
import { isDebugEnabled } from '../utils/debug';
import { getErrorMessage } from '../utils/errors';
import { parseIntParam } from '../utils/parseIntParam';
import { POLICY_SEVERITIES } from '../utils/severity';
import { sanitizeForLog } from '../utils/safeLog';
import { CloudBackupService } from '../services/CloudBackupService';
import { NotificationService } from '../services/NotificationService';
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
import { containerActionForStack } from './stacks';
import { activeBulkActions } from './labels';
import { buildLocalConfigurationStatus, type ConfigurationStatus } from './dashboard';
import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from '../services/license-headers';
import { LicenseService } from '../services/LicenseService';

const updateTracker = FleetUpdateTrackerService.getInstance();
const UPDATE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
const UPDATE_TIMEOUT_MSG = 'Node did not come back online within 5 minutes.';
const EARLY_FAIL_MS = 180 * 1000; // 3 minutes before declaring a probable pull failure

const CVE_ID_RE = /^(CVE-\d{4}-\d{4,}|GHSA-[\w-]{14,})$/;

const isIntFlag = (v: unknown): v is 0 | 1 => v === 0 || v === 1;

function validateScanPolicyRow(row: unknown): string | null {
  if (!row || typeof row !== 'object') return 'row must be an object';
  const r = row as Record<string, unknown>;
  if (typeof r.name !== 'string' || r.name.length === 0 || r.name.length > 200) return 'name must be a non-empty string';
  if (typeof r.max_severity !== 'string' || !POLICY_SEVERITIES.has(r.max_severity)) return 'max_severity must be CRITICAL, HIGH, MEDIUM, or LOW';
  if (r.stack_pattern !== null && typeof r.stack_pattern !== 'string') return 'stack_pattern must be a string or null';
  if (typeof r.stack_pattern === 'string' && r.stack_pattern.length > 200) return 'stack_pattern is too long';
  if (typeof r.node_identity !== 'string') return 'node_identity must be a string';
  if (r.node_identity.length > 500) return 'node_identity is too long';
  if (!isIntFlag(r.block_on_deploy)) return 'block_on_deploy must be 0 or 1';
  if (!isIntFlag(r.enabled)) return 'enabled must be 0 or 1';
  return null;
}

function validateCveSuppressionRow(row: unknown): string | null {
  if (!row || typeof row !== 'object') return 'row must be an object';
  const r = row as Record<string, unknown>;
  if (typeof r.cve_id !== 'string' || !CVE_ID_RE.test(r.cve_id)) return 'cve_id must be a valid CVE or GHSA identifier';
  if (r.pkg_name !== null && typeof r.pkg_name !== 'string') return 'pkg_name must be a string or null';
  if (typeof r.pkg_name === 'string' && r.pkg_name.length > 200) return 'pkg_name is too long';
  if (r.image_pattern !== null && typeof r.image_pattern !== 'string') return 'image_pattern must be a string or null';
  if (typeof r.image_pattern === 'string' && r.image_pattern.length > 300) return 'image_pattern is too long';
  if (typeof r.reason !== 'string') return 'reason must be a string';
  if (r.reason.length > 2000) return 'reason is too long';
  if (typeof r.created_by !== 'string' || r.created_by.length > 200) return 'created_by must be a string';
  if (typeof r.created_at !== 'number') return 'created_at must be a number';
  if (r.expires_at !== null && typeof r.expires_at !== 'number') return 'expires_at must be a number or null';
  return null;
}

interface FleetNodeOverview {
  id: number;
  name: string;
  type: 'local' | 'remote';
  mode?: string;
  status: 'online' | 'offline' | 'unknown';
  stats: {
    active: number;
    managed: number;
    unmanaged: number;
    exited: number;
    total: number;
  } | null;
  systemStats: {
    cpu: { usage: string; cores: number };
    memory: { total: number; used: number; free: number; usagePercent: string };
    disk: { total: number; used: number; free: number; usagePercent: string } | null;
  } | null;
  stacks: string[] | null;
  latency_ms?: number;
  last_successful_contact?: number | null;
  pilot_last_seen?: number | null;
  cordoned: boolean;
  cordoned_at: number | null;
  cordoned_reason: string | null;
}

/** Resolve the version to compare nodes against (latest from GitHub, or gateway fallback). */
async function getCompareTarget(gatewayVersion: string | null) {
  const latestVersion = await getLatestVersion();
  const latestValid = latestVersion !== null && isValidVersion(latestVersion);
  const result = {
    latestVersion,
    latestValid,
    compareVersion: latestValid ? latestVersion : gatewayVersion,
    compareValid: latestValid || isValidVersion(gatewayVersion),
  };
  if (isDebugEnabled()) {
    console.debug('[Fleet:debug] Compare target resolved:', { gatewayVersion, latestVersion, using: result.compareVersion, valid: result.compareValid });
  }
  return result;
}

async function fetchLocalNodeOverview(node: Node): Promise<FleetNodeOverview> {
  try {
    const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(node.id));
    const [allContainers, stacks, currentLoad, mem, fsSize] = await Promise.all([
      DockerController.getInstance(node.id).getAllContainers(),
      FileSystemService.getInstance(node.id).getStacks(),
      si.currentLoad(),
      si.mem(),
      si.fsSize(),
    ]);

    const isManagedByComposeDir = (c: Dockerode.ContainerInfo): boolean => {
      const workingDir: string | undefined = c.Labels?.['com.docker.compose.project.working_dir'];
      if (!workingDir) return false;
      const resolved = path.resolve(workingDir);
      return resolved === composeDir || resolved.startsWith(composeDir + path.sep);
    };

    const containers = allContainers as Dockerode.ContainerInfo[];
    const active = containers.filter(c => c.State === 'running').length;
    const exited = containers.filter(c => c.State === 'exited').length;
    const total = containers.length;
    const managed = containers.filter(c => c.State === 'running' && isManagedByComposeDir(c)).length;
    const unmanaged = containers.filter(c => c.State === 'running' && !isManagedByComposeDir(c)).length;

    const mainDisk = fsSize.find(fs => fs.mount === '/' || fs.mount === 'C:') || fsSize[0];

    return {
      id: node.id,
      name: node.name,
      type: node.type,
      status: 'online',
      stats: { active, managed, unmanaged, exited, total },
      systemStats: {
        cpu: { usage: currentLoad.currentLoad.toFixed(1), cores: currentLoad.cpus.length },
        memory: {
          total: mem.total,
          used: mem.used,
          free: mem.free,
          usagePercent: ((mem.used / mem.total) * 100).toFixed(1),
        },
        disk: mainDisk ? {
          total: mainDisk.size,
          used: mainDisk.used,
          free: mainDisk.available,
          usagePercent: mainDisk.use ? mainDisk.use.toFixed(1) : '0',
        } : null,
      },
      stacks,
      last_successful_contact: node.last_successful_contact ?? null,
      cordoned: node.cordoned,
      cordoned_at: node.cordoned_at,
      cordoned_reason: node.cordoned_reason,
    };
  } catch (error) {
    console.error(`[Fleet] Local node ${node.name} error:`, error);
    return {
      id: node.id, name: node.name, type: node.type, status: 'offline',
      stats: null, systemStats: null, stacks: null,
      last_successful_contact: node.last_successful_contact ?? null,
      cordoned: node.cordoned,
      cordoned_at: node.cordoned_at,
      cordoned_reason: node.cordoned_reason,
    };
  }
}

async function fetchRemoteNodeOverview(node: Node, db: DatabaseService): Promise<FleetNodeOverview> {
  // Pilot-agent nodes: use pilot_last_seen as the contact signal; no HTTP fetch.
  if (node.mode === 'pilot_agent') {
    return {
      id: node.id,
      name: node.name,
      type: node.type,
      mode: node.mode,
      status: node.pilot_last_seen ? 'online' : 'offline',
      stats: null,
      systemStats: null,
      stacks: null,
      last_successful_contact: node.pilot_last_seen ? Math.floor(node.pilot_last_seen / 1000) : null,
      pilot_last_seen: node.pilot_last_seen ? Math.floor(node.pilot_last_seen / 1000) : null,
      cordoned: node.cordoned,
      cordoned_at: node.cordoned_at,
      cordoned_reason: node.cordoned_reason,
    };
  }

  if (!node.api_url || !node.api_token) {
    return {
      id: node.id, name: node.name, type: node.type, status: 'offline',
      stats: null, systemStats: null, stacks: null,
      last_successful_contact: node.last_successful_contact ?? null,
      cordoned: node.cordoned,
      cordoned_at: node.cordoned_at,
      cordoned_reason: node.cordoned_reason,
    };
  }

  const baseUrl = node.api_url.replace(/\/$/, '');
  const headers = { Authorization: `Bearer ${node.api_token}` };
  const t0 = Date.now();

  try {
    const [statsRes, systemStatsRes, stacksRes] = await Promise.allSettled([
      fetch(`${baseUrl}/api/stats`, { headers, signal: AbortSignal.timeout(10000) }),
      fetch(`${baseUrl}/api/system/stats`, { headers, signal: AbortSignal.timeout(10000) }),
      fetch(`${baseUrl}/api/stacks`, { headers, signal: AbortSignal.timeout(10000) }),
    ]);

    interface RemoteSystemStats {
      cpu: { usage: string; cores: number };
      memory: { total: number; used: number; free: number; usagePercent: string };
      disk?: { total: number; used: number; free: number; usagePercent: string } | null;
    }

    const stats: FleetNodeOverview['stats'] | null = statsRes.status === 'fulfilled' && statsRes.value.ok
      ? await statsRes.value.json() as FleetNodeOverview['stats'] : null;
    const systemStatsRaw: RemoteSystemStats | null = systemStatsRes.status === 'fulfilled' && systemStatsRes.value.ok
      ? await systemStatsRes.value.json() as RemoteSystemStats : null;
    const stacks: string[] | null = stacksRes.status === 'fulfilled' && stacksRes.value.ok
      ? await stacksRes.value.json() as string[] : null;

    const systemStats: FleetNodeOverview['systemStats'] | null = systemStatsRaw ? {
      cpu: systemStatsRaw.cpu,
      memory: systemStatsRaw.memory,
      disk: systemStatsRaw.disk ? {
        total: systemStatsRaw.disk.total,
        used: systemStatsRaw.disk.used,
        free: systemStatsRaw.disk.free,
        usagePercent: systemStatsRaw.disk.usagePercent,
      } : null,
    } : null;

    const completedAt = Date.now();
    const latency_ms = completedAt - t0;
    const isOnline = !!(stats || systemStats);

    if (isOnline) {
      db.updateNodeLastContact(node.id);
    }

    return {
      id: node.id,
      name: node.name,
      type: node.type,
      mode: node.mode,
      status: isOnline ? 'online' : 'offline',
      stats,
      systemStats,
      stacks,
      latency_ms,
      last_successful_contact: isOnline
        ? Math.floor(completedAt / 1000)
        : node.last_successful_contact ?? null,
      cordoned: node.cordoned,
      cordoned_at: node.cordoned_at,
      cordoned_reason: node.cordoned_reason,
    };
  } catch (error) {
    console.error(`[Fleet] Remote node ${node.name} error:`, error);
    return {
      id: node.id, name: node.name, type: node.type, mode: node.mode, status: 'offline',
      stats: null, systemStats: null, stacks: null,
      last_successful_contact: node.last_successful_contact ?? null,
      cordoned: node.cordoned,
      cordoned_at: node.cordoned_at,
      cordoned_reason: node.cordoned_reason,
    };
  }
}

export const fleetRouter = Router();

// Fleet role: tells the frontend whether this Sencho is the control or a
// replica. The control serves read+write for security rules. Replicas are
// read-only and managed upstream.
fleetRouter.get('/role', authMiddleware, (req: Request, res: Response): void => {
  if (!requireAdmin(req, res)) return;
  res.json({ role: FleetSyncService.getRole() });
});

// Receive a full replacement of a replicated resource from the control.
// Restricted to node_proxy Bearer tokens so only a sibling Sencho can push.
//
// No requirePaid here: the control instance has already enforced its tier
// before issuing the push. The replica trusts a valid node_proxy bearer
// signed against THIS instance's secret and applies the payload regardless
// of the replica's own tier.
fleetRouter.post('/sync/:resource', authMiddleware, (req: Request, res: Response): void => {
  if (!requireNodeProxy(req, res)) return;
  const resource = req.params.resource;
  if (resource !== 'scan_policies' && resource !== 'cve_suppressions') {
    res.status(400).json({ error: `Unsupported sync resource: ${resource}` });
    return;
  }
  const body = req.body ?? {};
  const rows = Array.isArray(body.rows) ? body.rows : null;
  const targetIdentity = typeof body.targetIdentity === 'string' ? body.targetIdentity : '';
  // pushedAt is optional for back-compat with older controls that predate the
  // versioning protocol. When present and strictly older than the most recent
  // applied push for this resource, reject with 409 STALE_SYNC_PUSH so the
  // control's retry logic can fall back to the next write. Negative or zero
  // values are treated as absent: the sender always uses Date.now().
  const pushedAt = typeof body.pushedAt === 'number' && Number.isFinite(body.pushedAt) && body.pushedAt > 0
    ? body.pushedAt
    : null;
  // controlIdentity is optional for back-compat. The receiver anchors to the
  // first non-empty fingerprint it sees and rejects mismatches afterward.
  const controlIdentity = typeof body.controlIdentity === 'string' ? body.controlIdentity : '';
  if (!rows) {
    res.status(400).json({ error: 'rows array is required' });
    return;
  }
  if (rows.length > MAX_SYNC_ROWS) {
    res.status(413).json({ error: `Too many rows (max ${MAX_SYNC_ROWS})` });
    return;
  }
  const validator = resource === 'scan_policies' ? validateScanPolicyRow : validateCveSuppressionRow;
  for (let i = 0; i < rows.length; i++) {
    const err = validator(rows[i]);
    if (err) {
      res.status(400).json({ error: `Invalid row at index ${i}: ${err}` });
      return;
    }
  }
  try {
    FleetSyncService.getInstance().applyIncomingSync(
      resource,
      rows,
      targetIdentity,
      pushedAt ?? undefined,
      controlIdentity || undefined,
    );
    res.json({ success: true, applied: rows.length });
  } catch (error) {
    if (error instanceof StaleSyncPushError) {
      res.status(409).json({
        error: error.message,
        code: SYNC_ERROR_CODES.staleSyncPush,
      });
      return;
    }
    if (error instanceof ControlIdentityMismatchError) {
      res.status(409).json({
        error: error.message,
        code: SYNC_ERROR_CODES.controlIdentityMismatch,
        expected: error.expected,
        got: error.got,
      });
      return;
    }
    console.error('[FleetSync] Failed to apply incoming sync:', error);
    res.status(500).json({ error: 'Failed to apply sync' });
  }
});

// Demote this replica back to a standalone control. Wipes all replicated
// security rules and the cached fingerprint, then flips `fleet_role` to
// 'control'. The local UI's write controls become available again.
// `{confirm: true}` body is required so a misclick cannot destroy mirrored
// state.
fleetRouter.post('/role/demote', authMiddleware, (req: Request, res: Response): void => {
  if (!requireAdmin(req, res)) return;
  const body = req.body ?? {};
  if (body.confirm !== true) {
    res.status(400).json({
      error: 'Demote requires explicit confirmation. Send { "confirm": true } to proceed.',
    });
    return;
  }
  try {
    const demoted = FleetSyncService.getInstance().demote();
    if (!demoted) {
      res.status(409).json({
        error: 'This instance is already a control; nothing to demote.',
        code: 'ALREADY_CONTROL',
      });
      return;
    }
    res.json({ success: true, role: 'control' });
  } catch (error) {
    console.error('[FleetSync] Demote failed:', error);
    res.status(500).json({ error: 'Failed to demote replica' });
  }
});

// Reset the control anchor on this replica. An admin must opt in explicitly
// with `{override: true}` because reanchor wipes all replicated rows; the
// next push from a different control will re-populate them. Used when a
// control is permanently rebuilt or replaced and must be re-bound to its
// existing replicas.
fleetRouter.post('/role/reanchor', authMiddleware, (req: Request, res: Response): void => {
  if (!requireAdmin(req, res)) return;
  const body = req.body ?? {};
  if (body.override !== true) {
    res.status(400).json({
      error: 'Reanchor requires explicit override. Send { "override": true } to confirm.',
    });
    return;
  }
  try {
    FleetSyncService.getInstance().reanchor();
    res.json({ success: true });
  } catch (error) {
    console.error('[FleetSync] Reanchor failed:', error);
    res.status(500).json({ error: 'Failed to reset control anchor' });
  }
});

fleetRouter.get('/sync-status', authMiddleware, (req: Request, res: Response): void => {
  if (!requireAdmin(req, res)) return;
  if (!requirePaid(req, res)) return;
  res.json(DatabaseService.getInstance().getFleetSyncStatuses());
});

fleetRouter.get('/overview', authMiddleware, async (_req: Request, res: Response): Promise<void> => {
  try {
    const debug = isDebugEnabled();
    const db = DatabaseService.getInstance();
    const nodes = db.getNodes();
    if (debug) console.debug('[Fleet:debug] Overview requested, fetching', nodes.length, 'nodes');

    const results = await Promise.allSettled(
      nodes.map(async (node): Promise<FleetNodeOverview> => {
        if (node.type === 'remote') {
          return fetchRemoteNodeOverview(node, db);
        }
        return fetchLocalNodeOverview(node);
      }),
    );

    const overview: FleetNodeOverview[] = results.map((result, i) => {
      if (result.status === 'fulfilled') return result.value;
      console.error(`[Fleet] Failed to fetch node ${nodes[i].name}:`, result.reason);
      return {
        id: nodes[i].id,
        name: nodes[i].name,
        type: nodes[i].type,
        status: 'offline' as const,
        stats: null,
        systemStats: null,
        stacks: null,
        cordoned: nodes[i].cordoned,
        cordoned_at: nodes[i].cordoned_at,
        cordoned_reason: nodes[i].cordoned_reason,
      };
    });

    if (debug) {
      const online = overview.filter(n => n.status === 'online').length;
      console.debug('[Fleet:debug] Overview complete:', online, 'online,', overview.length - online, 'offline');
    }
    res.json(overview);
  } catch (error) {
    console.error('[Fleet] Overview error:', error);
    res.status(500).json({ error: 'Failed to fetch fleet overview' });
  }
});

interface FleetNodeConfiguration {
  id: number;
  name: string;
  type: 'local' | 'remote';
  status: 'online' | 'offline';
  configuration: ConfigurationStatus | null;
}

fleetRouter.get('/configuration', authMiddleware, async (req: Request, res: Response): Promise<void> => {
  try {
    const db = DatabaseService.getInstance();
    const nodes = db.getNodes();
    const userId = req.user?.userId ?? 0;
    const ls = LicenseService.getInstance();
    const localTier = ls.getTier();
    const localVariant = ls.getVariant();

    const results = await Promise.allSettled(
      nodes.map(async (node: Node): Promise<FleetNodeConfiguration> => {
        if (node.type === 'local') {
          return {
            id: node.id,
            name: node.name,
            type: 'local',
            status: 'online',
            configuration: buildLocalConfigurationStatus(node.id, userId, localTier, localVariant),
          };
        }

        if (!node.api_url || !node.api_token) {
          return { id: node.id, name: node.name, type: 'remote', status: 'offline', configuration: null };
        }

        try {
          const resp = await fetch(
            `${node.api_url.replace(/\/$/, '')}/api/dashboard/configuration`,
            {
              headers: {
                Authorization: `Bearer ${node.api_token}`,
                [PROXY_TIER_HEADER]: localTier,
                [PROXY_VARIANT_HEADER]: localVariant ?? '',
              },
              signal: AbortSignal.timeout(10000),
            },
          );
          const configuration = resp.ok ? (await resp.json() as ConfigurationStatus) : null;
          return {
            id: node.id,
            name: node.name,
            type: 'remote',
            status: configuration ? 'online' : 'offline',
            configuration,
          };
        } catch {
          return { id: node.id, name: node.name, type: 'remote', status: 'offline', configuration: null };
        }
      }),
    );

    const fleet: FleetNodeConfiguration[] = results.map((result, i) => {
      if (result.status === 'fulfilled') return result.value;
      console.error(`[Fleet] Configuration fetch failed for node ${nodes[i].name}:`, result.reason);
      return { id: nodes[i].id, name: nodes[i].name, type: nodes[i].type, status: 'offline', configuration: null };
    });

    res.json(fleet);
  } catch (error) {
    console.error('[Fleet] Configuration overview error:', error);
    res.status(500).json({ error: 'Failed to fetch fleet configuration' });
  }
});

fleetRouter.get('/node/:nodeId/stacks', authMiddleware, async (req: Request, res: Response): Promise<void> => {
  if (!requirePaid(req, res)) return;

  try {
    const nodeId = parseIntParam(req, res, 'nodeId', 'node ID');
    if (nodeId === null) return;
    const node = DatabaseService.getInstance().getNode(nodeId);
    if (!node) {
      res.status(404).json({ error: 'Node not found' });
      return;
    }

    if (node.type === 'remote') {
      if (!node.api_url || !node.api_token) {
        res.status(503).json({ error: 'Remote node not configured' });
        return;
      }
      const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/stacks`, {
        headers: { Authorization: `Bearer ${node.api_token}` },
        signal: AbortSignal.timeout(10000),
      });
      if (!response.ok) {
        res.status(502).json({ error: 'Failed to fetch stacks from remote node' });
        return;
      }
      const stacks = await response.json();
      if (isDebugEnabled()) console.debug('[Fleet:debug] Node stacks:', nodeId, node.type, Array.isArray(stacks) ? stacks.length : 0, 'stacks');
      res.json(stacks);
      return;
    }

    const stacks = await FileSystemService.getInstance(nodeId).getStacks();
    if (isDebugEnabled()) console.debug('[Fleet:debug] Node stacks:', nodeId, node.type, stacks.length, 'stacks');
    res.json(stacks);
  } catch (error) {
    console.error('[Fleet] Node stacks error:', error);
    res.status(500).json({ error: 'Failed to fetch node stacks' });
  }
});

fleetRouter.get('/node/:nodeId/stacks/:stackName/containers', authMiddleware, async (req: Request, res: Response): Promise<void> => {
  if (!requirePaid(req, res)) return;

  try {
    const nodeId = parseIntParam(req, res, 'nodeId', 'node ID');
    if (nodeId === null) return;
    const stackName = req.params.stackName as string;
    if (!isValidStackName(stackName)) {
      res.status(400).json({ error: 'Invalid stack name' });
      return;
    }
    const node = DatabaseService.getInstance().getNode(nodeId);
    if (!node) {
      res.status(404).json({ error: 'Node not found' });
      return;
    }

    if (node.type === 'remote') {
      if (!node.api_url || !node.api_token) {
        res.status(503).json({ error: 'Remote node not configured' });
        return;
      }
      const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(stackName)}/containers`, {
        headers: { Authorization: `Bearer ${node.api_token}` },
        signal: AbortSignal.timeout(10000),
      });
      if (!response.ok) {
        res.status(502).json({ error: 'Failed to fetch containers from remote node' });
        return;
      }
      const containers = await response.json();
      res.json(containers);
      return;
    }

    const dockerController = DockerController.getInstance(nodeId);
    const containers = await dockerController.getContainersByStack(stackName);
    if (isDebugEnabled()) console.debug('[Fleet:debug] Stack containers:', nodeId, stackName, containers.length, 'containers');
    res.json(containers);
  } catch (error) {
    console.error('[Fleet] Node stack containers error:', error);
    res.status(500).json({ error: 'Failed to fetch stack containers' });
  }
});

fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Response): Promise<void> => {
  try {
    const db = DatabaseService.getInstance();
    const nodes = db.getNodes();
    const gatewayVersion = getSenchoVersion();
    const gatewayValid = isValidVersion(gatewayVersion);

    const { latestVersion, latestValid, compareVersion, compareValid } = await getCompareTarget(gatewayVersion);
    const debug = isDebugEnabled();

    const results = await Promise.allSettled(
      nodes.map(async (node) => {
        const tracker = updateTracker.get(node.id);

        let version: string | null = null;
        let remoteStartedAt: number | null = null;
        let remoteUpdateError: string | null = null;
        let remoteOnline = false;
        if (node.type === 'local') {
          version = gatewayVersion;
        } else if (node.api_url && node.api_token) {
          const meta = await fetchRemoteMeta(node.api_url, node.api_token);
          version = meta.version;
          remoteStartedAt = meta.startedAt;
          remoteUpdateError = meta.updateError;
          remoteOnline = meta.online;
        }

        if (tracker?.status === 'updating') {
          const elapsed = Date.now() - tracker.startedAt;

          if (debug) {
            console.debug('[Fleet:debug] Polling update status for node', node.id, node.name, '- elapsed:', Math.round(elapsed / 1000) + 's', 'version:', version, 'wasOffline:', tracker.wasOffline, 'remoteOnline:', remoteOnline);
          }

          if (elapsed > UPDATE_TIMEOUT_MS) {
            if (debug) console.debug('[Fleet:debug] Node', node.id, 'timed out after', Math.round(elapsed / 1000) + 's');
            updateTracker.set(node.id, updateTracker.resolve(tracker, 'timeout', UPDATE_TIMEOUT_MSG));
          } else if (node.type === 'remote') {
            if (remoteUpdateError) {
              if (debug) console.debug('[Fleet:debug] Node', node.id, 'reported pull failure:', remoteUpdateError);
              updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', remoteUpdateError));
            } else if (!remoteOnline) {
              if (!tracker.wasOffline) {
                if (debug) console.debug('[Fleet:debug] Node', node.id, 'went offline (restarting)');
                updateTracker.set(node.id, { ...tracker, wasOffline: true });
              }
            } else if (version !== tracker.previousVersion) {
              if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 1 (version changed):', tracker.previousVersion, '->', version);
              updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed'));
            } else if (
              remoteStartedAt !== null &&
              tracker.previousProcessStart !== null &&
              remoteStartedAt !== tracker.previousProcessStart
            ) {
              if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 2 (process restarted):', tracker.previousProcessStart, '->', remoteStartedAt);
              updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed'));
            } else if (tracker.wasOffline && remoteOnline) {
              if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 3 (offline then online)');
              updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed'));
            } else if (
              elapsed > 15_000 &&
              isValidVersion(version) &&
              gatewayValid &&
              !semver.lt(version, compareVersion!)
            ) {
              // Signal 4: remote is now at or above gateway version (after
              // minimum processing time). Catches fast restarts where the 5s
              // polling interval misses the offline window and startedAt
              // hasn't been observed to change yet.
              if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 4 (version >= compare target):', version, '>=', compareVersion);
              updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed'));
            } else if (elapsed > EARLY_FAIL_MS) {
              if (debug) console.debug('[Fleet:debug] Node', node.id, 'early fail after', Math.round(elapsed / 1000) + 's - no signals detected');
              updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', 'Update may have failed. The node is still running and its version has not changed.'));
            }
          } else if (node.type === 'local') {
            // Local node has only two failure signals: an explicit pull/spawn
            // error, or the early-fail heuristic. Success is observed by the
            // frontend overlay (it reloads when /api/health reports a new
            // startedAt), at which point the new process starts with an empty
            // tracker map.
            const selfUpdate = SelfUpdateService.getInstance();
            const localError = selfUpdate.getLastError();
            if (localError) {
              if (debug) console.debug('[Fleet:debug] Local node', node.id, 'update failed:', localError);
              updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', localError));
              selfUpdate.clearLastError();
            } else if (elapsed > EARLY_FAIL_MS) {
              if (debug) console.debug('[Fleet:debug] Local node', node.id, 'early fail after', Math.round(elapsed / 1000) + 's');
              updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', 'Local update did not complete. The container may not have restarted; check Docker logs on the host.'));
            }
          }
        }

        // Auto-expire completed entries 60s after they resolved so the badge
        // is visible briefly after completion.
        if (tracker?.status === 'completed' && tracker.resolvedAt && Date.now() - tracker.resolvedAt > 60_000) {
          updateTracker.delete(node.id);
        }

        let updateAvailable = false;
        if (!isValidVersion(version)) {
          // Assume remote nodes are outdated when their version is unresolvable.
          updateAvailable = node.type === 'remote';
        } else if (compareValid) {
          updateAvailable = semver.lt(version, compareVersion!);
        }

        const currentTracker = updateTracker.get(node.id);
        return {
          nodeId: node.id,
          name: node.name,
          type: node.type,
          version,
          latestVersion: latestValid ? latestVersion : gatewayVersion,
          updateAvailable,
          updateStatus: currentTracker?.status ?? null,
          error: currentTracker?.error ?? null,
        };
      }),
    );

    const nodeStatuses = results.map((r, i) => {
      if (r.status === 'fulfilled') return r.value;
      return {
        nodeId: nodes[i].id,
        name: nodes[i].name,
        type: nodes[i].type,
        version: null,
        latestVersion: latestValid ? latestVersion : gatewayVersion,
        updateAvailable: false,
        updateStatus: null,
        error: null,
      };
    });

    if (isDebugEnabled()) {
      const trackerStates = Array.from(updateTracker.entries()).map(([nid, t]) => `${nid}:${t.status}`);
      console.debug('[Fleet:debug] Update status:', nodeStatuses.length, 'nodes, trackers:', trackerStates.join(', ') || 'none');
    }
    res.json({ nodes: nodeStatuses });
  } catch (error) {
    console.error('[Fleet] Update status error:', error);
    res.status(500).json({ error: 'Failed to fetch update status' });
  }
});

fleetRouter.post('/nodes/:nodeId/update', authMiddleware, async (req: Request, res: Response): Promise<void> => {
  if (!requireAdmin(req, res)) return;
  try {
    const nodeId = parseIntParam(req, res, 'nodeId', 'node ID');
    if (nodeId === null) return;
    const db = DatabaseService.getInstance();
    const node = db.getNode(nodeId);
    if (!node) {
      res.status(404).json({ error: 'Node not found' });
      return;
    }

    const existing = updateTracker.get(nodeId);
    if (existing?.status === 'updating') {
      if (Date.now() - existing.startedAt > UPDATE_TIMEOUT_MS) {
        updateTracker.set(nodeId, updateTracker.resolve(existing, 'timeout', UPDATE_TIMEOUT_MSG));
      } else {
        res.status(409).json({ error: 'Update already in progress for this node.' });
        return;
      }
    }
    // Clear terminal states to allow retry.
    if (existing && (existing.status === 'timeout' || existing.status === 'failed' || existing.status === 'completed')) {
      updateTracker.delete(nodeId);
    }

    console.log('[Fleet] Update triggered for node', node.name, node.type);
    if (isDebugEnabled()) {
      console.debug('[Fleet:debug] Update trigger details:', { nodeId, name: node.name, type: node.type, hasUrl: !!node.api_url, hasToken: !!node.api_token });
    }

    if (node.type === 'local') {
      if (!SelfUpdateService.getInstance().isAvailable()) {
        res.status(503).json({ error: 'Self-update unavailable on the local node.' });
        return;
      }
      updateTracker.set(nodeId, updateTracker.create('updating', getSenchoVersion(), null));
      scheduleLocalUpdate(res, 'Update initiated on local node. The server will restart shortly.');
      return;
    }

    if (!node.api_url || !node.api_token) {
      res.status(503).json({ error: 'Remote node not configured.' });
      return;
    }

    const meta = await fetchRemoteMeta(node.api_url, node.api_token);
    if (isDebugEnabled()) {
      console.debug('[Fleet:debug] Remote meta for update:', { nodeId, online: meta.online, version: meta.version, capabilities: meta.capabilities, startedAt: meta.startedAt });
    }
    if (!meta.online) {
      res.status(503).json({ error: 'Remote node is unreachable. Verify the node is running and the API URL is correct.' });
      return;
    }
    if (!meta.capabilities.includes('self-update')) {
      res.status(503).json({ error: 'Remote node does not support self-update. It may need to be updated manually first.' });
      return;
    }

    const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/system/update`, {
      method: 'POST',
      headers: {
        Authorization: `Bearer ${node.api_token}`,
        'Content-Type': 'application/json',
      },
      signal: AbortSignal.timeout(10000),
    });

    if (!response.ok) {
      const err = await response.json().catch(() => ({}));
      const errorMsg = (err as Record<string, string>)?.error || 'Remote node rejected update request.';
      updateTracker.set(nodeId, updateTracker.create('failed', meta.version, meta.startedAt, errorMsg));
      res.status(502).json({ error: errorMsg });
      return;
    }

    updateTracker.set(nodeId, updateTracker.create('updating', meta.version, meta.startedAt));
    res.status(202).json({ message: `Update initiated on ${node.name}.` });
  } catch (error) {
    console.error('[Fleet] Node update error:', error);
    const errorMsg = getErrorMessage(error, 'Failed to trigger node update.');
    const failedNodeId = parseInt(req.params.nodeId as string, 10);
    if (!isNaN(failedNodeId)) {
      updateTracker.set(failedNodeId, updateTracker.create('failed', null, null, errorMsg));
    }
    res.status(500).json({ error: 'Failed to trigger node update.' });
  }
});

fleetRouter.post('/update-all', authMiddleware, async (req: Request, res: Response): Promise<void> => {
  if (!requirePaid(req, res)) return;
  if (!requireAdmin(req, res)) return;
  try {
    const db = DatabaseService.getInstance();
    const nodes = db.getNodes();
    const gatewayVersion = getSenchoVersion();
    const { compareVersion, compareValid } = await getCompareTarget(gatewayVersion);

    const debug = isDebugEnabled();
    console.log('[Fleet] Update-all triggered,', nodes.length, 'nodes registered');
    if (debug) console.debug('[Fleet:debug] Update-all compare target:', { gatewayVersion, compareVersion, compareValid });

    const candidates = nodes.filter(node => {
      if (node.type === 'local') return false;
      const tracker = updateTracker.get(node.id);
      if (tracker?.status === 'updating') return false;
      if (!node.api_url || !node.api_token) return false;
      // Clear terminal states so they can be re-triggered.
      if (tracker && (tracker.status === 'timeout' || tracker.status === 'failed' || tracker.status === 'completed')) {
        updateTracker.delete(node.id);
      }
      return true;
    });

    const results = await Promise.allSettled(candidates.map(async (node) => {
      const meta = await fetchRemoteMeta(node.api_url!, node.api_token!);
      if (!meta.online) {
        return { name: node.name, triggered: false };
      }
      if (!meta.capabilities.includes('self-update')) {
        return { name: node.name, triggered: false };
      }
      if (isValidVersion(meta.version) && compareValid && !semver.lt(meta.version, compareVersion!)) {
        return { name: node.name, triggered: false };
      }
      const response = await fetch(`${node.api_url!.replace(/\/$/, '')}/api/system/update`, {
        method: 'POST',
        headers: { Authorization: `Bearer ${node.api_token}`, 'Content-Type': 'application/json' },
        signal: AbortSignal.timeout(10000),
      });
      if (response.ok) {
        updateTracker.set(node.id, updateTracker.create('updating', meta.version, meta.startedAt));
        return { name: node.name, triggered: true };
      }
      return { name: node.name, triggered: false };
    }));

    const updating: string[] = [];
    const skipped = nodes.filter(n => !candidates.includes(n)).map(n => n.name);
    for (let i = 0; i < results.length; i++) {
      const r = results[i];
      const val = r.status === 'fulfilled' ? r.value : { name: candidates[i].name, triggered: false };
      (val.triggered ? updating : skipped).push(val.name);
    }

    if (debug) console.debug('[Fleet:debug] Update-all results:', { updating, skippedCount: skipped.length, candidateCount: candidates.length });
    res.status(202).json({ updating, skipped });
  } catch (error) {
    console.error('[Fleet] Update all error:', error);
    res.status(500).json({ error: 'Failed to trigger fleet update.' });
  }
});

fleetRouter.delete('/nodes/:nodeId/update-status', authMiddleware, async (req: Request, res: Response): Promise<void> => {
  try {
    const nodeId = parseIntParam(req, res, 'nodeId', 'node ID');
    if (nodeId === null) return;
    const node = DatabaseService.getInstance().getNode(nodeId);
    if (!node) {
      res.status(404).json({ error: 'Node not found' });
      return;
    }
    updateTracker.delete(nodeId);
    res.status(204).send();
  } catch (error) {
    console.error('[Fleet] Clear update status error:', error);
    res.status(500).json({ error: 'Failed to clear update status.' });
  }
});

fleetRouter.delete('/update-status', authMiddleware, async (req: Request, res: Response): Promise<void> => {
  // Pre-fetch fresh latest version so the next GET has up-to-date data.
  if (req.query.recheck === 'true') {
    await getLatestVersion(true);
  }
  for (const [nodeId, tracker] of updateTracker.entries()) {
    if (tracker.status === 'timeout' || tracker.status === 'failed' || tracker.status === 'completed') {
      updateTracker.delete(nodeId);
    }
  }
  res.status(204).send();
});

// ─── Fleet Actions: gateway-orchestrated endpoints (multi-node) ───
//
// Per-node fleet-action endpoints (run on the target node via the proxy) live
// in `routes/fleetActions.ts`. The endpoint below is gateway-orchestrated and
// lives here so it sits behind the `/api/fleet/` proxy-exempt prefix.

// Fleet-wide stop by label name. Matches each node's labels by name and runs
// container stops on each matching stack.
// Tier: requirePaid + requireAdmin.
fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res: Response): Promise<void> => {
  if (!requirePaid(req, res)) return;
  if (!requireAdmin(req, res)) return;
  const body = req.body as { labelName?: unknown } | undefined;
  if (!body || typeof body !== 'object') {
    res.status(400).json({ error: 'Request body is required' });
    return;
  }
  const { labelName } = body;
  if (typeof labelName !== 'string' || labelName.trim().length === 0) {
    res.status(400).json({ error: 'labelName is required' });
    return;
  }
  const trimmed = labelName.trim();
  try {
    const db = DatabaseService.getInstance();
    const nodes = db.getNodes();
    const results = await Promise.all(nodes.map(async (node) => {
      const label = db.getLabels(node.id).find(l => l.name === trimmed);
      if (!label) {
        return { nodeId: node.id, nodeName: node.name, matched: false, stackResults: [] };
      }
      const stackNames = db.getStacksForLabel(label.id, node.id);
      if (stackNames.length === 0) {
        return { nodeId: node.id, nodeName: node.name, matched: true, stackResults: [] };
      }

      if (node.type === 'local') {
        // Share the per-node bulk lock with `POST /api/labels/:id/action` so
        // a fleet-stop and a per-label action cannot double-stop the same
        // containers concurrently on the same local node.
        const lockKey = `bulk:${node.id}`;
        if (activeBulkActions.has(lockKey)) {
          return {
            nodeId: node.id, nodeName: node.name, matched: true,
            stackResults: stackNames.map(stackName => ({ stackName, success: false, error: 'A bulk action is already running on this node' })),
          };
        }
        activeBulkActions.add(lockKey);
        try {
          const fsStacks = await FileSystemService.getInstance(node.id).getStacks();
          const fsStackSet = new Set(fsStacks);
          const validStacks = stackNames.filter(name => fsStackSet.has(name));
          const stackResults: { stackName: string; success: boolean; error?: string }[] = [];
          for (const stackName of validStacks) {
            const outcome = await containerActionForStack(node.id, stackName, 'stop');
            if (outcome.kind === 'ok') stackResults.push({ stackName, success: true });
            else if (outcome.kind === 'no-containers') stackResults.push({ stackName, success: false, error: 'No containers found for this stack' });
            else stackResults.push({ stackName, success: false, error: outcome.message });
          }
          if (stackResults.some(r => r.success)) invalidateNodeCaches(node.id);
          return { nodeId: node.id, nodeName: node.name, matched: true, stackResults };
        } finally {
          activeBulkActions.delete(lockKey);
        }
      }

      if (!node.api_url || !node.api_token) {
        return {
          nodeId: node.id, nodeName: node.name, matched: true,
          stackResults: stackNames.map(stackName => ({ stackName, success: false, error: 'Remote node not configured' })),
        };
      }
      try {
        const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/labels/${label.id}/action`, {
          method: 'POST',
          headers: { Authorization: `Bearer ${node.api_token}`, 'Content-Type': 'application/json' },
          body: JSON.stringify({ action: 'stop' }),
          signal: AbortSignal.timeout(60000),
        });
        if (!response.ok) {
          const err = (await response.json().catch(() => ({}))) as { error?: string };
          const message = err.error || `Remote returned ${response.status}`;
          return {
            nodeId: node.id, nodeName: node.name, matched: true,
            stackResults: stackNames.map(stackName => ({ stackName, success: false, error: message })),
          };
        }
        const remote = (await response.json()) as { results?: { stackName: string; success: boolean; error?: string }[] };
        return { nodeId: node.id, nodeName: node.name, matched: true, stackResults: remote.results ?? [] };
      } catch (err) {
        const errorMsg = getErrorMessage(err, 'Failed to reach remote node');
        return {
          nodeId: node.id, nodeName: node.name, matched: true,
          stackResults: stackNames.map(stackName => ({ stackName, success: false, error: errorMsg })),
        };
      }
    }));
    res.json({ results });
  } catch (error) {
    console.error('[Fleet] fleet-stop error:', error);
    res.status(500).json({ error: getErrorMessage(error, 'Failed to run fleet stop') });
  }
});

// ─── Fleet Snapshots (manual: Community; scheduled: Skipper+) ───

fleetRouter.post('/snapshots', authMiddleware, async (req: Request, res: Response): Promise<void> => {
  if (!requireAdmin(req, res)) return;

  try {
    const { description = '' } = req.body;
    if (typeof description === 'string' && description.length > 500) {
      res.status(400).json({ error: 'Description must be 500 characters or less' });
      return;
    }
    const db = DatabaseService.getInstance();
    const nodes = db.getNodes();
    const username = req.user?.username || 'admin';

    const captureStart = Date.now();
    const results = await Promise.allSettled(
      nodes.map(async (node) => {
        if (node.type === 'remote') {
          return captureRemoteNodeFiles(node);
        }
        return captureLocalNodeFiles(node);
      }),
    );

    const capturedNodes: SnapshotNodeData[] = [];
    const skippedNodes: Array<{ nodeId: number; nodeName: string; reason: string }> = [];

    results.forEach((result, i) => {
      if (result.status === 'fulfilled') {
        capturedNodes.push(result.value);
      } else {
        console.error(`[Fleet Snapshot] Failed to capture node ${nodes[i].name}:`, result.reason);
        skippedNodes.push({
          nodeId: nodes[i].id,
          nodeName: nodes[i].name,
          reason: result.reason instanceof Error ? result.reason.message : 'Unknown error',
        });
      }
    });

    let totalStacks = 0;
    const allFiles: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }> = [];

    for (const nodeData of capturedNodes) {
      totalStacks += nodeData.stacks.length;
      for (const stack of nodeData.stacks) {
        for (const file of stack.files) {
          allFiles.push({
            nodeId: nodeData.nodeId,
            nodeName: nodeData.nodeName,
            stackName: stack.stackName,
            filename: file.filename,
            content: file.content,
          });
        }
      }
    }

    const snapshotId = db.createSnapshot(
      description,
      username,
      capturedNodes.length,
      totalStacks,
      JSON.stringify(skippedNodes),
    );

    if (allFiles.length > 0) {
      db.insertSnapshotFiles(snapshotId, allFiles);
    }

    const cloudSvc = CloudBackupService.getInstance();
    if (cloudSvc.isEnabled() && cloudSvc.isAutoUploadOn()) {
      void cloudSvc.uploadSnapshot(snapshotId).catch(uploadErr => {
        const message = uploadErr instanceof Error ? uploadErr.message : String(uploadErr);
        console.error('[Fleet Snapshot] Cloud upload failed:', message);
        void NotificationService.getInstance()
          .dispatchAlert('error', 'system', `Cloud backup upload failed for snapshot ${snapshotId}: ${message}`)
          .catch(() => { /* notification dispatch is best-effort */ });
      });
    }

    console.log('[Fleet] Snapshot created:', capturedNodes.length, 'nodes,', totalStacks, 'stacks');
    if (isDebugEnabled()) {
      console.debug(`[Fleet:debug] Snapshot ${snapshotId} capture completed in ${Date.now() - captureStart}ms, ${allFiles.length} file(s) stored`);
      for (const skip of skippedNodes) {
        console.debug(`[Fleet:debug] Skipped node "${skip.nodeName}" (id=${skip.nodeId}): ${skip.reason}`);
      }
    }
    const snapshot = db.getSnapshot(snapshotId);
    res.status(201).json(snapshot);
  } catch (error) {
    console.error('[Fleet Snapshot] Create error:', error);
    res.status(500).json({ error: 'Failed to create fleet snapshot' });
  }
});

fleetRouter.get('/snapshots', authMiddleware, async (req: Request, res: Response): Promise<void> => {
  try {
    const limit = Math.min(parseInt(req.query.limit as string, 10) || 50, 100);
    const offset = parseInt(req.query.offset as string, 10) || 0;
    const db = DatabaseService.getInstance();
    const snapshots = db.getSnapshots(limit, offset);
    const total = db.getSnapshotCount();
    if (isDebugEnabled()) console.debug('[Fleet:debug] Snapshots list: limit=', limit, 'offset=', offset, 'total=', total);
    res.json({ snapshots, total });
  } catch (error) {
    console.error('[Fleet Snapshot] List error:', error);
    res.status(500).json({ error: 'Failed to list fleet snapshots' });
  }
});

fleetRouter.get('/snapshots/:id', authMiddleware, async (req: Request, res: Response): Promise<void> => {
  try {
    const id = parseIntParam(req, res, 'id', 'snapshot ID');
    if (id === null) return;
    const db = DatabaseService.getInstance();
    const snapshot = db.getSnapshot(id);
    if (!snapshot) {
      res.status(404).json({ error: 'Snapshot not found' });
      return;
    }

    const files = db.getSnapshotFiles(id);

    // Group files by node and stack.
    const nodesMap = new Map<number, { nodeId: number; nodeName: string; stacks: Map<string, Array<{ filename: string; content: string }>> }>();
    for (const file of files) {
      if (!nodesMap.has(file.node_id)) {
        nodesMap.set(file.node_id, { nodeId: file.node_id, nodeName: file.node_name, stacks: new Map() });
      }
      const nodeEntry = nodesMap.get(file.node_id)!;
      if (!nodeEntry.stacks.has(file.stack_name)) {
        nodeEntry.stacks.set(file.stack_name, []);
      }
      nodeEntry.stacks.get(file.stack_name)!.push({ filename: file.filename, content: file.content });
    }

    const nodes = Array.from(nodesMap.values()).map(n => ({
      nodeId: n.nodeId,
      nodeName: n.nodeName,
      stacks: Array.from(n.stacks.entries()).map(([stackName, stackFiles]) => ({
        stackName,
        files: stackFiles,
      })),
    }));

    if (isDebugEnabled()) console.debug('[Fleet:debug] Snapshot detail:', id, files.length, 'files');
    res.json({ ...snapshot, nodes });
  } catch (error) {
    console.error('[Fleet Snapshot] Detail error:', error);
    res.status(500).json({ error: 'Failed to fetch snapshot details' });
  }
});

fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request, res: Response): Promise<void> => {
  if (!requireAdmin(req, res)) return;

  try {
    const snapshotId = parseIntParam(req, res, 'id', 'snapshot ID');
    if (snapshotId === null) return;
    const { nodeId, stackName, redeploy = false } = req.body;

    if (!nodeId || !stackName) {
      res.status(400).json({ error: 'nodeId and stackName are required' });
      return;
    }
    if (!isValidStackName(stackName)) {
      res.status(400).json({ error: 'Invalid stack name' });
      return;
    }

    const db = DatabaseService.getInstance();
    const snapshot = db.getSnapshot(snapshotId);
    if (!snapshot) {
      res.status(404).json({ error: 'Snapshot not found' });
      return;
    }

    const files = db.getSnapshotStackFiles(snapshotId, nodeId, stackName);
    if (files.length === 0) {
      res.status(404).json({ error: 'No files found for this stack in the snapshot' });
      return;
    }

    if (isDebugEnabled()) {
      const fileNames = files.map(f => f.filename).join(', ');
      console.debug('[Fleet:debug] Restore: snapshot=%s, node=%s, stack="%s", files=[%s], redeploy=%s', sanitizeForLog(snapshotId), sanitizeForLog(nodeId), sanitizeForLog(stackName), sanitizeForLog(fileNames), sanitizeForLog(redeploy));
    }

    const node = db.getNode(nodeId);
    if (!node) {
      res.status(404).json({ error: 'Target node no longer exists' });
      return;
    }

    if (node.type === 'local') {
      const fsService = FileSystemService.getInstance(node.id);

      try {
        await fsService.backupStackFiles(stackName);
      } catch (e) {
        // Stack may not exist yet before first restore; that is ok.
        console.warn(`[Fleet Snapshot] Pre-restore backup failed for stack "${stackName}" (may not exist yet):`, getErrorMessage(e, 'unknown'));
      }

      for (const file of files) {
        if (file.filename === 'compose.yaml') {
          await fsService.saveStackContent(stackName, file.content);
        } else if (file.filename === '.env') {
          await fsService.saveEnvContent(stackName, file.content);
        }
      }

      if (redeploy) {
        const composeService = ComposeService.getInstance(node.id);
        await composeService.deployStack(stackName);
      }
    } else {
      if (!node.api_url || !node.api_token) {
        res.status(503).json({ error: 'Remote node not configured' });
        return;
      }

      const baseUrl = node.api_url.replace(/\/$/, '');
      const headers: Record<string, string> = {
        Authorization: `Bearer ${node.api_token}`,
        'Content-Type': 'application/json',
      };

      for (const file of files) {
        if (file.filename === 'compose.yaml') {
          const putRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, {
            method: 'PUT',
            headers,
            body: JSON.stringify({ content: file.content }),
            signal: AbortSignal.timeout(15000),
          });
          if (!putRes.ok) throw new Error('Failed to restore compose file on remote node');
        } else if (file.filename === '.env') {
          const putRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, {
            method: 'PUT',
            headers,
            body: JSON.stringify({ content: file.content }),
            signal: AbortSignal.timeout(15000),
          });
          if (!putRes.ok) throw new Error('Failed to restore env file on remote node');
        }
      }

      if (redeploy) {
        await fetch(`${baseUrl}/api/compose/${encodeURIComponent(stackName)}/up`, {
          method: 'POST',
          headers,
          signal: AbortSignal.timeout(30000),
        });
      }
    }

    console.log('[Fleet] Snapshot restore: snapshot=%s node=%s stack=%s', snapshotId, sanitizeForLog(nodeId), sanitizeForLog(stackName));
    res.json({ message: 'Stack restored successfully', redeployed: redeploy });
  } catch (error) {
    console.error('[Fleet Snapshot] Restore error:', error);
    res.status(500).json({ error: 'Failed to restore stack from snapshot' });
  }
});

fleetRouter.delete('/snapshots/:id', authMiddleware, async (req: Request, res: Response): Promise<void> => {
  if (!requireAdmin(req, res)) return;

  try {
    const id = parseIntParam(req, res, 'id', 'snapshot ID');
    if (id === null) return;
    const db = DatabaseService.getInstance();
    const snapshot = db.getSnapshot(id);
    if (!snapshot) {
      res.status(404).json({ error: 'Snapshot not found' });
      return;
    }
    if (isDebugEnabled()) {
      console.debug(`[Fleet:debug] Deleting snapshot ${id} (${snapshot.node_count} node(s), ${snapshot.stack_count} stack(s))`);
    }
    db.deleteSnapshot(id);
    console.log('[Fleet] Snapshot deleted:', id);
    res.json({ message: 'Snapshot deleted' });
  } catch (error) {
    console.error('[Fleet Snapshot] Delete error:', error);
    res.status(500).json({ error: 'Failed to delete snapshot' });
  }
});