sencho/backend/src/bootstrap/startup.ts

import type { Server } from 'http';
import crypto from 'crypto';
import os from 'os';
import { FileSystemService } from '../services/FileSystemService';
import { NodeRegistry } from '../services/NodeRegistry';
import { DatabaseService } from '../services/DatabaseService';
import { LicenseService } from '../services/LicenseService';
import SelfUpdateService from '../services/SelfUpdateService';
import { ImageOperationService } from '../services/ImageOperationService';
import SelfIdentityService from '../services/SelfIdentityService';
import { MonitorService } from '../services/MonitorService';
import { AutoHealService } from '../services/AutoHealService';
import { HealthGateService } from '../services/HealthGateService';
import { FleetSyncRetryService } from '../services/FleetSyncRetryService';
import { DockerEventManager } from '../services/DockerEventManager';
import TrivyService, { sweepStaleTrivyTempDirs } from '../services/TrivyService';
import { ImageUpdateService } from '../services/ImageUpdateService';
import { SchedulerService } from '../services/SchedulerService';
import { MfaService } from '../services/MfaService';
import { MeshService } from '../services/MeshService';
import { BlueprintReconciler } from '../services/BlueprintReconciler';
import { CveIntelService } from '../services/CveIntelService';
import { applyPilotModeCapabilityFilter } from '../services/CapabilityRegistry';
import { PilotTunnelManager } from '../services/PilotTunnelManager';
import { PilotMetrics } from '../services/PilotMetrics';
import { invalidateRemoteMetaCache } from '../helpers/cacheInvalidation';
import { sweepStaleTempDirs as sweepStaleGitTempDirs } from '../services/GitSourceService';
import { PORT } from '../helpers/constants';
import { LOW_MEMORY_FLOOR_BYTES } from '../utils/spawnErrors';

function isPilotMode(): boolean {
  return process.env.SENCHO_MODE === 'pilot';
}

/**
 * Pilot-agent hosts never run the first-run setup wizard, so the wizard
 * path that normally generates `auth_jwt_secret` (routes/auth.ts) never
 * fires. Without that secret, the agent-side loopback auth helper
 * (`pilot/agent.ts::getLoopbackAuthHeader`) cannot mint the
 * `pilot_tunnel`-scoped JWT it injects on every forwarded HTTP/WS request,
 * and the local Sencho's `authMiddleware` rejects every proxied call with
 * 401 "Authentication required". Generate the secret here on first boot in
 * pilot mode; subsequent boots reuse the persisted value. No-op outside
 * pilot mode (the wizard owns the lifecycle there).
 *
 * Returns true when a fresh secret was written, false otherwise.
 */
export function ensurePilotJwtSecret(): boolean {
  if (!isPilotMode()) return false;
  const dbSvc = DatabaseService.getInstance();
  if (dbSvc.getGlobalSettings().auth_jwt_secret) return false;
  const generated = crypto.randomBytes(64).toString('hex');
  dbSvc.updateGlobalSetting('auth_jwt_secret', generated);
  console.log('[Startup] pilot-agent: generated local auth_jwt_secret');
  return true;
}

/** Keep the pilot's persisted local node aligned with its configured mount. */
export function reconcilePilotComposeDir(): boolean {
  if (!isPilotMode()) return false;
  const configuredDir = process.env.COMPOSE_DIR?.trim();
  if (!configuredDir) return false;

  const db = DatabaseService.getInstance();
  const localNode = db.getDefaultNode();
  if (!localNode || localNode.compose_dir === configuredDir) return false;

  db.updateNode(localNode.id, { compose_dir: configuredDir });
  console.log(`[Startup] pilot-agent: compose directory set to ${configuredDir}`);
  return true;
}

function clearSelfContainerNotificationRouting(): void {
  const identity = SelfIdentityService.getInstance().getIdentity();
  const changed = DatabaseService.getInstance().clearSelfContainerNotificationRouting(
    NodeRegistry.getInstance().getDefaultNodeId(),
    {
      containerName: identity.containerName,
      composeProjectName: identity.composeProjectName,
    },
  );
  if (changed > 0) {
    console.log(`[Startup] Cleared stack routing from ${changed} Sencho self-container notification(s)`);
  }
}

/**
 * Run the startup sequence: stack-directory migration, service initialization,
 * background watchdogs, then bind the HTTP server. The caller passes the
 * already-constructed server so tests can import the module without binding a
 * port.
 */
export async function startServer(server: Server): Promise<void> {
  const freeBytes = os.freemem();
  const freeMiB = Math.round(freeBytes / (1024 * 1024));
  const totalMiB = Math.round(os.totalmem() / (1024 * 1024));
  const floorMiB = Math.round(LOW_MEMORY_FLOOR_BYTES / (1024 * 1024));
  console.log(`[Startup] Host memory: ${freeMiB} MiB free of ${totalMiB} MiB`);
  if (freeBytes < LOW_MEMORY_FLOOR_BYTES) {
    console.warn(
      `[Startup] Free host memory is ${freeMiB} MiB (below ${floorMiB} MiB floor). ` +
      'Sencho operations that spawn child processes (docker, /bin/sh) may fail ' +
      'with misleading ENOENT errors under memory pressure.'
    );
  }

  reconcilePilotComposeDir();

  try {
    console.log('Running stack migration check...');
    const defaultFsService = FileSystemService.getInstance(NodeRegistry.getInstance().getDefaultNodeId());
    await defaultFsService.migrateFlatToDirectory();
    console.log('Migration check completed');
  } catch (error) {
    console.error('Migration failed:', error);
  }

  ensurePilotJwtSecret();

  if (isPilotMode()) {
    applyPilotModeCapabilityFilter();
  }

  // Hydrate pilot/mesh counters from the persisted snapshot before any
  // service that increments them (MeshService, PilotTunnelManager) starts.
  // Failures fall through to zero-initialized counters; do not block boot.
  try {
    PilotMetrics.load(DatabaseService.getInstance());
  } catch (err) {
    console.warn('[Startup] PilotMetrics load failed:', (err as Error).message);
  }

  // Initialize the license service before any tier-gated code can run.
  LicenseService.getInstance().initialize();

  // Synchronous starts: schedule background timers and continue. None of
  // these fire their first tick for at least a few seconds, so they
  // safely run alongside the async initializers below.
  MonitorService.getInstance().start();
  AutoHealService.getInstance().start();
  HealthGateService.getInstance().start();
  FleetSyncRetryService.getInstance().start();
  ImageUpdateService.getInstance().start();
  SchedulerService.getInstance().start();
  MfaService.getInstance().start();
  MeshService.getInstance().start().catch((err) => {
    console.warn('[Startup] MeshService start failed:', (err as Error).message);
  });
  BlueprintReconciler.getInstance().start();
  CveIntelService.getInstance().start();

  // Drop the cached /api/meta entry on tunnel reconnect so the next
  // /api/nodes/:id/meta refetches fresh capabilities and version through
  // the live loopback bridge instead of waiting for the 3-minute TTL.
  PilotTunnelManager.getInstance().on('tunnel-up', invalidateRemoteMetaCache);

  // Most async initializers still run in parallel. Docker event monitoring
  // is sequenced after self identity so it never classifies Sencho's own
  // container as a routeable stack event.
  await Promise.all([
    SelfUpdateService.getInstance().initialize(),
    (async () => {
      await SelfIdentityService.getInstance().initialize();
      clearSelfContainerNotificationRouting();
      await DockerEventManager.getInstance().start();
    })(),
    TrivyService.getInstance().initialize(),
  ]);
  void ImageOperationService.getInstance().reconcileOnStartup().catch((error) => {
    console.error('[ImageOperation] Startup reconciliation failed:', error);
  });

  // Fire-and-forget housekeeping; logged but never awaited.
  sweepStaleGitTempDirs().catch((err) => {
    console.warn('[GitSource] Temp dir sweep failed:', (err as Error).message);
  });
  sweepStaleTrivyTempDirs().catch((err) => {
    console.warn('[Trivy] Temp dir sweep failed:', (err as Error).message);
  });

  const isPilotAgent = isPilotMode();
  const listenHost = isPilotAgent ? '127.0.0.1' : undefined;

  server.listen(PORT, listenHost, () => {
    console.log(`Server running on ${listenHost || '0.0.0.0'}:${PORT}${isPilotAgent ? ' (pilot-agent mode)' : ''}`);
    if (isPilotAgent) {
      import('../pilot/agent').then((m) => m.startPilotAgent(PORT)).catch((err) => {
        console.error('[Pilot] Agent startup failed:', err);
      });
    }
  });
}