From e9fce150108fba79877707d72699cf52a41e35f9 Mon Sep 17 00:00:00 2001 From: Anso Date: Thu, 23 Apr 2026 23:44:00 -0400 Subject: [PATCH] refactor(backend): extract bootstrap into startup/shutdown modules (phase 5) (#745) Move the startup and shutdown lifecycles out of index.ts: - bootstrap/startup.ts exports startServer(server) - migration check, service initialization, background watchdogs, HTTP listen, pilot-agent loopback bind. - bootstrap/shutdown.ts exports installShutdownHandlers(server) - SIGTERM/SIGINT handlers, in-order service stop chain, 10s force-exit guard, SQLite close. Restructure MfaService to add an instance + lifecycle so the replay purge timer no longer lives as a module-scope setInterval in index.ts. MfaService keeps all existing static methods (generateSecret, verifyTotp, currentWindow, generateBackupCodes, hashBackupCodes, verifyBackupCode, formatBackupCodeForDisplay, normalizeBackupCode, buildOtpauthUri) so every existing caller stays unchanged. The new start() / stop() pair is idempotent and calls .unref() so test shutdown is not blocked. bootstrap/startup calls MfaService.getInstance().start(). bootstrap/shutdown calls MfaService.getInstance().stop(). index.ts drops from 305 to 147 lines and now contains only the Express app composition: createApp, route mounts, remote proxy, createServer, attachUpgrade, static/SPA fallback, errorHandler, installShutdownHandlers, and the require.main guard that boots the server when run directly. Behavior is byte-for-byte identical: shutdown service order, log strings, force-exit timer, pilot-agent loopback logic, and the MFA purge cadence and debug logging all preserved verbatim. --- backend/src/bootstrap/shutdown.ts | 57 ++++++++++ backend/src/bootstrap/startup.ts | 64 +++++++++++ backend/src/index.ts | 172 ++--------------------------- backend/src/services/MfaService.ts | 38 +++++++ 4 files changed, 166 insertions(+), 165 deletions(-) create mode 100644 backend/src/bootstrap/shutdown.ts create mode 100644 backend/src/bootstrap/startup.ts diff --git a/backend/src/bootstrap/shutdown.ts b/backend/src/bootstrap/shutdown.ts new file mode 100644 index 00000000..ff71b5ae --- /dev/null +++ b/backend/src/bootstrap/shutdown.ts @@ -0,0 +1,57 @@ +import type { Server } from 'http'; +import { DatabaseService } from '../services/DatabaseService'; +import { LicenseService } from '../services/LicenseService'; +import { MonitorService } from '../services/MonitorService'; +import { AutoHealService } from '../services/AutoHealService'; +import { DockerEventManager } from '../services/DockerEventManager'; +import { ImageUpdateService } from '../services/ImageUpdateService'; +import { SchedulerService } from '../services/SchedulerService'; +import { MfaService } from '../services/MfaService'; + +/** + * Wire graceful shutdown handlers. Docker sends SIGTERM when the container + * stops; Ctrl-C sends SIGINT in dev. We allow in-flight requests to finish, + * then cleanly stop background services and close the SQLite connection + * before exiting. A 10 s force-exit timer guards against hung connections. + */ +export function installShutdownHandlers(server: Server): void { + const gracefulShutdown = (signal: string): void => { + console.log(`[Shutdown] ${signal} received - shutting down gracefully…`); + + server.close(() => { + console.log('[Shutdown] HTTP server closed'); + try { LicenseService.getInstance().destroy(); } catch (e) { + console.warn('[Shutdown] LicenseService cleanup failed:', (e as Error).message); + } + try { MonitorService.getInstance().stop(); } catch (e) { + console.warn('[Shutdown] MonitorService cleanup failed:', (e as Error).message); + } + try { AutoHealService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] AutoHealService cleanup failed:', (e as Error).message); } + try { DockerEventManager.getInstance().stop(); } catch (e) { + console.warn('[Shutdown] DockerEventManager cleanup failed:', (e as Error).message); + } + try { ImageUpdateService.getInstance().stop(); } catch (e) { + console.warn('[Shutdown] ImageUpdateService cleanup failed:', (e as Error).message); + } + try { SchedulerService.getInstance().stop(); } catch (e) { + console.warn('[Shutdown] SchedulerService cleanup failed:', (e as Error).message); + } + try { MfaService.getInstance().stop(); } catch (e) { + console.warn('[Shutdown] MfaService cleanup failed:', (e as Error).message); + } + try { DatabaseService.getInstance().getDb().close(); } catch (e) { + console.warn('[Shutdown] Database close failed:', (e as Error).message); + } + console.log('[Shutdown] Done - exiting'); + process.exit(0); + }); + + setTimeout(() => { + console.error('[Shutdown] Timed out waiting for connections - forcing exit'); + process.exit(1); + }, 10_000).unref(); + }; + + process.on('SIGTERM', () => gracefulShutdown('SIGTERM')); + process.on('SIGINT', () => gracefulShutdown('SIGINT')); +} diff --git a/backend/src/bootstrap/startup.ts b/backend/src/bootstrap/startup.ts new file mode 100644 index 00000000..a55c7eeb --- /dev/null +++ b/backend/src/bootstrap/startup.ts @@ -0,0 +1,64 @@ +import type { Server } from 'http'; +import { FileSystemService } from '../services/FileSystemService'; +import { NodeRegistry } from '../services/NodeRegistry'; +import { LicenseService } from '../services/LicenseService'; +import SelfUpdateService from '../services/SelfUpdateService'; +import { MonitorService } from '../services/MonitorService'; +import { AutoHealService } from '../services/AutoHealService'; +import { DockerEventManager } from '../services/DockerEventManager'; +import TrivyService from '../services/TrivyService'; +import { ImageUpdateService } from '../services/ImageUpdateService'; +import { SchedulerService } from '../services/SchedulerService'; +import { MfaService } from '../services/MfaService'; +import { sweepStaleTempDirs as sweepStaleGitTempDirs } from '../services/GitSourceService'; +import { PORT } from '../helpers/constants'; + +/** + * Run the startup sequence: stack-directory migration, service initialization, + * background watchdogs, then bind the HTTP server. The caller passes the + * already-constructed server so tests can import the module without binding a + * port. + */ +export async function startServer(server: Server): Promise { + try { + console.log('Running stack migration check...'); + const defaultFsService = FileSystemService.getInstance(NodeRegistry.getInstance().getDefaultNodeId()); + await defaultFsService.migrateFlatToDirectory(); + console.log('Migration check completed'); + } catch (error) { + console.error('Migration failed:', error); + } + + LicenseService.getInstance().initialize(); + + await SelfUpdateService.getInstance().initialize(); + + MonitorService.getInstance().start(); + AutoHealService.getInstance().start(); + + await DockerEventManager.getInstance().start(); + + await TrivyService.getInstance().initialize(); + + ImageUpdateService.getInstance().start(); + + SchedulerService.getInstance().start(); + + sweepStaleGitTempDirs().catch((err) => { + console.warn('[GitSource] Temp dir sweep failed:', (err as Error).message); + }); + + MfaService.getInstance().start(); + + const isPilotAgent = process.env.SENCHO_MODE === 'pilot'; + const listenHost = isPilotAgent ? '127.0.0.1' : undefined; + + server.listen(PORT, listenHost, () => { + console.log(`Server running on ${listenHost || '0.0.0.0'}:${PORT}${isPilotAgent ? ' (pilot-agent mode)' : ''}`); + if (isPilotAgent) { + import('../pilot/agent').then((m) => m.startPilotAgent(PORT)).catch((err) => { + console.error('[Pilot] Agent startup failed:', err); + }); + } + }); +} diff --git a/backend/src/index.ts b/backend/src/index.ts index 3454e487..928e378d 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -1,20 +1,5 @@ import express, { Request, Response } from 'express'; -import { FileSystemService } from './services/FileSystemService'; -import { DatabaseService } from './services/DatabaseService'; -import { MonitorService } from './services/MonitorService'; -import { AutoHealService } from './services/AutoHealService'; -import { DockerEventManager } from './services/DockerEventManager'; -import { ImageUpdateService } from './services/ImageUpdateService'; -import { NodeRegistry } from './services/NodeRegistry'; -import { LicenseService } from './services/LicenseService'; -import { SchedulerService } from './services/SchedulerService'; -import { sweepStaleTempDirs as sweepStaleGitTempDirs } from './services/GitSourceService'; import './types/express'; -import { - PORT, - MFA_REPLAY_TTL_MS, - MFA_REPLAY_PURGE_INTERVAL_MS, -} from './helpers/constants'; import { authGate, auditLog } from './middleware/authGate'; import { enforceApiTokenScope } from './middleware/apiTokenScope'; import { errorHandler } from './middleware/errorHandler'; @@ -22,6 +7,8 @@ import { createApp } from './app'; import { createRemoteProxyMiddleware } from './proxy/remoteNodeProxy'; import { createServer } from './server'; import { attachUpgrade } from './websocket/upgradeHandler'; +import { startServer } from './bootstrap/startup'; +import { installShutdownHandlers } from './bootstrap/shutdown'; import { metaRouter } from './routes/meta'; import { authRouter } from './routes/auth'; import { mfaRouter } from './routes/mfa'; @@ -54,10 +41,6 @@ import { containersRouter, portsRouter } from './routes/containers'; import { nodesRouter } from './routes/nodes'; import { stacksRouter } from './routes/stacks'; -import SelfUpdateService from './services/SelfUpdateService'; -import TrivyService from './services/TrivyService'; -import { isDebugEnabled } from './utils/debug'; - // Suppress [DEP0060] DeprecationWarning emitted by http-proxy@1.18.1 which calls // util._extend internally. The warning fires at runtime when createProxyServer() is // first invoked (NOT at import time), so intercepting process.emitWarning here - @@ -70,25 +53,18 @@ const _origEmitWarning = process.emitWarning.bind(process); _origEmitWarning(warning, ...args); }; -// Build the Express app with the canonical middleware pipeline (steps 1-9 of -// the order documented in app.ts). Steps 10-16 (authGate, auditLog, -// apiTokenScope, remote proxy, routes, static, errorHandler) are registered -// below as routes and handlers are extracted in later phases. const app = createApp(); -// FileSystemService and ComposeService are instantiated per-request via .getInstance(nodeId) - // Public /api/health and /api/meta (no auth). Mounted before authGate. app.use('/api', metaRouter); // Auth / MFA / SSO routers. Mounted before authGate because some paths are -// public (login, setup, SSO callbacks); handlers that need auth use the +// public (login, setup, SSO callbacks); handlers that need auth use // authMiddleware directly. app.use('/api/auth', authRouter); app.use('/api/auth', mfaRouter); app.use('/api/auth/sso', ssoRouter); - // Auth gate on all /api/* routes (exempts /auth/* and webhook triggers). app.use('/api', authGate); @@ -97,10 +73,6 @@ app.use('/api', auditLog); app.use('/api', enforceApiTokenScope); -// Phase 4A-1 route mounts. These live behind authGate + auditLog + -// apiTokenScope but ahead of any group still inlined below. As each -// remaining group gets extracted the inline block comes out and a new -// app.use slots in here. app.use('/api/license', licenseRouter); app.use('/api/system', systemUpdateRouter); app.use('/api/permissions', permissionsRouter); @@ -140,22 +112,13 @@ app.use('/api/stacks', stacksRouter); // the proxy then takes over for remote-targeted requests. app.use('/api/', createRemoteProxyMiddleware()); -// HTTP server + WebSocket servers (see server.ts for shape). const { server, wss, pilotTunnelWss } = createServer(app); - -// Dispatch WebSocket upgrades (see websocket/upgradeHandler.ts for the full -// dispatch order). Also wires the main wss's connection handler for -// container-exec / streamStats actions. attachUpgrade(server, { wss, pilotTunnelWss }); - - -// Serve static files in production (for Docker deployment) +// Static / SPA fallback. Production serves the built frontend; dev returns a +// JSON 404 for unmatched /api paths to prevent fetch hangs. if (process.env.NODE_ENV === 'production') { app.use(express.static('public')); - - // Handle SPA routing - serve index.html for non-API routes - // Using app.use middleware instead of app.get('*') for path-to-regexp compatibility app.use((req: Request, res: Response) => { if (!req.path.startsWith('/api')) { res.sendFile('index.html', { root: 'public' }); @@ -164,7 +127,6 @@ if (process.env.NODE_ENV === 'production') { } }); } else { - // In development, still need to catch 404s for API to prevent hangs app.use((req: Request, res: Response) => { if (req.path.startsWith('/api')) { res.status(404).json({ error: 'API endpoint not found' }); @@ -175,131 +137,11 @@ if (process.env.NODE_ENV === 'production') { // Central error handler: must be registered after all routes and static. app.use(errorHandler); -// Start server with migration -let mfaReplayPurgeTimer: NodeJS.Timeout | null = null; +installShutdownHandlers(server); -async function startServer() { - try { - // Run migration before starting server - console.log('Running stack migration check...'); - const defaultFsService = FileSystemService.getInstance(NodeRegistry.getInstance().getDefaultNodeId()); - await defaultFsService.migrateFlatToDirectory(); - console.log('Migration check completed'); - } catch (error) { - console.error('Migration failed:', error); - // Continue starting server even if migration fails - } - - // Initialize License Service (starts trial on first boot, periodic validation) - LicenseService.getInstance().initialize(); - - // Detect whether this instance can self-update (Docker Compose container inspection) - await SelfUpdateService.getInstance().initialize(); - - // Start Background Watchdog - MonitorService.getInstance().start(); - AutoHealService.getInstance().start(); - - // Start Docker Event Stream (causal crash/OOM/health detection per local node) - await DockerEventManager.getInstance().start(); - - // Detect Trivy binary so the vulnerability-scanning capability reflects - // reality before any request hits and so the first scan does not pay - // detection latency. - await TrivyService.getInstance().initialize(); - - // Start Background Image Update Checker - ImageUpdateService.getInstance().start(); - - // Start Scheduled Operations Service - SchedulerService.getInstance().start(); - - // Sweep any leftover git-source temp clones from a crashed prior run - sweepStaleGitTempDirs().catch((err) => { - console.warn('[GitSource] Temp dir sweep failed:', (err as Error).message); - }); - - // Periodic purge of used-MFA-code rows so the replay blacklist stays - // bounded even without verification traffic. The table holds (user, code, - // window) tuples for the last ~2 minutes; older rows are safe to drop. - mfaReplayPurgeTimer = setInterval(() => { - try { - const deleted = DatabaseService.getInstance().purgeOldMfaCodes(Date.now() - MFA_REPLAY_TTL_MS); - if (isDebugEnabled() && deleted > 0) { - console.log('[MFA:diag] replay purge deleted=', deleted); - } - } catch (err) { - console.warn('[MFA] Replay purge failed:', (err as Error).message); - } - }, MFA_REPLAY_PURGE_INTERVAL_MS); - mfaReplayPurgeTimer.unref(); - - // Pilot-agent mode: bind only to loopback so no external port is exposed. - // All traffic is demultiplexed from the primary via the pilot tunnel. - const isPilotAgent = process.env.SENCHO_MODE === 'pilot'; - const listenHost = isPilotAgent ? '127.0.0.1' : undefined; - - server.listen(PORT, listenHost, () => { - console.log(`Server running on ${listenHost || '0.0.0.0'}:${PORT}${isPilotAgent ? ' (pilot-agent mode)' : ''}`); - if (isPilotAgent) { - // Start the outbound tunnel client once the local HTTP server is ready - // to accept loopback traffic from the tunnel. - import('./pilot/agent').then((m) => m.startPilotAgent(PORT)).catch((err) => { - console.error('[Pilot] Agent startup failed:', err); - }); - } - }); -} - -// Only start the server when this file is the entry point (not when imported by tests). if (require.main === module) { - startServer(); + void startServer(server); } // Exports used by tests (supertest requires the http.Server instance). export { app, server }; - -// Graceful shutdown - allows in-flight requests to finish, then cleanly stops -// background services and closes the SQLite connection before the process exits. -// Docker sends SIGTERM when the container stops; Ctrl-C sends SIGINT in dev. -const gracefulShutdown = (signal: string) => { - console.log(`[Shutdown] ${signal} received - shutting down gracefully…`); - - server.close(() => { - console.log('[Shutdown] HTTP server closed'); - try { LicenseService.getInstance().destroy(); } catch (e) { - console.warn('[Shutdown] LicenseService cleanup failed:', (e as Error).message); - } - try { MonitorService.getInstance().stop(); } catch (e) { - console.warn('[Shutdown] MonitorService cleanup failed:', (e as Error).message); - } - try { AutoHealService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] AutoHealService cleanup failed:', (e as Error).message); } - try { DockerEventManager.getInstance().stop(); } catch (e) { - console.warn('[Shutdown] DockerEventManager cleanup failed:', (e as Error).message); - } - try { ImageUpdateService.getInstance().stop(); } catch (e) { - console.warn('[Shutdown] ImageUpdateService cleanup failed:', (e as Error).message); - } - try { SchedulerService.getInstance().stop(); } catch (e) { - console.warn('[Shutdown] SchedulerService cleanup failed:', (e as Error).message); - } - if (mfaReplayPurgeTimer) { - clearInterval(mfaReplayPurgeTimer); - mfaReplayPurgeTimer = null; - } - try { DatabaseService.getInstance().getDb().close(); } catch (e) { - console.warn('[Shutdown] Database close failed:', (e as Error).message); - } - console.log('[Shutdown] Done - exiting'); - process.exit(0); - }); - - // Force-exit after 10 s if connections refuse to drain - setTimeout(() => { - console.error('[Shutdown] Timed out waiting for connections - forcing exit'); - process.exit(1); - }, 10_000).unref(); -}; - -process.on('SIGTERM', () => gracefulShutdown('SIGTERM')); -process.on('SIGINT', () => gracefulShutdown('SIGINT')); diff --git a/backend/src/services/MfaService.ts b/backend/src/services/MfaService.ts index 33a105c3..43331ca6 100644 --- a/backend/src/services/MfaService.ts +++ b/backend/src/services/MfaService.ts @@ -2,6 +2,9 @@ import crypto from 'crypto'; import bcrypt from 'bcrypt'; import { authenticator } from 'otplib'; import { HashAlgorithms } from '@otplib/core'; +import { DatabaseService } from './DatabaseService'; +import { MFA_REPLAY_TTL_MS, MFA_REPLAY_PURGE_INTERVAL_MS } from '../helpers/constants'; +import { isDebugEnabled } from '../utils/debug'; // Configure otplib for the default TOTP contract we present to users: // - 6 digits @@ -27,6 +30,41 @@ export interface BackupVerifyResult { } export class MfaService { + private static instance: MfaService; + private purgeTimer: NodeJS.Timeout | null = null; + + public static getInstance(): MfaService { + if (!MfaService.instance) MfaService.instance = new MfaService(); + return MfaService.instance; + } + + /** + * Start the periodic purge of used-MFA-code rows. The replay blacklist + * holds (user, code, window) tuples for the last ~2 minutes; older rows + * are safe to drop. Idempotent: calling start() twice is a no-op. + */ + public start(): void { + if (this.purgeTimer) return; + this.purgeTimer = setInterval(() => { + try { + const deleted = DatabaseService.getInstance().purgeOldMfaCodes(Date.now() - MFA_REPLAY_TTL_MS); + if (isDebugEnabled() && deleted > 0) { + console.log('[MFA:diag] replay purge deleted=', deleted); + } + } catch (err) { + console.warn('[MFA] Replay purge failed:', (err as Error).message); + } + }, MFA_REPLAY_PURGE_INTERVAL_MS); + this.purgeTimer.unref(); + } + + public stop(): void { + if (this.purgeTimer) { + clearInterval(this.purgeTimer); + this.purgeTimer = null; + } + } + /** * Generate a fresh base32 TOTP secret ready for `buildOtpauthUri` and * `verifyTotp`. Each user should receive a unique secret.