mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-05 16:37:46 +00:00
refactor(backend): extract bootstrap into startup/shutdown modules (phase 5) (#745)
Move the startup and shutdown lifecycles out of index.ts: - bootstrap/startup.ts exports startServer(server) - migration check, service initialization, background watchdogs, HTTP listen, pilot-agent loopback bind. - bootstrap/shutdown.ts exports installShutdownHandlers(server) - SIGTERM/SIGINT handlers, in-order service stop chain, 10s force-exit guard, SQLite close. Restructure MfaService to add an instance + lifecycle so the replay purge timer no longer lives as a module-scope setInterval in index.ts. MfaService keeps all existing static methods (generateSecret, verifyTotp, currentWindow, generateBackupCodes, hashBackupCodes, verifyBackupCode, formatBackupCodeForDisplay, normalizeBackupCode, buildOtpauthUri) so every existing caller stays unchanged. The new start() / stop() pair is idempotent and calls .unref() so test shutdown is not blocked. bootstrap/startup calls MfaService.getInstance().start(). bootstrap/shutdown calls MfaService.getInstance().stop(). index.ts drops from 305 to 147 lines and now contains only the Express app composition: createApp, route mounts, remote proxy, createServer, attachUpgrade, static/SPA fallback, errorHandler, installShutdownHandlers, and the require.main guard that boots the server when run directly. Behavior is byte-for-byte identical: shutdown service order, log strings, force-exit timer, pilot-agent loopback logic, and the MFA purge cadence and debug logging all preserved verbatim.
This commit is contained in:
Anso
@@ -0,0 +1,57 @@
|
||||
import type { Server } from 'http';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { LicenseService } from '../services/LicenseService';
|
||||
import { MonitorService } from '../services/MonitorService';
|
||||
import { AutoHealService } from '../services/AutoHealService';
|
||||
import { DockerEventManager } from '../services/DockerEventManager';
|
||||
import { ImageUpdateService } from '../services/ImageUpdateService';
|
||||
import { SchedulerService } from '../services/SchedulerService';
|
||||
import { MfaService } from '../services/MfaService';
|
||||
|
||||
/**
|
||||
* Wire graceful shutdown handlers. Docker sends SIGTERM when the container
|
||||
* stops; Ctrl-C sends SIGINT in dev. We allow in-flight requests to finish,
|
||||
* then cleanly stop background services and close the SQLite connection
|
||||
* before exiting. A 10 s force-exit timer guards against hung connections.
|
||||
*/
|
||||
export function installShutdownHandlers(server: Server): void {
|
||||
const gracefulShutdown = (signal: string): void => {
|
||||
console.log(`[Shutdown] ${signal} received - shutting down gracefully…`);
|
||||
|
||||
server.close(() => {
|
||||
console.log('[Shutdown] HTTP server closed');
|
||||
try { LicenseService.getInstance().destroy(); } catch (e) {
|
||||
console.warn('[Shutdown] LicenseService cleanup failed:', (e as Error).message);
|
||||
}
|
||||
try { MonitorService.getInstance().stop(); } catch (e) {
|
||||
console.warn('[Shutdown] MonitorService cleanup failed:', (e as Error).message);
|
||||
}
|
||||
try { AutoHealService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] AutoHealService cleanup failed:', (e as Error).message); }
|
||||
try { DockerEventManager.getInstance().stop(); } catch (e) {
|
||||
console.warn('[Shutdown] DockerEventManager cleanup failed:', (e as Error).message);
|
||||
}
|
||||
try { ImageUpdateService.getInstance().stop(); } catch (e) {
|
||||
console.warn('[Shutdown] ImageUpdateService cleanup failed:', (e as Error).message);
|
||||
}
|
||||
try { SchedulerService.getInstance().stop(); } catch (e) {
|
||||
console.warn('[Shutdown] SchedulerService cleanup failed:', (e as Error).message);
|
||||
}
|
||||
try { MfaService.getInstance().stop(); } catch (e) {
|
||||
console.warn('[Shutdown] MfaService cleanup failed:', (e as Error).message);
|
||||
}
|
||||
try { DatabaseService.getInstance().getDb().close(); } catch (e) {
|
||||
console.warn('[Shutdown] Database close failed:', (e as Error).message);
|
||||
}
|
||||
console.log('[Shutdown] Done - exiting');
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
setTimeout(() => {
|
||||
console.error('[Shutdown] Timed out waiting for connections - forcing exit');
|
||||
process.exit(1);
|
||||
}, 10_000).unref();
|
||||
};
|
||||
|
||||
process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
|
||||
process.on('SIGINT', () => gracefulShutdown('SIGINT'));
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import type { Server } from 'http';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
import { NodeRegistry } from '../services/NodeRegistry';
|
||||
import { LicenseService } from '../services/LicenseService';
|
||||
import SelfUpdateService from '../services/SelfUpdateService';
|
||||
import { MonitorService } from '../services/MonitorService';
|
||||
import { AutoHealService } from '../services/AutoHealService';
|
||||
import { DockerEventManager } from '../services/DockerEventManager';
|
||||
import TrivyService from '../services/TrivyService';
|
||||
import { ImageUpdateService } from '../services/ImageUpdateService';
|
||||
import { SchedulerService } from '../services/SchedulerService';
|
||||
import { MfaService } from '../services/MfaService';
|
||||
import { sweepStaleTempDirs as sweepStaleGitTempDirs } from '../services/GitSourceService';
|
||||
import { PORT } from '../helpers/constants';
|
||||
|
||||
/**
|
||||
* Run the startup sequence: stack-directory migration, service initialization,
|
||||
* background watchdogs, then bind the HTTP server. The caller passes the
|
||||
* already-constructed server so tests can import the module without binding a
|
||||
* port.
|
||||
*/
|
||||
export async function startServer(server: Server): Promise<void> {
|
||||
try {
|
||||
console.log('Running stack migration check...');
|
||||
const defaultFsService = FileSystemService.getInstance(NodeRegistry.getInstance().getDefaultNodeId());
|
||||
await defaultFsService.migrateFlatToDirectory();
|
||||
console.log('Migration check completed');
|
||||
} catch (error) {
|
||||
console.error('Migration failed:', error);
|
||||
}
|
||||
|
||||
LicenseService.getInstance().initialize();
|
||||
|
||||
await SelfUpdateService.getInstance().initialize();
|
||||
|
||||
MonitorService.getInstance().start();
|
||||
AutoHealService.getInstance().start();
|
||||
|
||||
await DockerEventManager.getInstance().start();
|
||||
|
||||
await TrivyService.getInstance().initialize();
|
||||
|
||||
ImageUpdateService.getInstance().start();
|
||||
|
||||
SchedulerService.getInstance().start();
|
||||
|
||||
sweepStaleGitTempDirs().catch((err) => {
|
||||
console.warn('[GitSource] Temp dir sweep failed:', (err as Error).message);
|
||||
});
|
||||
|
||||
MfaService.getInstance().start();
|
||||
|
||||
const isPilotAgent = process.env.SENCHO_MODE === 'pilot';
|
||||
const listenHost = isPilotAgent ? '127.0.0.1' : undefined;
|
||||
|
||||
server.listen(PORT, listenHost, () => {
|
||||
console.log(`Server running on ${listenHost || '0.0.0.0'}:${PORT}${isPilotAgent ? ' (pilot-agent mode)' : ''}`);
|
||||
if (isPilotAgent) {
|
||||
import('../pilot/agent').then((m) => m.startPilotAgent(PORT)).catch((err) => {
|
||||
console.error('[Pilot] Agent startup failed:', err);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
+7
-165
@@ -1,20 +1,5 @@
|
||||
import express, { Request, Response } from 'express';
|
||||
import { FileSystemService } from './services/FileSystemService';
|
||||
import { DatabaseService } from './services/DatabaseService';
|
||||
import { MonitorService } from './services/MonitorService';
|
||||
import { AutoHealService } from './services/AutoHealService';
|
||||
import { DockerEventManager } from './services/DockerEventManager';
|
||||
import { ImageUpdateService } from './services/ImageUpdateService';
|
||||
import { NodeRegistry } from './services/NodeRegistry';
|
||||
import { LicenseService } from './services/LicenseService';
|
||||
import { SchedulerService } from './services/SchedulerService';
|
||||
import { sweepStaleTempDirs as sweepStaleGitTempDirs } from './services/GitSourceService';
|
||||
import './types/express';
|
||||
import {
|
||||
PORT,
|
||||
MFA_REPLAY_TTL_MS,
|
||||
MFA_REPLAY_PURGE_INTERVAL_MS,
|
||||
} from './helpers/constants';
|
||||
import { authGate, auditLog } from './middleware/authGate';
|
||||
import { enforceApiTokenScope } from './middleware/apiTokenScope';
|
||||
import { errorHandler } from './middleware/errorHandler';
|
||||
@@ -22,6 +7,8 @@ import { createApp } from './app';
|
||||
import { createRemoteProxyMiddleware } from './proxy/remoteNodeProxy';
|
||||
import { createServer } from './server';
|
||||
import { attachUpgrade } from './websocket/upgradeHandler';
|
||||
import { startServer } from './bootstrap/startup';
|
||||
import { installShutdownHandlers } from './bootstrap/shutdown';
|
||||
import { metaRouter } from './routes/meta';
|
||||
import { authRouter } from './routes/auth';
|
||||
import { mfaRouter } from './routes/mfa';
|
||||
@@ -54,10 +41,6 @@ import { containersRouter, portsRouter } from './routes/containers';
|
||||
import { nodesRouter } from './routes/nodes';
|
||||
import { stacksRouter } from './routes/stacks';
|
||||
|
||||
import SelfUpdateService from './services/SelfUpdateService';
|
||||
import TrivyService from './services/TrivyService';
|
||||
import { isDebugEnabled } from './utils/debug';
|
||||
|
||||
// Suppress [DEP0060] DeprecationWarning emitted by http-proxy@1.18.1 which calls
|
||||
// util._extend internally. The warning fires at runtime when createProxyServer() is
|
||||
// first invoked (NOT at import time), so intercepting process.emitWarning here -
|
||||
@@ -70,25 +53,18 @@ const _origEmitWarning = process.emitWarning.bind(process);
|
||||
_origEmitWarning(warning, ...args);
|
||||
};
|
||||
|
||||
// Build the Express app with the canonical middleware pipeline (steps 1-9 of
|
||||
// the order documented in app.ts). Steps 10-16 (authGate, auditLog,
|
||||
// apiTokenScope, remote proxy, routes, static, errorHandler) are registered
|
||||
// below as routes and handlers are extracted in later phases.
|
||||
const app = createApp();
|
||||
|
||||
// FileSystemService and ComposeService are instantiated per-request via .getInstance(nodeId)
|
||||
|
||||
// Public /api/health and /api/meta (no auth). Mounted before authGate.
|
||||
app.use('/api', metaRouter);
|
||||
|
||||
// Auth / MFA / SSO routers. Mounted before authGate because some paths are
|
||||
// public (login, setup, SSO callbacks); handlers that need auth use the
|
||||
// public (login, setup, SSO callbacks); handlers that need auth use
|
||||
// authMiddleware directly.
|
||||
app.use('/api/auth', authRouter);
|
||||
app.use('/api/auth', mfaRouter);
|
||||
app.use('/api/auth/sso', ssoRouter);
|
||||
|
||||
|
||||
// Auth gate on all /api/* routes (exempts /auth/* and webhook triggers).
|
||||
app.use('/api', authGate);
|
||||
|
||||
@@ -97,10 +73,6 @@ app.use('/api', auditLog);
|
||||
|
||||
app.use('/api', enforceApiTokenScope);
|
||||
|
||||
// Phase 4A-1 route mounts. These live behind authGate + auditLog +
|
||||
// apiTokenScope but ahead of any group still inlined below. As each
|
||||
// remaining group gets extracted the inline block comes out and a new
|
||||
// app.use slots in here.
|
||||
app.use('/api/license', licenseRouter);
|
||||
app.use('/api/system', systemUpdateRouter);
|
||||
app.use('/api/permissions', permissionsRouter);
|
||||
@@ -140,22 +112,13 @@ app.use('/api/stacks', stacksRouter);
|
||||
// the proxy then takes over for remote-targeted requests.
|
||||
app.use('/api/', createRemoteProxyMiddleware());
|
||||
|
||||
// HTTP server + WebSocket servers (see server.ts for shape).
|
||||
const { server, wss, pilotTunnelWss } = createServer(app);
|
||||
|
||||
// Dispatch WebSocket upgrades (see websocket/upgradeHandler.ts for the full
|
||||
// dispatch order). Also wires the main wss's connection handler for
|
||||
// container-exec / streamStats actions.
|
||||
attachUpgrade(server, { wss, pilotTunnelWss });
|
||||
|
||||
|
||||
|
||||
// Serve static files in production (for Docker deployment)
|
||||
// Static / SPA fallback. Production serves the built frontend; dev returns a
|
||||
// JSON 404 for unmatched /api paths to prevent fetch hangs.
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
app.use(express.static('public'));
|
||||
|
||||
// Handle SPA routing - serve index.html for non-API routes
|
||||
// Using app.use middleware instead of app.get('*') for path-to-regexp compatibility
|
||||
app.use((req: Request, res: Response) => {
|
||||
if (!req.path.startsWith('/api')) {
|
||||
res.sendFile('index.html', { root: 'public' });
|
||||
@@ -164,7 +127,6 @@ if (process.env.NODE_ENV === 'production') {
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// In development, still need to catch 404s for API to prevent hangs
|
||||
app.use((req: Request, res: Response) => {
|
||||
if (req.path.startsWith('/api')) {
|
||||
res.status(404).json({ error: 'API endpoint not found' });
|
||||
@@ -175,131 +137,11 @@ if (process.env.NODE_ENV === 'production') {
|
||||
// Central error handler: must be registered after all routes and static.
|
||||
app.use(errorHandler);
|
||||
|
||||
// Start server with migration
|
||||
let mfaReplayPurgeTimer: NodeJS.Timeout | null = null;
|
||||
installShutdownHandlers(server);
|
||||
|
||||
async function startServer() {
|
||||
try {
|
||||
// Run migration before starting server
|
||||
console.log('Running stack migration check...');
|
||||
const defaultFsService = FileSystemService.getInstance(NodeRegistry.getInstance().getDefaultNodeId());
|
||||
await defaultFsService.migrateFlatToDirectory();
|
||||
console.log('Migration check completed');
|
||||
} catch (error) {
|
||||
console.error('Migration failed:', error);
|
||||
// Continue starting server even if migration fails
|
||||
}
|
||||
|
||||
// Initialize License Service (starts trial on first boot, periodic validation)
|
||||
LicenseService.getInstance().initialize();
|
||||
|
||||
// Detect whether this instance can self-update (Docker Compose container inspection)
|
||||
await SelfUpdateService.getInstance().initialize();
|
||||
|
||||
// Start Background Watchdog
|
||||
MonitorService.getInstance().start();
|
||||
AutoHealService.getInstance().start();
|
||||
|
||||
// Start Docker Event Stream (causal crash/OOM/health detection per local node)
|
||||
await DockerEventManager.getInstance().start();
|
||||
|
||||
// Detect Trivy binary so the vulnerability-scanning capability reflects
|
||||
// reality before any request hits and so the first scan does not pay
|
||||
// detection latency.
|
||||
await TrivyService.getInstance().initialize();
|
||||
|
||||
// Start Background Image Update Checker
|
||||
ImageUpdateService.getInstance().start();
|
||||
|
||||
// Start Scheduled Operations Service
|
||||
SchedulerService.getInstance().start();
|
||||
|
||||
// Sweep any leftover git-source temp clones from a crashed prior run
|
||||
sweepStaleGitTempDirs().catch((err) => {
|
||||
console.warn('[GitSource] Temp dir sweep failed:', (err as Error).message);
|
||||
});
|
||||
|
||||
// Periodic purge of used-MFA-code rows so the replay blacklist stays
|
||||
// bounded even without verification traffic. The table holds (user, code,
|
||||
// window) tuples for the last ~2 minutes; older rows are safe to drop.
|
||||
mfaReplayPurgeTimer = setInterval(() => {
|
||||
try {
|
||||
const deleted = DatabaseService.getInstance().purgeOldMfaCodes(Date.now() - MFA_REPLAY_TTL_MS);
|
||||
if (isDebugEnabled() && deleted > 0) {
|
||||
console.log('[MFA:diag] replay purge deleted=', deleted);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('[MFA] Replay purge failed:', (err as Error).message);
|
||||
}
|
||||
}, MFA_REPLAY_PURGE_INTERVAL_MS);
|
||||
mfaReplayPurgeTimer.unref();
|
||||
|
||||
// Pilot-agent mode: bind only to loopback so no external port is exposed.
|
||||
// All traffic is demultiplexed from the primary via the pilot tunnel.
|
||||
const isPilotAgent = process.env.SENCHO_MODE === 'pilot';
|
||||
const listenHost = isPilotAgent ? '127.0.0.1' : undefined;
|
||||
|
||||
server.listen(PORT, listenHost, () => {
|
||||
console.log(`Server running on ${listenHost || '0.0.0.0'}:${PORT}${isPilotAgent ? ' (pilot-agent mode)' : ''}`);
|
||||
if (isPilotAgent) {
|
||||
// Start the outbound tunnel client once the local HTTP server is ready
|
||||
// to accept loopback traffic from the tunnel.
|
||||
import('./pilot/agent').then((m) => m.startPilotAgent(PORT)).catch((err) => {
|
||||
console.error('[Pilot] Agent startup failed:', err);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Only start the server when this file is the entry point (not when imported by tests).
|
||||
if (require.main === module) {
|
||||
startServer();
|
||||
void startServer(server);
|
||||
}
|
||||
|
||||
// Exports used by tests (supertest requires the http.Server instance).
|
||||
export { app, server };
|
||||
|
||||
// Graceful shutdown - allows in-flight requests to finish, then cleanly stops
|
||||
// background services and closes the SQLite connection before the process exits.
|
||||
// Docker sends SIGTERM when the container stops; Ctrl-C sends SIGINT in dev.
|
||||
const gracefulShutdown = (signal: string) => {
|
||||
console.log(`[Shutdown] ${signal} received - shutting down gracefully…`);
|
||||
|
||||
server.close(() => {
|
||||
console.log('[Shutdown] HTTP server closed');
|
||||
try { LicenseService.getInstance().destroy(); } catch (e) {
|
||||
console.warn('[Shutdown] LicenseService cleanup failed:', (e as Error).message);
|
||||
}
|
||||
try { MonitorService.getInstance().stop(); } catch (e) {
|
||||
console.warn('[Shutdown] MonitorService cleanup failed:', (e as Error).message);
|
||||
}
|
||||
try { AutoHealService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] AutoHealService cleanup failed:', (e as Error).message); }
|
||||
try { DockerEventManager.getInstance().stop(); } catch (e) {
|
||||
console.warn('[Shutdown] DockerEventManager cleanup failed:', (e as Error).message);
|
||||
}
|
||||
try { ImageUpdateService.getInstance().stop(); } catch (e) {
|
||||
console.warn('[Shutdown] ImageUpdateService cleanup failed:', (e as Error).message);
|
||||
}
|
||||
try { SchedulerService.getInstance().stop(); } catch (e) {
|
||||
console.warn('[Shutdown] SchedulerService cleanup failed:', (e as Error).message);
|
||||
}
|
||||
if (mfaReplayPurgeTimer) {
|
||||
clearInterval(mfaReplayPurgeTimer);
|
||||
mfaReplayPurgeTimer = null;
|
||||
}
|
||||
try { DatabaseService.getInstance().getDb().close(); } catch (e) {
|
||||
console.warn('[Shutdown] Database close failed:', (e as Error).message);
|
||||
}
|
||||
console.log('[Shutdown] Done - exiting');
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
// Force-exit after 10 s if connections refuse to drain
|
||||
setTimeout(() => {
|
||||
console.error('[Shutdown] Timed out waiting for connections - forcing exit');
|
||||
process.exit(1);
|
||||
}, 10_000).unref();
|
||||
};
|
||||
|
||||
process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
|
||||
process.on('SIGINT', () => gracefulShutdown('SIGINT'));
|
||||
|
||||
@@ -2,6 +2,9 @@ import crypto from 'crypto';
|
||||
import bcrypt from 'bcrypt';
|
||||
import { authenticator } from 'otplib';
|
||||
import { HashAlgorithms } from '@otplib/core';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import { MFA_REPLAY_TTL_MS, MFA_REPLAY_PURGE_INTERVAL_MS } from '../helpers/constants';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
|
||||
// Configure otplib for the default TOTP contract we present to users:
|
||||
// - 6 digits
|
||||
@@ -27,6 +30,41 @@ export interface BackupVerifyResult {
|
||||
}
|
||||
|
||||
export class MfaService {
|
||||
private static instance: MfaService;
|
||||
private purgeTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
public static getInstance(): MfaService {
|
||||
if (!MfaService.instance) MfaService.instance = new MfaService();
|
||||
return MfaService.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the periodic purge of used-MFA-code rows. The replay blacklist
|
||||
* holds (user, code, window) tuples for the last ~2 minutes; older rows
|
||||
* are safe to drop. Idempotent: calling start() twice is a no-op.
|
||||
*/
|
||||
public start(): void {
|
||||
if (this.purgeTimer) return;
|
||||
this.purgeTimer = setInterval(() => {
|
||||
try {
|
||||
const deleted = DatabaseService.getInstance().purgeOldMfaCodes(Date.now() - MFA_REPLAY_TTL_MS);
|
||||
if (isDebugEnabled() && deleted > 0) {
|
||||
console.log('[MFA:diag] replay purge deleted=', deleted);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('[MFA] Replay purge failed:', (err as Error).message);
|
||||
}
|
||||
}, MFA_REPLAY_PURGE_INTERVAL_MS);
|
||||
this.purgeTimer.unref();
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
if (this.purgeTimer) {
|
||||
clearInterval(this.purgeTimer);
|
||||
this.purgeTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a fresh base32 TOTP secret ready for `buildOtpauthUri` and
|
||||
* `verifyTotp`. Each user should receive a unique secret.
|
||||
|
||||
Reference in New Issue
Block a user