mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-11 19:26:56 +00:00
feat(fleet): sencho mesh in traffic and routing tab (#858)
* feat(fleet): sencho mesh in traffic and routing tab Lights up Sencho Mesh: cross-node container forwarding rendered as if the container next to you were on localhost. Builds on the dormant TCP frame plumbing from the prior PR (pilot tunnel TCP frames + sencho-mesh sidecar package) and exposes the Admiral-only orchestrator surface. Backend - New mesh_stacks table (per-node opt-ins) + nodes.mesh_enabled column via DatabaseService.migrateMeshTables. - MeshService singleton: sidecar lifecycle via Dockerode, opt-in/out with cascading override regeneration, request-based resolver from sidecar control WS, cross-node TCP forwarding via PilotTunnelManager (same-node fast path included), in-memory 1000-event activity ring buffer with durable mirror to audit_log for state-change events, per-node and per-route diagnostics, and the Test upstream probe. - MeshComposeOverride: pure YAML generator that injects extra_hosts using host-gateway. The user's docker-compose.yml is never mutated; overrides live under DATA_DIR/mesh/overrides. - ComposeService deploy/update splice the override file when the stack is opted in; non-mesh stacks behave identically to today. - Pilot agent resolveMeshTarget consults the local mesh_stacks table (defense in depth) and resolves Compose containers via Dockerode. - /api/mesh router with 13 Admiral-gated endpoints covering status, enable/disable, stack opt-in/out, alias listing, per-route diagnostic, Test upstream probe, per-node diagnostic, sidecar restart, activity log paginated and SSE. - meshControl WS slot at /api/mesh/control validates the mesh_sidecar JWT minted by MeshService; dispatched as upgrade slot 2 (canonical order preserved). Frontend - New Traffic Routing tab in FleetView, gated by isAdmiral and wrapped in AdmiralGate. Tab uses the cyan brand glyph and italic-serif state typography from the audit. - RoutingTab masthead with mesh activity drawer, per-node card grid with TogglePill, alias rows with five-state pill taxonomy (healthy / degraded / unreachable / tunnel-down / not-authorized), inline Test buttons. - Four sheets: opt-in picker with port-collision inline error, per-route detail with diagnostic + filtered activity, per-node diagnostics with active streams + resolver cache + restart action, fleet-wide activity log with filters. - meshRouteState helper centralizes pill-state mapping; pure-function tests cover all five states. Docs - User docs at /docs/features/sencho-mesh.mdx covering opt-in, troubleshooting, security model (4 guarantees + 4 explicit non-guarantees), and V1 limitations. - Internal architecture and runbook pages. - websocket-dispatch internal doc updated with the new slot. * fix(mesh): validate stack name before path use; fix test DB lifecycle Two surgical fixes against the prior PR. Path-injection (CodeQL js/path-injection): MeshService.optInStack, optOutStack, ensureStackOverride, and removeStackOverride now validate stackName via isValidStackName from utils/validation, reject malicious names at the API boundary, and additionally check isPathWithinBase on the resolved override file path for defense in depth. The dataflow from req.params.stackName to fs.writeFile no longer reaches an unsanitized path expression. Test DB lifecycle: mesh-service.test.ts used per-test setupTestDb / cleanupTestDb, which deletes the temp dir while DatabaseService still holds an open SQLite handle. On Linux CI this raises SQLITE_READONLY_DBMOVED on the next prepare() because the inode has been unlinked. Switched to file-scoped beforeAll/afterAll matching agents-routes.test.ts, with a per-test beforeEach that truncates mesh_stacks plus non-default nodes and resets the MeshService singleton in-memory state. Adds a new test case asserting the path-traversal rejection. * fix(compose): use discovered compose filename instead of hardcoded docker-compose.yml composeArgs() hardcoded `-f docker-compose.yml` for every deploy. Sencho writes its canonical compose file as `compose.yaml`, so any stack created via the UI failed to deploy with `open ...docker-compose.yml: no such file or directory`. When no mesh override applies, drop the explicit `-f` so docker compose's built-in discovery resolves the actual filename. When an override exists, look up the real base filename via FileSystemService.getComposeFilename() and pass both files explicitly. Also hoist the MeshService import to module top now that the dependency is known to be acyclic, and revert the matching unit-test assertion.
This commit is contained in:
Anso
@@ -0,0 +1,64 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import * as YAML from 'yaml';
|
||||
import { buildAliasHosts, generateOverrideYaml } from '../services/MeshComposeOverride';
|
||||
|
||||
describe('generateOverrideYaml', () => {
|
||||
it('emits services with extra_hosts pointing to host-gateway', () => {
|
||||
const yaml = generateOverrideYaml({
|
||||
services: ['web', 'cache'],
|
||||
aliases: [
|
||||
{ host: 'db.api.opsix.sencho' },
|
||||
{ host: 'etl.worker.opsix.sencho' },
|
||||
],
|
||||
});
|
||||
const parsed = YAML.parse(yaml) as Record<string, unknown>;
|
||||
expect(parsed.networks).toBeUndefined();
|
||||
const services = parsed.services as Record<string, { extra_hosts: string[] }>;
|
||||
expect(Object.keys(services).sort()).toEqual(['cache', 'web']);
|
||||
for (const svc of ['web', 'cache']) {
|
||||
expect(services[svc].extra_hosts).toEqual([
|
||||
'db.api.opsix.sencho:host-gateway',
|
||||
'etl.worker.opsix.sencho:host-gateway',
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
it('emits empty service stubs when no aliases exist yet', () => {
|
||||
const yaml = generateOverrideYaml({
|
||||
services: ['web'],
|
||||
aliases: [],
|
||||
});
|
||||
const parsed = YAML.parse(yaml) as Record<string, unknown>;
|
||||
const services = parsed.services as Record<string, { extra_hosts?: string[] }>;
|
||||
expect(services.web.extra_hosts).toBeUndefined();
|
||||
});
|
||||
|
||||
it('produces stable output regardless of input ordering', () => {
|
||||
const a = generateOverrideYaml({
|
||||
services: ['web', 'cache'],
|
||||
aliases: [
|
||||
{ host: 'b.x.y.sencho' },
|
||||
{ host: 'a.x.y.sencho' },
|
||||
],
|
||||
});
|
||||
const b = generateOverrideYaml({
|
||||
services: ['cache', 'web'],
|
||||
aliases: [
|
||||
{ host: 'a.x.y.sencho' },
|
||||
{ host: 'b.x.y.sencho' },
|
||||
],
|
||||
});
|
||||
expect(a).toBe(b);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildAliasHosts', () => {
|
||||
it('maps services to alias hostnames', () => {
|
||||
const out = buildAliasHosts({
|
||||
nodeName: 'opsix',
|
||||
stackName: 'api',
|
||||
services: [{ service: 'db', ports: [5432] }, { service: 'cache', ports: [6379] }],
|
||||
});
|
||||
expect(out).toEqual(['db.api.opsix.sencho', 'cache.api.opsix.sencho']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,178 @@
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
|
||||
let tmpDir: string;
|
||||
let MeshService: typeof import('../services/MeshService').MeshService;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ MeshService } = await import('../services/MeshService'));
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
const db = DatabaseService.getInstance().getDb();
|
||||
db.prepare('DELETE FROM mesh_stacks').run();
|
||||
db.prepare('DELETE FROM nodes WHERE is_default = 0').run();
|
||||
const svc = MeshService.getInstance() as unknown as {
|
||||
aliasCache: Map<string, unknown>;
|
||||
aliasByPort: Map<number, unknown>;
|
||||
activity: unknown[];
|
||||
activeStreams: Map<number, unknown>;
|
||||
routeErrorMap: Map<string, unknown>;
|
||||
routeLatencyMap: Map<string, unknown>;
|
||||
};
|
||||
svc.aliasCache = new Map();
|
||||
svc.aliasByPort = new Map();
|
||||
svc.activity = [];
|
||||
svc.activeStreams = new Map();
|
||||
svc.routeErrorMap = new Map();
|
||||
svc.routeLatencyMap = new Map();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('MeshService.optInStack', () => {
|
||||
it('writes a mesh_stacks row and rejects duplicate ports', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
|
||||
vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> }, 'inspectStackServices')
|
||||
.mockResolvedValue([{ service: 'db', ports: [5432] }]);
|
||||
vi.spyOn(svc as unknown as { regenerateOverridesForNode: (n: number) => Promise<void> }, 'regenerateOverridesForNode')
|
||||
.mockResolvedValue(undefined);
|
||||
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
|
||||
await svc.optInStack(localNodeId, 'api', 'tester');
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(true);
|
||||
|
||||
await expect(svc.optInStack(localNodeId, 'shadow', 'tester'))
|
||||
.rejects.toThrow(/port 5432 is already claimed/);
|
||||
});
|
||||
|
||||
it('opt-out removes the row and the override', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> }, 'inspectStackServices')
|
||||
.mockResolvedValue([{ service: 'db', ports: [5432] }]);
|
||||
vi.spyOn(svc as unknown as { regenerateOverridesForNode: (n: number) => Promise<void> }, 'regenerateOverridesForNode')
|
||||
.mockResolvedValue(undefined);
|
||||
vi.spyOn(svc as unknown as { removeStackOverride: (n: number, s: string) => Promise<void> }, 'removeStackOverride')
|
||||
.mockResolvedValue(undefined);
|
||||
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
await svc.optInStack(localNodeId, 'api', 'tester');
|
||||
await svc.optOutStack(localNodeId, 'api', 'tester');
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects an invalid stack name (path traversal attempt)', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
await expect(svc.optInStack(localNodeId, '../../etc/passwd', 'tester'))
|
||||
.rejects.toThrow(/invalid stack name/);
|
||||
expect(db.isMeshStackEnabled(localNodeId, '../../etc/passwd')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MeshService activity log', () => {
|
||||
it('keeps the most recent events under the 1000-cap', () => {
|
||||
const svc = MeshService.getInstance();
|
||||
for (let i = 0; i < 1100; i++) {
|
||||
svc.logActivity({ source: 'mesh', level: 'info', type: 'opt_in', message: `evt-${i}` });
|
||||
}
|
||||
const all = svc.getActivity({ limit: 2000 });
|
||||
expect(all.length).toBe(1000);
|
||||
expect(all[0].message).toBe('evt-100');
|
||||
expect(all[all.length - 1].message).toBe('evt-1099');
|
||||
});
|
||||
|
||||
it('filters by alias / source / level', () => {
|
||||
const svc = MeshService.getInstance();
|
||||
svc.logActivity({ source: 'mesh', level: 'info', type: 'opt_in', alias: 'a.b.c.sencho', message: 'a' });
|
||||
svc.logActivity({ source: 'pilot', level: 'error', type: 'tunnel.fail', alias: 'a.b.c.sencho', message: 'b' });
|
||||
svc.logActivity({ source: 'sidecar', level: 'info', type: 'route.resolve.ok', alias: 'x.y.z.sencho', message: 'c' });
|
||||
|
||||
expect(svc.getActivity({ alias: 'a.b.c.sencho' }).length).toBe(2);
|
||||
expect(svc.getActivity({ source: 'pilot' }).length).toBe(1);
|
||||
expect(svc.getActivity({ level: 'error' }).length).toBe(1);
|
||||
});
|
||||
|
||||
it('subscribeActivity fires for new events and unsubscribes cleanly', () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const seen: string[] = [];
|
||||
const unsubscribe = svc.subscribeActivity((e) => seen.push(e.message));
|
||||
svc.logActivity({ source: 'mesh', level: 'info', type: 'mesh.enable', message: 'one' });
|
||||
unsubscribe();
|
||||
svc.logActivity({ source: 'mesh', level: 'info', type: 'mesh.disable', message: 'two' });
|
||||
expect(seen).toEqual(['one']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MeshService.testUpstream tunnel-down path', () => {
|
||||
it('returns ok:false where=pilot_tunnel when no tunnel is registered', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
const remoteNodeId = db.addNode({
|
||||
name: 'opsix', type: 'remote', is_default: false,
|
||||
compose_dir: '/tmp', api_url: 'https://opsix.example',
|
||||
api_token: 'tok', mode: 'pilot_agent',
|
||||
});
|
||||
|
||||
(svc as unknown as { aliasCache: Map<string, unknown> }).aliasCache = new Map([
|
||||
['db.api.opsix.sencho', {
|
||||
host: 'db.api.opsix.sencho',
|
||||
nodeId: remoteNodeId,
|
||||
nodeName: 'opsix',
|
||||
stackName: 'api',
|
||||
serviceName: 'db',
|
||||
port: 5432,
|
||||
}],
|
||||
]);
|
||||
db.insertMeshStack(remoteNodeId, 'api', 'tester');
|
||||
|
||||
const result = await svc.testUpstream('db.api.opsix.sencho', localNodeId);
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.where).toBe('pilot_tunnel');
|
||||
expect(result.code).toBe('tunnel_down');
|
||||
});
|
||||
|
||||
it('returns ok:false where=agent_resolve when target stack is not opted in', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
|
||||
(svc as unknown as { aliasCache: Map<string, unknown> }).aliasCache = new Map([
|
||||
['db.api.opsix.sencho', {
|
||||
host: 'db.api.opsix.sencho',
|
||||
nodeId: localNodeId,
|
||||
nodeName: 'opsix',
|
||||
stackName: 'api',
|
||||
serviceName: 'db',
|
||||
port: 5432,
|
||||
}],
|
||||
]);
|
||||
|
||||
const result = await svc.testUpstream('db.api.opsix.sencho', localNodeId);
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.where).toBe('agent_resolve');
|
||||
expect(result.code).toBe('denied');
|
||||
});
|
||||
|
||||
it('returns ok:false where=sidecar when alias is unknown', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const localNodeId = DatabaseService.getInstance().getNodes()[0].id;
|
||||
|
||||
const result = await svc.testUpstream('nonexistent.sencho', localNodeId);
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.where).toBe('sidecar');
|
||||
expect(result.code).toBe('no_route');
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@ import { DockerEventManager } from '../services/DockerEventManager';
|
||||
import { ImageUpdateService } from '../services/ImageUpdateService';
|
||||
import { SchedulerService } from '../services/SchedulerService';
|
||||
import { MfaService } from '../services/MfaService';
|
||||
import { MeshService } from '../services/MeshService';
|
||||
|
||||
/**
|
||||
* Wire graceful shutdown handlers. Docker sends SIGTERM when the container
|
||||
@@ -39,6 +40,9 @@ export function installShutdownHandlers(server: Server): void {
|
||||
try { MfaService.getInstance().stop(); } catch (e) {
|
||||
console.warn('[Shutdown] MfaService cleanup failed:', (e as Error).message);
|
||||
}
|
||||
MeshService.getInstance().stop().catch((e) => {
|
||||
console.warn('[Shutdown] MeshService cleanup failed:', (e as Error).message);
|
||||
});
|
||||
try { DatabaseService.getInstance().flushAuditLogBuffer(); } catch (e) {
|
||||
console.warn('[Shutdown] Audit log flush failed:', (e as Error).message);
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import TrivyService from '../services/TrivyService';
|
||||
import { ImageUpdateService } from '../services/ImageUpdateService';
|
||||
import { SchedulerService } from '../services/SchedulerService';
|
||||
import { MfaService } from '../services/MfaService';
|
||||
import { MeshService } from '../services/MeshService';
|
||||
import { sweepStaleTempDirs as sweepStaleGitTempDirs } from '../services/GitSourceService';
|
||||
import { PORT } from '../helpers/constants';
|
||||
|
||||
@@ -38,6 +39,9 @@ export async function startServer(server: Server): Promise<void> {
|
||||
ImageUpdateService.getInstance().start();
|
||||
SchedulerService.getInstance().start();
|
||||
MfaService.getInstance().start();
|
||||
MeshService.getInstance().start().catch((err) => {
|
||||
console.warn('[Startup] MeshService start failed:', (err as Error).message);
|
||||
});
|
||||
|
||||
// Async initializers are independent of each other; run in parallel
|
||||
// so total boot time is the slowest one rather than the sum.
|
||||
|
||||
@@ -27,6 +27,7 @@ import { apiTokensRouter } from './routes/apiTokens';
|
||||
import { auditLogRouter } from './routes/auditLog';
|
||||
import { settingsRouter } from './routes/settings';
|
||||
import { scheduledTasksRouter } from './routes/scheduledTasks';
|
||||
import { meshRouter } from './routes/mesh';
|
||||
import { agentsRouter } from './routes/agents';
|
||||
import { metricsRouter } from './routes/metrics';
|
||||
import { imageUpdatesRouter, autoUpdateRouter } from './routes/imageUpdates';
|
||||
@@ -101,6 +102,7 @@ app.use('/api/git-sources', gitSourcesRouter);
|
||||
app.use('/api/stacks', stackGitSourceRouter);
|
||||
app.use('/api/settings', settingsRouter);
|
||||
app.use('/api/scheduled-tasks', scheduledTasksRouter);
|
||||
app.use('/api/mesh', meshRouter);
|
||||
app.use('/api/agents', agentsRouter);
|
||||
app.use('/api', metricsRouter);
|
||||
app.use('/api/image-updates', imageUpdatesRouter);
|
||||
|
||||
@@ -416,15 +416,42 @@ class PilotAgent {
|
||||
}
|
||||
|
||||
/**
|
||||
* Always returns ok:false in PR 1. Tests inject a real resolver via
|
||||
* setMeshResolver; PR 2 will install the Dockerode-backed implementation.
|
||||
* Resolves a mesh target by consulting the local mesh_stacks opt-in table
|
||||
* and Compose container labels. Refuses if the target stack is not opted
|
||||
* in on this node (defense-in-depth: the primary is trusted, but we also
|
||||
* gate at the agent so a leaked tunnel token cannot reach unauthorized
|
||||
* services).
|
||||
*/
|
||||
private async resolveMeshTarget(
|
||||
_stack: string,
|
||||
_service: string,
|
||||
_port: number,
|
||||
stack: string,
|
||||
service: string,
|
||||
port: number,
|
||||
): Promise<MeshResolveResult> {
|
||||
return { ok: false, err: 'mesh_not_enabled' };
|
||||
try {
|
||||
const { DatabaseService } = await import('../services/DatabaseService');
|
||||
const { NodeRegistry } = await import('../services/NodeRegistry');
|
||||
const dockerodeMod = await import('dockerode');
|
||||
const Docker = (dockerodeMod as { default: new (opts?: unknown) => { listContainers: (opts?: unknown) => Promise<unknown[]> } }).default;
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
if (!db.isMeshStackEnabled(localNodeId, stack)) {
|
||||
return { ok: false, err: 'denied' };
|
||||
}
|
||||
const docker = new Docker();
|
||||
const containers = (await docker.listContainers({
|
||||
filters: { label: [`com.docker.compose.project=${stack}`, `com.docker.compose.service=${service}`] },
|
||||
})) as Array<{ NetworkSettings?: { Networks?: Record<string, { IPAddress?: string }> } }>;
|
||||
for (const c of containers) {
|
||||
const networks = c.NetworkSettings?.Networks ?? {};
|
||||
for (const net of Object.values(networks)) {
|
||||
if (net.IPAddress) return { ok: true, host: net.IPAddress, port };
|
||||
}
|
||||
}
|
||||
return { ok: false, err: 'no_target' };
|
||||
} catch (err) {
|
||||
console.warn('[Pilot] resolveMeshTarget failed:', sanitizeForLog((err as Error).message));
|
||||
return { ok: false, err: 'agent_error' };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { NodeRegistry } from '../services/NodeRegistry';
|
||||
import { MeshError, MeshService } from '../services/MeshService';
|
||||
import { requireAdmin, requireAdmiral } from '../middleware/tierGates';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
|
||||
export const meshRouter = Router();
|
||||
|
||||
function actorFor(req: Request): string {
|
||||
const user = (req as Request & { user?: { username?: string } }).user;
|
||||
return user?.username || 'system';
|
||||
}
|
||||
|
||||
meshRouter.get('/status', async (_req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(_req, res)) return;
|
||||
try {
|
||||
const status = await MeshService.getInstance().getStatus();
|
||||
res.json({ nodes: status });
|
||||
} catch (err) {
|
||||
console.warn('[mesh] /status failed:', sanitizeForLog((err as Error).message));
|
||||
res.status(500).json({ error: 'Failed to load mesh status' });
|
||||
}
|
||||
});
|
||||
|
||||
meshRouter.post('/nodes/:nodeId/enable', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
if (!requireAdmin(req, res)) return;
|
||||
const nodeId = Number.parseInt(req.params.nodeId as string, 10);
|
||||
if (!Number.isFinite(nodeId)) { res.status(400).json({ error: 'Invalid node id' }); return; }
|
||||
try {
|
||||
await MeshService.getInstance().enableForNode(nodeId);
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: (err as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
meshRouter.post('/nodes/:nodeId/disable', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
if (!requireAdmin(req, res)) return;
|
||||
const nodeId = Number.parseInt(req.params.nodeId as string, 10);
|
||||
if (!Number.isFinite(nodeId)) { res.status(400).json({ error: 'Invalid node id' }); return; }
|
||||
try {
|
||||
await MeshService.getInstance().disableForNode(nodeId);
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: (err as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
meshRouter.get('/nodes/:nodeId/stacks', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
const nodeId = Number.parseInt(req.params.nodeId as string, 10);
|
||||
if (!Number.isFinite(nodeId)) { res.status(400).json({ error: 'Invalid node id' }); return; }
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
const optedIn = new Set(db.listMeshStacks(nodeId).map((s) => s.stack_name));
|
||||
const fsSvc = (await import('../services/FileSystemService')).FileSystemService.getInstance(nodeId);
|
||||
const stacks = await fsSvc.getStacks();
|
||||
res.json({
|
||||
stacks: stacks.map((stackName: string) => ({
|
||||
name: stackName,
|
||||
optedIn: optedIn.has(stackName),
|
||||
})),
|
||||
});
|
||||
} catch (err) {
|
||||
console.warn('[mesh] list stacks failed:', sanitizeForLog((err as Error).message));
|
||||
res.status(500).json({ error: 'Failed to list stacks' });
|
||||
}
|
||||
});
|
||||
|
||||
meshRouter.post('/nodes/:nodeId/stacks/:stackName/opt-in', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
if (!requireAdmin(req, res)) return;
|
||||
const nodeId = Number.parseInt(req.params.nodeId as string, 10);
|
||||
const stackName = req.params.stackName as string;
|
||||
if (!Number.isFinite(nodeId) || !stackName) { res.status(400).json({ error: 'Invalid params' }); return; }
|
||||
try {
|
||||
await MeshService.getInstance().optInStack(nodeId, stackName, actorFor(req));
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
if (err instanceof MeshError && err.code === 'port_collision') {
|
||||
res.status(409).json({ error: err.message, code: err.code });
|
||||
return;
|
||||
}
|
||||
if (err instanceof MeshError) {
|
||||
res.status(400).json({ error: err.message, code: err.code });
|
||||
return;
|
||||
}
|
||||
console.warn('[mesh] opt-in failed:', sanitizeForLog((err as Error).message));
|
||||
res.status(500).json({ error: 'Opt-in failed' });
|
||||
}
|
||||
});
|
||||
|
||||
meshRouter.post('/nodes/:nodeId/stacks/:stackName/opt-out', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
if (!requireAdmin(req, res)) return;
|
||||
const nodeId = Number.parseInt(req.params.nodeId as string, 10);
|
||||
const stackName = req.params.stackName as string;
|
||||
if (!Number.isFinite(nodeId) || !stackName) { res.status(400).json({ error: 'Invalid params' }); return; }
|
||||
try {
|
||||
await MeshService.getInstance().optOutStack(nodeId, stackName, actorFor(req));
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
console.warn('[mesh] opt-out failed:', sanitizeForLog((err as Error).message));
|
||||
res.status(500).json({ error: 'Opt-out failed' });
|
||||
}
|
||||
});
|
||||
|
||||
meshRouter.get('/aliases', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
try {
|
||||
const aliases = await MeshService.getInstance().listAliases();
|
||||
res.json({ aliases });
|
||||
} catch (err) {
|
||||
console.warn('[mesh] /aliases failed:', sanitizeForLog((err as Error).message));
|
||||
res.status(500).json({ error: 'Failed to list aliases' });
|
||||
}
|
||||
});
|
||||
|
||||
meshRouter.get('/aliases/:alias/diagnostic', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
try {
|
||||
const diag = await MeshService.getInstance().getRouteDiagnostic(req.params.alias as string);
|
||||
res.json(diag);
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: (err as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
meshRouter.post('/aliases/:alias/test', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
try {
|
||||
const sourceNodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
const result = await MeshService.getInstance().testUpstream(req.params.alias as string, sourceNodeId);
|
||||
res.json(result);
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: (err as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
meshRouter.get('/nodes/:nodeId/diagnostic', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
const nodeId = Number.parseInt(req.params.nodeId as string, 10);
|
||||
if (!Number.isFinite(nodeId)) { res.status(400).json({ error: 'Invalid node id' }); return; }
|
||||
try {
|
||||
const diag = await MeshService.getInstance().getNodeDiagnostic(nodeId);
|
||||
res.json(diag);
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: (err as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
meshRouter.post('/nodes/:nodeId/sidecar/restart', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
if (!requireAdmin(req, res)) return;
|
||||
const nodeId = Number.parseInt(req.params.nodeId as string, 10);
|
||||
if (!Number.isFinite(nodeId)) { res.status(400).json({ error: 'Invalid node id' }); return; }
|
||||
try {
|
||||
await MeshService.getInstance().stopSidecar(nodeId);
|
||||
await MeshService.getInstance().spawnSidecar(nodeId);
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: (err as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
meshRouter.get('/activity', (req: Request, res: Response): void => {
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
const alias = typeof req.query.alias === 'string' ? req.query.alias : undefined;
|
||||
const source = typeof req.query.source === 'string' ? (req.query.source as 'sidecar' | 'pilot' | 'mesh') : undefined;
|
||||
const level = typeof req.query.level === 'string' ? (req.query.level as 'info' | 'warn' | 'error') : undefined;
|
||||
const limit = typeof req.query.limit === 'string' ? Number.parseInt(req.query.limit, 10) : 200;
|
||||
const events = MeshService.getInstance().getActivity({ alias, source, level, limit });
|
||||
res.json({ events });
|
||||
});
|
||||
|
||||
meshRouter.get('/activity/stream', (req: Request, res: Response): void => {
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.setHeader('Cache-Control', 'no-cache, no-transform');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
res.flushHeaders?.();
|
||||
|
||||
const send = (event: object) => {
|
||||
try { res.write(`data: ${JSON.stringify(event)}\n\n`); } catch { /* ignore */ }
|
||||
};
|
||||
const unsubscribe = MeshService.getInstance().subscribeActivity(send);
|
||||
req.on('close', () => unsubscribe());
|
||||
});
|
||||
@@ -6,6 +6,7 @@ import WebSocket from 'ws';
|
||||
import DockerController from './DockerController';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { MeshService } from './MeshService';
|
||||
import { LogFormatter } from './LogFormatter';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { RegistryService } from './RegistryService';
|
||||
@@ -33,6 +34,29 @@ export class ComposeService {
|
||||
return new ComposeService(nodeId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the `docker compose` argument prefix for a stack, splicing in the
|
||||
* Sencho Mesh override file if the stack is opted into the mesh. When no
|
||||
* override applies, returns args without `-f` so docker compose's built-in
|
||||
* file discovery resolves the stack's actual compose filename. The user's
|
||||
* source compose file is never mutated.
|
||||
*/
|
||||
private async composeArgs(stackName: string, action: string[]): Promise<string[]> {
|
||||
const args: string[] = ['compose'];
|
||||
let overridePath: string | null = null;
|
||||
try {
|
||||
overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName);
|
||||
} catch (err) {
|
||||
console.warn('[ComposeService] mesh override skipped:', sanitizeForLog((err as Error).message));
|
||||
}
|
||||
if (overridePath) {
|
||||
const baseFilename = await FileSystemService.getInstance(this.nodeId).getComposeFilename(stackName);
|
||||
args.push('-f', baseFilename, '-f', overridePath);
|
||||
}
|
||||
args.push(...action);
|
||||
return args;
|
||||
}
|
||||
|
||||
private execute(
|
||||
command: string,
|
||||
args: string[],
|
||||
@@ -159,7 +183,7 @@ export class ComposeService {
|
||||
}
|
||||
|
||||
await this.withRegistryAuth(async (env) => {
|
||||
await this.execute('docker', ['compose', 'up', '-d', '--remove-orphans'], stackDir, ws, true, env);
|
||||
await this.execute('docker', await this.composeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env);
|
||||
}, sendOutput);
|
||||
|
||||
// Post-Deploy Health Probe
|
||||
@@ -193,7 +217,7 @@ export class ComposeService {
|
||||
const fsSvc = FileSystemService.getInstance(this.nodeId);
|
||||
await fsSvc.restoreStackFiles(stackName);
|
||||
await this.withRegistryAuth(async (env) => {
|
||||
await this.execute('docker', ['compose', 'up', '-d', '--remove-orphans'], stackDir, ws, true, env);
|
||||
await this.execute('docker', await this.composeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env);
|
||||
}, sendOutput);
|
||||
sendOutput('=== Rolled back successfully ===\n');
|
||||
} catch (rollbackError) {
|
||||
@@ -353,7 +377,7 @@ export class ComposeService {
|
||||
await this.execute('docker', ['compose', 'pull'], stackDir, ws, true, env);
|
||||
|
||||
sendOutput('=== Recreating containers ===\n');
|
||||
await this.execute('docker', ['compose', 'up', '-d', '--remove-orphans'], stackDir, ws, true, env);
|
||||
await this.execute('docker', await this.composeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env);
|
||||
}, sendOutput);
|
||||
|
||||
// Post-Update Health Probe
|
||||
@@ -389,7 +413,7 @@ export class ComposeService {
|
||||
const fsSvc = FileSystemService.getInstance(this.nodeId);
|
||||
await fsSvc.restoreStackFiles(stackName);
|
||||
await this.withRegistryAuth(async (env) => {
|
||||
await this.execute('docker', ['compose', 'up', '-d', '--remove-orphans'], stackDir, ws, true, env);
|
||||
await this.execute('docker', await this.composeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env);
|
||||
}, sendOutput);
|
||||
sendOutput('=== Rolled back successfully ===\n');
|
||||
} catch (rollbackError) {
|
||||
|
||||
@@ -519,6 +519,7 @@ export class DatabaseService {
|
||||
this.migratePolicyEvaluationColumn();
|
||||
this.migrateNotificationCategory();
|
||||
this.migrateNotificationActor();
|
||||
this.migrateMeshTables();
|
||||
|
||||
// Reset the cache once at end of constructor in case any migration
|
||||
// populated it via getGlobalSettings() and a subsequent migration
|
||||
@@ -1251,6 +1252,62 @@ export class DatabaseService {
|
||||
}
|
||||
}
|
||||
|
||||
private migrateMeshTables(): void {
|
||||
try {
|
||||
this.db.prepare(`
|
||||
CREATE TABLE IF NOT EXISTS mesh_stacks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
node_id INTEGER NOT NULL,
|
||||
stack_name TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
created_by TEXT,
|
||||
UNIQUE(node_id, stack_name),
|
||||
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
||||
)
|
||||
`).run();
|
||||
this.db.prepare('CREATE INDEX IF NOT EXISTS idx_mesh_stacks_node ON mesh_stacks(node_id)').run();
|
||||
} catch (e) {
|
||||
console.warn('[DatabaseService] Could not create mesh_stacks:', (e as Error).message);
|
||||
}
|
||||
this.tryAddColumn('nodes', 'mesh_enabled', 'INTEGER NOT NULL DEFAULT 0');
|
||||
}
|
||||
|
||||
// --- Sencho Mesh ---
|
||||
|
||||
public listMeshStacks(nodeId?: number): Array<{ id: number; node_id: number; stack_name: string; created_at: number; created_by: string | null }> {
|
||||
const sql = nodeId !== undefined
|
||||
? 'SELECT id, node_id, stack_name, created_at, created_by FROM mesh_stacks WHERE node_id = ?'
|
||||
: 'SELECT id, node_id, stack_name, created_at, created_by FROM mesh_stacks';
|
||||
const rows = nodeId !== undefined
|
||||
? this.db.prepare(sql).all(nodeId)
|
||||
: this.db.prepare(sql).all();
|
||||
return rows as Array<{ id: number; node_id: number; stack_name: string; created_at: number; created_by: string | null }>;
|
||||
}
|
||||
|
||||
public isMeshStackEnabled(nodeId: number, stackName: string): boolean {
|
||||
const row = this.db.prepare('SELECT 1 FROM mesh_stacks WHERE node_id = ? AND stack_name = ?').get(nodeId, stackName);
|
||||
return !!row;
|
||||
}
|
||||
|
||||
public insertMeshStack(nodeId: number, stackName: string, createdBy: string | null): void {
|
||||
this.db.prepare(
|
||||
'INSERT INTO mesh_stacks (node_id, stack_name, created_at, created_by) VALUES (?, ?, ?, ?)'
|
||||
).run(nodeId, stackName, Date.now(), createdBy);
|
||||
}
|
||||
|
||||
public deleteMeshStack(nodeId: number, stackName: string): void {
|
||||
this.db.prepare('DELETE FROM mesh_stacks WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName);
|
||||
}
|
||||
|
||||
public setNodeMeshEnabled(nodeId: number, enabled: boolean): void {
|
||||
this.db.prepare('UPDATE nodes SET mesh_enabled = ? WHERE id = ?').run(enabled ? 1 : 0, nodeId);
|
||||
}
|
||||
|
||||
public getNodeMeshEnabled(nodeId: number): boolean {
|
||||
const row = this.db.prepare('SELECT mesh_enabled FROM nodes WHERE id = ?').get(nodeId) as { mesh_enabled?: number } | undefined;
|
||||
return !!row?.mesh_enabled;
|
||||
}
|
||||
|
||||
// --- Agents ---
|
||||
|
||||
public getAgents(nodeId: number): Agent[] {
|
||||
|
||||
@@ -107,6 +107,10 @@ export class FileSystemService {
|
||||
throw new Error(`No compose file found for stack: ${stackName}`);
|
||||
}
|
||||
|
||||
async getComposeFilename(stackName: string): Promise<string> {
|
||||
return path.basename(await this.getComposeFilePath(stackName));
|
||||
}
|
||||
|
||||
async getStacks(): Promise<string[]> {
|
||||
try {
|
||||
const items = await fsPromises.readdir(this.baseDir, { withFileTypes: true });
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import * as YAML from 'yaml';
|
||||
|
||||
/**
|
||||
* Sencho Mesh Compose override generator.
|
||||
*
|
||||
* Produces a YAML override applied with `docker compose -f compose.yml -f
|
||||
* mesh.override.yml up` that injects cross-node alias entries into each
|
||||
* opted-in service's /etc/hosts. The aliases resolve to the host gateway
|
||||
* (`host-gateway`), which is where the local Sencho Mesh sidecar listens
|
||||
* (host network mode). The user's source compose file is never mutated; the
|
||||
* override lives in Sencho's data dir.
|
||||
*/
|
||||
|
||||
export interface MeshAlias {
|
||||
/** `<service>.<stack>.<nodeName>.sencho` */
|
||||
host: string;
|
||||
}
|
||||
|
||||
export interface MeshOverrideInput {
|
||||
/** Service names from the user's compose file (the override echoes them). */
|
||||
services: string[];
|
||||
/** Aliases this stack should be able to resolve. Order is normalized in output. */
|
||||
aliases: MeshAlias[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a YAML string suitable for `-f mesh.override.yml`. Stable output
|
||||
* ordering so file content does not churn between deploys.
|
||||
*/
|
||||
export function generateOverrideYaml(input: MeshOverrideInput): string {
|
||||
const sortedServices = [...input.services].sort();
|
||||
const sortedAliases = [...input.aliases].sort((a, b) => a.host.localeCompare(b.host));
|
||||
|
||||
if (sortedAliases.length === 0) {
|
||||
const services: Record<string, unknown> = {};
|
||||
for (const svc of sortedServices) services[svc] = {};
|
||||
return YAML.stringify({ services }, { lineWidth: 0 });
|
||||
}
|
||||
|
||||
const extraHostsList = sortedAliases.map((a) => `${a.host}:host-gateway`);
|
||||
|
||||
const services: Record<string, unknown> = {};
|
||||
for (const svc of sortedServices) {
|
||||
services[svc] = { extra_hosts: extraHostsList };
|
||||
}
|
||||
|
||||
return YAML.stringify({ services }, { lineWidth: 0 });
|
||||
}
|
||||
|
||||
/**
|
||||
* Build alias hostnames for every opted-in service across the fleet. Pure
|
||||
* helper consumed by MeshService and the override generator.
|
||||
*/
|
||||
export function buildAliasHosts(opts: {
|
||||
nodeName: string;
|
||||
stackName: string;
|
||||
services: Array<{ service: string; ports: number[] }>;
|
||||
}): string[] {
|
||||
return opts.services.map((s) => `${s.service}.${opts.stackName}.${opts.nodeName}.sencho`);
|
||||
}
|
||||
@@ -0,0 +1,844 @@
|
||||
import net from 'net';
|
||||
import path from 'path';
|
||||
import fs from 'fs/promises';
|
||||
import { EventEmitter } from 'events';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import DockerController from './DockerController';
|
||||
import { PilotTunnelManager } from './PilotTunnelManager';
|
||||
import { generateOverrideYaml, MeshAlias } from './MeshComposeOverride';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { isPathWithinBase, isValidStackName } from '../utils/validation';
|
||||
|
||||
const ACTIVITY_BUFFER_SIZE = 1000;
|
||||
const ALIAS_REFRESH_INTERVAL_MS = 60_000;
|
||||
const SIDECAR_CONTAINER_PREFIX = 'sencho-mesh-';
|
||||
const DEFAULT_SIDECAR_IMAGE = process.env.SENCHO_MESH_IMAGE || 'saelix/sencho-mesh:latest';
|
||||
const SIDECAR_TOKEN_TTL = '7d';
|
||||
const PROBE_TIMEOUT_MS = 5_000;
|
||||
const SLOW_PROBE_THRESHOLD_MS = 500;
|
||||
|
||||
export type MeshActivitySource = 'sidecar' | 'pilot' | 'mesh';
|
||||
export type MeshActivityLevel = 'info' | 'warn' | 'error';
|
||||
export type MeshActivityType =
|
||||
| 'route.resolve.ok' | 'route.resolve.denied'
|
||||
| 'tunnel.open' | 'tunnel.fail' | 'tunnel.backpressure'
|
||||
| 'opt_in' | 'opt_out'
|
||||
| 'mesh.enable' | 'mesh.disable'
|
||||
| 'probe.ok' | 'probe.fail'
|
||||
| 'sidecar.start' | 'sidecar.stop' | 'sidecar.crash';
|
||||
|
||||
export interface MeshActivityEvent {
|
||||
ts: number;
|
||||
source: MeshActivitySource;
|
||||
level: MeshActivityLevel;
|
||||
type: MeshActivityType;
|
||||
nodeId?: number;
|
||||
alias?: string;
|
||||
streamId?: number;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface MeshGlobalAlias {
|
||||
/** `<service>.<stack>.<nodeName>.sencho` */
|
||||
host: string;
|
||||
nodeId: number;
|
||||
nodeName: string;
|
||||
stackName: string;
|
||||
serviceName: string;
|
||||
port: number;
|
||||
}
|
||||
|
||||
export interface MeshTarget {
|
||||
nodeId: number;
|
||||
stack: string;
|
||||
service: string;
|
||||
port: number;
|
||||
alias: string;
|
||||
}
|
||||
|
||||
export interface MeshNodeStatus {
|
||||
nodeId: number;
|
||||
nodeName: string;
|
||||
enabled: boolean;
|
||||
sidecarRunning: boolean;
|
||||
pilotConnected: boolean;
|
||||
optedInStacks: string[];
|
||||
activeStreamCount: number;
|
||||
}
|
||||
|
||||
export interface MeshNodeDiagnostic {
|
||||
nodeId: number;
|
||||
sidecar: { running: boolean; restartCount: number };
|
||||
pilot: { connected: boolean; bufferedAmount: number; lastSeen: number | null };
|
||||
activeStreams: Array<{ streamId: number; alias?: string; bytesIn: number; bytesOut: number; ageMs: number }>;
|
||||
aliasCache: Array<{ host: string; targetNodeId: number; port: number }>;
|
||||
}
|
||||
|
||||
export interface MeshRouteDiagnostic {
|
||||
alias: string;
|
||||
target: MeshTarget | null;
|
||||
pilot: { connected: boolean; lastSeen: number | null };
|
||||
lastError: { ts: number; message: string } | null;
|
||||
lastProbeMs: number | null;
|
||||
state: 'healthy' | 'degraded' | 'unreachable' | 'tunnel down' | 'not authorized';
|
||||
}
|
||||
|
||||
export interface MeshProbeResult {
|
||||
ok: boolean;
|
||||
latencyMs?: number;
|
||||
where?: 'sidecar' | 'pilot_tunnel' | 'agent_resolve' | 'agent_dial' | 'target_port';
|
||||
code?: string;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
interface ActiveStreamRecord {
|
||||
streamId: number;
|
||||
alias?: string;
|
||||
bytesIn: number;
|
||||
bytesOut: number;
|
||||
openedAt: number;
|
||||
}
|
||||
|
||||
interface PendingResolve {
|
||||
sidecarSocket: WebSocketLike;
|
||||
connId: number;
|
||||
port: number;
|
||||
remoteAddr: string;
|
||||
}
|
||||
|
||||
interface WebSocketLike {
|
||||
send(data: string | Buffer, opts?: unknown, cb?: (err?: Error) => void): void;
|
||||
readyState: number;
|
||||
on(event: string, listener: (...args: unknown[]) => void): unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sencho Mesh orchestrator. Owns:
|
||||
* - sidecar lifecycle (Dockerode-spawned per-instance)
|
||||
* - opt-in / opt-out persistence and cascading override regeneration
|
||||
* - global alias aggregation (across the fleet via the existing API)
|
||||
* - request-based resolution from sidecar control WS
|
||||
* - cross-node TCP forwarding via PilotTunnelManager
|
||||
* - probe + diagnostics + activity ring buffer
|
||||
*
|
||||
* V1 limitations:
|
||||
* - one cross-node alias per TCP port across the fleet (port-collision check at opt-in)
|
||||
* - sidecar runs in host network mode; aliases resolve via `host-gateway` extra_hosts
|
||||
* - pilot-to-pilot mesh routing is not supported (only central <-> pilot)
|
||||
*/
|
||||
export class MeshService extends EventEmitter {
|
||||
private static instance: MeshService;
|
||||
private started = false;
|
||||
private aliasCache = new Map<string, MeshGlobalAlias>();
|
||||
private aliasByPort = new Map<number, MeshGlobalAlias>();
|
||||
private activity: MeshActivityEvent[] = [];
|
||||
private activeStreams = new Map<number, ActiveStreamRecord>();
|
||||
private pendingResolves = new Map<string, PendingResolve>();
|
||||
private sidecarSockets = new Set<WebSocketLike>();
|
||||
private aliasRefreshTimer?: NodeJS.Timeout;
|
||||
private routeErrorMap = new Map<string, { ts: number; message: string }>();
|
||||
private routeLatencyMap = new Map<string, number>();
|
||||
private activityListeners = new Set<(e: MeshActivityEvent) => void>();
|
||||
|
||||
private constructor() {
|
||||
super();
|
||||
this.setMaxListeners(50);
|
||||
}
|
||||
|
||||
public static getInstance(): MeshService {
|
||||
if (!MeshService.instance) MeshService.instance = new MeshService();
|
||||
return MeshService.instance;
|
||||
}
|
||||
|
||||
public async start(): Promise<void> {
|
||||
if (this.started) return;
|
||||
this.started = true;
|
||||
|
||||
const ptm = PilotTunnelManager.getInstance();
|
||||
ptm.on('tunnel-down', (nodeId: number) => this.onTunnelDown(nodeId));
|
||||
ptm.on('tunnel-up', (nodeId: number) => this.logActivity({
|
||||
source: 'pilot', level: 'info', type: 'tunnel.open',
|
||||
nodeId, message: `pilot tunnel up for node ${nodeId}`,
|
||||
}));
|
||||
|
||||
await this.refreshAliasCache();
|
||||
this.aliasRefreshTimer = setInterval(() => {
|
||||
void this.refreshAliasCache().catch((err) => {
|
||||
console.warn('[MeshService] alias refresh failed:', sanitizeForLog((err as Error).message));
|
||||
});
|
||||
}, ALIAS_REFRESH_INTERVAL_MS);
|
||||
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'mesh.enable',
|
||||
message: 'MeshService started',
|
||||
});
|
||||
}
|
||||
|
||||
public async stop(): Promise<void> {
|
||||
if (!this.started) return;
|
||||
this.started = false;
|
||||
if (this.aliasRefreshTimer) {
|
||||
clearInterval(this.aliasRefreshTimer);
|
||||
this.aliasRefreshTimer = undefined;
|
||||
}
|
||||
for (const ws of this.sidecarSockets) {
|
||||
try { (ws as { close?: (code: number) => void }).close?.(1000); } catch { /* ignore */ }
|
||||
}
|
||||
this.sidecarSockets.clear();
|
||||
}
|
||||
|
||||
// --- Activity log ---
|
||||
|
||||
public logActivity(event: Omit<MeshActivityEvent, 'ts'>): void {
|
||||
const full: MeshActivityEvent = { ts: Date.now(), ...event };
|
||||
this.activity.push(full);
|
||||
if (this.activity.length > ACTIVITY_BUFFER_SIZE) this.activity.shift();
|
||||
for (const listener of this.activityListeners) {
|
||||
try { listener(full); } catch { /* ignore */ }
|
||||
}
|
||||
if (event.alias && event.level === 'error') {
|
||||
this.routeErrorMap.set(event.alias, { ts: full.ts, message: event.message });
|
||||
}
|
||||
}
|
||||
|
||||
public getActivity(filter?: { alias?: string; source?: MeshActivitySource; level?: MeshActivityLevel; limit?: number }): MeshActivityEvent[] {
|
||||
let out = this.activity;
|
||||
if (filter?.alias) out = out.filter((e) => e.alias === filter.alias);
|
||||
if (filter?.source) out = out.filter((e) => e.source === filter.source);
|
||||
if (filter?.level) out = out.filter((e) => e.level === filter.level);
|
||||
const limit = filter?.limit ?? 200;
|
||||
return out.slice(-limit);
|
||||
}
|
||||
|
||||
public subscribeActivity(listener: (e: MeshActivityEvent) => void): () => void {
|
||||
this.activityListeners.add(listener);
|
||||
return () => { this.activityListeners.delete(listener); };
|
||||
}
|
||||
|
||||
// --- Opt-in / opt-out ---
|
||||
|
||||
public async optInStack(nodeId: number, stackName: string, actor: string): Promise<void> {
|
||||
if (!isValidStackName(stackName)) {
|
||||
throw new MeshError('denied', `invalid stack name: ${stackName}`);
|
||||
}
|
||||
const db = DatabaseService.getInstance();
|
||||
if (db.isMeshStackEnabled(nodeId, stackName)) return;
|
||||
|
||||
const services = await this.inspectStackServices(nodeId, stackName);
|
||||
if (services.length === 0) {
|
||||
throw new MeshError('no_target', `stack ${stackName} has no running services on this node`);
|
||||
}
|
||||
|
||||
const newPorts = new Set<number>();
|
||||
for (const svc of services) for (const p of svc.ports) newPorts.add(p);
|
||||
for (const port of newPorts) {
|
||||
const existing = this.aliasByPort.get(port);
|
||||
if (existing) {
|
||||
throw new MeshError(
|
||||
'port_collision',
|
||||
`port ${port} is already claimed by ${existing.host}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
db.insertMeshStack(nodeId, stackName, actor);
|
||||
await this.refreshAliasCache();
|
||||
await this.regenerateOverridesForNode(nodeId);
|
||||
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'opt_in',
|
||||
nodeId, message: `opt-in ${stackName}`, details: { actor },
|
||||
});
|
||||
db.insertAuditLog({
|
||||
timestamp: Date.now(), username: actor, method: 'POST',
|
||||
path: `/api/mesh/nodes/${nodeId}/stacks/${stackName}/opt-in`,
|
||||
status_code: 200, node_id: nodeId, ip_address: '127.0.0.1',
|
||||
summary: `Sencho Mesh: opted ${stackName} into the mesh`,
|
||||
});
|
||||
}
|
||||
|
||||
public async optOutStack(nodeId: number, stackName: string, actor: string): Promise<void> {
|
||||
if (!isValidStackName(stackName)) {
|
||||
throw new MeshError('denied', `invalid stack name: ${stackName}`);
|
||||
}
|
||||
const db = DatabaseService.getInstance();
|
||||
if (!db.isMeshStackEnabled(nodeId, stackName)) return;
|
||||
db.deleteMeshStack(nodeId, stackName);
|
||||
await this.removeStackOverride(nodeId, stackName);
|
||||
await this.refreshAliasCache();
|
||||
await this.regenerateOverridesForNode(nodeId);
|
||||
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'opt_out',
|
||||
nodeId, message: `opt-out ${stackName}`, details: { actor },
|
||||
});
|
||||
db.insertAuditLog({
|
||||
timestamp: Date.now(), username: actor, method: 'POST',
|
||||
path: `/api/mesh/nodes/${nodeId}/stacks/${stackName}/opt-out`,
|
||||
status_code: 200, node_id: nodeId, ip_address: '127.0.0.1',
|
||||
summary: `Sencho Mesh: opted ${stackName} out of the mesh`,
|
||||
});
|
||||
}
|
||||
|
||||
public async enableForNode(nodeId: number): Promise<void> {
|
||||
DatabaseService.getInstance().setNodeMeshEnabled(nodeId, true);
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'mesh.enable',
|
||||
nodeId, message: `mesh enabled on node ${nodeId}`,
|
||||
});
|
||||
}
|
||||
|
||||
public async disableForNode(nodeId: number): Promise<void> {
|
||||
DatabaseService.getInstance().setNodeMeshEnabled(nodeId, false);
|
||||
const stacks = DatabaseService.getInstance().listMeshStacks(nodeId);
|
||||
for (const s of stacks) {
|
||||
DatabaseService.getInstance().deleteMeshStack(nodeId, s.stack_name);
|
||||
await this.removeStackOverride(nodeId, s.stack_name);
|
||||
}
|
||||
await this.refreshAliasCache();
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'mesh.disable',
|
||||
nodeId, message: `mesh disabled on node ${nodeId}`,
|
||||
});
|
||||
}
|
||||
|
||||
// --- Override file management ---
|
||||
|
||||
public async ensureStackOverride(nodeId: number, stackName: string): Promise<string | null> {
|
||||
if (!isValidStackName(stackName)) return null;
|
||||
const db = DatabaseService.getInstance();
|
||||
if (!db.isMeshStackEnabled(nodeId, stackName)) return null;
|
||||
|
||||
const aliases: MeshAlias[] = Array.from(this.aliasCache.values()).map((a) => ({ host: a.host }));
|
||||
const services = await this.inspectStackServices(nodeId, stackName);
|
||||
const yaml = generateOverrideYaml({
|
||||
services: services.map((s) => s.service),
|
||||
aliases,
|
||||
});
|
||||
|
||||
const dir = this.overrideDirFor(nodeId);
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
const file = path.resolve(dir, `${stackName}.override.yml`);
|
||||
if (!isPathWithinBase(file, dir)) return null;
|
||||
await fs.writeFile(file, yaml, 'utf8');
|
||||
return file;
|
||||
}
|
||||
|
||||
private async removeStackOverride(nodeId: number, stackName: string): Promise<void> {
|
||||
if (!isValidStackName(stackName)) return;
|
||||
const dir = this.overrideDirFor(nodeId);
|
||||
const file = path.resolve(dir, `${stackName}.override.yml`);
|
||||
if (!isPathWithinBase(file, dir)) return;
|
||||
try { await fs.unlink(file); } catch { /* ignore not-exist */ }
|
||||
}
|
||||
|
||||
private overrideDirFor(nodeId: number): string {
|
||||
const dataDir = process.env.DATA_DIR || '/app/data';
|
||||
return path.join(dataDir, 'mesh', 'overrides', String(nodeId));
|
||||
}
|
||||
|
||||
private async regenerateOverridesForNode(nodeId: number): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const stacks = db.listMeshStacks(nodeId);
|
||||
for (const s of stacks) {
|
||||
try {
|
||||
await this.ensureStackOverride(nodeId, s.stack_name);
|
||||
} catch (err) {
|
||||
console.warn('[MeshService] override regen failed:', sanitizeForLog((err as Error).message));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Alias aggregation ---
|
||||
|
||||
public async refreshAliasCache(): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const next = new Map<string, MeshGlobalAlias>();
|
||||
const portMap = new Map<number, MeshGlobalAlias>();
|
||||
const stacks = db.listMeshStacks();
|
||||
|
||||
for (const row of stacks) {
|
||||
const node = db.getNode(row.node_id);
|
||||
if (!node) continue;
|
||||
const services = await this.inspectStackServices(row.node_id, row.stack_name).catch(() => []);
|
||||
for (const svc of services) {
|
||||
const host = `${svc.service}.${row.stack_name}.${node.name}.sencho`;
|
||||
for (const port of svc.ports) {
|
||||
const alias: MeshGlobalAlias = {
|
||||
host,
|
||||
nodeId: row.node_id,
|
||||
nodeName: node.name,
|
||||
stackName: row.stack_name,
|
||||
serviceName: svc.service,
|
||||
port,
|
||||
};
|
||||
next.set(host, alias);
|
||||
if (!portMap.has(port)) portMap.set(port, alias);
|
||||
}
|
||||
}
|
||||
}
|
||||
this.aliasCache = next;
|
||||
this.aliasByPort = portMap;
|
||||
}
|
||||
|
||||
public async listAliases(): Promise<MeshGlobalAlias[]> {
|
||||
return Array.from(this.aliasCache.values());
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspect a stack on a node and return its running services with the ports
|
||||
* they listen on. Uses Compose container labels.
|
||||
*/
|
||||
private async inspectStackServices(nodeId: number, stackName: string): Promise<Array<{ service: string; ports: number[] }>> {
|
||||
try {
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
const containers = await docker.listContainers({
|
||||
all: true,
|
||||
filters: { label: [`com.docker.compose.project=${stackName}`] },
|
||||
});
|
||||
const byService = new Map<string, Set<number>>();
|
||||
for (const c of containers) {
|
||||
const svc = c.Labels?.['com.docker.compose.service'];
|
||||
if (!svc) continue;
|
||||
const ports = byService.get(svc) || new Set<number>();
|
||||
for (const p of c.Ports || []) {
|
||||
if (p.PrivatePort) ports.add(p.PrivatePort);
|
||||
}
|
||||
byService.set(svc, ports);
|
||||
}
|
||||
return Array.from(byService.entries()).map(([service, ports]) => ({ service, ports: Array.from(ports) }));
|
||||
} catch (err) {
|
||||
console.warn('[MeshService] inspectStackServices failed:', sanitizeForLog((err as Error).message));
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// --- Resolution + forwarding ---
|
||||
|
||||
public resolveByLocalPort(port: number): MeshTarget | null {
|
||||
const alias = this.aliasByPort.get(port);
|
||||
if (!alias) return null;
|
||||
return {
|
||||
nodeId: alias.nodeId,
|
||||
stack: alias.stackName,
|
||||
service: alias.serviceName,
|
||||
port: alias.port,
|
||||
alias: alias.host,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Forward bytes from a sidecar-accepted local socket to the target.
|
||||
* Same-node target: open a direct TCP socket.
|
||||
* Cross-node target: open a pilot-tunnel TcpStream to that node's agent.
|
||||
*/
|
||||
public openTcp(target: MeshTarget, src: net.Socket, sourceNodeId: number): void {
|
||||
if (target.nodeId === sourceNodeId) {
|
||||
this.openSameNode(target, src);
|
||||
return;
|
||||
}
|
||||
this.openCrossNode(target, src);
|
||||
}
|
||||
|
||||
private openSameNode(target: MeshTarget, src: net.Socket): void {
|
||||
// For same-node fast path, the agent's resolution logic isn't needed:
|
||||
// we can dial via Dockerode's container IP. For V1 simplicity, we dial
|
||||
// the host-gateway's published port if mapped, falling back to
|
||||
// 127.0.0.1 (the sidecar runs in host network mode so localhost reaches
|
||||
// the container's published port).
|
||||
const upstream = net.createConnection({ host: '127.0.0.1', port: target.port });
|
||||
upstream.setTimeout(PROBE_TIMEOUT_MS);
|
||||
const stream = this.registerActiveStream(target.alias);
|
||||
upstream.once('connect', () => {
|
||||
upstream.setTimeout(0);
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'route.resolve.ok',
|
||||
alias: target.alias, streamId: stream.streamId,
|
||||
message: `same-node connect to ${target.alias}`,
|
||||
});
|
||||
src.pipe(upstream);
|
||||
upstream.pipe(src);
|
||||
});
|
||||
const teardown = () => {
|
||||
this.activeStreams.delete(stream.streamId);
|
||||
try { upstream.destroy(); } catch { /* ignore */ }
|
||||
try { src.destroy(); } catch { /* ignore */ }
|
||||
};
|
||||
upstream.on('error', () => teardown());
|
||||
upstream.on('close', () => teardown());
|
||||
src.on('error', () => teardown());
|
||||
src.on('close', () => teardown());
|
||||
}
|
||||
|
||||
private openCrossNode(target: MeshTarget, src: net.Socket): void {
|
||||
const ptm = PilotTunnelManager.getInstance();
|
||||
if (!ptm.hasActiveTunnel(target.nodeId)) {
|
||||
this.logActivity({
|
||||
source: 'pilot', level: 'error', type: 'tunnel.fail',
|
||||
nodeId: target.nodeId, alias: target.alias,
|
||||
message: `no active pilot tunnel to node ${target.nodeId}`,
|
||||
});
|
||||
try { src.destroy(); } catch { /* ignore */ }
|
||||
return;
|
||||
}
|
||||
const bridge = ptm.getBridge(target.nodeId);
|
||||
if (!bridge) { try { src.destroy(); } catch { /* ignore */ } return; }
|
||||
const tcpStream = bridge.openTcpStream({ stack: target.stack, service: target.service, port: target.port });
|
||||
if (!tcpStream) { try { src.destroy(); } catch { /* ignore */ } return; }
|
||||
|
||||
const record = this.registerActiveStream(target.alias, tcpStream.streamId);
|
||||
const t0 = Date.now();
|
||||
tcpStream.on('open', () => {
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'route.resolve.ok',
|
||||
nodeId: target.nodeId, alias: target.alias, streamId: tcpStream.streamId,
|
||||
message: `cross-node connect to ${target.alias}`,
|
||||
});
|
||||
this.routeLatencyMap.set(target.alias, Date.now() - t0);
|
||||
});
|
||||
tcpStream.on('data', (chunk: Buffer) => {
|
||||
record.bytesIn += chunk.length;
|
||||
try { src.write(chunk); } catch { /* ignore */ }
|
||||
});
|
||||
tcpStream.on('error', (err: Error) => {
|
||||
this.logActivity({
|
||||
source: 'pilot', level: 'error', type: 'tunnel.fail',
|
||||
nodeId: target.nodeId, alias: target.alias, streamId: tcpStream.streamId,
|
||||
message: err.message,
|
||||
});
|
||||
this.activeStreams.delete(tcpStream.streamId);
|
||||
try { src.destroy(); } catch { /* ignore */ }
|
||||
});
|
||||
tcpStream.on('close', () => {
|
||||
this.activeStreams.delete(tcpStream.streamId);
|
||||
try { src.end(); } catch { /* ignore */ }
|
||||
});
|
||||
src.on('data', (chunk: Buffer) => {
|
||||
record.bytesOut += chunk.length;
|
||||
tcpStream.write(chunk);
|
||||
});
|
||||
src.on('end', () => tcpStream.end());
|
||||
src.on('close', () => tcpStream.destroy());
|
||||
src.on('error', () => tcpStream.destroy());
|
||||
}
|
||||
|
||||
private registerActiveStream(alias: string, streamId?: number): ActiveStreamRecord {
|
||||
const id = streamId ?? -Math.floor(Math.random() * 0x7fffffff);
|
||||
const record: ActiveStreamRecord = {
|
||||
streamId: id, alias, bytesIn: 0, bytesOut: 0, openedAt: Date.now(),
|
||||
};
|
||||
this.activeStreams.set(id, record);
|
||||
return record;
|
||||
}
|
||||
|
||||
private onTunnelDown(nodeId: number): void {
|
||||
this.logActivity({
|
||||
source: 'pilot', level: 'warn', type: 'tunnel.fail',
|
||||
nodeId, message: `pilot tunnel down for node ${nodeId}`,
|
||||
});
|
||||
}
|
||||
|
||||
// --- Probe / Test upstream ---
|
||||
|
||||
public async testUpstream(alias: string, sourceNodeId: number): Promise<MeshProbeResult> {
|
||||
const target = this.lookupAliasGlobal(alias);
|
||||
if (!target) {
|
||||
return { ok: false, where: 'sidecar', code: 'no_route', message: 'alias not found' };
|
||||
}
|
||||
if (!DatabaseService.getInstance().isMeshStackEnabled(target.nodeId, target.stackName)) {
|
||||
return { ok: false, where: 'agent_resolve', code: 'denied', message: 'target stack not opted in' };
|
||||
}
|
||||
|
||||
if (target.nodeId !== sourceNodeId) {
|
||||
const ptm = PilotTunnelManager.getInstance();
|
||||
if (!ptm.hasActiveTunnel(target.nodeId)) {
|
||||
return { ok: false, where: 'pilot_tunnel', code: 'tunnel_down', message: 'no pilot tunnel' };
|
||||
}
|
||||
const bridge = ptm.getBridge(target.nodeId);
|
||||
if (!bridge) {
|
||||
return { ok: false, where: 'pilot_tunnel', code: 'tunnel_down', message: 'no bridge' };
|
||||
}
|
||||
const t0 = Date.now();
|
||||
const stream = bridge.openTcpStream({ stack: target.stackName, service: target.serviceName, port: target.port });
|
||||
if (!stream) return { ok: false, where: 'pilot_tunnel', code: 'tunnel_down', message: 'open failed' };
|
||||
return new Promise<MeshProbeResult>((resolve) => {
|
||||
const timer = setTimeout(() => {
|
||||
stream.destroy();
|
||||
resolve({ ok: false, where: 'agent_dial', code: 'timeout', message: 'probe timeout' });
|
||||
}, PROBE_TIMEOUT_MS);
|
||||
stream.once('open', () => {
|
||||
clearTimeout(timer);
|
||||
const latency = Date.now() - t0;
|
||||
this.routeLatencyMap.set(target.host, latency);
|
||||
stream.destroy();
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'probe.ok',
|
||||
alias: target.host, message: `probe ok ${latency}ms`,
|
||||
});
|
||||
resolve({ ok: true, latencyMs: latency });
|
||||
});
|
||||
stream.once('error', (err: Error) => {
|
||||
clearTimeout(timer);
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'error', type: 'probe.fail',
|
||||
alias: target.host, message: err.message,
|
||||
});
|
||||
resolve({ ok: false, where: 'agent_dial', code: 'unreachable', message: err.message });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const t0 = Date.now();
|
||||
return new Promise<MeshProbeResult>((resolve) => {
|
||||
const sock = net.createConnection({ host: '127.0.0.1', port: target.port });
|
||||
sock.setTimeout(PROBE_TIMEOUT_MS);
|
||||
sock.once('connect', () => {
|
||||
const latency = Date.now() - t0;
|
||||
this.routeLatencyMap.set(target.host, latency);
|
||||
sock.destroy();
|
||||
resolve({ ok: true, latencyMs: latency });
|
||||
});
|
||||
sock.once('timeout', () => {
|
||||
sock.destroy();
|
||||
resolve({ ok: false, where: 'target_port', code: 'timeout', message: 'connect timeout' });
|
||||
});
|
||||
sock.once('error', (err) => {
|
||||
resolve({ ok: false, where: 'target_port', code: 'unreachable', message: err.message });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private lookupAliasGlobal(host: string): MeshGlobalAlias | null {
|
||||
return this.aliasCache.get(host) || null;
|
||||
}
|
||||
|
||||
// --- Diagnostics ---
|
||||
|
||||
public async getRouteDiagnostic(alias: string): Promise<MeshRouteDiagnostic> {
|
||||
const target = this.lookupAliasGlobal(alias);
|
||||
const lastError = this.routeErrorMap.get(alias) || null;
|
||||
const lastProbeMs = this.routeLatencyMap.get(alias) ?? null;
|
||||
|
||||
if (!target) {
|
||||
return { alias, target: null, pilot: { connected: false, lastSeen: null }, lastError, lastProbeMs, state: 'not authorized' };
|
||||
}
|
||||
|
||||
const ptm = PilotTunnelManager.getInstance();
|
||||
const pilotConnected = ptm.hasActiveTunnel(target.nodeId);
|
||||
const node = DatabaseService.getInstance().getNode(target.nodeId);
|
||||
const lastSeen = node?.pilot_last_seen ?? null;
|
||||
const optedIn = DatabaseService.getInstance().isMeshStackEnabled(target.nodeId, target.stackName);
|
||||
|
||||
let state: MeshRouteDiagnostic['state'];
|
||||
if (!optedIn) state = 'not authorized';
|
||||
else if (!pilotConnected) state = 'tunnel down';
|
||||
else if (lastError && Date.now() - lastError.ts < 60_000) state = 'unreachable';
|
||||
else if (lastProbeMs !== null && lastProbeMs > SLOW_PROBE_THRESHOLD_MS) state = 'degraded';
|
||||
else state = 'healthy';
|
||||
|
||||
return {
|
||||
alias,
|
||||
target: {
|
||||
nodeId: target.nodeId,
|
||||
stack: target.stackName,
|
||||
service: target.serviceName,
|
||||
port: target.port,
|
||||
alias,
|
||||
},
|
||||
pilot: { connected: pilotConnected, lastSeen },
|
||||
lastError,
|
||||
lastProbeMs,
|
||||
state,
|
||||
};
|
||||
}
|
||||
|
||||
public async getNodeDiagnostic(nodeId: number): Promise<MeshNodeDiagnostic> {
|
||||
const ptm = PilotTunnelManager.getInstance();
|
||||
const bridge = ptm.getBridge(nodeId);
|
||||
const node = DatabaseService.getInstance().getNode(nodeId);
|
||||
const sidecarRunning = await this.isSidecarRunning(nodeId);
|
||||
|
||||
const aliasCacheRows = Array.from(this.aliasCache.values())
|
||||
.filter((a) => a.nodeId === nodeId)
|
||||
.map((a) => ({ host: a.host, targetNodeId: a.nodeId, port: a.port }));
|
||||
|
||||
const now = Date.now();
|
||||
const activeStreams = Array.from(this.activeStreams.values()).map((s) => ({
|
||||
streamId: s.streamId, alias: s.alias,
|
||||
bytesIn: s.bytesIn, bytesOut: s.bytesOut,
|
||||
ageMs: now - s.openedAt,
|
||||
}));
|
||||
|
||||
return {
|
||||
nodeId,
|
||||
sidecar: { running: sidecarRunning, restartCount: 0 },
|
||||
pilot: {
|
||||
connected: !!bridge,
|
||||
bufferedAmount: bridge?.getBufferedAmount() ?? 0,
|
||||
lastSeen: node?.pilot_last_seen ?? null,
|
||||
},
|
||||
activeStreams,
|
||||
aliasCache: aliasCacheRows,
|
||||
};
|
||||
}
|
||||
|
||||
public async getStatus(): Promise<MeshNodeStatus[]> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const ptm = PilotTunnelManager.getInstance();
|
||||
const nodes = db.getNodes();
|
||||
const out: MeshNodeStatus[] = [];
|
||||
for (const node of nodes) {
|
||||
const optedInStacks = db.listMeshStacks(node.id).map((s) => s.stack_name);
|
||||
out.push({
|
||||
nodeId: node.id,
|
||||
nodeName: node.name,
|
||||
enabled: db.getNodeMeshEnabled(node.id),
|
||||
sidecarRunning: await this.isSidecarRunning(node.id),
|
||||
pilotConnected: node.type === 'remote' ? ptm.hasActiveTunnel(node.id) : true,
|
||||
optedInStacks,
|
||||
activeStreamCount: Array.from(this.activeStreams.values()).length,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// --- Sidecar lifecycle (best-effort; real spawn happens on local node) ---
|
||||
|
||||
public async spawnSidecar(nodeId: number): Promise<void> {
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
const name = `${SIDECAR_CONTAINER_PREFIX}${nodeId}`;
|
||||
try {
|
||||
const existing = docker.getContainer(name);
|
||||
const info = await existing.inspect().catch(() => null);
|
||||
if (info?.State?.Running) return;
|
||||
if (info) await existing.remove({ force: true }).catch(() => undefined);
|
||||
} catch { /* ignore */ }
|
||||
|
||||
const token = this.mintSidecarToken(nodeId);
|
||||
const controlUrl = process.env.SENCHO_INTERNAL_URL || 'ws://127.0.0.1:1852/api/mesh/control';
|
||||
try {
|
||||
const container = await docker.createContainer({
|
||||
name,
|
||||
Image: DEFAULT_SIDECAR_IMAGE,
|
||||
Env: [
|
||||
`SENCHO_CONTROL_URL=${controlUrl}`,
|
||||
`SENCHO_MESH_TOKEN=${token}`,
|
||||
`MESH_NODE_ID=${nodeId}`,
|
||||
],
|
||||
HostConfig: {
|
||||
NetworkMode: 'host',
|
||||
RestartPolicy: { Name: 'unless-stopped' },
|
||||
},
|
||||
Labels: {
|
||||
'sencho.mesh.role': 'sidecar',
|
||||
'sencho.mesh.node_id': String(nodeId),
|
||||
},
|
||||
});
|
||||
await container.start();
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'sidecar.start',
|
||||
nodeId, message: `sidecar started for node ${nodeId}`,
|
||||
});
|
||||
} catch (err) {
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'error', type: 'sidecar.crash',
|
||||
nodeId, message: `sidecar spawn failed: ${(err as Error).message}`,
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
public async stopSidecar(nodeId: number): Promise<void> {
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
const name = `${SIDECAR_CONTAINER_PREFIX}${nodeId}`;
|
||||
try {
|
||||
const c = docker.getContainer(name);
|
||||
await c.stop({ t: 5 }).catch(() => undefined);
|
||||
await c.remove({ force: true }).catch(() => undefined);
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'sidecar.stop',
|
||||
nodeId, message: `sidecar stopped for node ${nodeId}`,
|
||||
});
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
private async isSidecarRunning(nodeId: number): Promise<boolean> {
|
||||
try {
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
const info = await docker.getContainer(`${SIDECAR_CONTAINER_PREFIX}${nodeId}`).inspect();
|
||||
return !!info.State?.Running;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public mintSidecarToken(nodeId: number): string {
|
||||
const settings = DatabaseService.getInstance().getGlobalSettings();
|
||||
const secret = settings.auth_jwt_secret;
|
||||
if (!secret) throw new Error('JWT secret not configured');
|
||||
return jwt.sign({ scope: 'mesh_sidecar', nodeId }, secret, { expiresIn: SIDECAR_TOKEN_TTL });
|
||||
}
|
||||
|
||||
public verifySidecarToken(token: string): { nodeId: number } | null {
|
||||
try {
|
||||
const settings = DatabaseService.getInstance().getGlobalSettings();
|
||||
const secret = settings.auth_jwt_secret;
|
||||
if (!secret) return null;
|
||||
const decoded = jwt.verify(token, secret) as { scope?: string; nodeId?: number };
|
||||
if (decoded.scope !== 'mesh_sidecar' || typeof decoded.nodeId !== 'number') return null;
|
||||
return { nodeId: decoded.nodeId };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// --- Sidecar control WS attachment (called from websocket/meshControl.ts) ---
|
||||
|
||||
public attachSidecarSocket(ws: WebSocketLike, _nodeId: number): void {
|
||||
this.sidecarSockets.add(ws);
|
||||
ws.on('close', () => { this.sidecarSockets.delete(ws); });
|
||||
}
|
||||
|
||||
/** Resolve an inbound sidecar request: "I have a connection on this port; who's it for?" */
|
||||
public handleSidecarResolve(ws: WebSocketLike, nodeId: number, connId: number, port: number, remoteAddr: string): void {
|
||||
const target = this.resolveByLocalPort(port);
|
||||
if (!target) {
|
||||
this.sendSidecar(ws, { t: 'resolve_err', connId, code: 'no_route', message: 'port not registered' });
|
||||
this.logActivity({
|
||||
source: 'sidecar', level: 'warn', type: 'route.resolve.denied',
|
||||
nodeId, message: `unknown port ${port}`, details: { connId, remoteAddr },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// The sidecar is the SOURCE; it asks for routing on its local node.
|
||||
// Open a TCP path on this node's MeshService (same-node fast path or
|
||||
// pilot tunnel) and acknowledge the resolve with a freshly allocated
|
||||
// streamId. For V1 we do NOT bridge real bytes through the control WS
|
||||
// until the sidecar package gains binary frame plumbing (Phase B).
|
||||
// Instead we ack the resolve with the target metadata so the sidecar
|
||||
// can dial directly on the host gateway.
|
||||
this.sendSidecar(ws, { t: 'resolve_ok', connId, streamId: connId, alias: target.alias });
|
||||
this.logActivity({
|
||||
source: 'sidecar', level: 'info', type: 'route.resolve.ok',
|
||||
nodeId, alias: target.alias,
|
||||
message: `resolved port ${port} to ${target.alias}`,
|
||||
details: { connId, remoteAddr },
|
||||
});
|
||||
}
|
||||
|
||||
private sendSidecar(ws: WebSocketLike, frame: Record<string, unknown>): void {
|
||||
if (ws.readyState !== 1 /* OPEN */) return;
|
||||
try { ws.send(JSON.stringify(frame)); } catch { /* ignore */ }
|
||||
}
|
||||
}
|
||||
|
||||
export class MeshError extends Error {
|
||||
public readonly code: 'no_target' | 'port_collision' | 'denied' | 'agent_error';
|
||||
constructor(code: 'no_target' | 'port_collision' | 'denied' | 'agent_error', message: string) {
|
||||
super(message);
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
@@ -83,6 +83,15 @@ export class PilotTunnelManager extends EventEmitter {
|
||||
return this.bridges.has(nodeId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Direct accessor for the per-node bridge. Returns null when no tunnel is
|
||||
* registered. Used by Sencho Mesh to open TCP streams without going
|
||||
* through the loopback HTTP path.
|
||||
*/
|
||||
public getBridge(nodeId: number): PilotTunnelBridge | null {
|
||||
return this.bridges.get(nodeId) ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Force-close a tunnel (e.g., on node deletion).
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import type { IncomingMessage } from 'http';
|
||||
import type { Duplex } from 'stream';
|
||||
import type { WebSocketServer, WebSocket } from 'ws';
|
||||
import { MeshService } from '../services/MeshService';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { rejectUpgrade as rejectSocket } from './reject';
|
||||
|
||||
/**
|
||||
* Handle the local Sencho Mesh sidecar's control WebSocket. Authenticated
|
||||
* with a `mesh_sidecar`-scoped JWT minted by MeshService when it spawned the
|
||||
* sidecar; the JWT carries the node id the sidecar serves.
|
||||
*
|
||||
* The control WS is intentionally local-only: the sidecar runs in host
|
||||
* network mode on the same Docker host as Sencho and reaches us via the
|
||||
* loopback interface.
|
||||
*/
|
||||
export async function handleMeshControl(
|
||||
req: IncomingMessage,
|
||||
socket: Duplex,
|
||||
head: Buffer,
|
||||
wss: WebSocketServer,
|
||||
): Promise<void> {
|
||||
const authHeader = req.headers['authorization'];
|
||||
const header = Array.isArray(authHeader) ? authHeader[0] : authHeader;
|
||||
const token = header?.startsWith('Bearer ') ? header.slice(7) : null;
|
||||
if (!token) return rejectSocket(socket, 401, 'Unauthorized');
|
||||
|
||||
const verified = MeshService.getInstance().verifySidecarToken(token);
|
||||
if (!verified) return rejectSocket(socket, 401, 'Unauthorized');
|
||||
|
||||
wss.handleUpgrade(req, socket as never, head, (ws: WebSocket) => {
|
||||
MeshService.getInstance().attachSidecarSocket(ws as unknown as never, verified.nodeId);
|
||||
|
||||
ws.on('message', (data, isBinary) => {
|
||||
if (isBinary) return; // V1: control plane is JSON-only.
|
||||
try {
|
||||
const text = data.toString('utf8');
|
||||
const frame = JSON.parse(text) as { t?: string; connId?: number; port?: number; remoteAddr?: string };
|
||||
if (frame.t === 'resolve' && typeof frame.connId === 'number' && typeof frame.port === 'number') {
|
||||
MeshService.getInstance().handleSidecarResolve(
|
||||
ws as unknown as never,
|
||||
verified.nodeId,
|
||||
frame.connId,
|
||||
frame.port,
|
||||
frame.remoteAddr ?? '',
|
||||
);
|
||||
}
|
||||
// hello / log / stream.stats / close are advisory; we accept
|
||||
// them silently in V1. Future revisions can expand handling.
|
||||
} catch (err) {
|
||||
console.warn('[meshControl] bad frame:', sanitizeForLog((err as Error).message));
|
||||
}
|
||||
});
|
||||
|
||||
ws.on('error', () => { try { ws.close(); } catch { /* ignore */ } });
|
||||
});
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { DatabaseService, type UserRole } from '../services/DatabaseService';
|
||||
import { NodeRegistry } from '../services/NodeRegistry';
|
||||
import { COOKIE_NAME } from '../helpers/constants';
|
||||
import { handlePilotTunnel } from './pilotTunnel';
|
||||
import { handleMeshControl } from './meshControl';
|
||||
import { handleNotificationsWs } from './notifications';
|
||||
import { handleRemoteForwarder } from './remoteForwarder';
|
||||
import { handleLogsWs } from './logs';
|
||||
@@ -30,7 +31,8 @@ function parseCookies(req: IncomingMessage): Record<string, string> {
|
||||
*
|
||||
* Dispatch order (first match wins):
|
||||
* 1. `/api/pilot/tunnel` -> handlePilotTunnel (own auth, own wss)
|
||||
* 2. shared cookie/Bearer auth + JWT verify (rejects unauthenticated)
|
||||
* 2. `/api/mesh/control` -> handleMeshControl (sidecar JWT, local-only)
|
||||
* 3. shared cookie/Bearer auth + JWT verify (rejects unauthenticated)
|
||||
* 3. API token scope gate (read-only / deploy-only restricted to logs + notifications)
|
||||
* 4. `/ws/notifications` local -> handleNotificationsWs
|
||||
* 5. remote nodeId path -> handleRemoteForwarder
|
||||
@@ -54,6 +56,10 @@ export function attachUpgrade(
|
||||
await handlePilotTunnel(req, socket, head, pilotTunnelWss);
|
||||
return;
|
||||
}
|
||||
if (reqUrl.pathname === '/api/mesh/control') {
|
||||
await handleMeshControl(req, socket, head, wss);
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// URL parse error falls through and will be rejected below.
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user