feat(mesh): collapse sidecar into Sencho process via in-process forwarder (#1000)

The separate saelix/sencho-mesh sidecar container is gone. The
forwarder logic that previously lived in mesh-sidecar/src/forwarder.ts
moves into the Sencho process as backend/src/services/MeshForwarder.ts,
a thin per-port net.Server lifecycle wrapper. MeshService implements
the host interface and owns resolve plus splice; MeshForwarder owns
listener boilerplate. One container per node, no separate image to
publish, no control WebSocket.

Operator-facing change: the Sencho container now runs in
network_mode: host so the forwarder can bind alias ports on the host
network where meshed containers' extra_hosts host-gateway entries
point. Without host network mode the listeners would land in the
container's namespace and inbound traffic from peers would never
reach them. The 1852:1852 port publish becomes a no-op under host
mode and is commented out in the operator template.

Same-node forward path now dials the target container's bridge IP
via Dockerode (preferring the compose default network for
deterministic selection across daemon versions) instead of 127.0.0.1.
The legacy 127.0.0.1 path only worked when the target service
published its port to the host; the IP path works regardless.

Cross-node mesh routing in this phase is central -> pilot direction
only via PilotTunnelManager.openTcpStream. Pilot -> central and
pilot <-> pilot via central relay land in Phase B with the
tcp_open_reverse frame.

Deletions:
- mesh-sidecar/ package entirely (Dockerfile, package, sources, tests)
- backend/src/websocket/meshControl.ts
- MeshService sidecar lifecycle: spawnSidecar, stopSidecar,
  isSidecarRunning, mintSidecarToken, verifySidecarToken,
  attachSidecarSocket, handleSidecarResolve, sendSidecar
- POST /api/mesh/nodes/:id/sidecar/restart route
- /api/mesh/control WS dispatch in upgradeHandler

Type cleanup: 'sidecar' literal removed from MeshActivitySource and
MeshProbeResult.where (also the frontend mirror). MeshNodeStatus
sidecarRunning becomes localForwarderListening (boolean | null) so
non-local nodes get a null instead of an unconditional false; the
honest semantic is "this view only knows the local forwarder state;
remote forwarder status lands in Phase B." MeshNodeDiagnostic
sidecar object becomes forwarder { listening, listenerCount }.

Frontend MeshDiagnosticsSheet drops the restart-sidecar action and
sidecar liveness card; surfaces forwarder state plus a "runs
in-process; no separate container" caption.

Resolves audit findings C-1 (data plane non-functional), C-2 (sidecar
control WS not loopback-enforced), C-4 (sidecar lifecycle Dockerode-
on-remote, PR #999 closed), and C-5 (saelix/sencho-mesh:latest
unreachable). C-3 (PR #992) is unchanged. M-12 (PR #994) is
unchanged.
This commit is contained in:
Anso
2026-05-08 15:51:39 -04:00
committed by GitHub
parent b5463e1771
commit f599110386
23 changed files with 496 additions and 2558 deletions
@@ -0,0 +1,166 @@
/**
* Unit tests for MeshForwarder. Exercises the per-port listener lifecycle
* (listen/unlisten/shutdown) and the accept dispatch into the host
* (MeshService surrogate). MeshForwarder itself does not splice bytes —
* the host's `handleAccept` does — so the test injects a recording
* surrogate and asserts the call shape.
*/
import net from 'net';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { MeshForwarder, type MeshForwarderHost } from '../services/MeshForwarder';
interface Accept {
port: number;
socket: net.Socket;
}
function makeRecordingHost(): { host: MeshForwarderHost; accepts: Accept[]; resolveAfter: (cb: (a: Accept) => void) => void } {
const accepts: Accept[] = [];
const subscribers: Array<(a: Accept) => void> = [];
const host: MeshForwarderHost = {
async handleAccept(port, socket) {
const a = { port, socket };
accepts.push(a);
for (const s of subscribers.splice(0)) s(a);
},
};
return {
host,
accepts,
resolveAfter: (cb) => { subscribers.push(cb); },
};
}
async function getEphemeralPort(): Promise<number> {
return new Promise((resolve, reject) => {
const s = net.createServer();
s.unref();
s.listen(0, '127.0.0.1', () => {
const addr = s.address();
if (!addr || typeof addr === 'string') {
reject(new Error('no address'));
return;
}
const port = addr.port;
s.close(() => resolve(port));
});
});
}
async function dial(port: number, host = '127.0.0.1'): Promise<net.Socket> {
return new Promise((resolve, reject) => {
const s = net.createConnection({ host, port });
s.once('connect', () => resolve(s));
s.once('error', reject);
});
}
async function waitFor<T>(check: () => T | undefined, timeoutMs = 1000): Promise<T> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
const v = check();
if (v !== undefined && v !== null && (Array.isArray(v) ? (v as unknown[]).length > 0 : true)) {
return v as T;
}
await new Promise((r) => setTimeout(r, 10));
}
throw new Error('timeout waiting for condition');
}
describe('MeshForwarder', () => {
let forwarder: MeshForwarder | null = null;
afterEach(async () => {
if (forwarder) await forwarder.shutdown();
forwarder = null;
});
beforeEach(() => { /* fresh per test */ });
it('binds a listener on the requested port and reports it via getListenerPorts', async () => {
const { host } = makeRecordingHost();
forwarder = new MeshForwarder(host);
const port = await getEphemeralPort();
await forwarder.listen(port);
expect(forwarder.getListenerPorts()).toEqual([port]);
expect(forwarder.isListening(port)).toBe(true);
});
it('hands accepted sockets to the host with the original destination port', async () => {
const { host, accepts } = makeRecordingHost();
forwarder = new MeshForwarder(host);
const port = await getEphemeralPort();
await forwarder.listen(port);
const client = await dial(port);
const seen = await waitFor(() => accepts.length ? accepts : undefined);
expect(seen[0].port).toBe(port);
expect(seen[0].socket).toBeInstanceOf(net.Socket);
client.destroy();
seen[0].socket.destroy();
});
it('listen is idempotent (repeated calls on the same port no-op)', async () => {
const { host } = makeRecordingHost();
forwarder = new MeshForwarder(host);
const port = await getEphemeralPort();
await forwarder.listen(port);
await forwarder.listen(port);
expect(forwarder.getListenerPorts()).toEqual([port]);
});
it('two concurrent listen() calls on the same port produce a single bind, not EADDRINUSE', async () => {
const { host } = makeRecordingHost();
forwarder = new MeshForwarder(host);
const port = await getEphemeralPort();
// Fire both calls before either resolves. Without the in-flight
// dedup map, the second call would race past the listeners.has()
// check and fail with EADDRINUSE on its own bind attempt.
const [a, b] = await Promise.all([forwarder.listen(port), forwarder.listen(port)]);
expect(a).toBeUndefined();
expect(b).toBeUndefined();
expect(forwarder.getListenerPorts()).toEqual([port]);
});
it('unlisten releases the port so a new listener can bind it', async () => {
const { host } = makeRecordingHost();
forwarder = new MeshForwarder(host);
const port = await getEphemeralPort();
await forwarder.listen(port);
await forwarder.unlisten(port);
expect(forwarder.isListening(port)).toBe(false);
// Verify the port is genuinely free by binding a fresh net.Server.
await new Promise<void>((resolve, reject) => {
const probe = net.createServer();
probe.once('error', reject);
probe.listen(port, '127.0.0.1', () => probe.close(() => resolve()));
});
});
it('rejects new connections after shutdown', async () => {
const { host } = makeRecordingHost();
forwarder = new MeshForwarder(host);
const port = await getEphemeralPort();
await forwarder.listen(port);
await forwarder.shutdown();
await expect(dial(port)).rejects.toThrow();
forwarder = null;
});
it('destroys the source socket when the host handler throws', async () => {
const host: MeshForwarderHost = {
async handleAccept() { throw new Error('host blew up'); },
};
forwarder = new MeshForwarder(host);
const port = await getEphemeralPort();
await forwarder.listen(port);
const client = await dial(port);
const closed = new Promise<void>((resolve) => client.once('close', () => resolve()));
await closed;
});
});
+2 -2
View File
@@ -97,7 +97,7 @@ describe('MeshService activity log', () => {
const svc = MeshService.getInstance();
svc.logActivity({ source: 'mesh', level: 'info', type: 'opt_in', alias: 'a.b.c.sencho', message: 'a' });
svc.logActivity({ source: 'pilot', level: 'error', type: 'tunnel.fail', alias: 'a.b.c.sencho', message: 'b' });
svc.logActivity({ source: 'sidecar', level: 'info', type: 'route.resolve.ok', alias: 'x.y.z.sencho', message: 'c' });
svc.logActivity({ source: 'mesh', level: 'info', type: 'route.resolve.ok', alias: 'x.y.z.sencho', message: 'c' });
expect(svc.getActivity({ alias: 'a.b.c.sencho' }).length).toBe(2);
expect(svc.getActivity({ source: 'pilot' }).length).toBe(1);
@@ -172,7 +172,7 @@ describe('MeshService.testUpstream tunnel-down path', () => {
const result = await svc.testUpstream('nonexistent.sencho', localNodeId);
expect(result.ok).toBe(false);
expect(result.where).toBe('sidecar');
expect(result.where).toBe('no_route');
expect(result.code).toBe('no_route');
});
});
+1 -15
View File
@@ -173,24 +173,10 @@ meshRouter.get('/nodes/:nodeId/diagnostic', async (req: Request, res: Response):
}
});
meshRouter.post('/nodes/:nodeId/sidecar/restart', async (req: Request, res: Response): Promise<void> => {
if (!requireAdmiral(req, res)) return;
if (!requireAdmin(req, res)) return;
const nodeId = Number.parseInt(req.params.nodeId as string, 10);
if (!Number.isFinite(nodeId)) { res.status(400).json({ error: 'Invalid node id' }); return; }
try {
await MeshService.getInstance().stopSidecar(nodeId);
await MeshService.getInstance().spawnSidecar(nodeId);
res.json({ ok: true });
} catch (err) {
res.status(500).json({ error: (err as Error).message });
}
});
meshRouter.get('/activity', (req: Request, res: Response): void => {
if (!requireAdmiral(req, res)) return;
const alias = typeof req.query.alias === 'string' ? req.query.alias : undefined;
const source = typeof req.query.source === 'string' ? (req.query.source as 'sidecar' | 'pilot' | 'mesh') : undefined;
const source = typeof req.query.source === 'string' ? (req.query.source as 'pilot' | 'mesh') : undefined;
const level = typeof req.query.level === 'string' ? (req.query.level as 'info' | 'warn' | 'error') : undefined;
const limit = typeof req.query.limit === 'string' ? Number.parseInt(req.query.limit, 10) : 200;
const events = MeshService.getInstance().getActivity({ alias, source, level, limit });
+101
View File
@@ -0,0 +1,101 @@
import net from 'net';
import { sanitizeForLog } from '../utils/safeLog';
/**
* In-process mesh TCP forwarder. Owns per-port `net.Server` listeners on the
* host network and delegates accepted sockets to the host (MeshService) for
* resolve + splice. Replaces the separate `saelix/sencho-mesh` sidecar
* container that previously did this job over a control WebSocket. The
* resolve step is now a sync map lookup rather than a round-trip, so
* MeshForwarder is just a thin lifecycle layer; all routing + splicing
* lives on MeshService.
*
* Sencho's container must run in `network_mode: host` (Linux) for the
* listeners to bind on the host's network where meshed containers'
* `extra_hosts: <alias>:host-gateway` entries point. Without host network
* mode, `net.createServer().listen(port)` lands inside the container's
* namespace and inbound traffic from peers never reaches it.
*/
export interface MeshForwarderHost {
/** Called on each accepted inbound socket. The host owns the splice
* lifecycle; MeshForwarder only manages listener boilerplate. */
handleAccept(port: number, source: net.Socket): Promise<void>;
}
export class MeshForwarder {
private readonly listeners = new Map<number, net.Server>();
/**
* In-flight `listen(port)` promises so concurrent callers race-safely
* deduplicate. Without this guard, two concurrent calls to listen on
* the same port would both pass the `listeners.has(port)` check (which
* is only populated after the listening event resolves) and the second
* would fail with EADDRINUSE.
*/
private readonly pending = new Map<number, Promise<void>>();
private shuttingDown = false;
constructor(private readonly host: MeshForwarderHost) {}
public async listen(port: number): Promise<void> {
if (this.shuttingDown) return;
if (this.listeners.has(port)) return;
const inflight = this.pending.get(port);
if (inflight) return inflight;
const promise = (async () => {
const server = net.createServer((socket) => this.acceptConnection(port, socket));
try {
await new Promise<void>((resolve, reject) => {
const onError = (err: Error) => { server.removeListener('listening', onListening); reject(err); };
const onListening = () => { server.removeListener('error', onError); resolve(); };
server.once('error', onError);
server.once('listening', onListening);
// Bind on all interfaces. Under host network mode this is
// the host's own network; under bridge mode (mesh disabled
// at boot) this would be the container's namespace.
server.listen(port, '0.0.0.0');
});
this.listeners.set(port, server);
} finally {
this.pending.delete(port);
}
})();
this.pending.set(port, promise);
return promise;
}
public async unlisten(port: number): Promise<void> {
const server = this.listeners.get(port);
if (!server) return;
this.listeners.delete(port);
await new Promise<void>((resolve) => server.close(() => resolve()));
}
public async shutdown(): Promise<void> {
this.shuttingDown = true;
const ports = Array.from(this.listeners.keys());
await Promise.all(ports.map((p) => this.unlisten(p)));
}
public getListenerPorts(): number[] {
return Array.from(this.listeners.keys());
}
public isListening(port: number): boolean {
return this.listeners.has(port);
}
private acceptConnection(port: number, source: net.Socket): void {
if (this.shuttingDown) {
try { source.destroy(); } catch { /* ignore */ }
return;
}
// Defer to the host for resolve + splice. MeshForwarder itself does
// not look at the source bytes; routing lives on MeshService where
// the alias map and the cross-node bridge dispatch are.
this.host.handleAccept(port, source).catch((err) => {
console.warn('[MeshForwarder] accept handler failed:', sanitizeForLog((err as Error).message));
try { source.destroy(); } catch { /* ignore */ }
});
}
}
+200 -197
View File
@@ -2,11 +2,11 @@ import net from 'net';
import path from 'path';
import fs from 'fs/promises';
import { EventEmitter } from 'events';
import jwt from 'jsonwebtoken';
import { DatabaseService } from './DatabaseService';
import DockerController from './DockerController';
import { LicenseService } from './LicenseService';
import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from './license-headers';
import { MeshForwarder, type MeshForwarderHost } from './MeshForwarder';
import { NodeRegistry } from './NodeRegistry';
import { PilotTunnelManager } from './PilotTunnelManager';
import { generateOverrideYaml, MeshAlias } from './MeshComposeOverride';
@@ -15,13 +15,10 @@ import { isPathWithinBase, isValidStackName } from '../utils/validation';
const ACTIVITY_BUFFER_SIZE = 1000;
const ALIAS_REFRESH_INTERVAL_MS = 60_000;
const SIDECAR_CONTAINER_PREFIX = 'sencho-mesh-';
const DEFAULT_SIDECAR_IMAGE = process.env.SENCHO_MESH_IMAGE || 'saelix/sencho-mesh:latest';
const SIDECAR_TOKEN_TTL = '7d';
const PROBE_TIMEOUT_MS = 5_000;
const SLOW_PROBE_THRESHOLD_MS = 500;
export type MeshActivitySource = 'sidecar' | 'pilot' | 'mesh';
export type MeshActivitySource = 'pilot' | 'mesh';
export type MeshActivityLevel = 'info' | 'warn' | 'error';
export type MeshActivityType =
| 'route.resolve.ok' | 'route.resolve.denied'
@@ -29,7 +26,7 @@ export type MeshActivityType =
| 'opt_in' | 'opt_out'
| 'mesh.enable' | 'mesh.disable'
| 'probe.ok' | 'probe.fail'
| 'sidecar.start' | 'sidecar.stop' | 'sidecar.crash';
| 'forwarder.listen' | 'forwarder.unlisten' | 'forwarder.error';
export interface MeshActivityEvent {
ts: number;
@@ -65,7 +62,8 @@ export interface MeshNodeStatus {
nodeId: number;
nodeName: string;
enabled: boolean;
sidecarRunning: boolean;
/** Forwarder state for the LOCAL node (the Sencho instance answering this request). Always `null` for any non-local node — fetching the remote forwarder state requires a cross-node call which lands in Phase B. */
localForwarderListening: boolean | null;
pilotConnected: boolean;
optedInStacks: string[];
activeStreamCount: number;
@@ -73,7 +71,7 @@ export interface MeshNodeStatus {
export interface MeshNodeDiagnostic {
nodeId: number;
sidecar: { running: boolean; restartCount: number };
forwarder: { listening: boolean; listenerCount: number };
pilot: { connected: boolean; bufferedAmount: number; lastSeen: number | null };
activeStreams: Array<{ streamId: number; alias?: string; bytesIn: number; bytesOut: number; ageMs: number }>;
aliasCache: Array<{ host: string; targetNodeId: number; port: number }>;
@@ -91,7 +89,7 @@ export interface MeshRouteDiagnostic {
export interface MeshProbeResult {
ok: boolean;
latencyMs?: number;
where?: 'sidecar' | 'pilot_tunnel' | 'agent_resolve' | 'agent_dial' | 'target_port';
where?: 'no_route' | 'pilot_tunnel' | 'agent_resolve' | 'agent_dial' | 'target_port';
code?: string;
message?: string;
}
@@ -104,50 +102,44 @@ interface ActiveStreamRecord {
openedAt: number;
}
interface PendingResolve {
sidecarSocket: WebSocketLike;
connId: number;
port: number;
remoteAddr: string;
}
interface WebSocketLike {
send(data: string | Buffer, opts?: unknown, cb?: (err?: Error) => void): void;
readyState: number;
on(event: string, listener: (...args: unknown[]) => void): unknown;
}
/**
* Sencho Mesh orchestrator. Owns:
* - sidecar lifecycle (Dockerode-spawned per-instance)
* - in-process TCP forwarder (`MeshForwarder`) that binds host-network
* listeners on alias ports. Replaces the prior separate sidecar
* container; one container per node now.
* - opt-in / opt-out persistence and cascading override regeneration
* - global alias aggregation (across the fleet via the existing API)
* - request-based resolution from sidecar control WS
* - cross-node TCP forwarding via PilotTunnelManager
* - global alias aggregation (across the fleet via the existing HTTP
* proxy chain — see `inspectStackServices`)
* - cross-node TCP forwarding via `PilotTunnelManager` (central-side)
* - probe + diagnostics + activity ring buffer
*
* V1 limitations:
* - one cross-node alias per TCP port across the fleet (port-collision check at opt-in)
* - sidecar runs in host network mode; aliases resolve via `host-gateway` extra_hosts
* - pilot-to-pilot mesh routing is not supported (only central <-> pilot)
* - one cross-node alias per TCP port across the fleet (port-collision
* check at opt-in)
* - aliases resolve via `host-gateway` extra_hosts; Sencho's container
* must run with `network_mode: host` for the forwarder's listeners to
* bind on the host's network where meshed containers' `host-gateway`
* entries point
* - cross-node mesh routing is central → pilot in this phase. Pilot →
* central and pilot ↔ pilot via central relay land in Phase B.
*/
export class MeshService extends EventEmitter {
export class MeshService extends EventEmitter implements MeshForwarderHost {
private static instance: MeshService;
private started = false;
private aliasCache = new Map<string, MeshGlobalAlias>();
private aliasByPort = new Map<number, MeshGlobalAlias>();
private activity: MeshActivityEvent[] = [];
private activeStreams = new Map<number, ActiveStreamRecord>();
private pendingResolves = new Map<string, PendingResolve>();
private sidecarSockets = new Set<WebSocketLike>();
private aliasRefreshTimer?: NodeJS.Timeout;
private routeErrorMap = new Map<string, { ts: number; message: string }>();
private routeLatencyMap = new Map<string, number>();
private activityListeners = new Set<(e: MeshActivityEvent) => void>();
private readonly forwarder: MeshForwarder;
private constructor() {
super();
this.setMaxListeners(50);
this.forwarder = new MeshForwarder(this);
}
public static getInstance(): MeshService {
@@ -167,10 +159,16 @@ export class MeshService extends EventEmitter {
}));
await this.refreshAliasCache();
await this.syncForwarderListeners();
this.aliasRefreshTimer = setInterval(() => {
void this.refreshAliasCache().catch((err) => {
console.warn('[MeshService] alias refresh failed:', sanitizeForLog((err as Error).message));
});
void (async () => {
try {
await this.refreshAliasCache();
await this.syncForwarderListeners();
} catch (err) {
console.warn('[MeshService] alias refresh failed:', sanitizeForLog((err as Error).message));
}
})();
}, ALIAS_REFRESH_INTERVAL_MS);
this.logActivity({
@@ -186,10 +184,49 @@ export class MeshService extends EventEmitter {
clearInterval(this.aliasRefreshTimer);
this.aliasRefreshTimer = undefined;
}
for (const ws of this.sidecarSockets) {
try { (ws as { close?: (code: number) => void }).close?.(1000); } catch { /* ignore */ }
await this.forwarder.shutdown();
}
/**
* Bind the forwarder's listeners to the local-owned alias ports and
* release any listeners no longer in the alias set. Called from
* `start`, after each `refreshAliasCache` tick, and after every
* opt-in / opt-out / disable on the local node so the bound port set
* follows the DB state.
*/
private async syncForwarderListeners(): Promise<void> {
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
const wantPorts = new Set<number>();
for (const alias of this.aliasByPort.values()) {
if (alias.nodeId === localNodeId) wantPorts.add(alias.port);
}
const havePorts = new Set(this.forwarder.getListenerPorts());
for (const port of havePorts) {
if (!wantPorts.has(port)) {
await this.forwarder.unlisten(port);
this.logActivity({
source: 'mesh', level: 'info', type: 'forwarder.unlisten',
nodeId: localNodeId, message: `forwarder released port ${port}`,
});
}
}
for (const port of wantPorts) {
if (havePorts.has(port)) continue;
try {
await this.forwarder.listen(port);
this.logActivity({
source: 'mesh', level: 'info', type: 'forwarder.listen',
nodeId: localNodeId, message: `forwarder listening on port ${port}`,
});
} catch (err) {
this.logActivity({
source: 'mesh', level: 'error', type: 'forwarder.error',
nodeId: localNodeId,
message: `forwarder bind failed on port ${port}: ${sanitizeForLog((err as Error).message)}`,
details: { port },
});
}
}
this.sidecarSockets.clear();
}
// --- Activity log ---
@@ -248,6 +285,7 @@ export class MeshService extends EventEmitter {
db.insertMeshStack(nodeId, stackName, actor);
await this.refreshAliasCache();
await this.syncForwarderListeners();
await this.regenerateOverridesForNode(nodeId);
this.logActivity({
@@ -271,6 +309,7 @@ export class MeshService extends EventEmitter {
db.deleteMeshStack(nodeId, stackName);
await this.removeStackOverride(nodeId, stackName);
await this.refreshAliasCache();
await this.syncForwarderListeners();
await this.regenerateOverridesForNode(nodeId);
this.logActivity({
@@ -301,6 +340,7 @@ export class MeshService extends EventEmitter {
await this.removeStackOverride(nodeId, s.stack_name);
}
await this.refreshAliasCache();
await this.syncForwarderListeners();
this.logActivity({
source: 'mesh', level: 'info', type: 'mesh.disable',
nodeId, message: `mesh disabled on node ${nodeId}`,
@@ -483,25 +523,50 @@ export class MeshService extends EventEmitter {
}
/**
* Forward bytes from a sidecar-accepted local socket to the target.
* Same-node target: open a direct TCP socket.
* Cross-node target: open a pilot-tunnel TcpStream to that node's agent.
* MeshForwarder calls this on every accepted inbound socket. Resolves
* the alias by destination port, then dispatches to the same-node fast
* path or the cross-node bridge path.
*/
public openTcp(target: MeshTarget, src: net.Socket, sourceNodeId: number): void {
if (target.nodeId === sourceNodeId) {
this.openSameNode(target, src);
public async handleAccept(port: number, src: net.Socket): Promise<void> {
const target = this.resolveByLocalPort(port);
if (!target) {
this.logActivity({
source: 'mesh', level: 'warn', type: 'route.resolve.denied',
message: `inbound on port ${port} has no registered alias`,
details: { port, remoteAddr: src.remoteAddress ?? '' },
});
try { src.destroy(); } catch { /* ignore */ }
return;
}
this.openCrossNode(target, src);
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
if (target.nodeId === localNodeId) {
await this.openSameNode(target, src);
} else {
this.openCrossNode(target, src);
}
}
private openSameNode(target: MeshTarget, src: net.Socket): void {
// For same-node fast path, the agent's resolution logic isn't needed:
// we can dial via Dockerode's container IP. For V1 simplicity, we dial
// the host-gateway's published port if mapped, falling back to
// 127.0.0.1 (the sidecar runs in host network mode so localhost reaches
// the container's published port).
const upstream = net.createConnection({ host: '127.0.0.1', port: target.port });
/**
* Same-node forward: dial the target container's bridge IP directly.
* Sencho runs in `network_mode: host` so it sees the docker bridge
* networks and can reach container IPs without going through any
* host-port publish. Looks up the container by Compose's
* `<project>-<service>-<index>` naming convention; falls back to a
* label-filtered listContainers if the conventional name is absent
* (e.g. when the operator overrode the project name).
*/
private async openSameNode(target: MeshTarget, src: net.Socket): Promise<void> {
const ip = await this.resolveContainerIp(target);
if (!ip) {
this.logActivity({
source: 'mesh', level: 'error', type: 'route.resolve.denied',
alias: target.alias,
message: `cannot resolve container IP for ${target.alias}`,
});
try { src.destroy(); } catch { /* ignore */ }
return;
}
const upstream = net.createConnection({ host: ip, port: target.port });
upstream.setTimeout(PROBE_TIMEOUT_MS);
const stream = this.registerActiveStream(target.alias);
upstream.once('connect', () => {
@@ -509,7 +574,7 @@ export class MeshService extends EventEmitter {
this.logActivity({
source: 'mesh', level: 'info', type: 'route.resolve.ok',
alias: target.alias, streamId: stream.streamId,
message: `same-node connect to ${target.alias}`,
message: `same-node connect to ${target.alias} (${ip}:${target.port})`,
});
src.pipe(upstream);
upstream.pipe(src);
@@ -525,6 +590,60 @@ export class MeshService extends EventEmitter {
src.on('close', () => teardown());
}
/** Find the bridge-network IP of the first container of `<stack>/<service>`. */
private async resolveContainerIp(target: MeshTarget): Promise<string | null> {
try {
const docker = DockerController.getInstance().getDocker();
// Compose default container name pattern; -1 is the first replica.
const conventionalName = `${target.stack}-${target.service}-1`;
const info = await docker.getContainer(conventionalName).inspect().catch(() => null);
const fromInspect = info ? this.extractContainerIp(target.stack, info) : null;
if (fromInspect) return fromInspect;
// Fallback: filter by compose labels in case of a non-conventional
// container name (operator overrode `container_name` or compose
// project).
const containers = await docker.listContainers({
all: true,
filters: {
label: [
`com.docker.compose.project=${target.stack}`,
`com.docker.compose.service=${target.service}`,
],
},
});
if (containers.length === 0) return null;
const fallbackInfo = await docker.getContainer(containers[0].Id).inspect().catch(() => null);
return fallbackInfo ? this.extractContainerIp(target.stack, fallbackInfo) : null;
} catch (err) {
console.warn('[MeshService] container IP lookup failed:', sanitizeForLog((err as Error).message));
return null;
}
}
/**
* Pick a deterministic IP. Prefer the compose default network
* (`<stack>_default` or any network whose name starts with `<stack>_`),
* then any other declared network, then the legacy bridge `IPAddress`.
* Without this preference order, `Object.values(Networks)` ordering on
* containers attached to multiple networks varies across daemon
* versions and can make same-node forwarding flaky on a redeploy.
*/
private extractContainerIp(
stackName: string,
info: { NetworkSettings?: { Networks?: Record<string, { IPAddress?: string }>; IPAddress?: string } },
): string | null {
const networks = info.NetworkSettings?.Networks ?? {};
const composeDefault = networks[`${stackName}_default`];
if (composeDefault?.IPAddress) return composeDefault.IPAddress;
for (const [name, net] of Object.entries(networks)) {
if (name.startsWith(`${stackName}_`) && net?.IPAddress) return net.IPAddress;
}
for (const net of Object.values(networks)) {
if (net?.IPAddress) return net.IPAddress;
}
return info.NetworkSettings?.IPAddress || null;
}
private openCrossNode(target: MeshTarget, src: net.Socket): void {
const ptm = PilotTunnelManager.getInstance();
if (!ptm.hasActiveTunnel(target.nodeId)) {
@@ -598,7 +717,7 @@ export class MeshService extends EventEmitter {
public async testUpstream(alias: string, sourceNodeId: number): Promise<MeshProbeResult> {
const target = this.lookupAliasGlobal(alias);
if (!target) {
return { ok: false, where: 'sidecar', code: 'no_route', message: 'alias not found' };
return { ok: false, where: 'no_route', code: 'no_route', message: 'alias not found' };
}
if (!DatabaseService.getInstance().isMeshStackEnabled(target.nodeId, target.stackName)) {
return { ok: false, where: 'agent_resolve', code: 'denied', message: 'target stack not opted in' };
@@ -726,7 +845,8 @@ export class MeshService extends EventEmitter {
const ptm = PilotTunnelManager.getInstance();
const bridge = ptm.getBridge(nodeId);
const node = DatabaseService.getInstance().getNode(nodeId);
const sidecarRunning = await this.isSidecarRunning(nodeId);
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
const isLocal = nodeId === localNodeId;
const aliasCacheRows = Array.from(this.aliasCache.values())
.filter((a) => a.nodeId === nodeId)
@@ -739,9 +859,13 @@ export class MeshService extends EventEmitter {
ageMs: now - s.openedAt,
}));
const listenerCount = isLocal ? this.forwarder.getListenerPorts().length : 0;
return {
nodeId,
sidecar: { running: sidecarRunning, restartCount: 0 },
forwarder: {
listening: isLocal && this.isLocalForwarderActive(),
listenerCount,
},
pilot: {
connected: !!bridge,
bufferedAmount: bridge?.getBufferedAmount() ?? 0,
@@ -755,151 +879,30 @@ export class MeshService extends EventEmitter {
public async getStatus(): Promise<MeshNodeStatus[]> {
const db = DatabaseService.getInstance();
const nodes = db.getNodes();
const out: MeshNodeStatus[] = [];
for (const node of nodes) {
const optedInStacks = db.listMeshStacks(node.id).map((s) => s.stack_name);
out.push({
nodeId: node.id,
nodeName: node.name,
enabled: db.getNodeMeshEnabled(node.id),
sidecarRunning: await this.isSidecarRunning(node.id),
pilotConnected: this.isMeshReachable(node.id),
optedInStacks,
activeStreamCount: Array.from(this.activeStreams.values()).length,
});
}
return out;
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
const localListening = this.isLocalForwarderActive();
return nodes.map((node) => ({
nodeId: node.id,
nodeName: node.name,
enabled: db.getNodeMeshEnabled(node.id),
localForwarderListening: node.id === localNodeId ? localListening : null,
pilotConnected: this.isMeshReachable(node.id),
optedInStacks: db.listMeshStacks(node.id).map((s) => s.stack_name),
activeStreamCount: this.activeStreams.size,
}));
}
// --- Sidecar lifecycle (best-effort; real spawn happens on local node) ---
public async spawnSidecar(nodeId: number): Promise<void> {
const docker = DockerController.getInstance(nodeId).getDocker();
const name = `${SIDECAR_CONTAINER_PREFIX}${nodeId}`;
try {
const existing = docker.getContainer(name);
const info = await existing.inspect().catch(() => null);
if (info?.State?.Running) return;
if (info) await existing.remove({ force: true }).catch(() => undefined);
} catch { /* ignore */ }
const token = this.mintSidecarToken(nodeId);
const controlUrl = process.env.SENCHO_INTERNAL_URL || 'ws://127.0.0.1:1852/api/mesh/control';
try {
const container = await docker.createContainer({
name,
Image: DEFAULT_SIDECAR_IMAGE,
Env: [
`SENCHO_CONTROL_URL=${controlUrl}`,
`SENCHO_MESH_TOKEN=${token}`,
`MESH_NODE_ID=${nodeId}`,
],
HostConfig: {
NetworkMode: 'host',
RestartPolicy: { Name: 'unless-stopped' },
},
Labels: {
'sencho.mesh.role': 'sidecar',
'sencho.mesh.node_id': String(nodeId),
},
});
await container.start();
this.logActivity({
source: 'mesh', level: 'info', type: 'sidecar.start',
nodeId, message: `sidecar started for node ${nodeId}`,
});
} catch (err) {
this.logActivity({
source: 'mesh', level: 'error', type: 'sidecar.crash',
nodeId, message: `sidecar spawn failed: ${(err as Error).message}`,
});
throw err;
}
/** True when the local Sencho's forwarder is started and bound to at least one alias port. */
private isLocalForwarderActive(): boolean {
return this.started && this.forwarder.getListenerPorts().length > 0;
}
public async stopSidecar(nodeId: number): Promise<void> {
const docker = DockerController.getInstance(nodeId).getDocker();
const name = `${SIDECAR_CONTAINER_PREFIX}${nodeId}`;
try {
const c = docker.getContainer(name);
await c.stop({ t: 5 }).catch(() => undefined);
await c.remove({ force: true }).catch(() => undefined);
this.logActivity({
source: 'mesh', level: 'info', type: 'sidecar.stop',
nodeId, message: `sidecar stopped for node ${nodeId}`,
});
} catch { /* ignore */ }
}
private async isSidecarRunning(nodeId: number): Promise<boolean> {
try {
const docker = DockerController.getInstance(nodeId).getDocker();
const info = await docker.getContainer(`${SIDECAR_CONTAINER_PREFIX}${nodeId}`).inspect();
return !!info.State?.Running;
} catch {
return false;
}
}
public mintSidecarToken(nodeId: number): string {
const settings = DatabaseService.getInstance().getGlobalSettings();
const secret = settings.auth_jwt_secret;
if (!secret) throw new Error('JWT secret not configured');
return jwt.sign({ scope: 'mesh_sidecar', nodeId }, secret, { expiresIn: SIDECAR_TOKEN_TTL });
}
public verifySidecarToken(token: string): { nodeId: number } | null {
try {
const settings = DatabaseService.getInstance().getGlobalSettings();
const secret = settings.auth_jwt_secret;
if (!secret) return null;
const decoded = jwt.verify(token, secret) as { scope?: string; nodeId?: number };
if (decoded.scope !== 'mesh_sidecar' || typeof decoded.nodeId !== 'number') return null;
return { nodeId: decoded.nodeId };
} catch {
return null;
}
}
// --- Sidecar control WS attachment (called from websocket/meshControl.ts) ---
public attachSidecarSocket(ws: WebSocketLike, _nodeId: number): void {
this.sidecarSockets.add(ws);
ws.on('close', () => { this.sidecarSockets.delete(ws); });
}
/** Resolve an inbound sidecar request: "I have a connection on this port; who's it for?" */
public handleSidecarResolve(ws: WebSocketLike, nodeId: number, connId: number, port: number, remoteAddr: string): void {
const target = this.resolveByLocalPort(port);
if (!target) {
this.sendSidecar(ws, { t: 'resolve_err', connId, code: 'no_route', message: 'port not registered' });
this.logActivity({
source: 'sidecar', level: 'warn', type: 'route.resolve.denied',
nodeId, message: `unknown port ${port}`, details: { connId, remoteAddr },
});
return;
}
// The sidecar is the SOURCE; it asks for routing on its local node.
// Open a TCP path on this node's MeshService (same-node fast path or
// pilot tunnel) and acknowledge the resolve with a freshly allocated
// streamId. For V1 we do NOT bridge real bytes through the control WS
// until the sidecar package gains binary frame plumbing (Phase B).
// Instead we ack the resolve with the target metadata so the sidecar
// can dial directly on the host gateway.
this.sendSidecar(ws, { t: 'resolve_ok', connId, streamId: connId, alias: target.alias });
this.logActivity({
source: 'sidecar', level: 'info', type: 'route.resolve.ok',
nodeId, alias: target.alias,
message: `resolved port ${port} to ${target.alias}`,
details: { connId, remoteAddr },
});
}
private sendSidecar(ws: WebSocketLike, frame: Record<string, unknown>): void {
if (ws.readyState !== 1 /* OPEN */) return;
try { ws.send(JSON.stringify(frame)); } catch { /* ignore */ }
}
// mintSidecarToken / verifySidecarToken / spawnSidecar / stopSidecar /
// isSidecarRunning / handleSidecarResolve / sendSidecar /
// attachSidecarSocket are gone: the in-process MeshForwarder replaces
// the entire sidecar layer. Routing decisions happen via direct
// MeshService calls — no JWT minting, no separate container, no control
// WebSocket. See `docs/internal/architecture/mesh.md` for the new flow.
}
export class MeshError extends Error {
-57
View File
@@ -1,57 +0,0 @@
import type { IncomingMessage } from 'http';
import type { Duplex } from 'stream';
import type { WebSocketServer, WebSocket } from 'ws';
import { MeshService } from '../services/MeshService';
import { sanitizeForLog } from '../utils/safeLog';
import { rejectUpgrade as rejectSocket } from './reject';
/**
* Handle the local Sencho Mesh sidecar's control WebSocket. Authenticated
* with a `mesh_sidecar`-scoped JWT minted by MeshService when it spawned the
* sidecar; the JWT carries the node id the sidecar serves.
*
* The control WS is intentionally local-only: the sidecar runs in host
* network mode on the same Docker host as Sencho and reaches us via the
* loopback interface.
*/
export async function handleMeshControl(
req: IncomingMessage,
socket: Duplex,
head: Buffer,
wss: WebSocketServer,
): Promise<void> {
const authHeader = req.headers['authorization'];
const header = Array.isArray(authHeader) ? authHeader[0] : authHeader;
const token = header?.startsWith('Bearer ') ? header.slice(7) : null;
if (!token) return rejectSocket(socket, 401, 'Unauthorized');
const verified = MeshService.getInstance().verifySidecarToken(token);
if (!verified) return rejectSocket(socket, 401, 'Unauthorized');
wss.handleUpgrade(req, socket as never, head, (ws: WebSocket) => {
MeshService.getInstance().attachSidecarSocket(ws as unknown as never, verified.nodeId);
ws.on('message', (data, isBinary) => {
if (isBinary) return; // V1: control plane is JSON-only.
try {
const text = data.toString('utf8');
const frame = JSON.parse(text) as { t?: string; connId?: number; port?: number; remoteAddr?: string };
if (frame.t === 'resolve' && typeof frame.connId === 'number' && typeof frame.port === 'number') {
MeshService.getInstance().handleSidecarResolve(
ws as unknown as never,
verified.nodeId,
frame.connId,
frame.port,
frame.remoteAddr ?? '',
);
}
// hello / log / stream.stats / close are advisory; we accept
// them silently in V1. Future revisions can expand handling.
} catch (err) {
console.warn('[meshControl] bad frame:', sanitizeForLog((err as Error).message));
}
});
ws.on('error', () => { try { ws.close(); } catch { /* ignore */ } });
});
}
+1 -7
View File
@@ -7,7 +7,6 @@ import { DatabaseService, type UserRole } from '../services/DatabaseService';
import { NodeRegistry } from '../services/NodeRegistry';
import { COOKIE_NAME } from '../helpers/constants';
import { handlePilotTunnel } from './pilotTunnel';
import { handleMeshControl } from './meshControl';
import { handleNotificationsWs } from './notifications';
import { handleRemoteForwarder } from './remoteForwarder';
import { handleLogsWs } from './logs';
@@ -31,8 +30,7 @@ function parseCookies(req: IncomingMessage): Record<string, string> {
*
* Dispatch order (first match wins):
* 1. `/api/pilot/tunnel` -> handlePilotTunnel (own auth, own wss)
* 2. `/api/mesh/control` -> handleMeshControl (sidecar JWT, local-only)
* 3. shared cookie/Bearer auth + JWT verify (rejects unauthenticated)
* 2. shared cookie/Bearer auth + JWT verify (rejects unauthenticated)
* 3. API token scope gate (read-only / deploy-only restricted to logs + notifications)
* 4. `/ws/notifications` local -> handleNotificationsWs
* 5. remote nodeId path -> handleRemoteForwarder
@@ -56,10 +54,6 @@ export function attachUpgrade(
await handlePilotTunnel(req, socket, head, pilotTunnelWss);
return;
}
if (reqUrl.pathname === '/api/mesh/control') {
await handleMeshControl(req, socket, head, wss);
return;
}
} catch {
// URL parse error falls through and will be rejected below.
}