gsadmin/sencho
1
0
Fork 0
mirror of https://github.com/Studio-Saelix/sencho.git synced 2026-07-26 11:49:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(mesh): route peer→central traffic over the existing forward WS (#1094)

Browse Source 
* fix(mesh): route peer→central traffic over the existing forward WS

The reverse mesh callback path (`/api/mesh/proxy-tunnel-from-peer`) needed
SENCHO_PRIMARY_URL on central plus a publicly reachable origin from the
peer's perspective. In a typical homelab where central sits behind NAT,
peer→central dispatch silently failed at the dialer's short-circuit and
the headline "call any service on any node by hostname" worked one way
only.

The forward WS at `/api/mesh/proxy-tunnel` is already bidirectional end
to end. Make the bridge a persistent control-plane primitive: dial every
mesh-enabled proxy peer at startup, reconcile every 60 s, never idle-close.
Peer→central traffic multiplexes over the same WS via `tcp_open_reverse`.

Removed:
- `meshProxyTunnelFromPeer.ts` WS handler and dispatch
- `MeshCentralRegistry`, `PeerToCentralMeshSessionDialer`
- `mesh_handshake` first-frame state machine in `meshProxyTunnel.ts`
- `maybeSendBootstrap`, `buildHandshakeFrame` in the dialer
- `mesh_proxy_callback_bootstrap` capability and `maybeWarnUnsetPrimaryUrl`
- `mesh_centrals` table (drop migration; greenfield, no users)
- `PilotTunnelManager.replaceOrRegisterProxyBridge` (dead after handler removal)
- twelve associated unit/integration tests plus the peer-recovery branch
  in `MeshService.openCrossNode`

Added:
- `MeshService.proactiveBridgeFanout` selects every mesh-enabled proxy
  peer (no longer gated on `mesh_stacks` rows)
- `startBridgeReconcileLoop` runs the fanout every 60 s (override via
  `SENCHO_MESH_RECONCILE_INTERVAL_MS`)
- `MeshProxyTunnelDialer` default idle TTL is now `0` and exposes
  `isDialing(nodeId)` for the status surface
- `MeshNodeStatus.reverseCallbackStatus` discriminator
  (`connected | connecting | unavailable | not_applicable`) surfaced via
  `/api/mesh/status` and rendered as a pill in the Routing tab
- `openCrossNode` error message distinguishes "no proxy target" from
  "waiting for central to dial the reverse bridge"
- New tests: `mesh-service-proxy-tunnel-reconcile`,
  `mesh-status-reverse-callback`, `mesh-proxy-tunnel-dialer-no-idle-close`

SENCHO_PRIMARY_URL is no longer required for any mesh function.

* fix(mesh): rewrite proxy-tunnel reconcile test contents

The previous commit renamed the file but the rewritten test bodies stayed
unstaged on top of the rename. This commit lands the actual rewrite: the
fanout assertion now requires every mesh-enabled proxy peer to be dialed,
not just those with `mesh_stacks` rows, and adds a reconcile-tick
repeated-call test.
This commit is contained in:
Anso
2026-05-17 22:00:31 -04:00
committed by GitHub
parent 54c07d4930
commit f6e42535c8
37 changed files with 414 additions and 3107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/__tests__/capability-mesh-callback-bootstrap.test.ts
-37
View File
@@ -1,37 +0,0 @@
import { describe, it, expect, afterEach } from 'vitest';
import {
getActiveCapabilities,
applyPilotModeCapabilityFilter,
enableCapability,
disableCapability,
} from '../services/CapabilityRegistry';
describe('mesh_proxy_callback_bootstrap capability', () => {
afterEach(() => {
// Ensure the runtime override does not leak between tests.
enableCapability('mesh_proxy_callback_bootstrap');
});
it('is registered in the default CAPABILITIES list', () => {
expect(getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap');
});
it('is NOT filtered out in pilot mode (pilots can also be bootstrap targets)', () => {
applyPilotModeCapabilityFilter();
expect(getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap');
enableCapability('host-console');
enableCapability('self-update');
});
it('is stripped from the active capabilities list when the data plane is disabled', () => {
disableCapability('mesh_proxy_callback_bootstrap');
expect(getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
});
it('returns to the advertised list once enableCapability is called again', () => {
disableCapability('mesh_proxy_callback_bootstrap');
expect(getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
enableCapability('mesh_proxy_callback_bootstrap');
expect(getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap');
});
});
backend/src/__tests__/mesh-bootstrap-integration.test.ts
-187
View File
@@ -1,187 +0,0 @@
/**
* End-to-end protocol-role invariant for the symmetric mesh callback dial.
*
* The whole point of Task 8 + Task 10 is the asymmetry of registration even
* though the wire protocol becomes symmetric:
*
* - Central is the loopback HTTP host. Every bridge central registers
* (pilot-initiated or peer-initiated dial-back) is a
* `PilotTunnelBridge` parked in `PilotTunnelManager`.
* - The peer (proxy-mode remote) never runs a `PilotTunnelBridge`; on the
* peer side a `TcpStreamSwitchboard` rides the same WS and the local
* `MeshService.reverseDialer` slot is held while the WS is alive.
*
* This file exercises the central side of that invariant end-to-end using
* the same in-process fixture pattern as `mesh-proxy-tunnel-from-peer.test.ts`:
* a fully signed callback JWT lands at `/api/mesh/proxy-tunnel-from-peer`,
* and we assert what central produced is a `PilotTunnelBridge` (not a
* switchboard) in the manager. The matching mesh_handshake side (central
* pushing the JWT) is covered in `mesh-proxy-tunnel-dialer-handshake.test.ts`
* and `mesh-proxy-tunnel-first-frame.test.ts`; here we lock in the registry
* outcome that ties those two halves together.
*/
import http from 'http';
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import WebSocket from 'ws';
import jwt from 'jsonwebtoken';
import { createHash } from 'crypto';
import type { AddressInfo } from 'net';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import { MeshProxyTunnelDialer } from '../services/MeshProxyTunnelDialer';
import { MeshCentralRegistry } from '../services/MeshCentralRegistry';
import { PilotTunnelManager } from '../services/PilotTunnelManager';
import { PilotTunnelBridge } from '../services/PilotTunnelBridge';
let tmpDir: string;
let handleMeshProxyTunnelFromPeerUpgrade: typeof import('../websocket/meshProxyTunnelFromPeer').handleMeshProxyTunnelFromPeerUpgrade;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
interface ServerHandle {
server: http.Server;
port: number;
close: () => Promise<void>;
}
async function startCentralServer(): Promise<ServerHandle> {
const server = http.createServer();
server.on('upgrade', (req, socket, head) => {
const pathname = new URL(req.url ?? '/', 'http://localhost').pathname;
if (pathname === '/api/mesh/proxy-tunnel-from-peer') {
handleMeshProxyTunnelFromPeerUpgrade(req, socket, head);
} else {
socket.destroy();
}
});
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', () => resolve()));
const port = (server.address() as AddressInfo).port;
return {
server,
port,
close: () => new Promise<void>((resolve) => server.close(() => resolve())),
};
}
const CANONICAL_ORIGIN = 'https://central.example.com';
const INSTANCE_ID = 'bootstrap-integration-central';
const PEER_API_TOKEN = 'peer-token-bootstrap';
let secret: string;
let srv: ServerHandle;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ handleMeshProxyTunnelFromPeerUpgrade } = await import('../websocket/meshProxyTunnelFromPeer'));
({ DatabaseService } = await import('../services/DatabaseService'));
srv = await startCentralServer();
});
afterAll(async () => {
if (srv) await srv.close();
delete process.env.SENCHO_PRIMARY_URL;
cleanupTestDb(tmpDir);
});
beforeEach(() => {
process.env.SENCHO_PRIMARY_URL = CANONICAL_ORIGIN;
MeshProxyTunnelDialer.resetForTest();
MeshCentralRegistry.resetForTest();
PilotTunnelManager.resetForTest();
const db = DatabaseService.getInstance();
secret = db.getGlobalSettings().auth_jwt_secret;
db.setSystemState('instance_id', INSTANCE_ID);
});
function seedPeerNode(): number {
const db = DatabaseService.getInstance();
const id = db.addNode({
name: `peer-bs-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
type: 'remote',
mode: 'proxy',
api_url: 'https://peer.example.com',
api_token: PEER_API_TOKEN,
compose_dir: '/tmp',
is_default: false,
});
db.setNodeMeshEnabled(id, true);
return id;
}
function mintCallbackJwt(peerNodeId: number): string {
const fp = createHash('sha256').update(PEER_API_TOKEN).digest('hex').slice(0, 16);
const nowSec = Math.floor(Date.now() / 1000);
return jwt.sign(
{
sub: String(peerNodeId),
iss: INSTANCE_ID,
aud: CANONICAL_ORIGIN,
scope: 'mesh_tunnel',
iat: nowSec,
exp: nowSec + 3600,
kid: 'v1',
peer_token_fp: fp,
},
secret,
{ algorithm: 'HS256' },
);
}
describe('Mesh bootstrap E2E (protocol-role invariant)', () => {
it('after peer dial-back, central registers a PilotTunnelBridge in PilotTunnelManager', async () => {
// This is the symmetric callback half of the protocol: the peer has
// already received a bootstrap JWT in some prior session and is now
// dialing back into central. On central, the resulting WS must
// produce a PilotTunnelBridge (not a TcpStreamSwitchboard) and the
// manager must hold it under the peer's nodeId.
const peerNodeId = seedPeerNode();
const token = mintCallbackJwt(peerNodeId);
const ws = await new Promise<WebSocket>((resolve, reject) => {
const s = new WebSocket(`ws://127.0.0.1:${srv.port}/api/mesh/proxy-tunnel-from-peer`, {
headers: { authorization: `Bearer ${token}` },
});
s.once('open', () => resolve(s));
s.once('error', reject);
s.once('unexpected-response', (_req, res) => reject(new Error(`upgrade rejected ${res.statusCode}`)));
});
// Allow the handler's async bridge.start() + manager registration to land.
await new Promise((r) => setTimeout(r, 60));
const bridge = PilotTunnelManager.getInstance().getBridge(peerNodeId);
expect(bridge).not.toBeNull();
// The manager only narrows to MeshTunnelHandle in its public API,
// but the underlying object must be the central-side bridge class.
expect(bridge).toBeInstanceOf(PilotTunnelBridge);
try { ws.close(1000, 'test cleanup'); } catch { /* ignore */ }
await new Promise((r) => setTimeout(r, 30));
});
it('a successful dial-back persists the JWT scope on the bridge (mesh_tunnel scope)', async () => {
// Cross-check: the JWT carries scope=mesh_tunnel, the central
// ingress accepted it, so the bridge that lands in the manager is a
// proxy-mode bridge (not a pilot-agent tunnel). We assert via the
// dialer's hasBridge surface to confirm the proxy registry side
// sees nothing (the dial-back lives in the manager, not in
// MeshProxyTunnelDialer's outbound map).
const peerNodeId = seedPeerNode();
const token = mintCallbackJwt(peerNodeId);
const ws = await new Promise<WebSocket>((resolve, reject) => {
const s = new WebSocket(`ws://127.0.0.1:${srv.port}/api/mesh/proxy-tunnel-from-peer`, {
headers: { authorization: `Bearer ${token}` },
});
s.once('open', () => resolve(s));
s.once('error', reject);
s.once('unexpected-response', (_req, res) => reject(new Error(`upgrade rejected ${res.statusCode}`)));
});
await new Promise((r) => setTimeout(r, 60));
// The dialer is the central-initiated outbound side. A peer-initiated
// dial-back lands in PilotTunnelManager only; the dialer's own
// bridges map stays empty for this nodeId.
expect(MeshProxyTunnelDialer.getInstance().hasBridge(peerNodeId)).toBe(false);
expect(PilotTunnelManager.getInstance().getBridge(peerNodeId)).not.toBeNull();
try { ws.close(1000, 'test cleanup'); } catch { /* ignore */ }
await new Promise((r) => setTimeout(r, 30));
});
});
backend/src/__tests__/mesh-central-registry.test.ts
-122
View File
@@ -1,122 +0,0 @@
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import { DatabaseService } from '../services/DatabaseService';
import { MeshCentralRegistry } from '../services/MeshCentralRegistry';
let sharedTmpDir: string;
beforeAll(async () => { sharedTmpDir = await setupTestDb(); });
afterAll(() => cleanupTestDb(sharedTmpDir));
describe('mesh_centrals schema', () => {
it('creates the mesh_centrals table on initSchema', () => {
const db = DatabaseService.getInstance();
const row = db.getDb().prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='mesh_centrals'"
).get();
expect(row).toBeDefined();
});
it('mesh_centrals has the expected columns', () => {
const db = DatabaseService.getInstance();
const cols = db.getDb().prepare("PRAGMA table_info('mesh_centrals')").all() as Array<{ name: string }>;
const names = cols.map(c => c.name).sort();
expect(names).toEqual([
'callback_jwt',
'central_api_url',
'central_instance_id',
'jwt_expires_at',
'jwt_issued_at',
'last_bootstrap_at',
'last_reject_reason',
'last_rejected_at',
'last_used_at',
]);
});
});
describe('MeshCentralRegistry', () => {
beforeEach(() => {
MeshCentralRegistry.resetForTest();
// Per-test cleanup of the mesh_centrals table so tests are order-independent.
// The DB itself is created once per file in the file-scope beforeAll above;
// resetting setupTestDb per test would invalidate the DatabaseService
// singleton's open connection (SQLITE_READONLY_DBMOVED on Linux CI).
DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_centrals').run();
});
const sampleMaterial = (overrides: Partial<{
centralInstanceId: string;
centralApiUrl: string;
callbackJwt: string;
jwtIssuedAt: number;
jwtExpiresAt: number;
}> = {}) => ({
centralInstanceId: 'inst-uuid-1',
centralApiUrl: 'https://central.example.com',
callbackJwt: 'eyJhbGciOiJIUzI1NiJ9.fake.token',
jwtIssuedAt: 1_700_000_000,
jwtExpiresAt: 1_700_000_000 + 365 * 24 * 3600,
...overrides,
});
it('upsert inserts a new row when none exists', () => {
MeshCentralRegistry.getInstance().upsert(sampleMaterial());
const row = MeshCentralRegistry.getInstance().getActive();
expect(row?.centralInstanceId).toBe('inst-uuid-1');
expect(row?.centralApiUrl).toBe('https://central.example.com');
expect(row?.lastBootstrapAt).toBeGreaterThan(0);
});
it('upsert overwrites when same instance id is provided', () => {
const reg = MeshCentralRegistry.getInstance();
reg.upsert(sampleMaterial({ callbackJwt: 'old.jwt' }));
reg.upsert(sampleMaterial({ callbackJwt: 'new.jwt' }));
expect(reg.getActive()?.callbackJwt).toBe('new.jwt');
});
it('getActive returns most-recently-bootstrapped row when multiple exist and warns once', () => {
const reg = MeshCentralRegistry.getInstance();
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
reg.upsert(sampleMaterial({ centralInstanceId: 'old', jwtIssuedAt: 1 }));
DatabaseService.getInstance().getDb().prepare(`
INSERT INTO mesh_centrals VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, NULL)
`).run('new', 'https://other.example.com', 'jwt2', 2, 999, Date.now() + 1000);
expect(reg.getActive()?.centralInstanceId).toBe('new');
expect(warn).toHaveBeenCalledOnce();
expect(warn.mock.calls[0][0]).toMatch(/multiple central rows/);
reg.getActive();
expect(warn).toHaveBeenCalledOnce();
warn.mockRestore();
});
it('getActive returns null when table is empty', () => {
expect(MeshCentralRegistry.getInstance().getActive()).toBeNull();
});
it('clearForInstance removes only the named instance', () => {
const reg = MeshCentralRegistry.getInstance();
reg.upsert(sampleMaterial({ centralInstanceId: 'a' }));
reg.upsert(sampleMaterial({ centralInstanceId: 'b' }));
reg.clearForInstance('a');
const remaining = DatabaseService.getInstance().getDb().prepare(
"SELECT central_instance_id FROM mesh_centrals"
).all() as Array<{ central_instance_id: string }>;
expect(remaining.map(r => r.central_instance_id)).toEqual(['b']);
});
it('markUsed updates last_used_at, only on success', () => {
const reg = MeshCentralRegistry.getInstance();
reg.upsert(sampleMaterial());
reg.markUsed('inst-uuid-1');
expect(reg.getActive()?.lastUsedAt).toBeGreaterThan(0);
});
it('markRejected stores reason and timestamp', () => {
const reg = MeshCentralRegistry.getInstance();
reg.upsert(sampleMaterial());
reg.markRejected('inst-uuid-1', 'token_fingerprint_mismatch');
const row = reg.getActive();
expect(row?.lastRejectedAt).toBeGreaterThan(0);
expect(row?.lastRejectReason).toBe('token_fingerprint_mismatch');
});
});
backend/src/__tests__/mesh-instance-id-change-integration.test.ts
-84
View File
@@ -1,84 +0,0 @@
/**
* Central instance id rotation invariant.
*
* `MeshCentralRegistry.upsert` is the single source of truth for the
* locally-cached callback material. When central regenerates its
* `instance_id` (factory reset, DB swap, container rebuild without volume),
* the next bootstrap frame arrives with a different `centralInstanceId`. The
* registry MUST:
* - persist the new row,
* - emit a single `central-instance-changed` event so subscribers (loud
* log, dialer reset) can react.
*
* Failing this invariant means the peer would keep dialing back with the
* old JWT (rejected with `instance_mismatch`) without anyone noticing.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import { MeshCentralRegistry } from '../services/MeshCentralRegistry';
import { DatabaseService } from '../services/DatabaseService';
let tmpDir: string;
beforeAll(async () => { tmpDir = await setupTestDb(); });
afterAll(() => cleanupTestDb(tmpDir));
describe('Central instance id change', () => {
beforeEach(() => {
MeshCentralRegistry.resetForTest();
// Per-test cleanup of the mesh_centrals table. The DB lives once per
// file (beforeAll); resetting setupTestDb per test would invalidate
// the DatabaseService singleton's connection on Linux.
DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_centrals').run();
});
it('drops the old row and accepts the new one with a central-instance-changed event', () => {
const reg = MeshCentralRegistry.getInstance();
reg.upsert({
centralInstanceId: 'old-uuid',
centralApiUrl: 'https://central.example.com',
callbackJwt: 'jwt-old',
jwtIssuedAt: 1,
jwtExpiresAt: 999999,
});
const events: Array<{ previousInstanceId: string; newInstanceId: string }> = [];
reg.on('central-instance-changed', (e: { previousInstanceId: string; newInstanceId: string }) => events.push(e));
reg.upsert({
centralInstanceId: 'new-uuid',
centralApiUrl: 'https://central.example.com',
callbackJwt: 'jwt-new',
jwtIssuedAt: 2,
jwtExpiresAt: 999999,
});
expect(events).toHaveLength(1);
expect(events[0].previousInstanceId).toBe('old-uuid');
expect(events[0].newInstanceId).toBe('new-uuid');
// getActive returns the most-recently-bootstrapped row.
expect(reg.getActive()?.centralInstanceId).toBe('new-uuid');
expect(reg.getActive()?.callbackJwt).toBe('jwt-new');
});
it('an upsert with the same instance id does NOT emit central-instance-changed', () => {
const reg = MeshCentralRegistry.getInstance();
reg.upsert({
centralInstanceId: 'stable-uuid',
centralApiUrl: 'https://central.example.com',
callbackJwt: 'jwt-1',
jwtIssuedAt: 1,
jwtExpiresAt: 999999,
});
const events: Array<unknown> = [];
reg.on('central-instance-changed', (e: unknown) => events.push(e));
reg.upsert({
centralInstanceId: 'stable-uuid',
centralApiUrl: 'https://central.example.com',
callbackJwt: 'jwt-2-rotated',
jwtIssuedAt: 2,
jwtExpiresAt: 999999,
});
expect(events).toHaveLength(0);
// Same id, the JWT material is overwritten in place.
expect(reg.getActive()?.callbackJwt).toBe('jwt-2-rotated');
});
});
backend/src/__tests__/mesh-proxy-tunnel-dialer-handshake.test.ts
-164
View File
@@ -1,164 +0,0 @@
/**
* `MeshProxyTunnelDialer.maybeSendBootstrap`: capability-gated
* `mesh_handshake` send.
*
* Central mints a signed `mesh_tunnel`-scoped JWT bound to the peer's
* `api_token` fingerprint and pushes it as the first text frame on the
* fresh WS, but only when the peer advertises the
* `mesh_proxy_callback_bootstrap` capability AND a canonical central
* origin (`SENCHO_PRIMARY_URL`) is configured. All other paths must
* remain silent (no frame) so the legacy peer-side first-frame state
* machine can fall straight through to TCP traffic.
*/
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import jwt from 'jsonwebtoken';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import { MeshProxyTunnelDialer } from '../services/MeshProxyTunnelDialer';
import { NodeRegistry } from '../services/NodeRegistry';
import { OFFLINE_META } from '../services/CapabilityRegistry';
import { DatabaseService } from '../services/DatabaseService';
let tmpDir: string;
const CENTRAL_INSTANCE_ID = 'central-instance-for-handshake-test';
beforeAll(async () => {
tmpDir = await setupTestDb();
// Seed system_state.instance_id (LicenseService.initialize() is not
// invoked in the test harness; mesh handshake requires this value).
DatabaseService.getInstance().setSystemState('instance_id', CENTRAL_INSTANCE_ID);
});
afterAll(() => {
cleanupTestDb(tmpDir);
});
describe('MeshProxyTunnelDialer capability-gated handshake', () => {
let nodeSeq = 0;
beforeEach(() => {
MeshProxyTunnelDialer.resetForTest();
process.env.SENCHO_PRIMARY_URL = 'https://central.example.com';
nodeSeq += 1;
});
afterEach(() => {
delete process.env.SENCHO_PRIMARY_URL;
vi.restoreAllMocks();
});
function fakeWs(): { send: ReturnType<typeof vi.fn>; sentFrames: unknown[] } {
const sentFrames: unknown[] = [];
return {
send: vi.fn((data: string) => { sentFrames.push(JSON.parse(data)); }),
sentFrames,
};
}
function seedProxyNode(): number {
const db = DatabaseService.getInstance();
const id = db.addNode({
name: `peer-handshake-${nodeSeq}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
type: 'remote',
mode: 'proxy',
api_url: 'https://peer.example.com',
api_token: 'peer-token-for-fp',
compose_dir: '/tmp',
is_default: false,
});
db.setNodeMeshEnabled(id, true);
return id;
}
type DialerPrivate = { maybeSendBootstrap: (nodeId: number, ws: unknown) => Promise<void> };
it('sends mesh_handshake when peer advertises mesh_proxy_callback_bootstrap', async () => {
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue({
...OFFLINE_META,
version: '0.79.0',
capabilities: ['stacks', 'mesh_proxy_callback_bootstrap'],
online: true,
startedAt: Date.now(),
});
const id = seedProxyNode();
const ws = fakeWs();
await (MeshProxyTunnelDialer.getInstance() as unknown as DialerPrivate).maybeSendBootstrap(id, ws);
expect(ws.send).toHaveBeenCalledOnce();
expect(ws.sentFrames[0]).toMatchObject({ t: 'mesh_handshake', v: 1 });
});
it('skips mesh_handshake when peer lacks the capability', async () => {
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue({
...OFFLINE_META,
version: '0.78.0',
capabilities: ['stacks'],
online: true,
startedAt: Date.now(),
});
const id = seedProxyNode();
const ws = fakeWs();
await (MeshProxyTunnelDialer.getInstance() as unknown as DialerPrivate).maybeSendBootstrap(id, ws);
expect(ws.send).not.toHaveBeenCalled();
});
it('skips mesh_handshake when meta fetch returns offline shape', async () => {
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue({
...OFFLINE_META,
online: false,
});
const id = seedProxyNode();
const ws = fakeWs();
await (MeshProxyTunnelDialer.getInstance() as unknown as DialerPrivate).maybeSendBootstrap(id, ws);
expect(ws.send).not.toHaveBeenCalled();
});
it('skips mesh_handshake when SENCHO_PRIMARY_URL is unset (fail-safe)', async () => {
delete process.env.SENCHO_PRIMARY_URL;
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue({
...OFFLINE_META,
version: '0.79.0',
capabilities: ['mesh_proxy_callback_bootstrap'],
online: true,
startedAt: Date.now(),
});
const id = seedProxyNode();
const ws = fakeWs();
await (MeshProxyTunnelDialer.getInstance() as unknown as DialerPrivate).maybeSendBootstrap(id, ws);
expect(ws.send).not.toHaveBeenCalled();
});
it('signs the JWT with auth_jwt_secret using HS256 and includes the required claims', async () => {
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue({
...OFFLINE_META,
version: '0.79.0',
capabilities: ['mesh_proxy_callback_bootstrap'],
online: true,
startedAt: Date.now(),
});
const id = seedProxyNode();
const ws = fakeWs();
await (MeshProxyTunnelDialer.getInstance() as unknown as DialerPrivate).maybeSendBootstrap(id, ws);
const frame = ws.sentFrames[0] as {
meshTunnelJwt: string;
centralApiUrl: string;
centralInstanceId: string;
peerNodeId: number;
jwtExpiresAt: number;
};
expect(frame.centralApiUrl).toBe('https://central.example.com');
expect(frame.centralInstanceId).toBe(CENTRAL_INSTANCE_ID);
expect(frame.peerNodeId).toBe(id);
expect(typeof frame.jwtExpiresAt).toBe('number');
const decoded = jwt.decode(frame.meshTunnelJwt, { complete: true });
expect(decoded?.header.alg).toBe('HS256');
const payload = decoded?.payload as Record<string, unknown>;
expect(payload).toMatchObject({
sub: String(id),
iss: CENTRAL_INSTANCE_ID,
aud: 'https://central.example.com',
scope: 'mesh_tunnel',
kid: 'v1',
});
expect(typeof payload.peer_token_fp).toBe('string');
expect((payload.peer_token_fp as string).length).toBe(16);
});
});
backend/src/__tests__/mesh-proxy-tunnel-dialer-no-idle-close.test.ts
+61
View File
@@ -0,0 +1,61 @@
/**
* `MeshProxyTunnelDialer` default-TTL contract.
*
* The bridge is now a persistent bidirectional control-plane channel
* (peer→central reverse traffic multiplexes over the same WS that
* central uses to send forward `tcp_open` frames). The default
* `SENCHO_MESH_PROXY_TUNNEL_IDLE_MS` is `0`, which disables the idle
* sweeper entirely, so the dialer never tears the bridge down for
* inactivity. The env override is retained as an escape hatch for the
* legacy stream-scoped behavior.
*/
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { MeshProxyTunnelDialer } from '../services/MeshProxyTunnelDialer';
let prev: string | undefined;
beforeEach(() => {
prev = process.env.SENCHO_MESH_PROXY_TUNNEL_IDLE_MS;
delete process.env.SENCHO_MESH_PROXY_TUNNEL_IDLE_MS;
});
afterEach(() => {
if (prev === undefined) delete process.env.SENCHO_MESH_PROXY_TUNNEL_IDLE_MS;
else process.env.SENCHO_MESH_PROXY_TUNNEL_IDLE_MS = prev;
});
describe('MeshProxyTunnelDialer idle-close default', () => {
it('defaults to no idle close (TTL 0) and never starts the sweep timer', () => {
const dialer = MeshProxyTunnelDialer.resetForTest() as unknown as {
idleTtlMs: number;
idleCheckTimer: NodeJS.Timeout | null;
};
expect(dialer.idleTtlMs).toBe(0);
expect(dialer.idleCheckTimer).toBeNull();
});
it('respects an explicit env override to re-enable the legacy stream-scoped TTL', () => {
process.env.SENCHO_MESH_PROXY_TUNNEL_IDLE_MS = '300000';
const dialer = MeshProxyTunnelDialer.resetForTest() as unknown as {
idleTtlMs: number;
idleCheckTimer: NodeJS.Timeout | null;
};
expect(dialer.idleTtlMs).toBe(300_000);
expect(dialer.idleCheckTimer).not.toBeNull();
});
it('respects an explicit override of `0` (no idle close) without scheduling', () => {
process.env.SENCHO_MESH_PROXY_TUNNEL_IDLE_MS = '0';
const dialer = MeshProxyTunnelDialer.resetForTest() as unknown as {
idleTtlMs: number;
idleCheckTimer: NodeJS.Timeout | null;
};
expect(dialer.idleTtlMs).toBe(0);
expect(dialer.idleCheckTimer).toBeNull();
});
it('exposes isDialing(nodeId) which is false when no dial is in flight', () => {
const dialer = MeshProxyTunnelDialer.resetForTest();
expect(dialer.isDialing(42)).toBe(false);
});
});
backend/src/__tests__/mesh-proxy-tunnel-first-frame.test.ts
-227
View File
@@ -1,227 +0,0 @@
/**
* `meshProxyTunnel.ts` first-frame state machine.
*
* The PEER side of a proxy-mode mesh tunnel optionally consumes a
* `mesh_handshake` JSON frame as the FIRST text frame from central,
* persists the bootstrap material via MeshCentralRegistry, then yields
* control to the existing TcpStreamSwitchboard. Out-of-order or
* malformed handshake frames close the WS with code 1008.
*/
import http from 'http';
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import WebSocket from 'ws';
import type { AddressInfo } from 'net';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
let tmpDir: string;
let handleMeshProxyTunnel: typeof import('../websocket/meshProxyTunnel').handleMeshProxyTunnel;
let MeshService: typeof import('../services/MeshService').MeshService;
let MeshCentralRegistry: typeof import('../services/MeshCentralRegistry').MeshCentralRegistry;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
interface ServerHandle {
server: http.Server;
port: number;
close: () => Promise<void>;
}
async function startServer(): Promise<ServerHandle> {
const server = http.createServer();
server.on('upgrade', (req, socket, head) => {
const pathname = new URL(req.url ?? '/', 'http://localhost').pathname;
if (pathname === '/api/mesh/proxy-tunnel') {
void handleMeshProxyTunnel(req, socket, head);
} else {
socket.destroy();
}
});
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', () => resolve()));
const port = (server.address() as AddressInfo).port;
return {
server,
port,
close: () => new Promise<void>((resolve) => server.close(() => resolve())),
};
}
function dialTunnel(port: number, query: string = ''): Promise<WebSocket> {
return new Promise((resolve, reject) => {
const ws = new WebSocket(`ws://127.0.0.1:${port}/api/mesh/proxy-tunnel${query}`);
ws.once('open', () => resolve(ws));
ws.once('error', reject);
});
}
function makeHandshakeFrame(overrides: Partial<{
centralInstanceId: string;
centralApiUrl: string;
meshTunnelJwt: string;
jwtExpiresAt: number;
peerNodeId: number;
}> = {}): string {
return JSON.stringify({
t: 'mesh_handshake',
v: 1,
peerNodeId: overrides.peerNodeId ?? 7,
centralInstanceId: overrides.centralInstanceId ?? 'central-instance-abc',
centralApiUrl: overrides.centralApiUrl ?? 'https://central.example.test',
meshTunnelJwt: overrides.meshTunnelJwt ?? 'jwt.token.body',
jwtExpiresAt: overrides.jwtExpiresAt ?? Math.floor(Date.now() / 1000) + 3600,
});
}
beforeAll(async () => {
tmpDir = await setupTestDb();
({ handleMeshProxyTunnel } = await import('../websocket/meshProxyTunnel'));
({ MeshService } = await import('../services/MeshService'));
({ MeshCentralRegistry } = await import('../services/MeshCentralRegistry'));
({ DatabaseService } = await import('../services/DatabaseService'));
});
afterAll(() => {
cleanupTestDb(tmpDir);
});
beforeEach(() => {
delete process.env.SENCHO_MODE;
MeshService.getInstance().setReverseDialer(null);
MeshService.getInstance().setProxyTunnelSelfCentralNodeId(null);
MeshCentralRegistry.resetForTest();
DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_centrals').run();
});
describe('meshProxyTunnel first-frame state machine', () => {
it('consumes a mesh_handshake first frame and persists material via MeshCentralRegistry', async () => {
const srv = await startServer();
try {
const ws = await dialTunnel(srv.port, '?nodeId=7');
await new Promise((r) => setTimeout(r, 20));
const expiresAt = Math.floor(Date.now() / 1000) + 7200;
ws.send(makeHandshakeFrame({
centralInstanceId: 'central-instance-xyz',
centralApiUrl: 'https://central.example.test/',
meshTunnelJwt: 'callback.jwt.value',
jwtExpiresAt: expiresAt,
}));
await new Promise((r) => setTimeout(r, 30));
const row = MeshCentralRegistry.getInstance().getActive();
expect(row).not.toBeNull();
expect(row?.centralInstanceId).toBe('central-instance-xyz');
// Trailing slash should be stripped to keep the persisted URL canonical.
expect(row?.centralApiUrl).toBe('https://central.example.test');
expect(row?.callbackJwt).toBe('callback.jwt.value');
expect(row?.jwtExpiresAt).toBe(expiresAt);
ws.close(1000, 'test cleanup');
await new Promise((r) => setTimeout(r, 30));
} finally {
await srv.close();
}
});
it('passes through if first frame is not a mesh_handshake (delegates to switchboard)', async () => {
const srv = await startServer();
try {
const ws = await dialTunnel(srv.port, '?nodeId=7');
await new Promise((r) => setTimeout(r, 20));
// Send a known non-handshake JSON frame (tcp_close for a non-existent
// stream is a no-op the switchboard accepts without side effects).
ws.send(JSON.stringify({ t: 'tcp_close', s: 999999 }));
await new Promise((r) => setTimeout(r, 30));
// No handshake material should have been persisted.
expect(MeshCentralRegistry.getInstance().getActive()).toBeNull();
// WS should still be open (no protocol error close).
expect(ws.readyState).toBe(WebSocket.OPEN);
ws.close(1000, 'test cleanup');
await new Promise((r) => setTimeout(r, 30));
} finally {
await srv.close();
}
});
it('closes the WS as protocol error when mesh_handshake arrives after a non-handshake frame', async () => {
const srv = await startServer();
try {
const ws = await dialTunnel(srv.port, '?nodeId=7');
await new Promise((r) => setTimeout(r, 20));
// First a non-handshake frame, then the handshake.
ws.send(JSON.stringify({ t: 'tcp_close', s: 999999 }));
await new Promise((r) => setTimeout(r, 10));
const closeInfo = new Promise<{ code: number; reason: string }>((resolve) => {
ws.once('close', (code, reason) => resolve({ code, reason: reason.toString() }));
});
ws.send(makeHandshakeFrame());
const info = await closeInfo;
expect(info.code).toBe(1008);
expect(MeshCentralRegistry.getInstance().getActive()).toBeNull();
} finally {
await srv.close();
}
});
it('rejects malformed mesh_handshake (missing required fields) and closes WS as protocol error', async () => {
const srv = await startServer();
try {
const ws = await dialTunnel(srv.port, '?nodeId=7');
await new Promise((r) => setTimeout(r, 20));
const closeInfo = new Promise<{ code: number; reason: string }>((resolve) => {
ws.once('close', (code, reason) => resolve({ code, reason: reason.toString() }));
});
// Recognized 't' discriminator but missing required fields.
ws.send(JSON.stringify({ t: 'mesh_handshake', v: 1 }));
const info = await closeInfo;
expect(info.code).toBe(1008);
expect(MeshCentralRegistry.getInstance().getActive()).toBeNull();
} finally {
await srv.close();
}
});
it('persists mesh_handshake material even when reverse-dialer CAS rejects the bridge', async () => {
// Simulate a peer-initiated callback bridge that already owns the
// reverseDialer slot. Central's Trigger-2 dial then arrives with a
// rotated mesh_handshake frame; the CAS swap must refuse the new
// bridge, but the bootstrap material has to land on the peer
// anyway so subsequent peer-initiated callbacks authenticate with
// the rotated JWT.
MeshService.getInstance().setReverseDialer({
openMeshTcpStream: () => null,
});
const srv = await startServer();
try {
const ws = await dialTunnel(srv.port, '?nodeId=7');
const expiresAt = Math.floor(Date.now() / 1000) + 7200;
const closeInfo = new Promise<{ code: number }>((resolve) => {
ws.once('close', (code) => resolve({ code }));
});
ws.send(makeHandshakeFrame({
centralInstanceId: 'rotation-central-id',
centralApiUrl: 'https://central.example.test',
meshTunnelJwt: 'rotation.jwt.value',
jwtExpiresAt: expiresAt,
}));
const info = await closeInfo;
expect(info.code).toBe(1013);
const row = MeshCentralRegistry.getInstance().getActive();
expect(row).not.toBeNull();
expect(row?.centralInstanceId).toBe('rotation-central-id');
expect(row?.centralApiUrl).toBe('https://central.example.test');
expect(row?.callbackJwt).toBe('rotation.jwt.value');
expect(row?.jwtExpiresAt).toBe(expiresAt);
} finally {
await srv.close();
}
});
});
backend/src/__tests__/mesh-proxy-tunnel-from-peer.test.ts
-282
View File
@@ -1,282 +0,0 @@
/**
* `meshProxyTunnelFromPeer.ts`: central-side ingress for peer-initiated
* dial-back tunnels. Validates the chain of JWT claims and node-state
* preconditions that the peer's `mesh_tunnel` bootstrap token must satisfy
* before the upgrade succeeds and a proxy bridge is registered.
*
* The chain (in order): algorithm whitelist, signature, scope, audience
* (= SENCHO_PRIMARY_URL), issuer (= central instance_id), exp, iat clock
* skew (60s), node existence, mode==proxy, api_token fingerprint match.
* Every failure point returns HTTP 401 with a JSON `{reason}` body using
* a stable machine-readable code; the happy path constructs a
* `PilotTunnelBridge` and registers it via
* `PilotTunnelManager.replaceOrRegisterProxyBridge`.
*/
import http from 'http';
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import WebSocket from 'ws';
import jwt from 'jsonwebtoken';
import { createHash } from 'crypto';
import type { AddressInfo } from 'net';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
let tmpDir: string;
let handleMeshProxyTunnelFromPeerUpgrade: typeof import('../websocket/meshProxyTunnelFromPeer').handleMeshProxyTunnelFromPeerUpgrade;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let PilotTunnelManager: typeof import('../services/PilotTunnelManager').PilotTunnelManager;
interface ServerHandle {
server: http.Server;
port: number;
close: () => Promise<void>;
}
async function startServer(): Promise<ServerHandle> {
const server = http.createServer();
server.on('upgrade', (req, socket, head) => {
const pathname = new URL(req.url ?? '/', 'http://localhost').pathname;
if (pathname === '/api/mesh/proxy-tunnel-from-peer') {
handleMeshProxyTunnelFromPeerUpgrade(req, socket, head);
} else {
socket.destroy();
}
});
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', () => resolve()));
const port = (server.address() as AddressInfo).port;
return {
server,
port,
close: () => new Promise<void>((resolve) => server.close(() => resolve())),
};
}
interface UpgradeOutcome {
kind: 'open' | 'unexpected' | 'error';
status?: number;
body?: string;
ws?: WebSocket;
}
function attemptUpgrade(port: number, token: string): Promise<UpgradeOutcome> {
return new Promise((resolve) => {
const ws = new WebSocket(`ws://127.0.0.1:${port}/api/mesh/proxy-tunnel-from-peer`, {
headers: { authorization: `Bearer ${token}` },
});
const timer = setTimeout(() => resolve({ kind: 'error' }), 3000);
ws.once('open', () => {
clearTimeout(timer);
resolve({ kind: 'open', ws });
});
ws.once('unexpected-response', (_req, res) => {
const chunks: Buffer[] = [];
res.on('data', (c: Buffer) => chunks.push(c));
res.on('end', () => {
clearTimeout(timer);
resolve({ kind: 'unexpected', status: res.statusCode, body: Buffer.concat(chunks).toString('utf8') });
});
});
ws.once('error', () => {
// 'unexpected-response' fires first; 'error' here is the
// post-handshake failure ws raises after the server destroys
// the socket. Already resolved above.
});
});
}
function parseReason(body: string | undefined): string | null {
if (!body) return null;
try {
const parsed = JSON.parse(body) as { reason?: unknown };
return typeof parsed.reason === 'string' ? parsed.reason : null;
} catch {
return null;
}
}
const CANONICAL_ORIGIN = 'https://central.example.com';
const INSTANCE_ID = 'test-central-instance';
const PEER_API_TOKEN = 'peer-token-123';
let secret: string;
let peerNodeId: number;
let srv: ServerHandle;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ handleMeshProxyTunnelFromPeerUpgrade } = await import('../websocket/meshProxyTunnelFromPeer'));
({ DatabaseService } = await import('../services/DatabaseService'));
({ PilotTunnelManager } = await import('../services/PilotTunnelManager'));
});
beforeEach(async () => {
process.env.SENCHO_PRIMARY_URL = CANONICAL_ORIGIN;
const db = DatabaseService.getInstance();
secret = db.getGlobalSettings().auth_jwt_secret;
db.setSystemState('instance_id', INSTANCE_ID);
// Unique peer per test to avoid PilotTunnelManager collisions across
// the 12 cases (one happy path actually registers a bridge).
peerNodeId = db.addNode({
name: `peer-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
type: 'remote',
mode: 'proxy',
api_url: 'https://peer.example.com',
api_token: PEER_API_TOKEN,
compose_dir: '/tmp',
is_default: false,
});
db.setNodeMeshEnabled(peerNodeId, true);
if (!srv) srv = await startServer();
});
afterAll(async () => {
if (srv) await srv.close();
delete process.env.SENCHO_PRIMARY_URL;
cleanupTestDb(tmpDir);
});
function expectedFp(token: string = PEER_API_TOKEN): string {
return createHash('sha256').update(token).digest('hex').slice(0, 16);
}
function makeJwt(overrides: Partial<Record<string, unknown>> = {}, signOpts: { alg?: jwt.Algorithm; secret?: string } = {}): string {
const payload: Record<string, unknown> = {
sub: String(peerNodeId),
iss: INSTANCE_ID,
aud: CANONICAL_ORIGIN,
scope: 'mesh_tunnel',
iat: Math.floor(Date.now() / 1000),
exp: Math.floor(Date.now() / 1000) + 3600,
kid: 'v1',
peer_token_fp: expectedFp(),
...overrides,
};
return jwt.sign(payload, signOpts.secret ?? secret, { algorithm: signOpts.alg ?? 'HS256' });
}
describe('/api/mesh/proxy-tunnel-from-peer validation chain', () => {
it('rejects alg=none (algorithm_mismatch)', async () => {
// jsonwebtoken refuses to sign with alg=none unless explicitly
// enabled and given a null secret; we build the token manually so
// the test exercises the central's defence, not the library's.
const header = Buffer.from(JSON.stringify({ alg: 'none', typ: 'JWT' })).toString('base64url');
const body = Buffer.from(JSON.stringify({
sub: String(peerNodeId), iss: INSTANCE_ID, aud: CANONICAL_ORIGIN,
scope: 'mesh_tunnel', iat: Math.floor(Date.now() / 1000),
exp: Math.floor(Date.now() / 1000) + 3600, peer_token_fp: expectedFp(),
})).toString('base64url');
const token = `${header}.${body}.`;
const outcome = await attemptUpgrade(srv.port, token);
expect(outcome.kind).toBe('unexpected');
expect(outcome.status).toBe(401);
expect(parseReason(outcome.body)).toBe('algorithm_mismatch');
});
it('rejects alg=RS256 (algorithm_mismatch)', async () => {
const { generateKeyPairSync } = await import('crypto');
const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 });
const token = jwt.sign({
sub: String(peerNodeId), iss: INSTANCE_ID, aud: CANONICAL_ORIGIN,
scope: 'mesh_tunnel', exp: Math.floor(Date.now() / 1000) + 3600,
peer_token_fp: expectedFp(),
}, privateKey, { algorithm: 'RS256' });
const outcome = await attemptUpgrade(srv.port, token);
expect(outcome.kind).toBe('unexpected');
expect(outcome.status).toBe(401);
expect(parseReason(outcome.body)).toBe('algorithm_mismatch');
});
it('rejects bad signature (signature_invalid)', async () => {
const token = makeJwt({}, { secret: 'wrong-secret-not-the-real-one' });
const outcome = await attemptUpgrade(srv.port, token);
expect(outcome.kind).toBe('unexpected');
expect(outcome.status).toBe(401);
expect(parseReason(outcome.body)).toBe('signature_invalid');
});
it('rejects scope mismatch', async () => {
const token = makeJwt({ scope: 'pilot_tunnel' });
const outcome = await attemptUpgrade(srv.port, token);
expect(outcome.kind).toBe('unexpected');
expect(outcome.status).toBe(401);
expect(parseReason(outcome.body)).toBe('scope_mismatch');
});
it('rejects audience mismatch', async () => {
const token = makeJwt({ aud: 'https://other.example.com' });
const outcome = await attemptUpgrade(srv.port, token);
expect(outcome.kind).toBe('unexpected');
expect(outcome.status).toBe(401);
expect(parseReason(outcome.body)).toBe('audience_mismatch');
});
it('rejects issuer mismatch', async () => {
const token = makeJwt({ iss: 'wrong-instance-id' });
const outcome = await attemptUpgrade(srv.port, token);
expect(outcome.kind).toBe('unexpected');
expect(outcome.status).toBe(401);
expect(parseReason(outcome.body)).toBe('instance_mismatch');
});
it('rejects expired token (stale)', async () => {
const token = makeJwt({ exp: Math.floor(Date.now() / 1000) - 10 });
const outcome = await attemptUpgrade(srv.port, token);
expect(outcome.kind).toBe('unexpected');
expect(outcome.status).toBe(401);
// jsonwebtoken throws on expired tokens before our exp check sees it,
// so the rejection surfaces as signature_invalid via the verify catch.
// The contract: stale tokens are rejected with a 401 and some
// deterministic reason code; accept either of the two equivalent
// failures since both convey "stale credential" to the operator.
const reason = parseReason(outcome.body);
expect(['stale', 'signature_invalid']).toContain(reason);
});
it('rejects clock-skewed token (clock_skew)', async () => {
const token = makeJwt({ iat: Math.floor(Date.now() / 1000) + 600 });
const outcome = await attemptUpgrade(srv.port, token);
expect(outcome.kind).toBe('unexpected');
expect(outcome.status).toBe(401);
expect(parseReason(outcome.body)).toBe('clock_skew');
});
it('rejects missing node (node_deleted)', async () => {
const token = makeJwt({ sub: '999999' });
const outcome = await attemptUpgrade(srv.port, token);
expect(outcome.kind).toBe('unexpected');
expect(outcome.status).toBe(401);
expect(parseReason(outcome.body)).toBe('node_deleted');
});
it('rejects mode mismatch', async () => {
DatabaseService.getInstance().updateNode(peerNodeId, { mode: 'pilot_agent' });
const token = makeJwt();
const outcome = await attemptUpgrade(srv.port, token);
expect(outcome.kind).toBe('unexpected');
expect(outcome.status).toBe(401);
expect(parseReason(outcome.body)).toBe('mode_mismatch');
});
it('rejects token fingerprint mismatch', async () => {
const token = makeJwt();
// Rotate the api_token after minting; the JWT now carries the
// fingerprint of the old token, but getNode returns the new one.
DatabaseService.getInstance().updateNode(peerNodeId, { api_token: 'rotated-token-xyz' });
const outcome = await attemptUpgrade(srv.port, token);
expect(outcome.kind).toBe('unexpected');
expect(outcome.status).toBe(401);
expect(parseReason(outcome.body)).toBe('token_fingerprint_mismatch');
});
it('accepts a fully valid token and registers a proxy bridge', async () => {
const token = makeJwt();
const outcome = await attemptUpgrade(srv.port, token);
expect(outcome.kind).toBe('open');
// Give the bridge.start() microtasks a moment to land and register.
await new Promise((r) => setTimeout(r, 50));
const bridge = PilotTunnelManager.getInstance().getBridge(peerNodeId);
expect(bridge).not.toBeNull();
try { outcome.ws?.close(1000, 'test cleanup'); } catch { /* ignore */ }
// Allow the manager's 'closed' handler to remove the bridge entry.
await new Promise((r) => setTimeout(r, 30));
});
});
backend/src/__tests__/mesh-service-proactive-bootstrap-fanout.test.ts → backend/src/__tests__/mesh-service-proxy-tunnel-reconcile.test.ts
+37 -37
View File
@@ -1,15 +1,11 @@
/**
* `MeshService.proactiveBootstrapFanout` (Trigger 3).
* `MeshService.proactiveBridgeFanout` and the bridge-reconcile loop.
*
* At central startup, after `MeshService.start()` finishes, iterate the
* mesh-enabled proxy-mode nodes that have at least one `mesh_stacks` row
* and call `MeshProxyTunnelDialer.ensureBridge(nodeId)` on each. This
* proactively re-establishes the central->peer bridges so the
* capability-gated handshake can mint and ship bootstrap material to any
* peer that just came online or just got upgraded to v0.79+.
*
* Concurrency 4, 250ms stagger, fire-and-forget. Failures do not abort
* the fan-out.
* Central proactively dials every mesh-enabled proxy-mode peer at startup
* and on every reconcile tick so peer→central reverse traffic always has a
* live forward WS to multiplex through. The fanout selects ALL such peers
* regardless of whether they have any `mesh_stacks` rows yet, because the
* bridge is a control-plane primitive, not stack-scoped.
*/
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
@@ -61,7 +57,13 @@ function clearNodesAndMeshStacks(): void {
db.prepare('DELETE FROM nodes WHERE is_default = 0').run();
}
describe('MeshService.proactiveBootstrapFanout (Trigger 3)', () => {
function callFanout(): Promise<void> {
return (MeshService.getInstance() as unknown as {
proactiveBridgeFanout: () => Promise<void>;
}).proactiveBridgeFanout();
}
describe('MeshService.proactiveBridgeFanout', () => {
beforeEach(() => {
clearNodesAndMeshStacks();
MeshProxyTunnelDialer.resetForTest();
@@ -71,7 +73,7 @@ describe('MeshService.proactiveBootstrapFanout (Trigger 3)', () => {
vi.restoreAllMocks();
});
it('iterates only mesh-enabled proxy-mode nodes that have mesh_stacks rows', async () => {
it('iterates every mesh-enabled proxy-mode node regardless of mesh_stacks rows', async () => {
const calls: number[] = [];
vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge')
.mockImplementation(async (id: number) => {
@@ -79,22 +81,20 @@ describe('MeshService.proactiveBootstrapFanout (Trigger 3)', () => {
return null;
});
// Eligible: mesh-enabled proxy node with a mesh_stacks row.
const eligible = seedProxyNode(true);
insertMeshStack(eligible, `stack-eligible-${uniqueSuffix()}`);
const withStack = seedProxyNode(true);
insertMeshStack(withStack, `stack-${uniqueSuffix()}`);
// Ineligible: mesh-enabled proxy node WITHOUT a mesh_stacks row.
seedProxyNode(true);
// Mesh-enabled proxy node WITHOUT any mesh_stacks row. The old
// SQL excluded these; this assertion locks in the new behavior.
const withoutStack = seedProxyNode(true);
// Ineligible: mesh-disabled proxy node WITH a mesh_stacks row.
// Mesh-disabled proxy node WITH a mesh_stacks row. Must still be skipped.
const meshOff = seedProxyNode(false);
insertMeshStack(meshOff, `stack-meshoff-${uniqueSuffix()}`);
await (MeshService.getInstance() as unknown as {
proactiveBootstrapFanout: () => Promise<void>;
}).proactiveBootstrapFanout();
await callFanout();
expect(calls).toEqual([eligible]);
expect(calls.sort((a, b) => a - b)).toEqual([withStack, withoutStack].sort((a, b) => a - b));
});
it('throttles to concurrency 4', async () => {
@@ -109,14 +109,9 @@ describe('MeshService.proactiveBootstrapFanout (Trigger 3)', () => {
return null;
});
for (let i = 0; i < 12; i++) {
const id = seedProxyNode(true);
insertMeshStack(id, `stack-${i}-${uniqueSuffix()}`);
}
for (let i = 0; i < 12; i++) seedProxyNode(true);
await (MeshService.getInstance() as unknown as {
proactiveBootstrapFanout: () => Promise<void>;
}).proactiveBootstrapFanout();
await callFanout();
expect(maxInflight).toBeLessThanOrEqual(4);
expect(maxInflight).toBeGreaterThan(0);
@@ -124,11 +119,7 @@ describe('MeshService.proactiveBootstrapFanout (Trigger 3)', () => {
it('failures do not abort the fanout', async () => {
const ids: number[] = [];
for (let i = 0; i < 3; i++) {
const id = seedProxyNode(true);
insertMeshStack(id, `stack-fail-${i}-${uniqueSuffix()}`);
ids.push(id);
}
for (let i = 0; i < 3; i++) ids.push(seedProxyNode(true));
const failingId = ids[1];
const calls: number[] = [];
@@ -139,10 +130,19 @@ describe('MeshService.proactiveBootstrapFanout (Trigger 3)', () => {
return null;
});
await (MeshService.getInstance() as unknown as {
proactiveBootstrapFanout: () => Promise<void>;
}).proactiveBootstrapFanout();
await callFanout();
expect(calls.slice().sort((a, b) => a - b)).toEqual(ids.slice().sort((a, b) => a - b));
});
it('repeated fanout calls invoke ensureBridge each tick (reconcile semantics)', async () => {
const ensureSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge')
.mockResolvedValue(null);
const id = seedProxyNode(true);
await callFanout();
await callFanout();
expect(ensureSpy.mock.calls.filter(([n]) => n === id).length).toBe(2);
});
});
backend/src/__tests__/mesh-service.test.ts
+13 -69
View File
@@ -1021,10 +1021,9 @@ describe('MeshService.openCrossNode (BUG-4)', () => {
};
}
// Install a stub reverseDialer so openCrossNode skips the peer-side
// bootstrap path (PeerToCentralMeshSessionDialer.ensureSession). The
// dispatch behavior under test relies on dialMeshTcpStream being called
// directly; the bootstrap kick would short-circuit before that mock fires.
// Install a stub reverseDialer so openCrossNode routes through the
// multiplex path rather than depending on a live MeshProxyTunnelDialer
// bridge being open against a real peer.
const stubDialer = { openMeshTcpStream: vi.fn() };
beforeEach(() => { MeshService.getInstance().setReverseDialer(stubDialer); });
afterEach(() => { MeshService.getInstance().setReverseDialer(null); });
@@ -1092,16 +1091,13 @@ describe('MeshService.openCrossNode (BUG-4)', () => {
});
});
describe('MeshService.openCrossNode peer-recovery gating', () => {
// Regression cover for the v0.81.0 forward-direction break: on central
// (no mesh_centrals row, no reverseDialer because central is the bridge
// initiator side) the peer-recovery branch incorrectly fired, found no
// PeerToCentralMeshSessionDialer session, and destroyed the inbound
// socket with route.resolve.fail forward-from-peer no_session. The
// intent of the branch was peer cold-start; the guard
// `!this.reverseDialer` alone could not distinguish "I am central" from
// "I am a peer waiting for central to dial in", because both states
// present as reverseDialer===null.
describe('MeshService.openCrossNode without reverseDialer', () => {
// Forward + reverse mesh traffic now share the same central-initiated
// bridge. When openCrossNode runs without a reverseDialer installed it
// falls straight through to dialMeshTcpStream, which on central uses
// MeshProxyTunnelDialer.ensureBridge to reach a proxy peer. The
// previous peer-recovery branch (which depended on the now-removed
// mesh_centrals table) is gone.
function makeFakeStream(streamId: number): MeshTcpStreamLike & EventEmitter {
const ee = new EventEmitter() as MeshTcpStreamLike & EventEmitter & { destroyed: boolean };
ee.destroyed = false;
@@ -1115,17 +1111,11 @@ describe('MeshService.openCrossNode peer-recovery gating', () => {
return { destroy: vi.fn(), end: vi.fn(), on: vi.fn(), write: vi.fn() };
}
// No stub reverseDialer here: the whole point is to exercise the
// !reverseDialer code path. Reset the registry per test so the mesh_centrals
// row state is deterministic.
beforeEach(async () => {
const { MeshCentralRegistry } = await import('../services/MeshCentralRegistry');
MeshCentralRegistry.resetForTest();
DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_centrals').run();
beforeEach(() => {
MeshService.getInstance().setReverseDialer(null);
});
it('central (no mesh_centrals row) skips peer-recovery and calls dialMeshTcpStream', async () => {
it('central (no reverseDialer) falls through to dialMeshTcpStream without route.resolve.fail forward-from-peer', async () => {
const svc = MeshService.getInstance();
const target: MeshTarget = {
nodeId: 7, stack: 'audit-mesh-proxy', service: 'echo',
@@ -1142,11 +1132,7 @@ describe('MeshService.openCrossNode peer-recovery gating', () => {
.openCrossNode(target, fakeSrc);
const events = svc.getActivity({ limit: 50 });
const dispatch = events.find((e) => e.type === 'route.dispatch');
expect(dispatch).toBeDefined();
// The bug surface: a route.resolve.fail with direction=forward-from-peer
// would mean central wrongly went through the peer-recovery branch.
expect(events.some((e) => e.type === 'route.dispatch')).toBe(true);
const wrongFail = events.find((e) =>
e.type === 'route.resolve.fail'
&& (e.details as { direction?: string } | undefined)?.direction === 'forward-from-peer',
@@ -1156,48 +1142,6 @@ describe('MeshService.openCrossNode peer-recovery gating', () => {
expect(fakeSrc.destroy).not.toHaveBeenCalled();
fakeStream.emit('close');
});
it('proxy-peer (mesh_centrals row + no session) still emits route.resolve.fail forward-from-peer no_session', async () => {
const { MeshCentralRegistry } = await import('../services/MeshCentralRegistry');
const { PeerToCentralMeshSessionDialer } = await import('../services/PeerToCentralMeshSessionDialer');
MeshCentralRegistry.getInstance().upsert({
centralInstanceId: 'central-uuid-test',
centralApiUrl: 'https://central.example.com',
callbackJwt: 'eyJhbGciOiJIUzI1NiJ9.fake.token',
jwtIssuedAt: Math.floor(Date.now() / 1000),
jwtExpiresAt: Math.floor(Date.now() / 1000) + 90 * 24 * 3600,
});
const ensureSpy = vi.spyOn(
PeerToCentralMeshSessionDialer.getInstance(),
'ensureSession',
).mockResolvedValue(null);
const svc = MeshService.getInstance();
const target: MeshTarget = {
nodeId: 1, stack: 'audit-mesh-central', service: 'echo',
port: 9000, alias: 'echo.audit-mesh-central.Local.sencho',
};
const dialSpy = vi.spyOn(
svc as unknown as { dialMeshTcpStream: (t: MeshTarget) => MeshTcpStreamLike | null },
'dialMeshTcpStream',
);
const fakeSrc = makeFakeSocket();
await (svc as unknown as { openCrossNode: (t: MeshTarget, s: unknown) => Promise<void> })
.openCrossNode(target, fakeSrc);
const events = svc.getActivity({ limit: 50 });
const fail = events.find((e) =>
e.type === 'route.resolve.fail'
&& (e.details as { direction?: string; reason?: string } | undefined)?.direction === 'forward-from-peer'
&& (e.details as { reason?: string } | undefined)?.reason === 'no_session',
);
expect(fail).toBeDefined();
expect(ensureSpy).toHaveBeenCalled();
// Peer-recovery aborted dispatch before reaching dialMeshTcpStream.
expect(dialSpy).not.toHaveBeenCalled();
expect(fakeSrc.destroy).toHaveBeenCalled();
});
});
describe('MeshService pilot handleAccept dispatch', () => {
backend/src/__tests__/mesh-setup-error-classification.test.ts
+1 -13
View File
@@ -5,13 +5,11 @@ import type { MeshActivityEvent, MeshDataPlaneStatus } from '../services/MeshSer
let tmpDir: string;
let MeshService: typeof import('../services/MeshService').MeshService;
let DockerController: typeof import('../services/DockerController').default;
let capability: typeof import('../services/CapabilityRegistry');
beforeAll(async () => {
tmpDir = await setupTestDb();
({ MeshService } = await import('../services/MeshService'));
({ default: DockerController } = await import('../services/DockerController'));
capability = await import('../services/CapabilityRegistry');
});
afterAll(() => {
@@ -43,7 +41,6 @@ beforeEach(() => {
message: 'mesh data plane has not initialized yet',
subnet: '',
};
capability.enableCapability('mesh_proxy_callback_bootstrap');
});
afterEach(() => {
@@ -51,7 +48,6 @@ afterEach(() => {
else process.env.SENCHO_MESH_SUBNET = prevSubnetEnv;
if (prevHostnameEnv === undefined) delete process.env.HOSTNAME;
else process.env.HOSTNAME = prevHostnameEnv;
capability.enableCapability('mesh_proxy_callback_bootstrap');
vi.restoreAllMocks();
});
@@ -94,7 +90,6 @@ describe('MeshService.setupMeshNetwork failure classification', () => {
expect(status.ok).toBe(false);
expect(status.reason).toBe('subnet_invalid');
expect(status.subnet).toBe('not-a-cidr');
expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
const entry = lastDisable(svc);
expect(entry?.level).toBe('error');
expect(entry?.details?.reason).toBe('subnet_invalid');
@@ -114,7 +109,6 @@ describe('MeshService.setupMeshNetwork failure classification', () => {
expect(status.reason).toBe('subnet_overlap');
expect(status.subnet).toBe('10.42.0.0/24');
expect(status.message).toMatch(/overlap/i);
expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
const entry = lastDisable(svc);
expect(entry?.level).toBe('error');
expect(entry?.details?.reason).toBe('subnet_overlap');
@@ -169,9 +163,6 @@ describe('MeshService.setupMeshNetwork failure classification', () => {
const entry = lastDisable(svc);
expect(entry?.level).toBe('warn');
expect(entry?.details?.reason).toBe('not_in_docker');
// Capability is still stripped even in the warn-level not_in_docker
// case; pilot processes outside Docker must not advertise it.
expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
});
it('records the 404-on-inspect path as not_in_docker at level warn', async () => {
@@ -187,11 +178,9 @@ describe('MeshService.setupMeshNetwork failure classification', () => {
expect(lastDisable(svc)?.level).toBe('warn');
});
it('records success as ok and re-enables the capability', async () => {
it('records success as ok', async () => {
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
process.env.HOSTNAME = 'sencho';
// Force the capability off first to prove the success path flips it back on.
capability.disableCapability('mesh_proxy_callback_bootstrap');
mockDocker();
const svc = MeshService.getInstance();
await callSetup(svc);
@@ -200,7 +189,6 @@ describe('MeshService.setupMeshNetwork failure classification', () => {
expect(status.reason).toBe('ok');
expect(status.subnet).toBe('10.42.0.0/24');
expect(status.message).toBeNull();
expect(capability.getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap');
});
it('preserves the legacy networkSetupError getter on failure', async () => {
backend/src/__tests__/mesh-startup-primary-url-warning.test.ts
-73
View File
@@ -1,73 +0,0 @@
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import { MeshService } from '../services/MeshService';
import { DatabaseService } from '../services/DatabaseService';
function seedMeshedProxyNode(): void {
const db = DatabaseService.getInstance();
const id = db.addNode({
name: `p-${Date.now()}-${Math.random()}`,
type: 'remote',
mode: 'proxy',
api_url: 'https://p.example.com',
api_token: 't',
compose_dir: '/tmp',
is_default: false,
});
db.setNodeMeshEnabled(id, true);
}
let tmpDir: string;
beforeAll(async () => { tmpDir = await setupTestDb(); });
afterAll(() => cleanupTestDb(tmpDir));
describe('SENCHO_PRIMARY_URL preflight warning', () => {
let originalPrimaryUrl: string | undefined;
beforeEach(() => {
originalPrimaryUrl = process.env.SENCHO_PRIMARY_URL;
// The DB singleton persists across tests in the same process; wipe any
// non-default nodes seeded by a prior test so each case starts clean.
// setupTestDb itself runs once per file (beforeAll above) to avoid
// invalidating the DatabaseService singleton's open connection.
DatabaseService.getInstance().getDb().prepare('DELETE FROM nodes WHERE is_default = 0').run();
});
afterEach(() => {
if (originalPrimaryUrl === undefined) {
delete process.env.SENCHO_PRIMARY_URL;
} else {
process.env.SENCHO_PRIMARY_URL = originalPrimaryUrl;
}
});
it('warns when SENCHO_PRIMARY_URL is unset and a mesh-enabled proxy node exists', () => {
delete process.env.SENCHO_PRIMARY_URL;
seedMeshedProxyNode();
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
const svc = MeshService.getInstance() as unknown as { maybeWarnUnsetPrimaryUrl: () => void };
svc.maybeWarnUnsetPrimaryUrl();
expect(warn.mock.calls.some(c => /SENCHO_PRIMARY_URL is unset/.test(String(c[0])))).toBe(true);
warn.mockRestore();
});
it('does not warn when SENCHO_PRIMARY_URL is set', () => {
process.env.SENCHO_PRIMARY_URL = 'https://central.example.com';
seedMeshedProxyNode();
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
const svc = MeshService.getInstance() as unknown as { maybeWarnUnsetPrimaryUrl: () => void };
svc.maybeWarnUnsetPrimaryUrl();
expect(warn.mock.calls.some(c => /SENCHO_PRIMARY_URL is unset/.test(String(c[0])))).toBe(false);
warn.mockRestore();
});
it('does not warn when SENCHO_PRIMARY_URL is unset but no mesh-enabled proxy node exists', () => {
delete process.env.SENCHO_PRIMARY_URL;
// No seeding: zero mesh-enabled proxy nodes (only the default local node).
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
const svc = MeshService.getInstance() as unknown as { maybeWarnUnsetPrimaryUrl: () => void };
svc.maybeWarnUnsetPrimaryUrl();
expect(warn.mock.calls.some(c => /SENCHO_PRIMARY_URL is unset/.test(String(c[0])))).toBe(false);
warn.mockRestore();
});
});
backend/src/__tests__/mesh-status-reverse-callback.test.ts
+109
View File
@@ -0,0 +1,109 @@
/**
* `MeshService.getStatus().reverseCallbackStatus` discriminator.
*
* Surfaces the per-node state of the peer→central reverse path: forward
* bridge open (`connected`), dial in flight (`connecting`), no bridge and
* not dialing (`unavailable`), or non-proxy node (`not_applicable`). The
* Routing tab consumes this to render a transient pill while central
* reconciles a dropped bridge.
*/
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import { MeshService } from '../services/MeshService';
import { MeshProxyTunnelDialer } from '../services/MeshProxyTunnelDialer';
import { DatabaseService } from '../services/DatabaseService';
let tmpDir: string;
beforeAll(async () => { tmpDir = await setupTestDb(); });
afterAll(() => cleanupTestDb(tmpDir));
function uniqueSuffix(): string {
return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
}
function seedProxyNode(meshEnabled: boolean): number {
const db = DatabaseService.getInstance();
const id = db.addNode({
name: `peer-status-${uniqueSuffix()}`,
type: 'remote',
mode: 'proxy',
api_url: `https://peer-${uniqueSuffix()}.example.com`,
api_token: `tok-${uniqueSuffix()}`,
compose_dir: '/tmp',
is_default: false,
});
if (meshEnabled) db.setNodeMeshEnabled(id, true);
return id;
}
function seedPilotNode(): number {
const db = DatabaseService.getInstance();
return db.addNode({
name: `pilot-${uniqueSuffix()}`,
type: 'remote',
mode: 'pilot_agent',
api_url: `https://pilot-${uniqueSuffix()}.example.com`,
api_token: `tok-${uniqueSuffix()}`,
compose_dir: '/tmp',
is_default: false,
});
}
beforeEach(() => {
DatabaseService.getInstance().getDb().prepare('DELETE FROM nodes WHERE is_default = 0').run();
MeshProxyTunnelDialer.resetForTest();
vi.restoreAllMocks();
});
describe('MeshService.getStatus reverseCallbackStatus', () => {
it('returns connected when the dialer has a bridge for a mesh-enabled proxy peer', async () => {
const id = seedProxyNode(true);
vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'hasBridge')
.mockImplementation((n: number) => n === id);
const statuses = await MeshService.getInstance().getStatus();
const entry = statuses.find((s) => s.nodeId === id);
expect(entry?.reverseCallbackStatus).toBe('connected');
});
it('returns connecting when a dial is in flight for the peer', async () => {
const id = seedProxyNode(true);
vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'hasBridge').mockReturnValue(false);
vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'isDialing')
.mockImplementation((n: number) => n === id);
const statuses = await MeshService.getInstance().getStatus();
const entry = statuses.find((s) => s.nodeId === id);
expect(entry?.reverseCallbackStatus).toBe('connecting');
});
it('returns unavailable when no bridge and no dial in flight', async () => {
const id = seedProxyNode(true);
vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'hasBridge').mockReturnValue(false);
vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'isDialing').mockReturnValue(false);
const statuses = await MeshService.getInstance().getStatus();
const entry = statuses.find((s) => s.nodeId === id);
expect(entry?.reverseCallbackStatus).toBe('unavailable');
});
it('returns not_applicable for the local node', async () => {
const statuses = await MeshService.getInstance().getStatus();
const local = statuses.find((s) => s.nodeName === 'Local' || s.nodeId === 1);
expect(local?.reverseCallbackStatus).toBe('not_applicable');
});
it('returns not_applicable for pilot-mode peers', async () => {
const id = seedPilotNode();
const statuses = await MeshService.getInstance().getStatus();
const entry = statuses.find((s) => s.nodeId === id);
expect(entry?.reverseCallbackStatus).toBe('not_applicable');
});
it('returns not_applicable for mesh-disabled proxy peers', async () => {
const id = seedProxyNode(false);
const statuses = await MeshService.getInstance().getStatus();
const entry = statuses.find((s) => s.nodeId === id);
expect(entry?.reverseCallbackStatus).toBe('not_applicable');
});
});
backend/src/__tests__/mesh-token-rotation-integration.test.ts
-197
View File
@@ -1,197 +0,0 @@
/**
* Triggers 1 + 2: proactive mesh bridge bootstrap on mesh-enable and on
* api_token rotation.
*
* Trigger 1 (mesh-enable): after `setNodeMeshEnabled(true)`, the service
* fire-and-forgets `MeshProxyTunnelDialer.ensureBridge(nodeId)` for any
* remote/proxy peer. The capability-gated handshake then ships
* `mesh_handshake` material to peers that just came online or just got
* upgraded to a build that advertises `mesh_proxy_callback_bootstrap`.
*
* Trigger 2 (api_token rotation): when the nodes router persists a new
* `api_token` for a mesh-enabled proxy peer, it closes the existing bridge
* with reason 'peer token rotated' and re-dials. The next ensureBridge mints
* a JWT whose fingerprint matches the freshly stored token; without this
* trigger the remote would reject the upgrade with token_fingerprint_mismatch
* until the next idle re-dial.
*/
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import request from 'supertest';
import jwt from 'jsonwebtoken';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
import { MeshService } from '../services/MeshService';
import { MeshProxyTunnelDialer } from '../services/MeshProxyTunnelDialer';
import { DatabaseService } from '../services/DatabaseService';
let tmpDir: string;
let app: import('express').Express;
let authHeader: string;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ app } = await import('../index'));
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
authHeader = `Bearer ${token}`;
});
afterAll(() => {
cleanupTestDb(tmpDir);
});
function uniqueSuffix(): string {
return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
}
function seedProxyNode(meshEnabled: boolean): number {
const db = DatabaseService.getInstance();
const id = db.addNode({
name: `peer-rotate-${uniqueSuffix()}`,
type: 'remote',
mode: 'proxy',
api_url: `https://peer-${uniqueSuffix()}.example.com`,
api_token: `tok-${uniqueSuffix()}`,
compose_dir: '/tmp',
is_default: false,
});
if (meshEnabled) {
db.setNodeMeshEnabled(id, true);
}
return id;
}
function clearTestNodes(): void {
const db = DatabaseService.getInstance().getDb();
db.prepare('DELETE FROM mesh_stacks').run();
db.prepare('DELETE FROM nodes WHERE is_default = 0').run();
}
describe('Trigger 1: enableForNode triggers proactive bridge bootstrap', () => {
beforeEach(() => {
clearTestNodes();
MeshProxyTunnelDialer.resetForTest();
});
afterEach(() => {
vi.restoreAllMocks();
});
it('calls ensureBridge for a remote proxy-mode node', async () => {
const ensureSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge')
.mockResolvedValue(null);
const nodeId = seedProxyNode(false);
await MeshService.getInstance().enableForNode(nodeId);
// Trigger is fire-and-forget; wait a microtask for the void promise.
await new Promise((r) => setImmediate(r));
expect(ensureSpy).toHaveBeenCalledWith(nodeId);
});
it('skips local nodes', async () => {
const ensureSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge')
.mockResolvedValue(null);
const localId = DatabaseService.getInstance().getNodes()[0].id;
await MeshService.getInstance().enableForNode(localId);
await new Promise((r) => setImmediate(r));
expect(ensureSpy).not.toHaveBeenCalled();
});
it('swallows ensureBridge rejections without throwing', async () => {
vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge')
.mockRejectedValue(new Error('peer unreachable'));
const nodeId = seedProxyNode(false);
await expect(MeshService.getInstance().enableForNode(nodeId)).resolves.toBeUndefined();
await new Promise((r) => setImmediate(r));
});
});
describe('Trigger 2: api_token rotation forces re-bootstrap', () => {
beforeEach(() => {
clearTestNodes();
MeshProxyTunnelDialer.resetForTest();
});
afterEach(() => {
vi.restoreAllMocks();
});
it('calls closeBridge then ensureBridge when rotation handler runs', async () => {
const closeSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'closeBridge');
const ensureSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge')
.mockResolvedValue(null);
const nodeId = seedProxyNode(true);
const res = await request(app)
.put(`/api/nodes/${nodeId}`)
.set('Authorization', authHeader)
.send({ api_token: 'rotated-token-abc' });
expect(res.status).toBe(200);
await new Promise((r) => setImmediate(r));
expect(closeSpy).toHaveBeenCalledWith(nodeId, 'peer token rotated');
expect(ensureSpy).toHaveBeenCalledWith(nodeId);
const closeOrder = closeSpy.mock.invocationCallOrder[0];
const ensureOrder = ensureSpy.mock.invocationCallOrder[0];
expect(closeOrder).toBeLessThan(ensureOrder);
});
it('does not fire when api_token is not in the update payload', async () => {
const closeSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'closeBridge');
const ensureSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge')
.mockResolvedValue(null);
const nodeId = seedProxyNode(true);
const res = await request(app)
.put(`/api/nodes/${nodeId}`)
.set('Authorization', authHeader)
.send({ name: `renamed-${uniqueSuffix()}` });
expect(res.status).toBe(200);
await new Promise((r) => setImmediate(r));
expect(closeSpy).not.toHaveBeenCalled();
expect(ensureSpy).not.toHaveBeenCalled();
});
it('does not fire when mesh is disabled on the node', async () => {
const closeSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'closeBridge');
const ensureSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge')
.mockResolvedValue(null);
const nodeId = seedProxyNode(false);
const res = await request(app)
.put(`/api/nodes/${nodeId}`)
.set('Authorization', authHeader)
.send({ api_token: 'rotated-token-xyz' });
expect(res.status).toBe(200);
await new Promise((r) => setImmediate(r));
expect(closeSpy).not.toHaveBeenCalled();
expect(ensureSpy).not.toHaveBeenCalled();
});
it('does not fire when api_token in payload equals the current value', async () => {
const closeSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'closeBridge');
const ensureSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge')
.mockResolvedValue(null);
const nodeId = seedProxyNode(true);
const existingToken = DatabaseService.getInstance().getNode(nodeId)?.api_token;
expect(existingToken).toBeTruthy();
const res = await request(app)
.put(`/api/nodes/${nodeId}`)
.set('Authorization', authHeader)
.send({ name: `renamed-${uniqueSuffix()}`, api_token: existingToken });
expect(res.status).toBe(200);
await new Promise((r) => setImmediate(r));
expect(closeSpy).not.toHaveBeenCalled();
expect(ensureSpy).not.toHaveBeenCalled();
});
});
backend/src/__tests__/mesh-version-skew-integration.test.ts
-112
View File
@@ -1,112 +0,0 @@
/**
* Version skew invariant: central must NOT send a `mesh_handshake` to a peer
* that lacks the `mesh_proxy_callback_bootstrap` capability.
*
* The capability gate keeps central rolling forward without breaking older
* peers: a peer that does not understand the bootstrap frame would otherwise
* see a stray JSON text frame before the TCP frames it expects. The dialer's
* `maybeSendBootstrap` must read the capability list from the remote's meta
* endpoint and silently skip the send when the bit is absent.
*
* This is complementary to `mesh-proxy-tunnel-dialer-handshake.test.ts`,
* which exercises the same code path with a finer-grained set of inputs;
* this file locks in the same invariant as an integration so a future
* refactor that moves the capability check to a different layer does not
* silently regress.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import { NodeRegistry } from '../services/NodeRegistry';
import { MeshProxyTunnelDialer } from '../services/MeshProxyTunnelDialer';
import { OFFLINE_META } from '../services/CapabilityRegistry';
import { MeshCentralRegistry } from '../services/MeshCentralRegistry';
import { DatabaseService } from '../services/DatabaseService';
let tmpDir: string;
beforeAll(async () => { tmpDir = await setupTestDb(); });
afterAll(() => cleanupTestDb(tmpDir));
describe('Version skew: peer without mesh_proxy_callback_bootstrap', () => {
beforeEach(() => {
// Per-test reset of dialer + registry singletons + env. The DB lives
// once per file (beforeAll above); resetting setupTestDb per test
// would invalidate the DatabaseService singleton on Linux.
MeshProxyTunnelDialer.resetForTest();
MeshCentralRegistry.resetForTest();
process.env.SENCHO_PRIMARY_URL = 'https://central.example.com';
DatabaseService.getInstance().setSystemState('instance_id', 'test-central-instance');
});
afterEach(() => {
delete process.env.SENCHO_PRIMARY_URL;
vi.restoreAllMocks();
});
type DialerPrivate = {
maybeSendBootstrap: (nodeId: number, ws: unknown) => Promise<void>;
};
it('central does not send mesh_handshake when capability is absent', async () => {
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue({
...OFFLINE_META,
version: '0.78.0',
capabilities: ['stacks'],
online: true,
startedAt: Date.now(),
});
// Seed a proxy node so maybeSendBootstrap has something to look up.
const db = DatabaseService.getInstance();
const id = db.addNode({
name: `n-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
type: 'remote',
mode: 'proxy',
api_url: 'https://n',
api_token: 't',
compose_dir: '/tmp',
is_default: false,
});
db.setNodeMeshEnabled(id, true);
const sentFrames: unknown[] = [];
const fakeWs = { send: vi.fn((data: string) => sentFrames.push(JSON.parse(data))) };
await (MeshProxyTunnelDialer.getInstance() as unknown as DialerPrivate)
.maybeSendBootstrap(id, fakeWs);
expect(fakeWs.send).not.toHaveBeenCalled();
expect(sentFrames).toHaveLength(0);
// No persistence as a side effect either.
expect(MeshCentralRegistry.getInstance().getActive()).toBeNull();
});
it('central does send mesh_handshake when the capability flips to advertised', async () => {
// Sanity: the same code path with the capability present must take
// the affirmative branch, so we know the skip above is gated on the
// capability and not on some unrelated precondition.
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue({
...OFFLINE_META,
version: '0.79.0',
capabilities: ['stacks', 'mesh_proxy_callback_bootstrap'],
online: true,
startedAt: Date.now(),
});
const db = DatabaseService.getInstance();
const id = db.addNode({
name: `n-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
type: 'remote',
mode: 'proxy',
api_url: 'https://n',
api_token: 't',
compose_dir: '/tmp',
is_default: false,
});
db.setNodeMeshEnabled(id, true);
const sentFrames: unknown[] = [];
const fakeWs = { send: vi.fn((data: string) => sentFrames.push(JSON.parse(data))) };
await (MeshProxyTunnelDialer.getInstance() as unknown as DialerPrivate)
.maybeSendBootstrap(id, fakeWs);
expect(fakeWs.send).toHaveBeenCalledOnce();
expect(sentFrames[0]).toMatchObject({ t: 'mesh_handshake', v: 1 });
});
});
backend/src/__tests__/peer-to-central-mesh-session-dialer.test.ts
-275
View File
@@ -1,275 +0,0 @@
/**
* `PeerToCentralMeshSessionDialer`: peer-side counterpart to the central
* ingress at `/api/mesh/proxy-tunnel-from-peer` (Task 9). Reads cached
* central material from `MeshCentralRegistry`, opens a WebSocket to
* central carrying the bootstrapped JWT in the Authorization header, and
* branches on the upgrade outcome:
*
* - successful open: marks the cache row "used", arms the bridge.
* - 401 with a terminal reason code (stale, signature_invalid, ...):
* clears the cache so we stop dialing on a bad credential.
* - 401 with a transient reason (clock_skew, mode_mismatch, ...):
* keeps the cache, marks rejected so the operator can see why.
* - 404 endpoint_not_found: keeps the cache and arms a longer backoff
* so we do not hammer an older central that has not yet shipped the
* peer ingress endpoint.
*
* The dial logic is also rate-limited per dialer process (5 attempts /
* 60s window) so a misbehaving central or a misconfigured peer cannot
* flood either side with handshake traffic.
*/
import http from 'http';
import type { AddressInfo } from 'net';
import { WebSocketServer, type WebSocket as WsClient } from 'ws';
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
let tmpDir: string;
let PeerToCentralMeshSessionDialer: typeof import('../services/PeerToCentralMeshSessionDialer').PeerToCentralMeshSessionDialer;
let MeshCentralRegistry: typeof import('../services/MeshCentralRegistry').MeshCentralRegistry;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let TcpStreamSwitchboardCtor: typeof import('../mesh/tcpStreamSwitchboard').TcpStreamSwitchboard;
let MeshService: typeof import('../services/MeshService').MeshService;
interface RejectingServer {
server: http.Server;
url: string;
lastAuthHeader: string | null;
lastPath: string | null;
close: () => Promise<void>;
}
/**
* Spins up a minimal HTTP server that intercepts WebSocket upgrades and
* responds with a fixed status + JSON body, mimicking how the central
* ingress rejects bad credentials. Records the auth header and path so
* tests can assert the dialer is sending the right material.
*/
async function startRejectingServer(status: number, reason: string): Promise<RejectingServer> {
const handle: RejectingServer = {
server: http.createServer(),
url: '',
lastAuthHeader: null,
lastPath: null,
close: () => new Promise<void>((resolve) => handle.server.close(() => resolve())),
};
handle.server.on('upgrade', (req, socket) => {
handle.lastAuthHeader = (req.headers['authorization'] as string | undefined) ?? null;
handle.lastPath = req.url ?? null;
const body = JSON.stringify({ reason });
const statusText = status === 404 ? 'Not Found' : 'Unauthorized';
const head = [
`HTTP/1.1 ${status} ${statusText}`,
'Content-Type: application/json',
`Content-Length: ${Buffer.byteLength(body)}`,
'Connection: close',
'',
body,
].join('\r\n');
try { socket.write(head); } catch { /* ignore */ }
try { socket.destroy(); } catch { /* ignore */ }
});
await new Promise<void>((resolve) => handle.server.listen(0, '127.0.0.1', () => resolve()));
const port = (handle.server.address() as AddressInfo).port;
handle.url = `http://127.0.0.1:${port}`;
return handle;
}
beforeAll(async () => {
tmpDir = await setupTestDb();
({ PeerToCentralMeshSessionDialer } = await import('../services/PeerToCentralMeshSessionDialer'));
({ MeshCentralRegistry } = await import('../services/MeshCentralRegistry'));
({ DatabaseService } = await import('../services/DatabaseService'));
({ TcpStreamSwitchboard: TcpStreamSwitchboardCtor } = await import('../mesh/tcpStreamSwitchboard'));
({ MeshService } = await import('../services/MeshService'));
});
afterAll(() => {
cleanupTestDb(tmpDir);
});
beforeEach(() => {
MeshCentralRegistry.resetForTest();
PeerToCentralMeshSessionDialer.resetForTest();
DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_centrals').run();
});
afterEach(() => {
PeerToCentralMeshSessionDialer.resetForTest();
MeshCentralRegistry.resetForTest();
});
describe('PeerToCentralMeshSessionDialer', () => {
it('returns null when no central material is cached', async () => {
const result = await PeerToCentralMeshSessionDialer.getInstance().ensureSession();
expect(result).toBeNull();
});
it('dials using the cached URL + JWT bearer (asserted by server-side recording)', async () => {
const srv = await startRejectingServer(401, 'clock_skew');
try {
MeshCentralRegistry.getInstance().upsert({
centralInstanceId: 'inst-a',
centralApiUrl: srv.url,
callbackJwt: 'fake.jwt.token',
jwtIssuedAt: 1,
jwtExpiresAt: Math.floor(Date.now() / 1000) + 3600,
});
const result = await PeerToCentralMeshSessionDialer.getInstance().ensureSession();
expect(result).toBeNull();
expect(srv.lastPath).toBe('/api/mesh/proxy-tunnel-from-peer');
expect(srv.lastAuthHeader).toBe('Bearer fake.jwt.token');
} finally {
await srv.close();
}
});
it('clears cache on 401 reason=stale', async () => {
const srv = await startRejectingServer(401, 'stale');
try {
MeshCentralRegistry.getInstance().upsert({
centralInstanceId: 'inst-b',
centralApiUrl: srv.url,
callbackJwt: 'jwt',
jwtIssuedAt: 1,
jwtExpiresAt: Math.floor(Date.now() / 1000) + 3600,
});
await PeerToCentralMeshSessionDialer.getInstance().ensureSession();
expect(MeshCentralRegistry.getInstance().getActive()).toBeNull();
} finally {
await srv.close();
}
});
it('keeps cache on 401 reason=clock_skew but records the rejection', async () => {
const srv = await startRejectingServer(401, 'clock_skew');
try {
MeshCentralRegistry.getInstance().upsert({
centralInstanceId: 'inst-c',
centralApiUrl: srv.url,
callbackJwt: 'jwt',
jwtIssuedAt: 1,
jwtExpiresAt: Math.floor(Date.now() / 1000) + 3600,
});
await PeerToCentralMeshSessionDialer.getInstance().ensureSession();
const row = MeshCentralRegistry.getInstance().getActive();
expect(row).not.toBeNull();
expect(row?.lastRejectReason).toBe('clock_skew');
} finally {
await srv.close();
}
});
it('keeps cache on HTTP 404 endpoint_not_found and rate-limits future dials', async () => {
const srv = await startRejectingServer(404, 'whatever');
try {
MeshCentralRegistry.getInstance().upsert({
centralInstanceId: 'inst-d',
centralApiUrl: srv.url,
callbackJwt: 'jwt',
jwtIssuedAt: 1,
jwtExpiresAt: Math.floor(Date.now() / 1000) + 3600,
});
const dialer = PeerToCentralMeshSessionDialer.getInstance();
await dialer.ensureSession();
// Cache must survive a 404 (older central, transient infra).
expect(MeshCentralRegistry.getInstance().getActive()).not.toBeNull();
// The second call must not even hit the server: the endpoint
// backoff blocks new dials for the configured window.
srv.lastAuthHeader = null;
const second = await dialer.ensureSession();
expect(second).toBeNull();
expect(srv.lastAuthHeader).toBeNull();
} finally {
await srv.close();
}
});
it('rate-limits dials after the configured attempt cap', async () => {
const srv = await startRejectingServer(401, 'clock_skew');
try {
MeshCentralRegistry.getInstance().upsert({
centralInstanceId: 'inst-e',
centralApiUrl: srv.url,
callbackJwt: 'jwt',
jwtIssuedAt: 1,
jwtExpiresAt: Math.floor(Date.now() / 1000) + 3600,
});
const dialer = PeerToCentralMeshSessionDialer.getInstance();
// 5 dials should land; the 6th must be suppressed by the limiter.
for (let i = 0; i < 5; i++) {
await dialer.ensureSession();
}
srv.lastAuthHeader = null;
const sixth = await dialer.ensureSession();
expect(sixth).toBeNull();
expect(srv.lastAuthHeader).toBeNull();
} finally {
await srv.close();
}
});
it('marks the row used on successful WS open and installs a reverseDialer', async () => {
const wss = new WebSocketServer({ noServer: true });
const srv = http.createServer();
srv.on('upgrade', (req, socket, head) => {
if (req.url?.startsWith('/api/mesh/proxy-tunnel-from-peer')) {
wss.handleUpgrade(req, socket, head, () => { /* accept */ });
} else {
socket.destroy();
}
});
await new Promise<void>((resolve) => srv.listen(0, '127.0.0.1', () => resolve()));
const port = (srv.address() as AddressInfo).port;
const url = `http://127.0.0.1:${port}`;
// Start with a clean reverseDialer slot on the local MeshService so
// setReverseDialer's CAS swap succeeds inside attachSwitchboard.
MeshService.getInstance().setReverseDialer(null);
let switchboard: InstanceType<typeof TcpStreamSwitchboardCtor> | null = null;
try {
MeshCentralRegistry.getInstance().upsert({
centralInstanceId: 'inst-success',
centralApiUrl: url,
callbackJwt: 'fake.jwt',
jwtIssuedAt: 1,
jwtExpiresAt: 9999999999,
});
switchboard = await PeerToCentralMeshSessionDialer.getInstance().ensureSession();
expect(switchboard).not.toBeNull();
// The R1-A2 design puts a TcpStreamSwitchboard on the peer end of
// this WS, not a PilotTunnelBridge (which is central's role). A
// future regression that returns the wrong shape would surface
// here before the more abstract reverseDialer-installed check.
expect(switchboard).toBeInstanceOf(TcpStreamSwitchboardCtor);
expect(PeerToCentralMeshSessionDialer.getInstance().hasSession()).toBe(true);
// markUsed is called synchronously inside attachSwitchboard before
// ensureSession resolves, so the DB row reflects it immediately.
expect(MeshCentralRegistry.getInstance().getActive()?.lastUsedAt ?? 0).toBeGreaterThan(0);
// The R1-A2 wiring under test: MeshService.reverseDialer must be
// populated so MeshService.dialMeshTcpStream routes peer-side
// cross-fleet traffic through this callback bridge instead of
// falling through to PilotTunnelManager.ensureBridge(centralId),
// which has no record for central on a proxy peer.
const meshSvc = MeshService.getInstance() as unknown as { reverseDialer: unknown };
expect(meshSvc.reverseDialer).not.toBeNull();
} finally {
try { switchboard?.cleanup('test done'); } catch { /* ignore */ }
// The dialer owns the client-side WS and the switchboard doesn't
// close it on cleanup; tear it down explicitly so wss.close can
// resolve (it waits for all client connections to disconnect).
try {
const inst = PeerToCentralMeshSessionDialer.getInstance() as unknown as { currentWs: WsClient | null };
inst.currentWs?.close(1000, 'test done');
} catch { /* ignore */ }
MeshService.getInstance().setReverseDialer(null);
await new Promise<void>((resolve) => wss.close(() => resolve()));
await new Promise<void>((resolve) => srv.close(() => resolve()));
}
});
it('singleton returns the same instance', () => {
const a = PeerToCentralMeshSessionDialer.getInstance();
const b = PeerToCentralMeshSessionDialer.getInstance();
expect(a).toBe(b);
});
});
backend/src/__tests__/pilot-metrics.test.ts
+1 -6
View File
@@ -66,17 +66,12 @@ describe('PilotMetrics persistence', () => {
proxy_bridges_total: 7,
proxy_dials_failed: 5,
proxy_idle_closes: 2,
proxy_bridges_peer_initiated_total: 3,
mesh_central_bootstraps_total: 4,
mesh_callback_dials_failed_total: 0,
mesh_callback_auth_failures_total: 0,
});
PilotMetrics.load(db);
const snap = PilotMetrics.snapshot();
expect(snap.tunnels_total).toBe(11);
expect(snap.proxy_dials_failed).toBe(5);
expect(snap.proxy_bridges_total).toBe(7);
expect(snap.mesh_central_bootstraps_total).toBe(4);
});
it('threshold flush: increments past the threshold trigger a single persist', () => {
@@ -150,7 +145,7 @@ describe('PilotMetrics persistence', () => {
expect(snap.proxy_dials_failed).toBe(9);
expect(snap.proxy_bridges_total).toBe(0);
expect(snap.tunnels_total).toBe(0);
expect(snap.mesh_callback_auth_failures_total).toBe(0);
expect(snap.proxy_idle_closes).toBe(0);
});
it('stop() cancels a pending interval flush without persisting', () => {
backend/src/__tests__/pilot-mode-regression-integration.test.ts
-74
View File
@@ -1,74 +0,0 @@
/**
* Pilot-mode regression invariant: a pilot-agent tunnel always wins over a
* peer-initiated proxy-mode bridge for the same nodeId.
*
* The two registration paths share the `PilotTunnelManager.bridges` map.
* Without the `bridgeKinds` index, a peer-initiated dial-back could quietly
* replace a live pilot tunnel and break the agent's reverse-stream relay.
* `replaceOrRegisterProxyBridge` is the single point that has to refuse the
* replacement; this test locks that contract in.
*
* Mirrors the `injectBridgeForTest` pattern used elsewhere in the suite so
* we exercise the manager invariant without owning a real WebSocket.
*/
import { describe, it, expect, beforeEach, vi } from 'vitest';
import { EventEmitter } from 'events';
import { PilotTunnelManager } from '../services/PilotTunnelManager';
import type { PilotTunnelBridge } from '../services/PilotTunnelBridge';
function makeFakeBridge(): EventEmitter & {
close: ReturnType<typeof vi.fn>;
getActiveStreamCount: () => number;
} {
const ee = new EventEmitter() as EventEmitter & {
close: ReturnType<typeof vi.fn>;
getActiveStreamCount: () => number;
};
ee.close = vi.fn();
ee.getActiveStreamCount = () => 0;
return ee;
}
describe('Pilot-mode regression: pilot tunnel wins over peer-initiated proxy bridge', () => {
beforeEach(() => {
PilotTunnelManager.resetForTest();
});
it('refuses replaceOrRegisterProxyBridge when a pilot tunnel exists for the same nodeId', () => {
const mgr = PilotTunnelManager.getInstance();
const pilot = makeFakeBridge();
const proxy = makeFakeBridge();
mgr.injectBridgeForTest(7, pilot as unknown as PilotTunnelBridge, 'pilot');
expect(() => mgr.replaceOrRegisterProxyBridge(7, proxy as unknown as PilotTunnelBridge))
.toThrow(/pilot tunnel/);
// Pilot bridge is still the resident bridge for nodeId 7.
expect(mgr.getBridge(7)).toBe(pilot);
// The pilot bridge must not be closed by the rejected replacement.
expect(pilot.close).not.toHaveBeenCalled();
});
it('allows replaceOrRegisterProxyBridge to swap one proxy bridge for another', () => {
const mgr = PilotTunnelManager.getInstance();
const oldProxy = makeFakeBridge();
const newProxy = makeFakeBridge();
mgr.injectBridgeForTest(9, oldProxy as unknown as PilotTunnelBridge, 'proxy');
mgr.replaceOrRegisterProxyBridge(9, newProxy as unknown as PilotTunnelBridge);
// The new dial is the source of truth; old proxy is closed.
expect(oldProxy.close).toHaveBeenCalledOnce();
expect(mgr.getBridge(9)).toBe(newProxy);
});
it('replaceOrRegisterProxyBridge on an empty slot just registers (no refuse, no close)', () => {
const mgr = PilotTunnelManager.getInstance();
const proxy = makeFakeBridge();
mgr.replaceOrRegisterProxyBridge(11, proxy as unknown as PilotTunnelBridge);
expect(mgr.getBridge(11)).toBe(proxy);
expect(proxy.close).not.toHaveBeenCalled();
});
});
backend/src/__tests__/pilot-tunnel-manager-replace.test.ts
-54
View File
@@ -1,54 +0,0 @@
/**
* PilotTunnelManager.replaceOrRegisterProxyBridge: peer-initiated proxy
* bridges (Phase R1) need to be able to supersede a previous proxy bridge
* for the same nodeId, but must never shadow a live pilot tunnel (which
* always wins the bridge slot).
*/
import { describe, it, expect, beforeEach, vi } from 'vitest';
import { EventEmitter } from 'events';
import { PilotTunnelManager } from '../services/PilotTunnelManager';
import type { PilotTunnelBridge } from '../services/PilotTunnelBridge';
type FakeBridge = PilotTunnelBridge & {
close: ReturnType<typeof vi.fn<(code?: number, reason?: string) => void>>;
getActiveStreamCount: ReturnType<typeof vi.fn<() => number>>;
};
function makeFakeBridge(): FakeBridge {
const ee = new EventEmitter() as unknown as FakeBridge;
ee.close = vi.fn<(code?: number, reason?: string) => void>();
ee.getActiveStreamCount = vi.fn<() => number>().mockReturnValue(0);
return ee;
}
describe('PilotTunnelManager.replaceOrRegisterProxyBridge', () => {
beforeEach(() => PilotTunnelManager.resetForTest?.());
it('registers when no existing bridge', () => {
const mgr = PilotTunnelManager.getInstance();
const bridge = makeFakeBridge();
mgr.replaceOrRegisterProxyBridge(42, bridge);
expect(mgr.getBridge(42)).toBe(bridge);
});
it('replaces when existing bridge is a proxy bridge (closes the old one)', () => {
const mgr = PilotTunnelManager.getInstance();
const oldBridge = makeFakeBridge();
const newBridge = makeFakeBridge();
mgr.injectBridgeForTest(42, oldBridge, 'proxy');
mgr.replaceOrRegisterProxyBridge(42, newBridge);
expect(oldBridge.close).toHaveBeenCalledWith(1000, 'replaced-by-newer-proxy');
expect(mgr.getBridge(42)).toBe(newBridge);
});
it('throws when existing bridge is a pilot tunnel (preserves the pilot tunnel)', () => {
const mgr = PilotTunnelManager.getInstance();
const pilotBridge = makeFakeBridge();
const proxyBridge = makeFakeBridge();
mgr.injectBridgeForTest(42, pilotBridge, 'pilot');
expect(() => mgr.replaceOrRegisterProxyBridge(42, proxyBridge))
.toThrow(/pilot tunnel.*proxy bridge refused/);
expect(mgr.getBridge(42)).toBe(pilotBridge);
expect(pilotBridge.close).not.toHaveBeenCalled();
});
});
backend/src/__tests__/system-pilot-tunnels-central-callback.test.ts
-83
View File
@@ -1,83 +0,0 @@
/**
* Tests for the `centralCallback` diag block returned by
* GET /api/system/pilot-tunnels. The block reads from MeshCentralRegistry
* (cached central material + last-used/last-rejected timestamps) and from
* PeerToCentralMeshSessionDialer (live bridge presence). Counters for the
* peer-callback path live in PilotMetrics and are exposed via the existing
* `counters` field of the same response, so this block only adds the
* cached-row diagnostics that PilotMetrics does not track.
*/
import { describe, it, expect, beforeAll, beforeEach, afterAll } from 'vitest';
import request from 'supertest';
import type { Express } from 'express';
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
import { MeshCentralRegistry } from '../services/MeshCentralRegistry';
import { PeerToCentralMeshSessionDialer } from '../services/PeerToCentralMeshSessionDialer';
import { DatabaseService } from '../services/DatabaseService';
let tmpDir: string;
let app: Express;
let cookie: string;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ app } = await import('../index'));
cookie = await loginAsTestAdmin(app);
});
afterAll(() => cleanupTestDb(tmpDir));
beforeEach(() => {
MeshCentralRegistry.resetForTest();
PeerToCentralMeshSessionDialer.resetForTest();
DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_centrals').run();
});
describe('/api/system/pilot-tunnels centralCallback diag', () => {
it('returns null centralCallback fields when no central material is cached', async () => {
const res = await request(app).get('/api/system/pilot-tunnels').set('Cookie', cookie);
expect(res.status).toBe(200);
expect(res.body.centralCallback).toEqual({
bridgeOpen: false,
lastBootstrapAt: null,
lastDialOkAt: null,
lastDialFailAt: null,
lastDialFailReason: null,
});
});
it('returns populated centralCallback fields when material is present', async () => {
MeshCentralRegistry.getInstance().upsert({
centralInstanceId: 'inst-1',
centralApiUrl: 'https://central.example.com',
callbackJwt: 'jwt-value',
jwtIssuedAt: 1,
jwtExpiresAt: 9999999999,
});
MeshCentralRegistry.getInstance().markUsed('inst-1');
const res = await request(app).get('/api/system/pilot-tunnels').set('Cookie', cookie);
expect(res.status).toBe(200);
expect(res.body.centralCallback.bridgeOpen).toBe(false);
expect(res.body.centralCallback.lastBootstrapAt).toBeGreaterThan(0);
expect(res.body.centralCallback.lastDialOkAt).toBeGreaterThan(0);
expect(res.body.centralCallback.lastDialFailAt).toBeNull();
expect(res.body.centralCallback.lastDialFailReason).toBeNull();
});
it('surfaces last reject reason when central marks a dial failed', async () => {
MeshCentralRegistry.getInstance().upsert({
centralInstanceId: 'inst-2',
centralApiUrl: 'https://central.example.com',
callbackJwt: 'jwt-value',
jwtIssuedAt: 1,
jwtExpiresAt: 9999999999,
});
MeshCentralRegistry.getInstance().markRejected('inst-2', 'signature_invalid');
const res = await request(app).get('/api/system/pilot-tunnels').set('Cookie', cookie);
expect(res.status).toBe(200);
expect(res.body.centralCallback.lastDialFailAt).toBeGreaterThan(0);
expect(res.body.centralCallback.lastDialFailReason).toBe('signature_invalid');
});
});
backend/src/routes/metrics.ts
+1 -17
View File
@@ -6,8 +6,6 @@ import { DatabaseService } from '../services/DatabaseService';
import { CacheService } from '../services/CacheService';
import { NodeRegistry } from '../services/NodeRegistry';
import { PilotTunnelManager } from '../services/PilotTunnelManager';
import { MeshCentralRegistry } from '../services/MeshCentralRegistry';
import { PeerToCentralMeshSessionDialer } from '../services/PeerToCentralMeshSessionDialer';
import { authMiddleware } from '../middleware/auth';
import { requireAdmin } from '../middleware/tierGates';
import { STATS_CACHE_TTL_MS, SYSTEM_STATS_CACHE_TTL_MS } from '../helpers/constants';
@@ -263,21 +261,7 @@ metricsRouter.get('/system/pilot-tunnels', authMiddleware, async (req: Request,
if (!requireAdmin(req, res)) return;
try {
const snapshot = PilotTunnelManager.getInstance().getMetricsSnapshot();
// centralCallback exposes the peer-side view of the symmetric mesh
// callback path (cached central material plus the live bridge presence).
// PilotMetrics already tracks the attempt/success counters via
// `counters.mesh_central_bootstraps_total` and friends; this block adds
// the per-instance last-success / last-failure diagnostics that live in
// MeshCentralRegistry rather than PilotMetrics.
const cached = MeshCentralRegistry.getInstance().getActive();
const centralCallback = {
bridgeOpen: PeerToCentralMeshSessionDialer.getInstance().hasSession(),
lastBootstrapAt: cached?.lastBootstrapAt ?? null,
lastDialOkAt: cached?.lastUsedAt ?? null,
lastDialFailAt: cached?.lastRejectedAt ?? null,
lastDialFailReason: cached?.lastRejectReason ?? null,
};
res.json({ ...snapshot, centralCallback });
res.json(snapshot);
} catch (error) {
console.error('Failed to fetch pilot tunnel metrics:', error);
res.status(500).json({ error: 'Failed to fetch pilot tunnel metrics' });
backend/src/services/CapabilityRegistry.ts
-1
View File
@@ -34,7 +34,6 @@ export const CAPABILITIES = [
'registries',
'self-update',
'vulnerability-scanning',
'mesh_proxy_callback_bootstrap',
] as const;
export type Capability = (typeof CAPABILITIES)[number];
backend/src/services/DatabaseService.ts
+7 -17
View File
@@ -1489,23 +1489,13 @@ export class DatabaseService {
} catch (e) {
console.warn('[DatabaseService] mesh_stacks migration:', (e as Error).message);
}
try {
this.db.prepare(`
CREATE TABLE IF NOT EXISTS mesh_centrals (
central_instance_id TEXT PRIMARY KEY,
central_api_url TEXT NOT NULL,
callback_jwt TEXT NOT NULL,
jwt_issued_at INTEGER NOT NULL,
jwt_expires_at INTEGER NOT NULL,
last_bootstrap_at INTEGER NOT NULL,
last_used_at INTEGER,
last_rejected_at INTEGER,
last_reject_reason TEXT
)
`).run();
} catch (e) {
console.warn('[DatabaseService] Could not create mesh_centrals:', (e as Error).message);
}
// mesh_centrals was the peer-side cache of the reverse-callback JWT
// (central → peer bootstrap material). Peer→central traffic now
// multiplexes over the existing forward WS via `tcp_open_reverse`,
// so the table is no longer written or read. Drop it on every boot;
// idempotent.
try { this.db.prepare('DROP TABLE IF EXISTS mesh_centrals').run(); }
catch (e) { console.warn('[DatabaseService] Could not drop mesh_centrals:', (e as Error).message); }
this.tryAddColumn('nodes', 'mesh_enabled', 'INTEGER NOT NULL DEFAULT 0');
}
backend/src/services/MeshCentralRegistry.ts
-119
View File
@@ -1,119 +0,0 @@
import { EventEmitter } from 'events';
import { DatabaseService } from './DatabaseService';
export interface MeshCentralMaterial {
centralInstanceId: string;
centralApiUrl: string;
callbackJwt: string;
jwtIssuedAt: number;
jwtExpiresAt: number;
}
export interface MeshCentralRow extends MeshCentralMaterial {
lastBootstrapAt: number;
lastUsedAt: number | null;
lastRejectedAt: number | null;
lastRejectReason: string | null;
}
export class MeshCentralRegistry extends EventEmitter {
private static instance: MeshCentralRegistry | null = null;
private warnedMultiRow = false;
private constructor() { super(); }
public static getInstance(): MeshCentralRegistry {
if (!this.instance) this.instance = new MeshCentralRegistry();
return this.instance;
}
public static resetForTest(): void {
this.instance = null;
}
public upsert(material: MeshCentralMaterial): void {
const db = DatabaseService.getInstance().getDb();
const prior = this.getActive();
db.prepare(`
INSERT INTO mesh_centrals (
central_instance_id, central_api_url, callback_jwt,
jwt_issued_at, jwt_expires_at, last_bootstrap_at,
last_used_at, last_rejected_at, last_reject_reason
) VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, NULL)
ON CONFLICT(central_instance_id) DO UPDATE SET
central_api_url = excluded.central_api_url,
callback_jwt = excluded.callback_jwt,
jwt_issued_at = excluded.jwt_issued_at,
jwt_expires_at = excluded.jwt_expires_at,
last_bootstrap_at = excluded.last_bootstrap_at
`).run(
material.centralInstanceId,
material.centralApiUrl,
material.callbackJwt,
material.jwtIssuedAt,
material.jwtExpiresAt,
Date.now(),
);
const isInstanceChange = prior && prior.centralInstanceId !== material.centralInstanceId;
if (isInstanceChange) {
this.emit('central-instance-changed', {
previousInstanceId: prior!.centralInstanceId,
newInstanceId: material.centralInstanceId,
});
}
this.emit('central-bootstrap', { centralInstanceId: material.centralInstanceId });
}
public getActive(): MeshCentralRow | null {
const db = DatabaseService.getInstance().getDb();
const rows = db.prepare(`
SELECT * FROM mesh_centrals ORDER BY last_bootstrap_at DESC
`).all() as Array<{
central_instance_id: string;
central_api_url: string;
callback_jwt: string;
jwt_issued_at: number;
jwt_expires_at: number;
last_bootstrap_at: number;
last_used_at: number | null;
last_rejected_at: number | null;
last_reject_reason: string | null;
}>;
if (rows.length === 0) return null;
if (rows.length > 1 && !this.warnedMultiRow) {
console.warn(`[MeshCentralRegistry] multiple central rows present (${rows.length}), multi-central unsupported in v0.79`);
this.warnedMultiRow = true;
}
const r = rows[0];
return {
centralInstanceId: r.central_instance_id,
centralApiUrl: r.central_api_url,
callbackJwt: r.callback_jwt,
jwtIssuedAt: r.jwt_issued_at,
jwtExpiresAt: r.jwt_expires_at,
lastBootstrapAt: r.last_bootstrap_at,
lastUsedAt: r.last_used_at,
lastRejectedAt: r.last_rejected_at,
lastRejectReason: r.last_reject_reason,
};
}
public clearForInstance(centralInstanceId: string): void {
DatabaseService.getInstance().getDb()
.prepare(`DELETE FROM mesh_centrals WHERE central_instance_id = ?`)
.run(centralInstanceId);
}
public markUsed(centralInstanceId: string): void {
DatabaseService.getInstance().getDb()
.prepare(`UPDATE mesh_centrals SET last_used_at = ? WHERE central_instance_id = ?`)
.run(Date.now(), centralInstanceId);
}
public markRejected(centralInstanceId: string, reason: string): void {
DatabaseService.getInstance().getDb()
.prepare(`UPDATE mesh_centrals SET last_rejected_at = ?, last_reject_reason = ? WHERE central_instance_id = ?`)
.run(Date.now(), reason, centralInstanceId);
}
}
backend/src/services/MeshProxyTunnelDialer.ts
+11 -113
View File
@@ -1,12 +1,9 @@
import { EventEmitter } from 'events';
import { createHash } from 'crypto';
import jwt from 'jsonwebtoken';
import WebSocket from 'ws';
import { MAX_FRAME_SIZE_BYTES } from '../pilot/protocol';
import { PilotTunnelBridge, type MeshTunnelHandle } from './PilotTunnelBridge';
import { PilotTunnelManager } from './PilotTunnelManager';
import { NodeRegistry } from './NodeRegistry';
import { DatabaseService } from './DatabaseService';
import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog';
import { httpUrlToWs } from '../utils/wsUrl';
import { isDebugEnabled } from '../utils/debug';
@@ -29,15 +26,16 @@ import type { MeshActivityType } from './MeshService';
* Lifecycle:
* - `ensureBridge(nodeId)` dials if not connected; concurrent callers
* dedupe through an in-flight promise map.
* - A periodic check tears down bridges that have seen no active streams
* for `SENCHO_MESH_PROXY_TUNNEL_IDLE_MS` (default 5 minutes). Setting
* the env var to `0` disables idle close (tunnel persists until the
* remote drops it or central shuts down).
* - The bridge is a persistent bidirectional control-plane channel:
* central → peer `tcp_open` frames and peer → central `tcp_open_reverse`
* frames flow over the same WS. The default idle TTL is 0 (no idle
* close); `SENCHO_MESH_PROXY_TUNNEL_IDLE_MS` overrides the default for
* operators who still want stream-scoped tunnels.
* - Recent failures are cached for 60 seconds so a misconfigured remote
* does not cause a continuous redial storm; `MeshService.getStatus`
* consults the cache to surface a `reachableReason` to the UI.
*/
const DEFAULT_IDLE_TTL_MS = 5 * 60 * 1000;
const DEFAULT_IDLE_TTL_MS = 0;
const IDLE_CHECK_INTERVAL_MS = 60 * 1000;
const HANDSHAKE_TIMEOUT_MS = 15_000;
const FAILURE_CACHE_TTL_MS = 60 * 1000;
@@ -148,6 +146,11 @@ export class MeshProxyTunnelDialer extends EventEmitter {
return this.bridges.get(nodeId) ?? null;
}
/** True when a dial for this node is currently in flight. */
public isDialing(nodeId: number): boolean {
return this.inflight.has(nodeId);
}
public getRecentFailure(nodeId: number): DialFailure | null {
const entry = this.recentFailures.get(nodeId);
if (!entry) return null;
@@ -262,18 +265,6 @@ export class MeshProxyTunnelDialer extends EventEmitter {
return null;
}
// Capability-gated mesh_handshake: pushes a signed `mesh_tunnel` JWT
// and the central origin so the peer can dial central back for
// reverse cross-fleet TCP streams (R2). Skipped silently when the
// peer lacks the capability or `SENCHO_PRIMARY_URL` is unset so
// legacy peers keep their forward-only TCP path untouched.
await this.maybeSendBootstrap(nodeId, ws);
if (this.stopped) {
try { ws.close(1001, 'dialer shutdown'); } catch { /* ignore */ }
return null;
}
const bridge = new PilotTunnelBridge(nodeId, ws);
try {
await bridge.start();
@@ -451,47 +442,6 @@ export class MeshProxyTunnelDialer extends EventEmitter {
this.redialTimers.set(nodeId, timer);
}
/**
* If the peer advertises `mesh_proxy_callback_bootstrap` and central has
* a canonical origin (`SENCHO_PRIMARY_URL`), mint a `mesh_tunnel`-scoped
* JWT bound to the peer's `api_token` fingerprint and push it as the
* first text frame on the freshly-opened WS. The peer's
* `meshProxyTunnel` first-frame state machine persists the bootstrap
* material before yielding to TCP traffic. Every other path is a silent
* skip so legacy peers continue to receive raw TCP frames immediately.
*/
private async maybeSendBootstrap(nodeId: number, ws: WebSocket): Promise<void> {
const canonicalOrigin = (process.env.SENCHO_PRIMARY_URL ?? '').replace(/\/+$/, '');
if (!canonicalOrigin) return;
let meta;
try { meta = await NodeRegistry.getInstance().fetchMetaForNode(nodeId); }
catch { return; }
if (!meta?.online) return;
if (!meta.capabilities?.includes('mesh_proxy_callback_bootstrap')) return;
const db = DatabaseService.getInstance();
const node = db.getNode(nodeId);
if (!node || node.type !== 'remote' || node.mode !== 'proxy' || !node.api_token) return;
const authSecret = db.getGlobalSettings().auth_jwt_secret;
const centralInstanceId = db.getSystemState('instance_id');
if (!authSecret || !centralInstanceId) return;
const built = buildHandshakeFrame(nodeId, node.api_token, canonicalOrigin, centralInstanceId, authSecret);
try {
ws.send(JSON.stringify(built.frame));
void this.logActivity(nodeId, 'open.ok', {
bootstrapSent: true,
centralApiUrl: redactSensitiveText(canonicalOrigin),
kid: 'v1',
jwtExpiresAt: built.expSec,
});
} catch (err) {
console.warn(`[MeshProxyDialer] mesh_handshake send failed for node ${nodeId}: ${(err as Error).message}`.replace(/[\n\r]/g, ''));
}
}
private async logActivity(
nodeId: number,
event: ProxyTunnelEvent,
@@ -517,58 +467,6 @@ export class MeshProxyTunnelDialer extends EventEmitter {
}
}
interface HandshakeFrame {
t: 'mesh_handshake';
v: 1;
peerNodeId: number;
centralInstanceId: string;
centralApiUrl: string;
meshTunnelJwt: string;
jwtExpiresAt: number;
}
/**
* Build the mesh_handshake frame and the associated JWT. Pure helper so the
* dialer's `maybeSendBootstrap` stays under the 30-line ceiling.
*/
function buildHandshakeFrame(
nodeId: number,
apiToken: string,
canonicalOrigin: string,
centralInstanceId: string,
authSecret: string,
): { frame: HandshakeFrame; expSec: number } {
const peerTokenFp = createHash('sha256').update(apiToken).digest('hex').slice(0, 16);
const nowSec = Math.floor(Date.now() / 1000);
const expSec = nowSec + 365 * 24 * 3600;
const meshTunnelJwt = jwt.sign(
{
sub: String(nodeId),
iss: centralInstanceId,
aud: canonicalOrigin,
scope: 'mesh_tunnel',
iat: nowSec,
exp: expSec,
kid: 'v1',
peer_token_fp: peerTokenFp,
},
authSecret,
{ algorithm: 'HS256' },
);
return {
frame: {
t: 'mesh_handshake',
v: 1,
peerNodeId: nodeId,
centralInstanceId,
centralApiUrl: canonicalOrigin,
meshTunnelJwt,
jwtExpiresAt: expSec,
},
expSec,
};
}
function classifyDialError(err: unknown): { code: DialFailureCode; message: string } {
const message = sanitizeForLog((err as Error).message || String(err));
const httpStatus = (err as Error & { httpStatus?: number }).httpStatus;
backend/src/services/MeshService.ts
+107 -109
View File
@@ -4,7 +4,7 @@ import fs from 'fs/promises';
import { EventEmitter } from 'events';
import * as YAML from 'yaml';
import { ComposeService } from './ComposeService';
import { DatabaseService } from './DatabaseService';
import { DatabaseService, type NodeMode } from './DatabaseService';
import DockerController from './DockerController';
import { FileSystemService } from './FileSystemService';
import { LicenseService } from './LicenseService';
@@ -14,7 +14,6 @@ import { NodeRegistry } from './NodeRegistry';
import { PilotTunnelManager } from './PilotTunnelManager';
import { MeshProxyTunnelDialer, type DialFailureCode } from './MeshProxyTunnelDialer';
import { generateOverrideYaml, MeshAlias, SENCHO_MESH_NETWORK } from './MeshComposeOverride';
import { disableCapability, enableCapability } from './CapabilityRegistry';
import { lookupContainerIp } from '../mesh/containerLookup';
import { sanitizeForLog } from '../utils/safeLog';
import { isPathWithinBase, isValidStackName } from '../utils/validation';
@@ -97,7 +96,7 @@ export type MeshActivityType =
| 'forwarder.listen' | 'forwarder.unlisten' | 'forwarder.error'
| 'proxy-tunnel.open.ok' | 'proxy-tunnel.open.fail' | 'proxy-tunnel.close'
| 'mesh.proxy_tunnel.identify'
| 'mesh_handshake.received';
| 'mesh.reconcile.fail';
export interface MeshActivityEvent {
ts: number;
@@ -148,13 +147,28 @@ export interface MeshRegenSummary {
* - `pilot`: a remote with a pilot agent. Live-tunnel state is captured
* separately in `pilotConnected`.
* - `proxy`: a remote that central reaches via the long-lived api_token.
* Mesh tunnels are opened on demand by `MeshProxyTunnelDialer`; the
* operator sees no badge while the configuration is sound.
* Central maintains a persistent bidirectional WS to each mesh-enabled
* proxy peer, reconciled periodically; the operator sees no badge while
* the bridge is up.
* - `unreachable`: configuration or runtime problem keeps mesh traffic
* from flowing. `reachableReason` carries an actionable hint.
*/
export type MeshReachableMode = 'local' | 'pilot' | 'proxy' | 'unreachable';
/**
* State of the peer→central reverse path. The forward WS to a proxy-mode
* peer is bidirectional; peer→central traffic flows over the same WS via
* `tcp_open_reverse`. This discriminator surfaces whether that bridge is
* currently usable so the Routing tab can show a transient pill while the
* dialer is reconnecting.
* - `connected`: forward WS is open; peer can dispatch reverse streams.
* - `connecting`: dial in flight; transient.
* - `unavailable`: no bridge and no dial in flight (peer just rebooted, or
* last dial cached a failure).
* - `not_applicable`: not a proxy-mode peer, or mesh disabled on this node.
*/
export type MeshReverseCallbackStatus = 'connected' | 'connecting' | 'unavailable' | 'not_applicable';
export interface MeshNodeStatus {
nodeId: number;
nodeName: string;
@@ -174,6 +188,8 @@ export interface MeshNodeStatus {
reachableMode: MeshReachableMode;
/** Short, operator-facing reason when `reachableMode === 'unreachable'`. Null otherwise. */
reachableReason: string | null;
/** Peer→central reverse path state. `not_applicable` for non-proxy peers. */
reverseCallbackStatus: MeshReverseCallbackStatus;
optedInStacks: string[];
activeStreamCount: number;
}
@@ -245,6 +261,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
private activity: MeshActivityEvent[] = [];
private activeStreams = new Map<number, ActiveStreamRecord>();
private aliasRefreshTimer?: NodeJS.Timeout;
private bridgeReconcileTimer?: NodeJS.Timeout;
private routeErrorMap = new Map<string, { ts: number; message: string }>();
private routeLatencyMap = new Map<string, number>();
private activityListeners = new Set<(e: MeshActivityEvent) => void>();
@@ -335,8 +352,6 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
this.selfCentralNodeId = this.resolveSelfCentralNodeId();
this.maybeWarnUnsetPrimaryUrl();
await this.setupMeshNetwork();
try {
await this.refreshAliasCache();
@@ -355,10 +370,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
});
}
await this.regenerateAllOverrides();
// Trigger 3: proactively re-establish central->peer bridges so any
// peer that is online and capable receives bootstrap material on
// startup. Fire-and-forget so start() does not block on remote I/O.
void this.proactiveBootstrapFanout();
// Proactively dial every mesh-enabled proxy peer so the forward WS
// (which also carries peer→central reverse traffic) is up before any
// user request hits it. Fire-and-forget so start() does not block on
// remote I/O.
void this.proactiveBridgeFanout();
this.startBridgeReconcileLoop();
this.aliasRefreshTimer = setInterval(() => {
void (async () => {
try {
@@ -388,49 +405,25 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
clearInterval(this.aliasRefreshTimer);
this.aliasRefreshTimer = undefined;
}
this.stopBridgeReconcileLoop();
await this.forwarder.shutdown();
}
/**
* Operator-facing preflight: if `SENCHO_PRIMARY_URL` is unset at
* startup and the central has at least one mesh-enabled proxy-mode
* node, warn that the Task 8 capability-gated handshake will fall
* back to an inferred origin for the callback bootstrap. The matching
* fail-safe in `MeshProxyTunnelDialer` silently skips the bootstrap
* when the env is unset; this warn makes the consequence visible.
* Walk every mesh-enabled proxy-mode peer and call
* `MeshProxyTunnelDialer.ensureBridge(nodeId)` on each. The bridge is
* the persistent bidirectional control-plane channel: central→peer
* `tcp_open` and peer→central `tcp_open_reverse` both flow over the
* same WS. Bounded concurrency 4 with a 250 ms stagger; failures are
* logged and never abort the fan-out. Called both at startup and on
* every reconcile tick. `ensureBridge` short-circuits on already-open
* bridges, so steady-state cost is one Map lookup per peer.
*/
private maybeWarnUnsetPrimaryUrl(): void {
if (process.env.SENCHO_PRIMARY_URL) return;
const row = DatabaseService.getInstance().getDb().prepare(`
SELECT COUNT(*) as n FROM nodes
WHERE type='remote' AND mode='proxy' AND mesh_enabled=1
`).get() as { n: number };
if (row.n > 0) {
console.warn(
'[Mesh] SENCHO_PRIMARY_URL is unset; mesh callback bootstrap will use ' +
'inferred origin. Set SENCHO_PRIMARY_URL to make peer-initiated mesh ' +
'callbacks robust against reverse-proxy edge cases.'
);
}
}
/**
* Trigger 3: at central startup, walk every mesh-enabled proxy-mode
* peer that already has at least one `mesh_stacks` row and call
* `MeshProxyTunnelDialer.ensureBridge(nodeId)`. The dialer's
* capability-gated handshake then mints and ships bootstrap material
* to peers that just came online or just got upgraded to a build that
* advertises `mesh_proxy_callback_bootstrap`. Bounded concurrency 4
* with a 250ms stagger; failures are logged and never abort the
* fan-out.
*/
private async proactiveBootstrapFanout(): Promise<void> {
private async proactiveBridgeFanout(): Promise<void> {
const rows = DatabaseService.getInstance().getDb().prepare(`
SELECT DISTINCT n.id AS id
FROM nodes n
INNER JOIN mesh_stacks ms ON ms.node_id = n.id
WHERE n.type = 'remote' AND n.mode = 'proxy' AND n.mesh_enabled = 1
ORDER BY n.id
SELECT id FROM nodes
WHERE type = 'remote' AND mode = 'proxy' AND mesh_enabled = 1
ORDER BY id
`).all() as Array<{ id: number }>;
const queue = rows.map((r) => r.id);
@@ -445,8 +438,8 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
this.logActivity({
source: 'mesh', level: 'warn', type: 'proxy-tunnel.open.fail',
nodeId,
message: `boot proactive bootstrap failed: ${sanitizeForLog((err as Error).message)}`,
details: { trigger: 'startup_fanout' },
message: `proxy-tunnel reconcile dial failed: ${sanitizeForLog((err as Error).message)}`,
details: { trigger: 'reconcile' },
});
}
await new Promise((r) => setTimeout(r, 250));
@@ -456,6 +449,35 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
await Promise.all(Array.from({ length: workerCount }, () => worker()));
}
/**
* Schedule the proxy-tunnel reconcile tick. Interval is overridable via
* `SENCHO_MESH_RECONCILE_INTERVAL_MS` (default 60_000 ms) so an operator
* can tune the peer-reboot detection window. Idempotent: a second call
* is a no-op while the timer is live.
*/
private startBridgeReconcileLoop(): void {
if (this.bridgeReconcileTimer) return;
const raw = process.env.SENCHO_MESH_RECONCILE_INTERVAL_MS;
const parsed = raw === undefined ? Number.NaN : Number(raw);
const intervalMs = Number.isFinite(parsed) && parsed >= 1000 ? parsed : 60_000;
this.bridgeReconcileTimer = setInterval(() => {
void this.proactiveBridgeFanout().catch((err) => {
this.logActivity({
source: 'mesh', level: 'error', type: 'mesh.reconcile.fail',
message: `bridge reconcile threw: ${sanitizeForLog((err as Error).message)}`,
});
});
}, intervalMs);
this.bridgeReconcileTimer.unref?.();
}
private stopBridgeReconcileLoop(): void {
if (this.bridgeReconcileTimer) {
clearInterval(this.bridgeReconcileTimer);
this.bridgeReconcileTimer = undefined;
}
}
public getSenchoIp(): string | null {
return this.senchoIp;
}
@@ -481,7 +503,6 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
/**
* Single recording path for every mesh-setup failure. Keeps the legacy
* `networkSetupError` string in sync, sets the typed `dataPlaneStatus`,
* strips `mesh_proxy_callback_bootstrap` from advertised capabilities,
* and emits a `mesh.disable` activity entry. Callers pass `level: 'warn'`
* for expected conditions (`not_in_docker` in dev mode) and `'error'` for
* real failures.
@@ -500,7 +521,6 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
this.networkSetupError = message;
this.dataPlaneStatus = { ok: false, reason, message, subnet };
this.senchoIp = null;
disableCapability('mesh_proxy_callback_bootstrap');
this.logActivity({
source: 'mesh',
level,
@@ -580,7 +600,6 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
this.networkSetupError = null;
this.dataPlaneStatus = { ok: true, reason: 'ok', message: null, subnet };
enableCapability('mesh_proxy_callback_bootstrap');
}
/**
@@ -856,15 +875,14 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
source: 'mesh', level: 'info', type: 'mesh.enable',
nodeId, message: `mesh enabled on node ${nodeId}`,
});
// Trigger 1: proactive bootstrap. If the peer is a proxy-mode remote,
// dial the mesh callback bridge immediately so the capability-gated
// handshake can ship `mesh_handshake` material without waiting for
// the next forwarder dial. Fire-and-forget; failures are logged by
// the dialer.
// When mesh is enabled on a proxy peer, dial the persistent bridge
// immediately so the next forward (or peer-initiated reverse)
// request has the WS already up. Fire-and-forget; failures are
// logged by the dialer.
const node = DatabaseService.getInstance().getNode(nodeId);
if (node && node.type === 'remote' && node.mode === 'proxy') {
void MeshProxyTunnelDialer.getInstance().ensureBridge(nodeId).catch((err) => {
console.warn(`[Mesh] proactive bootstrap on mesh-enable failed for node ${nodeId}: ${(err as Error).message}`);
console.warn(`[Mesh] proxy-tunnel dial on mesh-enable failed for node ${nodeId}: ${(err as Error).message}`);
});
}
}
@@ -1812,59 +1830,23 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
message: `cross-node dispatch to ${target.alias} on node ${target.nodeId}`,
});
// Peer-side recovery: when this Sencho is acting as a proxy-mode peer
// (mesh_centrals has a row from a prior bootstrap) and the
// reverseDialer is not currently installed, the inbound bridge from
// central is either cold-start (peer just rebooted) or has been torn
// down (idle close, central restart). Kick the symmetric-dial path
// so the peer re-opens its callback WS to central before attempting
// the cross-node dispatch.
//
// Central instances never have a mesh_centrals row (central is not a
// peer of itself), so this branch is correctly skipped on central.
// Central falls straight through to dialMeshTcpStream which uses its
// own PilotTunnelManager + MeshProxyTunnelDialer to reach the target
// peer. Without this gate, central enters the branch on every
// forward dispatch, finds no session, and destroys the inbound
// socket with route.resolve.fail forward-from-peer no_session.
if (!this.reverseDialer) {
const { MeshCentralRegistry } = await import('./MeshCentralRegistry');
const isProxyPeer = MeshCentralRegistry.getInstance().getActive() !== null;
if (isProxyPeer) {
try {
const { PeerToCentralMeshSessionDialer } = await import('./PeerToCentralMeshSessionDialer');
const session = await PeerToCentralMeshSessionDialer.getInstance().ensureSession();
if (!session) {
this.logActivity({
source: 'mesh', level: 'warn', type: 'route.resolve.fail',
nodeId: target.nodeId, alias: target.alias,
message: `peer cross-node dispatch failed: no central callback session available`,
details: { direction: 'forward-from-peer', reason: 'no_session' },
});
try { src.destroy(); } catch { /* ignore */ }
return;
}
} catch (err) {
this.logActivity({
source: 'mesh', level: 'warn', type: 'route.resolve.fail',
nodeId: target.nodeId, alias: target.alias,
message: `peer cross-node dispatch failed: bootstrap threw`,
details: { direction: 'forward-from-peer', reason: 'bootstrap_threw' },
});
try { src.destroy(); } catch { /* ignore */ }
return;
}
}
}
const tcpStream = await this.dialMeshTcpStream(target);
if (!tcpStream) {
// Three failure shapes on this path:
// - central with reverseDialer somehow installed (unusual; relay edge case)
// - central without reverseDialer (normal): ensureBridge could not reach the target peer
// - peer side: target nodeId is central's id, which the peer's NodeRegistry does
// not know as a remote proxy target, so ensureBridge returns null. The actionable
// condition is "central has not dialed the bridge yet"; tell the operator.
const targetIsLocalKnownRemote = NodeRegistry.getInstance().getNode(target.nodeId)?.type === 'remote';
const message = this.reverseDialer
? `cannot open reverse mesh stream to node ${target.nodeId}`
: targetIsLocalKnownRemote
? `no mesh tunnel reachable for node ${target.nodeId}`
: `peer cross-node dispatch deferred: waiting for central to dial the reverse bridge`;
this.logActivity({
source: 'pilot', level: 'error', type: 'tunnel.fail',
nodeId: target.nodeId, alias: target.alias,
message: this.reverseDialer
? `cannot open reverse mesh stream to node ${target.nodeId}`
: `no mesh tunnel reachable for node ${target.nodeId}`,
nodeId: target.nodeId, alias: target.alias, message,
});
try { src.destroy(); } catch { /* ignore */ }
return;
@@ -2148,12 +2130,14 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
const localListening = this.isLocalForwarderActive();
const ptm = PilotTunnelManager.getInstance();
const dialer = MeshProxyTunnelDialer.getInstance();
return nodes.map((node) => {
const reach = this.computeReachable(node, localNodeId);
const meshEnabled = db.getNodeMeshEnabled(node.id);
return {
nodeId: node.id,
nodeName: node.name,
enabled: db.getNodeMeshEnabled(node.id),
enabled: meshEnabled,
localForwarderListening: node.id === localNodeId ? localListening : null,
// `pilotConnected` stays at its original meaning: a pilot
// tunnel is currently registered for this node. The
@@ -2163,12 +2147,26 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
pilotConnected: node.type !== 'remote' || ptm.hasActiveTunnel(node.id),
reachableMode: reach.mode,
reachableReason: reach.reason,
reverseCallbackStatus: this.computeReverseCallbackStatus(node, meshEnabled, dialer),
optedInStacks: db.listMeshStacks(node.id).map((s) => s.stack_name),
activeStreamCount: this.activeStreams.size,
};
});
}
private computeReverseCallbackStatus(
node: { id: number; type: 'local' | 'remote'; mode: NodeMode },
meshEnabled: boolean,
dialer: MeshProxyTunnelDialer,
): MeshReverseCallbackStatus {
if (node.type !== 'remote' || node.mode !== 'proxy' || !meshEnabled) {
return 'not_applicable';
}
if (dialer.hasBridge(node.id)) return 'connected';
if (dialer.isDialing(node.id)) return 'connecting';
return 'unavailable';
}
/** True when the local Sencho's forwarder is started and bound to at least one alias port. */
private isLocalForwarderActive(): boolean {
return this.started && this.forwarder.getListenerPorts().length > 0;
backend/src/services/PeerToCentralMeshSessionDialer.ts
-276
View File
@@ -1,276 +0,0 @@
/**
* PeerToCentralMeshSessionDialer: peer-side counterpart to the central
* ingress at `/api/mesh/proxy-tunnel-from-peer`. Reads cached central
* material from `MeshCentralRegistry`, opens a WebSocket to central
* carrying the bootstrapped JWT in the Authorization header, and on a
* successful upgrade installs a local `PilotTunnelBridge` whose loopback
* URL the local proxy can target as if central had dialed us.
*
* Failure handling branches on the central's machine-readable reason
* code returned in the 401 body: terminal codes (stale,
* signature_invalid, fingerprint, node_deleted, instance_mismatch,
* audience_mismatch) clear the cache so we stop dialing with a known-bad
* credential. Transient codes (clock_skew, mode_mismatch) keep the cache
* but record the rejection so an operator can see why we stalled.
* HTTP 404 (older central without the peer ingress endpoint) keeps the
* cache and arms a longer backoff window so we do not hammer the remote.
*
* Process-local rate limit: at most 5 dial attempts per 60s window.
* Concurrency: a single in-flight promise is shared across callers so
* parallel `ensureSession()` calls coalesce into one dial.
*/
import { EventEmitter } from 'events';
import WebSocket from 'ws';
import {
MAX_FRAME_SIZE_BYTES,
decodeBinaryFrame,
decodeJsonFrame,
wsDataToBuffer,
wsDataToString,
} from '../pilot/protocol';
import { MeshCentralRegistry } from './MeshCentralRegistry';
import {
attachTcpStreamSwitchboard,
resolveByComposeLabels,
type TcpStreamSwitchboard,
type ReverseTcpStreamHandle,
} from '../mesh/tcpStreamSwitchboard';
import { PilotMetrics } from './PilotMetrics';
import { httpUrlToWs } from '../utils/wsUrl';
import { sanitizeForLog } from '../utils/safeLog';
interface SwitchboardReverseDialer {
openMeshTcpStream(target: { nodeId: number; stack: string; service: string; port: number }): ReverseTcpStreamHandle | null;
}
const HANDSHAKE_TIMEOUT_MS = 15_000;
const RATE_LIMIT_WINDOW_MS = 60_000;
const RATE_LIMIT_MAX = 5;
const ENDPOINT_UNAVAILABLE_BACKOFF_MS = 5 * 60_000;
const CALLBACK_PATH = '/api/mesh/proxy-tunnel-from-peer';
/**
* Reasons that indicate the cached central material is permanently bad
* (wrong secret, wrong fingerprint, node removed on central, etc.) and
* should be evicted so the next mesh-enable cycle remints fresh
* material. `endpoint_not_found` is explicitly NOT in this set because
* a 404 typically means "central is older than the peer ingress",
* which is recoverable by waiting.
*/
const REJECT_REASONS_CLEAR = new Set([
'stale',
'token_fingerprint_mismatch',
'node_deleted',
'instance_mismatch',
'signature_invalid',
'audience_mismatch',
'unknown',
]);
interface DialError extends Error {
reason?: string;
httpStatus?: number;
}
export class PeerToCentralMeshSessionDialer extends EventEmitter {
private static instance: PeerToCentralMeshSessionDialer | null = null;
private currentSession: TcpStreamSwitchboard | null = null;
private currentWs: WebSocket | null = null;
private inflight: Promise<TcpStreamSwitchboard | null> | null = null;
private recentDials: number[] = [];
private endpointUnavailableUntil = 0;
private constructor() { super(); }
public static getInstance(): PeerToCentralMeshSessionDialer {
if (!this.instance) this.instance = new PeerToCentralMeshSessionDialer();
return this.instance;
}
public static resetForTest(): void {
if (this.instance) {
try { this.instance.currentSession?.cleanup('test reset'); } catch { /* ignore */ }
try { this.instance.currentWs?.close(1000, 'test reset'); } catch { /* ignore */ }
}
this.instance = null;
}
public hasSession(): boolean {
return this.currentSession !== null;
}
public async ensureSession(): Promise<TcpStreamSwitchboard | null> {
if (this.currentSession) return this.currentSession;
if (Date.now() < this.endpointUnavailableUntil) return null;
if (this.isRateLimited()) return null;
if (this.inflight) return this.inflight;
this.inflight = this.dial().finally(() => { this.inflight = null; });
return this.inflight;
}
private isRateLimited(): boolean {
const now = Date.now();
this.recentDials = this.recentDials.filter((ts) => now - ts < RATE_LIMIT_WINDOW_MS);
return this.recentDials.length >= RATE_LIMIT_MAX;
}
private async dial(): Promise<TcpStreamSwitchboard | null> {
const material = MeshCentralRegistry.getInstance().getActive();
if (!material) return null;
this.recentDials.push(Date.now());
const wsUrl = httpUrlToWs(material.centralApiUrl) + CALLBACK_PATH;
const ws = new WebSocket(wsUrl, {
headers: { Authorization: `Bearer ${material.callbackJwt}` },
handshakeTimeout: HANDSHAKE_TIMEOUT_MS,
maxPayload: MAX_FRAME_SIZE_BYTES,
});
try {
await this.awaitOpen(ws);
} catch (err) {
ws.on('error', () => { /* swallow tail */ });
try { ws.close(); } catch { /* ignore */ }
this.handleDialFailure(err, material.centralInstanceId);
return null;
}
return this.attachSwitchboard(ws, material.centralInstanceId);
}
/**
* Wire the peer-initiated callback WS into the local MeshService. The
* R1-A2 design puts the peer end of the bridge in TcpStreamSwitchboard
* mode (peer multiplexes streams; central side runs PilotTunnelBridge).
* Without this wiring the WS opens cleanly but MeshService.reverseDialer
* stays null, so MeshService.dialMeshTcpStream falls through to
* PilotTunnelManager.ensureBridge(centralNodeId), which has no record
* for central on a proxy-mode peer (peers do not enroll central) and
* fails with proxy-tunnel.open.fail reason=no_target. End state matches
* the v0.78.1 reverse-direction failure even though the callback bridge
* is alive.
*
* Wiring symmetric to the central-initiated handler at
* `meshProxyTunnel.ts:115-163`:
* - attachTcpStreamSwitchboard with the same compose-label resolver
* - SwitchboardReverseDialer that delegates to switchboard.openReverseStream
* - setReverseDialer(localDialer, null) with CAS so a concurrent
* central-initiated tunnel does not get silently overwritten
* - ws.on('message') dispatches JSON/binary frames to the switchboard
* - ws.on('close'/'error') tears down switchboard + clears reverseDialer
*/
private async attachSwitchboard(ws: WebSocket, instanceId: string): Promise<TcpStreamSwitchboard | null> {
let switchboard: TcpStreamSwitchboard;
try {
switchboard = attachTcpStreamSwitchboard({
ws,
resolveTarget: resolveByComposeLabels,
logLabel: 'MeshCallback',
});
} catch (err) {
try { ws.close(1011, 'switchboard attach failed'); } catch { /* ignore */ }
PilotMetrics.increment('mesh_callback_dials_failed_total');
console.warn(`[PeerToCentralMeshSessionDialer] attach failed: ${sanitizeForLog((err as Error).message)}`);
return null;
}
const { MeshService } = await import('./MeshService');
const meshService = MeshService.getInstance();
const localDialer: SwitchboardReverseDialer = {
openMeshTcpStream(target) {
return switchboard.openReverseStream(target);
},
};
const installed = meshService.setReverseDialer(localDialer, null);
if (!installed) {
console.warn('[PeerToCentralMeshSessionDialer] reverse dialer already installed; rejecting concurrent callback bridge');
switchboard.cleanup('reverse dialer already installed');
try { ws.close(1013, 'reverse dialer already installed'); } catch { /* ignore */ }
PilotMetrics.increment('mesh_callback_dials_failed_total');
return null;
}
const onMessage = (data: unknown, isBinary: boolean): void => {
try {
if (isBinary) {
const buf = wsDataToBuffer(data);
if (!buf) return;
switchboard.handleBinaryFrame(decodeBinaryFrame(buf));
return;
}
const text = wsDataToString(data);
if (text == null) return;
switchboard.handleJsonFrame(decodeJsonFrame(text));
} catch (err) {
console.warn(`[PeerToCentralMeshSessionDialer] malformed frame: ${sanitizeForLog((err as Error).message)}`);
}
};
let tornDown = false;
const teardown = (): void => {
if (tornDown) return;
tornDown = true;
ws.off('message', onMessage);
try { switchboard.cleanup('mesh callback bridge closed'); } catch { /* ignore */ }
meshService.setReverseDialer(null, localDialer);
if (this.currentSession === switchboard) this.currentSession = null;
if (this.currentWs === ws) this.currentWs = null;
};
ws.on('message', onMessage);
ws.once('close', teardown);
ws.once('error', (err) => {
console.warn(`[PeerToCentralMeshSessionDialer] ws error: ${sanitizeForLog(err.message)}`);
teardown();
});
this.currentSession = switchboard;
this.currentWs = ws;
PilotMetrics.increment('mesh_central_bootstraps_total');
MeshCentralRegistry.getInstance().markUsed(instanceId);
return switchboard;
}
private awaitOpen(ws: WebSocket): Promise<void> {
return new Promise<void>((resolve, reject) => {
const cleanup = (): void => {
ws.removeAllListeners('open');
ws.removeAllListeners('error');
ws.removeAllListeners('unexpected-response');
};
ws.once('open', () => { cleanup(); resolve(); });
ws.once('error', (err) => { cleanup(); reject(err); });
ws.once('unexpected-response', (_req, res) => {
cleanup();
let body = '';
res.on('data', (chunk: Buffer) => { body += chunk.toString('utf8'); });
res.on('end', () => reject(this.buildUpgradeError(res.statusCode, body)));
});
});
}
private buildUpgradeError(statusCode: number | undefined, body: string): DialError {
let reason = 'unknown';
try {
const parsed = JSON.parse(body) as { reason?: unknown };
if (typeof parsed.reason === 'string') reason = parsed.reason;
} catch { /* keep unknown */ }
if (statusCode === 404) reason = 'endpoint_not_found';
const err = new Error(`upgrade rejected: HTTP ${statusCode ?? 'unknown'} reason=${reason}`) as DialError;
err.reason = reason;
err.httpStatus = statusCode;
return err;
}
private handleDialFailure(err: unknown, instanceId: string): void {
const reason = (err as DialError).reason ?? 'unknown';
PilotMetrics.increment('mesh_callback_dials_failed_total');
if (REJECT_REASONS_CLEAR.has(reason)) {
MeshCentralRegistry.getInstance().clearForInstance(instanceId);
PilotMetrics.increment('mesh_callback_auth_failures_total');
} else {
MeshCentralRegistry.getInstance().markRejected(instanceId, reason);
if (reason === 'endpoint_not_found') {
this.endpointUnavailableUntil = Date.now() + ENDPOINT_UNAVAILABLE_BACKOFF_MS;
}
}
console.warn(`[PeerToCentralMeshSessionDialer] dial failed: ${sanitizeForLog(`reason=${reason}`)}`);
}
}
backend/src/services/PilotMetrics.ts
+1 -13
View File
@@ -28,16 +28,8 @@ export interface Counters {
proxy_bridges_total: number;
/** Failed proxy-tunnel dial attempts (all reasons). Pair with `proxy_bridges_total` for an attempt/success ratio; pair with `proxy_idle_closes` for retention. */
proxy_dials_failed: number;
/** Proxy-tunnel teardowns initiated by the dialer's idle sweep (zero active streams for the configured TTL). */
/** Proxy-tunnel teardowns initiated by the dialer's idle sweep (zero active streams for the configured TTL). Effectively 0 by default since the dialer no longer idle-closes the bridge. */
proxy_idle_closes: number;
/** Proxy bridges registered via the peer-initiated dial-back path (`/api/mesh/proxy-tunnel-from-peer`). Disjoint from `proxy_bridges_total`, which counts central-initiated dials. */
proxy_bridges_peer_initiated_total: number;
/** Successful peer-to-central dial-back sessions (counted on WS open + bridge start). Disjoint from `proxy_bridges_peer_initiated_total`, which is incremented by the central-side ingress at register time. Useful for operators investigating asymmetric counts (peer thinks it dialed, central never registered). */
mesh_central_bootstraps_total: number;
/** Peer-to-central dial-back attempts that did not complete a WS open. Covers transport errors, upgrade rejections, and bridge.start failures. */
mesh_callback_dials_failed_total: number;
/** Subset of `mesh_callback_dials_failed_total` where central responded 401 with a terminal reason code (stale, signature_invalid, ...) that caused the peer to clear its cached central material. */
mesh_callback_auth_failures_total: number;
}
const ZERO_COUNTERS: Counters = {
@@ -49,10 +41,6 @@ const ZERO_COUNTERS: Counters = {
proxy_bridges_total: 0,
proxy_dials_failed: 0,
proxy_idle_closes: 0,
proxy_bridges_peer_initiated_total: 0,
mesh_central_bootstraps_total: 0,
mesh_callback_dials_failed_total: 0,
mesh_callback_auth_failures_total: 0,
};
export const PILOT_METRICS_FLUSH_INTERVAL_MS = 1_000;
backend/src/services/PilotTunnelManager.ts
+8 -43
View File
@@ -36,11 +36,10 @@ export class PilotTunnelCapacityError extends Error {
/**
* Discriminator for the two flavors of bridge that share the `bridges`
* map: `'pilot'` is set by `registerTunnel` (agent-initiated long-lived
* tunnel); `'proxy'` is set by `registerProxyBridge` /
* `replaceOrRegisterProxyBridge` (central- or peer-initiated short-lived
* proxy bridge). Used by the rejection-message formatter and by
* `replaceOrRegisterProxyBridge` so a peer-initiated dial can supersede a
* previous proxy bridge but never shadow a live pilot tunnel.
* tunnel); `'proxy'` is set by `registerProxyBridge` (central-initiated
* persistent bridge dialed by `MeshProxyTunnelDialer`). Used by the
* rejection-message formatter so a pilot tunnel and a proxy bridge for the
* same nodeId cannot coexist silently.
*/
export type BridgeKind = 'pilot' | 'proxy';
@@ -282,46 +281,12 @@ export class PilotTunnelManager extends EventEmitter {
this.emit('proxy-bridge-up', nodeId);
}
/**
* Register a peer-initiated proxy bridge for an existing remote. If a proxy
* bridge already exists for this nodeId, close it (the new dial is the source
* of truth) and replace. If a pilot tunnel exists, refuse: pilot tunnels
* always win over peer-initiated proxy bridges.
*/
public replaceOrRegisterProxyBridge(nodeId: number, bridge: PilotTunnelBridge): void {
const existingKind = this.bridgeKinds.get(nodeId);
if (existingKind === 'pilot') {
throw new Error(this.formatBridgeConflict(nodeId, existingKind));
}
if (existingKind === 'proxy') {
const old = this.bridges.get(nodeId);
this.bridges.delete(nodeId);
this.bridgeKinds.delete(nodeId);
try { old?.close(1000, 'replaced-by-newer-proxy'); } catch { /* best-effort cleanup */ }
}
if (this.bridges.size >= PILOT_TUNNEL_HARD_LIMIT) {
PilotMetrics.increment('tunnels_rejected_capacity');
throw new PilotTunnelCapacityError(PILOT_TUNNEL_HARD_LIMIT);
}
bridge.once('closed', () => {
if (this.bridges.get(nodeId) === bridge) {
this.bridges.delete(nodeId);
this.bridgeKinds.delete(nodeId);
this.emit('proxy-bridge-down', nodeId, 'remote_closed');
}
});
this.bridges.set(nodeId, bridge);
this.bridgeKinds.set(nodeId, 'proxy');
}
/**
* Single source of truth for the "slot already held" rejection
* message thrown by `registerProxyBridge` and
* `replaceOrRegisterProxyBridge`. Picks the bridge-kind subject and
* the rejection tail so a stale call site cannot drift from the
* other. `undefined` collapses to the pilot branch defensively;
* production code always sets `bridgeKinds` whenever `bridges` is
* set, so this is unreachable in practice.
* message thrown by `registerProxyBridge`. Picks the bridge-kind
* subject and the rejection tail. `undefined` collapses to the pilot
* branch defensively; production code always sets `bridgeKinds`
* whenever `bridges` is set, so this is unreachable in practice.
*/
private formatBridgeConflict(nodeId: number, existingKind: BridgeKind | undefined): string {
return existingKind === 'proxy'
backend/src/websocket/meshProxyTunnel.ts
+5 -129
View File
@@ -11,44 +11,11 @@ import {
import { sanitizeForLog } from '../utils/safeLog';
import { isDebugEnabled } from '../utils/debug';
import { rejectUpgrade as reject } from './reject';
import { MeshCentralRegistry, type MeshCentralMaterial } from '../services/MeshCentralRegistry';
/**
* Bootstrap phase for the first-frame state machine.
*
* Central MAY send a `mesh_handshake` JSON frame as the FIRST text frame
* after upgrade. If it arrives we persist the callback material via
* MeshCentralRegistry and transition to `consumed`. Any frame other than
* the handshake (or no frame at all) moves us to `past`, after which a
* later handshake is a protocol error.
*/
type BootstrapPhase = 'awaiting' | 'consumed' | 'past';
interface MeshHandshakeFrame {
t: 'mesh_handshake';
v: number;
peerNodeId: number;
centralInstanceId: string;
centralApiUrl: string;
meshTunnelJwt: string;
jwtExpiresAt: number;
}
function isMeshHandshakeFrame(parsed: unknown): parsed is MeshHandshakeFrame {
return typeof parsed === 'object' && parsed !== null
&& (parsed as { t?: unknown }).t === 'mesh_handshake'
&& typeof (parsed as { v?: unknown }).v === 'number'
&& typeof (parsed as { peerNodeId?: unknown }).peerNodeId === 'number'
&& typeof (parsed as { centralInstanceId?: unknown }).centralInstanceId === 'string'
&& typeof (parsed as { centralApiUrl?: unknown }).centralApiUrl === 'string'
&& typeof (parsed as { meshTunnelJwt?: unknown }).meshTunnelJwt === 'string'
&& typeof (parsed as { jwtExpiresAt?: unknown }).jwtExpiresAt === 'number';
}
/**
* Mesh proxy-tunnel ingress.
*
* The remote side of a Phase C proxy-mode mesh tunnel. Central dials
* The peer side of a proxy-mode mesh tunnel. Central dials
* `WSS <api_url>/api/mesh/proxy-tunnel` using the fleet credential as a
* Bearer header (either the `node_proxy` JWT generated by the remote's
* Settings → Nodes → Generate Token flow, or a `full-admin` api_token
@@ -65,21 +32,12 @@ function isMeshHandshakeFrame(parsed: unknown): parsed is MeshHandshakeFrame {
* the reverse-dialer on the local MeshService so meshed containers on
* this Sencho can dial cross-node aliases via `tcp_open_reverse` over
* the same WS. On disconnect the reverse-dialer registration is
* cleared.
* cleared. Central maintains the bridge persistently (no idle close;
* reconciled on a timer), so peer→central traffic always has a live
* channel in steady state.
*/
const wss = new WebSocketServer({ noServer: true, maxPayload: MAX_FRAME_SIZE_BYTES });
/**
* Upper bound on the pre-CAS first-frame wait inside attachSwitchboard.
* Sized to cover localhost-loopback RTT plus a slim margin for the
* central-side ms-scale gap between `ws.send(mesh_handshake)` and the
* registerProxyBridge throw + ws.close that follows on a refused dial.
* On a dial that carries no bootstrap, this is pure added latency
* before the reverse-dialer install; 30ms is well below operator
* perception and well above realistic localhost RTT.
*/
const FIRST_FRAME_WAIT_MS = 30;
interface SwitchboardReverseDialer {
openMeshTcpStream(target: { nodeId: number; stack: string; service: string; port: number }): ReverseTcpStreamHandle | null;
}
@@ -126,74 +84,18 @@ export async function handleMeshProxyTunnel(req: IncomingMessage, socket: Duplex
async function attachSwitchboard(ws: WebSocket, peerNodeId: number | null): Promise<void> {
let switchboard: TcpStreamSwitchboard | null = null;
let meshServiceCleanup: (() => void) | null = null;
let phase: BootstrapPhase = 'awaiting';
const onMessage = (data: unknown, isBinary: boolean): void => {
if (!switchboard) return;
try {
if (isBinary) {
if (phase === 'awaiting') phase = 'past';
const buf = wsDataToBuffer(data);
if (!buf) return;
switchboard.handleBinaryFrame(decodeBinaryFrame(buf));
return;
}
const text = wsDataToString(data);
if (text == null) {
if (phase === 'awaiting') phase = 'past';
return;
}
let parsedForBootstrap: unknown = null;
let parseSucceeded = false;
try {
parsedForBootstrap = JSON.parse(text);
parseSucceeded = true;
} catch {
// Not valid JSON; defer to decodeJsonFrame's stricter error path below.
}
if (parseSucceeded && typeof parsedForBootstrap === 'object'
&& parsedForBootstrap !== null
&& (parsedForBootstrap as { t?: unknown }).t === 'mesh_handshake') {
if (phase !== 'awaiting') {
ws.close(1008, 'mesh_handshake out of order');
return;
}
if (!isMeshHandshakeFrame(parsedForBootstrap)) {
ws.close(1008, 'malformed mesh_handshake');
return;
}
const material: MeshCentralMaterial = {
centralInstanceId: parsedForBootstrap.centralInstanceId,
centralApiUrl: parsedForBootstrap.centralApiUrl.replace(/\/+$/, ''),
callbackJwt: parsedForBootstrap.meshTunnelJwt,
jwtIssuedAt: Math.floor(Date.now() / 1000),
jwtExpiresAt: parsedForBootstrap.jwtExpiresAt,
};
try {
MeshCentralRegistry.getInstance().upsert(material);
phase = 'consumed';
const centralInstanceId = parsedForBootstrap.centralInstanceId;
const jwtExpiresAt = parsedForBootstrap.jwtExpiresAt;
void (async () => {
try {
const { MeshService: MeshSvc } = await import('../services/MeshService');
MeshSvc.getInstance().logActivity({
source: 'mesh', level: 'info',
type: 'mesh_handshake.received',
message: `mesh callback bootstrap received from central ${centralInstanceId}`,
details: { centralInstanceId, jwtExpiresAt },
});
} catch { /* best-effort log; bootstrap success path must not depend on it */ }
})();
} catch (err) {
console.warn(`[meshProxyTunnel] mesh_handshake persist failed: ${sanitizeForLog((err as Error).message)}`);
try { ws.close(1011, 'bootstrap persist failed'); } catch { /* ignore */ }
}
return;
}
if (phase === 'awaiting') phase = 'past';
if (text == null) return;
switchboard.handleJsonFrame(decodeJsonFrame(text));
} catch (err) {
if (isDebugEnabled()) {
@@ -221,11 +123,6 @@ async function attachSwitchboard(ws: WebSocket, peerNodeId: number | null): Prom
logLabel: 'MeshProxy',
});
// Wire listeners synchronously so any first frame from central
// reaches onMessage even when the reverse-dialer CAS swap below
// refuses this bridge. Bootstrap material identifies central for
// the next peer-initiated callback and must persist independent
// of whether this particular WS becomes the active bridge.
ws.on('message', onMessage);
ws.once('close', teardown);
ws.once('error', (err) => {
@@ -251,23 +148,7 @@ async function attachSwitchboard(ws: WebSocket, peerNodeId: number | null): Prom
const installed = meshService.setReverseDialer(localDialer, null);
if (!installed) {
console.warn('[MeshProxy] reverse dialer already installed; rejecting concurrent tunnel');
// Hold the WS open briefly so any in-flight first frame can
// land in onMessage before we send the close. Central
// optionally sends a mesh_handshake as the first frame on
// Trigger 1 / Trigger 2 dials; that bootstrap material must
// persist via MeshCentralRegistry regardless of whether this
// WS becomes the active bridge. The wait resolves as soon as
// any 'message' arrives (the listener attached above runs
// synchronously on the event and updates the registry); the
// timeout bounds the close-latency overhead for refused
// dials that carry no bootstrap.
await new Promise<void>((resolve) => {
const onFirst = (): void => { clearTimeout(t); resolve(); };
const t = setTimeout(() => { ws.off('message', onFirst); resolve(); }, FIRST_FRAME_WAIT_MS);
ws.once('message', onFirst);
});
try { ws.close(1013, 'reverse dialer already installed'); } catch { /* ignore */ }
// teardown fires on the 'close' event and runs switchboard.cleanup.
return;
}
// Install central's view of this peer's nodeId so handleAccept
@@ -276,11 +157,6 @@ async function attachSwitchboard(ws: WebSocket, peerNodeId: number | null): Prom
//
// The install persists across bridge lifecycles: the peer's identity
// in central's namespace is stable for the enrollment, not per-WS.
// Null-clearing on teardown caused dispatch to fall back to
// getDefaultNodeId() after idle close, which collides with central's
// own nodeId for Local and misdispatches cross-fleet traffic to the
// same-node path. A subsequent install with a different nodeId
// (e.g. re-enrollment) is detected by the setter's overwrite-warn.
if (peerNodeId !== null) {
meshService.setProxyTunnelSelfCentralNodeId(peerNodeId);
}
backend/src/websocket/meshProxyTunnelFromPeer.ts
-142
View File
@@ -1,142 +0,0 @@
/**
* Central-side ingress for peer-initiated proxy-mode mesh tunnels.
*
* After the bootstrap exchange in `meshProxyTunnel.ts` (Task 7/8), a peer
* with the central's `mesh_tunnel` JWT can dial *back* to central at this
* endpoint. The validation chain enforces every claim the bootstrap mint
* produced and confirms the peer's stored fingerprint still matches its
* current api_token. Each failure path returns HTTP 401 with a stable
* machine-readable `reason` code in the JSON body, so the dialer side
* (PeerToCentralMeshSessionDialer, Task 10) can act on the reason without
* scraping prose.
*
* Auth runs MANUALLY in this handler. Express middleware does not run on
* WebSocket upgrades, and the credential here is a Bearer JWT minted by
* the central itself, not a user session.
*/
import type { IncomingMessage } from 'http';
import type { Duplex } from 'stream';
import jwt, { type Algorithm, type JwtPayload } from 'jsonwebtoken';
import { createHash } from 'crypto';
import { WebSocketServer } from 'ws';
import { DatabaseService } from '../services/DatabaseService';
import { PilotTunnelBridge } from '../services/PilotTunnelBridge';
import { PilotTunnelManager } from '../services/PilotTunnelManager';
import { PilotMetrics } from '../services/PilotMetrics';
import { sanitizeForLog } from '../utils/safeLog';
const wss = new WebSocketServer({ noServer: true });
const CLOCK_SKEW_SEC = 60;
type RejectReason =
| 'algorithm_mismatch' | 'signature_invalid'
| 'scope_mismatch' | 'audience_mismatch' | 'instance_mismatch'
| 'stale' | 'clock_skew' | 'node_deleted' | 'mode_mismatch'
| 'token_fingerprint_mismatch' | 'malformed';
function rejectUpgrade(socket: Duplex, reason: RejectReason): void {
const body = JSON.stringify({ reason });
const headers = [
'HTTP/1.1 401 Unauthorized',
'Content-Type: application/json',
`Content-Length: ${Buffer.byteLength(body)}`,
'Connection: close',
'',
body,
].join('\r\n');
try { socket.write(headers); } catch { /* socket already gone */ }
try { socket.destroy(); } catch { /* socket already gone */ }
}
function extractBearer(req: IncomingMessage): string | null {
const h = req.headers['authorization'];
if (typeof h !== 'string' || !h.startsWith('Bearer ')) return null;
return h.slice('Bearer '.length).trim() || null;
}
export function handleMeshProxyTunnelFromPeerUpgrade(
req: IncomingMessage,
socket: Duplex,
head: Buffer,
): void {
const token = extractBearer(req);
if (!token) { rejectUpgrade(socket, 'malformed'); return; }
let header: { alg?: string; kid?: string } = {};
try {
const decoded = jwt.decode(token, { complete: true });
header = (decoded?.header ?? {}) as typeof header;
} catch { rejectUpgrade(socket, 'malformed'); return; }
if (header.alg !== 'HS256') { rejectUpgrade(socket, 'algorithm_mismatch'); return; }
const settings = DatabaseService.getInstance().getGlobalSettings();
const secret = settings.auth_jwt_secret;
if (!secret) { rejectUpgrade(socket, 'malformed'); return; }
let payload: JwtPayload;
try {
payload = jwt.verify(token, secret, { algorithms: ['HS256' as Algorithm] }) as JwtPayload;
} catch { rejectUpgrade(socket, 'signature_invalid'); return; }
if (payload.scope !== 'mesh_tunnel') { rejectUpgrade(socket, 'scope_mismatch'); return; }
// SENCHO_PRIMARY_URL must be set on central for peer-initiated dial-back
// to operate. The Task 8 bootstrap-mint side already skips when it's
// unset; this is the matching fail-safe on the verify side. Reject all
// peer dials with audience_mismatch when central has no canonical
// origin to validate against.
const canonicalOrigin = (process.env.SENCHO_PRIMARY_URL ?? '').replace(/\/+$/, '');
if (!canonicalOrigin) { rejectUpgrade(socket, 'audience_mismatch'); return; }
if (payload.aud !== canonicalOrigin) { rejectUpgrade(socket, 'audience_mismatch'); return; }
// instance_id is operational state, not user-defined config, so it
// lives in system_state rather than global_settings.
const instanceId = DatabaseService.getInstance().getSystemState('instance_id');
if (payload.iss !== instanceId) { rejectUpgrade(socket, 'instance_mismatch'); return; }
const nowSec = Math.floor(Date.now() / 1000);
if (typeof payload.exp !== 'number' || payload.exp <= nowSec) {
rejectUpgrade(socket, 'stale'); return;
}
if (typeof payload.iat !== 'number' || payload.iat > nowSec + CLOCK_SKEW_SEC) {
rejectUpgrade(socket, 'clock_skew'); return;
}
const peerNodeId = Number(payload.sub);
if (!Number.isInteger(peerNodeId) || peerNodeId <= 0) {
rejectUpgrade(socket, 'malformed'); return;
}
const node = DatabaseService.getInstance().getNode(peerNodeId);
if (!node) { rejectUpgrade(socket, 'node_deleted'); return; }
if (node.type !== 'remote' || node.mode !== 'proxy') {
rejectUpgrade(socket, 'mode_mismatch'); return;
}
if (!node.api_token) { rejectUpgrade(socket, 'token_fingerprint_mismatch'); return; }
const expectedFp = createHash('sha256').update(node.api_token).digest('hex').slice(0, 16);
const claimedFp = payload['peer_token_fp'];
if (typeof claimedFp !== 'string' || claimedFp !== expectedFp) {
rejectUpgrade(socket, 'token_fingerprint_mismatch'); return;
}
wss.handleUpgrade(req, socket, head, (ws) => {
try {
const bridge = new PilotTunnelBridge(peerNodeId, ws);
bridge.start().then(() => {
try {
PilotTunnelManager.getInstance().replaceOrRegisterProxyBridge(peerNodeId, bridge);
PilotMetrics.increment('proxy_bridges_peer_initiated_total');
} catch (err) {
try { bridge.close(1013, 'manager rejected'); } catch { /* ignore */ }
console.warn(`[meshProxyTunnelFromPeer] register failed: ${sanitizeForLog((err as Error).message)}`);
}
}).catch((err) => {
try { bridge.close(1011, 'bridge start failed'); } catch { /* ignore */ }
console.warn(`[meshProxyTunnelFromPeer] bridge start failed: ${sanitizeForLog((err as Error).message)}`);
});
} catch (err) {
console.warn(`[meshProxyTunnelFromPeer] bridge construct failed: ${sanitizeForLog((err as Error).message)}`);
try { ws.close(1011, 'bridge init failed'); } catch { /* ignore */ }
}
});
}
backend/src/websocket/upgradeHandler.ts
+12 -20
View File
@@ -8,7 +8,6 @@ import { NodeRegistry } from '../services/NodeRegistry';
import { COOKIE_NAME } from '../helpers/constants';
import { handlePilotTunnel } from './pilotTunnel';
import { handleMeshProxyTunnel } from './meshProxyTunnel';
import { handleMeshProxyTunnelFromPeerUpgrade } from './meshProxyTunnelFromPeer';
import { handleNotificationsWs } from './notifications';
import { handleRemoteForwarder } from './remoteForwarder';
import { handleLogsWs } from './logs';
@@ -33,15 +32,14 @@ function parseCookies(req: IncomingMessage): Record<string, string> {
*
* Dispatch order (first match wins):
* 1. `/api/pilot/tunnel` -> handlePilotTunnel (own auth, own wss)
* 2. `/api/mesh/proxy-tunnel-from-peer` -> handleMeshProxyTunnelFromPeerUpgrade (own JWT chain, central-side dial-back)
* 3. shared cookie/Bearer auth + JWT verify (rejects unauthenticated)
* 4. API token scope gate (read-only / deploy-only restricted to logs + notifications)
* 5. `/api/mesh/proxy-tunnel` -> handleMeshProxyTunnel (machine-to-machine: node_proxy or full-admin api_token)
* 6. `/ws/notifications` local -> handleNotificationsWs
* 7. remote nodeId path -> handleRemoteForwarder
* 8. `/api/stacks/:name/logs` -> handleLogsWs
* 9. `/api/system/host-console` -> handleHostConsoleWs
* 10. fallback -> handleGenericWs (`/ws` exec + stats)
* 2. shared cookie/Bearer auth + JWT verify (rejects unauthenticated)
* 3. API token scope gate (read-only / deploy-only restricted to logs + notifications)
* 4. `/api/mesh/proxy-tunnel` -> handleMeshProxyTunnel (machine-to-machine: node_proxy or full-admin api_token; bidirectional bridge for both forward and reverse mesh traffic)
* 5. `/ws/notifications` local -> handleNotificationsWs
* 6. remote nodeId path -> handleRemoteForwarder
* 7. `/api/stacks/:name/logs` -> handleLogsWs
* 8. `/api/system/host-console` -> handleHostConsoleWs
* 9. fallback -> handleGenericWs (`/ws` exec + stats)
*/
export function attachUpgrade(
server: http.Server,
@@ -52,22 +50,16 @@ export function attachUpgrade(
attachGenericConnectionHandlers(wss);
server.on('upgrade', async (req, socket, head) => {
// Pilot-agent tunnel ingress: machine credentials, no cookies.
// Mesh proxy-tunnel-from-peer ingress: peer-initiated dial-back over
// a `mesh_tunnel`-scoped JWT (minted earlier during the bootstrap
// exchange). Both handlers run their own auth before the shared
// cookie/Bearer pipeline because their credentials are not user
// sessions and would fail the shared user-existence check.
// Pilot-agent tunnel ingress: machine credentials, no cookies. Runs its
// own auth before the shared cookie/Bearer pipeline because the
// credential is not a user session and would fail the shared
// user-existence check.
try {
const reqUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
if (reqUrl.pathname === '/api/pilot/tunnel') {
await handlePilotTunnel(req, socket, head, pilotTunnelWss);
return;
}
if (reqUrl.pathname === '/api/mesh/proxy-tunnel-from-peer') {
handleMeshProxyTunnelFromPeerUpgrade(req, socket, head);
return;
}
} catch {
// URL parse error falls through and will be rejected below.
}
docs/features/sencho-mesh.mdx
+9 -12
View File
@@ -71,19 +71,16 @@ services:
Sencho's static IP on the network is `<network address> + 2` (so `10.42.0.2` for the example above). Operators can configure each node independently; the override generator pushes alias hostnames that resolve to whichever IP the deploying node uses locally.
## Bidirectional secure callback
## Bidirectional traffic
Proxy-mode mesh nodes can re-establish their secure mesh callback to central
when traffic starts from that node. Central is the hub; the callback is signed
with a `mesh_tunnel`-scoped credential bound to the node's API token.
<Note>
**Prerequisite:** set `SENCHO_PRIMARY_URL` on central to its canonical public
origin. This value is used as the audience for the mesh callback credential
and as the callback URL on the node side. Without it, central infers an origin
from inbound request headers; the inference works in standard installs but is
not the intended deploy shape for fleets where reverse-proxy headers may differ.
</Note>
Mesh traffic flows in both directions over a single persistent WebSocket
between central and each proxy-mode peer. Central dials the bridge to every
mesh-enabled peer on startup and re-dials any dropped bridge on its
reconcile tick (every 60 seconds by default). When a container on the peer
needs to reach a service on central, the request multiplexes over that same
bridge: no extra inbound listener on central, no public URL required on
the central side. This is the path that makes "call a service on any node
by hostname" work in NAT'd homelab topologies out of the box.
## Test upstream
frontend/src/components/fleet/MeshStackTopologySheet.test.tsx
+1
View File
@@ -21,6 +21,7 @@ function makeNode(over: Partial<MeshNodeStatus> & Pick<MeshNodeStatus, 'nodeId'
pilotConnected: over.pilotConnected ?? false,
reachableMode: over.reachableMode ?? 'unreachable',
reachableReason: over.reachableReason ?? null,
reverseCallbackStatus: over.reverseCallbackStatus ?? 'not_applicable',
optedInStacks: over.optedInStacks ?? [],
activeStreamCount: over.activeStreamCount ?? 0,
};
frontend/src/components/fleet/RoutingNodeCard.tsx
+16
View File
@@ -72,6 +72,22 @@ export function RoutingNodeCard({
unreachable
</span>
)}
{status.reverseCallbackStatus === 'connecting' && (
<span
title="Central is dialing the reverse bridge to this peer."
className="inline-flex items-center px-1.5 py-0.5 rounded-sm border border-card-border bg-card text-[10px] leading-3 font-mono uppercase tracking-[0.18em] text-stat-subtitle"
>
reconnecting
</span>
)}
{status.reverseCallbackStatus === 'unavailable' && (
<span
title="Peer→central tunnel is between dials. Central will redial on its next reconcile tick."
className="inline-flex items-center px-1.5 py-0.5 rounded-sm border border-amber-500/40 bg-amber-500/10 text-[10px] leading-3 font-mono uppercase tracking-[0.18em] text-amber-600 dark:text-amber-400"
>
reverse unavailable
</span>
)}
</div>
<div className="flex items-center gap-2">
<TogglePill
frontend/src/lib/mesh-topology-layout.test.ts
+1
View File
@@ -22,6 +22,7 @@ function makeNode(over: Partial<MeshNodeStatus> & Pick<MeshNodeStatus, 'nodeId'
pilotConnected: over.pilotConnected ?? false,
reachableMode: over.reachableMode ?? 'unreachable',
reachableReason: over.reachableReason ?? null,
reverseCallbackStatus: over.reverseCallbackStatus ?? 'not_applicable',
optedInStacks: over.optedInStacks ?? [],
activeStreamCount: over.activeStreamCount ?? 0,
};
frontend/src/types/mesh.ts
+13
View File
@@ -28,6 +28,17 @@ export interface MeshAlias {
export type MeshReachableMode = 'local' | 'pilot' | 'proxy' | 'unreachable';
/**
* State of the peer→central reverse path for a proxy-mode peer. Central
* maintains a persistent forward WS to every mesh-enabled proxy peer;
* peer→central traffic multiplexes over that same bridge. The four values:
* - `connected`: bridge open, reverse multiplex available.
* - `connecting`: dial in flight (transient).
* - `unavailable`: bridge not open, no dial in flight; central will redial on its next reconcile tick.
* - `not_applicable`: local node, pilot-mode peer, or mesh disabled.
*/
export type MeshReverseCallbackStatus = 'connected' | 'connecting' | 'unavailable' | 'not_applicable';
export interface MeshNodeStatus {
nodeId: number;
nodeName: string;
@@ -40,6 +51,8 @@ export interface MeshNodeStatus {
reachableMode: MeshReachableMode;
/** Operator-facing reason when `reachableMode === 'unreachable'`. Null otherwise. */
reachableReason: string | null;
/** Peer→central reverse path state. `not_applicable` for non-proxy peers. */
reverseCallbackStatus: MeshReverseCallbackStatus;
optedInStacks: string[];
activeStreamCount: number;
}