diff --git a/backend/src/__tests__/capability-mesh-callback-bootstrap.test.ts b/backend/src/__tests__/capability-mesh-callback-bootstrap.test.ts deleted file mode 100644 index df5e4705..00000000 --- a/backend/src/__tests__/capability-mesh-callback-bootstrap.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { describe, it, expect, afterEach } from 'vitest'; -import { - getActiveCapabilities, - applyPilotModeCapabilityFilter, - enableCapability, - disableCapability, -} from '../services/CapabilityRegistry'; - -describe('mesh_proxy_callback_bootstrap capability', () => { - afterEach(() => { - // Ensure the runtime override does not leak between tests. - enableCapability('mesh_proxy_callback_bootstrap'); - }); - - it('is registered in the default CAPABILITIES list', () => { - expect(getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap'); - }); - - it('is NOT filtered out in pilot mode (pilots can also be bootstrap targets)', () => { - applyPilotModeCapabilityFilter(); - expect(getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap'); - enableCapability('host-console'); - enableCapability('self-update'); - }); - - it('is stripped from the active capabilities list when the data plane is disabled', () => { - disableCapability('mesh_proxy_callback_bootstrap'); - expect(getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap'); - }); - - it('returns to the advertised list once enableCapability is called again', () => { - disableCapability('mesh_proxy_callback_bootstrap'); - expect(getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap'); - enableCapability('mesh_proxy_callback_bootstrap'); - expect(getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap'); - }); -}); diff --git a/backend/src/__tests__/mesh-bootstrap-integration.test.ts b/backend/src/__tests__/mesh-bootstrap-integration.test.ts deleted file mode 100644 index be8dd6fb..00000000 --- a/backend/src/__tests__/mesh-bootstrap-integration.test.ts +++ /dev/null @@ -1,187 +0,0 @@ -/** - * End-to-end protocol-role invariant for the symmetric mesh callback dial. - * - * The whole point of Task 8 + Task 10 is the asymmetry of registration even - * though the wire protocol becomes symmetric: - * - * - Central is the loopback HTTP host. Every bridge central registers - * (pilot-initiated or peer-initiated dial-back) is a - * `PilotTunnelBridge` parked in `PilotTunnelManager`. - * - The peer (proxy-mode remote) never runs a `PilotTunnelBridge`; on the - * peer side a `TcpStreamSwitchboard` rides the same WS and the local - * `MeshService.reverseDialer` slot is held while the WS is alive. - * - * This file exercises the central side of that invariant end-to-end using - * the same in-process fixture pattern as `mesh-proxy-tunnel-from-peer.test.ts`: - * a fully signed callback JWT lands at `/api/mesh/proxy-tunnel-from-peer`, - * and we assert what central produced is a `PilotTunnelBridge` (not a - * switchboard) in the manager. The matching mesh_handshake side (central - * pushing the JWT) is covered in `mesh-proxy-tunnel-dialer-handshake.test.ts` - * and `mesh-proxy-tunnel-first-frame.test.ts`; here we lock in the registry - * outcome that ties those two halves together. - */ -import http from 'http'; -import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; -import WebSocket from 'ws'; -import jwt from 'jsonwebtoken'; -import { createHash } from 'crypto'; -import type { AddressInfo } from 'net'; -import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; -import { MeshProxyTunnelDialer } from '../services/MeshProxyTunnelDialer'; -import { MeshCentralRegistry } from '../services/MeshCentralRegistry'; -import { PilotTunnelManager } from '../services/PilotTunnelManager'; -import { PilotTunnelBridge } from '../services/PilotTunnelBridge'; - -let tmpDir: string; -let handleMeshProxyTunnelFromPeerUpgrade: typeof import('../websocket/meshProxyTunnelFromPeer').handleMeshProxyTunnelFromPeerUpgrade; -let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; - -interface ServerHandle { - server: http.Server; - port: number; - close: () => Promise; -} - -async function startCentralServer(): Promise { - const server = http.createServer(); - server.on('upgrade', (req, socket, head) => { - const pathname = new URL(req.url ?? '/', 'http://localhost').pathname; - if (pathname === '/api/mesh/proxy-tunnel-from-peer') { - handleMeshProxyTunnelFromPeerUpgrade(req, socket, head); - } else { - socket.destroy(); - } - }); - await new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve())); - const port = (server.address() as AddressInfo).port; - return { - server, - port, - close: () => new Promise((resolve) => server.close(() => resolve())), - }; -} - -const CANONICAL_ORIGIN = 'https://central.example.com'; -const INSTANCE_ID = 'bootstrap-integration-central'; -const PEER_API_TOKEN = 'peer-token-bootstrap'; - -let secret: string; -let srv: ServerHandle; - -beforeAll(async () => { - tmpDir = await setupTestDb(); - ({ handleMeshProxyTunnelFromPeerUpgrade } = await import('../websocket/meshProxyTunnelFromPeer')); - ({ DatabaseService } = await import('../services/DatabaseService')); - srv = await startCentralServer(); -}); - -afterAll(async () => { - if (srv) await srv.close(); - delete process.env.SENCHO_PRIMARY_URL; - cleanupTestDb(tmpDir); -}); - -beforeEach(() => { - process.env.SENCHO_PRIMARY_URL = CANONICAL_ORIGIN; - MeshProxyTunnelDialer.resetForTest(); - MeshCentralRegistry.resetForTest(); - PilotTunnelManager.resetForTest(); - const db = DatabaseService.getInstance(); - secret = db.getGlobalSettings().auth_jwt_secret; - db.setSystemState('instance_id', INSTANCE_ID); -}); - -function seedPeerNode(): number { - const db = DatabaseService.getInstance(); - const id = db.addNode({ - name: `peer-bs-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, - type: 'remote', - mode: 'proxy', - api_url: 'https://peer.example.com', - api_token: PEER_API_TOKEN, - compose_dir: '/tmp', - is_default: false, - }); - db.setNodeMeshEnabled(id, true); - return id; -} - -function mintCallbackJwt(peerNodeId: number): string { - const fp = createHash('sha256').update(PEER_API_TOKEN).digest('hex').slice(0, 16); - const nowSec = Math.floor(Date.now() / 1000); - return jwt.sign( - { - sub: String(peerNodeId), - iss: INSTANCE_ID, - aud: CANONICAL_ORIGIN, - scope: 'mesh_tunnel', - iat: nowSec, - exp: nowSec + 3600, - kid: 'v1', - peer_token_fp: fp, - }, - secret, - { algorithm: 'HS256' }, - ); -} - -describe('Mesh bootstrap E2E (protocol-role invariant)', () => { - it('after peer dial-back, central registers a PilotTunnelBridge in PilotTunnelManager', async () => { - // This is the symmetric callback half of the protocol: the peer has - // already received a bootstrap JWT in some prior session and is now - // dialing back into central. On central, the resulting WS must - // produce a PilotTunnelBridge (not a TcpStreamSwitchboard) and the - // manager must hold it under the peer's nodeId. - const peerNodeId = seedPeerNode(); - const token = mintCallbackJwt(peerNodeId); - - const ws = await new Promise((resolve, reject) => { - const s = new WebSocket(`ws://127.0.0.1:${srv.port}/api/mesh/proxy-tunnel-from-peer`, { - headers: { authorization: `Bearer ${token}` }, - }); - s.once('open', () => resolve(s)); - s.once('error', reject); - s.once('unexpected-response', (_req, res) => reject(new Error(`upgrade rejected ${res.statusCode}`))); - }); - // Allow the handler's async bridge.start() + manager registration to land. - await new Promise((r) => setTimeout(r, 60)); - - const bridge = PilotTunnelManager.getInstance().getBridge(peerNodeId); - expect(bridge).not.toBeNull(); - // The manager only narrows to MeshTunnelHandle in its public API, - // but the underlying object must be the central-side bridge class. - expect(bridge).toBeInstanceOf(PilotTunnelBridge); - - try { ws.close(1000, 'test cleanup'); } catch { /* ignore */ } - await new Promise((r) => setTimeout(r, 30)); - }); - - it('a successful dial-back persists the JWT scope on the bridge (mesh_tunnel scope)', async () => { - // Cross-check: the JWT carries scope=mesh_tunnel, the central - // ingress accepted it, so the bridge that lands in the manager is a - // proxy-mode bridge (not a pilot-agent tunnel). We assert via the - // dialer's hasBridge surface to confirm the proxy registry side - // sees nothing (the dial-back lives in the manager, not in - // MeshProxyTunnelDialer's outbound map). - const peerNodeId = seedPeerNode(); - const token = mintCallbackJwt(peerNodeId); - const ws = await new Promise((resolve, reject) => { - const s = new WebSocket(`ws://127.0.0.1:${srv.port}/api/mesh/proxy-tunnel-from-peer`, { - headers: { authorization: `Bearer ${token}` }, - }); - s.once('open', () => resolve(s)); - s.once('error', reject); - s.once('unexpected-response', (_req, res) => reject(new Error(`upgrade rejected ${res.statusCode}`))); - }); - await new Promise((r) => setTimeout(r, 60)); - - // The dialer is the central-initiated outbound side. A peer-initiated - // dial-back lands in PilotTunnelManager only; the dialer's own - // bridges map stays empty for this nodeId. - expect(MeshProxyTunnelDialer.getInstance().hasBridge(peerNodeId)).toBe(false); - expect(PilotTunnelManager.getInstance().getBridge(peerNodeId)).not.toBeNull(); - - try { ws.close(1000, 'test cleanup'); } catch { /* ignore */ } - await new Promise((r) => setTimeout(r, 30)); - }); -}); diff --git a/backend/src/__tests__/mesh-central-registry.test.ts b/backend/src/__tests__/mesh-central-registry.test.ts deleted file mode 100644 index 903dc152..00000000 --- a/backend/src/__tests__/mesh-central-registry.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; -import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; -import { DatabaseService } from '../services/DatabaseService'; -import { MeshCentralRegistry } from '../services/MeshCentralRegistry'; - -let sharedTmpDir: string; -beforeAll(async () => { sharedTmpDir = await setupTestDb(); }); -afterAll(() => cleanupTestDb(sharedTmpDir)); - -describe('mesh_centrals schema', () => { - it('creates the mesh_centrals table on initSchema', () => { - const db = DatabaseService.getInstance(); - const row = db.getDb().prepare( - "SELECT name FROM sqlite_master WHERE type='table' AND name='mesh_centrals'" - ).get(); - expect(row).toBeDefined(); - }); - - it('mesh_centrals has the expected columns', () => { - const db = DatabaseService.getInstance(); - const cols = db.getDb().prepare("PRAGMA table_info('mesh_centrals')").all() as Array<{ name: string }>; - const names = cols.map(c => c.name).sort(); - expect(names).toEqual([ - 'callback_jwt', - 'central_api_url', - 'central_instance_id', - 'jwt_expires_at', - 'jwt_issued_at', - 'last_bootstrap_at', - 'last_reject_reason', - 'last_rejected_at', - 'last_used_at', - ]); - }); -}); - -describe('MeshCentralRegistry', () => { - beforeEach(() => { - MeshCentralRegistry.resetForTest(); - // Per-test cleanup of the mesh_centrals table so tests are order-independent. - // The DB itself is created once per file in the file-scope beforeAll above; - // resetting setupTestDb per test would invalidate the DatabaseService - // singleton's open connection (SQLITE_READONLY_DBMOVED on Linux CI). - DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_centrals').run(); - }); - - const sampleMaterial = (overrides: Partial<{ - centralInstanceId: string; - centralApiUrl: string; - callbackJwt: string; - jwtIssuedAt: number; - jwtExpiresAt: number; - }> = {}) => ({ - centralInstanceId: 'inst-uuid-1', - centralApiUrl: 'https://central.example.com', - callbackJwt: 'eyJhbGciOiJIUzI1NiJ9.fake.token', - jwtIssuedAt: 1_700_000_000, - jwtExpiresAt: 1_700_000_000 + 365 * 24 * 3600, - ...overrides, - }); - - it('upsert inserts a new row when none exists', () => { - MeshCentralRegistry.getInstance().upsert(sampleMaterial()); - const row = MeshCentralRegistry.getInstance().getActive(); - expect(row?.centralInstanceId).toBe('inst-uuid-1'); - expect(row?.centralApiUrl).toBe('https://central.example.com'); - expect(row?.lastBootstrapAt).toBeGreaterThan(0); - }); - - it('upsert overwrites when same instance id is provided', () => { - const reg = MeshCentralRegistry.getInstance(); - reg.upsert(sampleMaterial({ callbackJwt: 'old.jwt' })); - reg.upsert(sampleMaterial({ callbackJwt: 'new.jwt' })); - expect(reg.getActive()?.callbackJwt).toBe('new.jwt'); - }); - - it('getActive returns most-recently-bootstrapped row when multiple exist and warns once', () => { - const reg = MeshCentralRegistry.getInstance(); - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); - reg.upsert(sampleMaterial({ centralInstanceId: 'old', jwtIssuedAt: 1 })); - DatabaseService.getInstance().getDb().prepare(` - INSERT INTO mesh_centrals VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, NULL) - `).run('new', 'https://other.example.com', 'jwt2', 2, 999, Date.now() + 1000); - expect(reg.getActive()?.centralInstanceId).toBe('new'); - expect(warn).toHaveBeenCalledOnce(); - expect(warn.mock.calls[0][0]).toMatch(/multiple central rows/); - reg.getActive(); - expect(warn).toHaveBeenCalledOnce(); - warn.mockRestore(); - }); - - it('getActive returns null when table is empty', () => { - expect(MeshCentralRegistry.getInstance().getActive()).toBeNull(); - }); - - it('clearForInstance removes only the named instance', () => { - const reg = MeshCentralRegistry.getInstance(); - reg.upsert(sampleMaterial({ centralInstanceId: 'a' })); - reg.upsert(sampleMaterial({ centralInstanceId: 'b' })); - reg.clearForInstance('a'); - const remaining = DatabaseService.getInstance().getDb().prepare( - "SELECT central_instance_id FROM mesh_centrals" - ).all() as Array<{ central_instance_id: string }>; - expect(remaining.map(r => r.central_instance_id)).toEqual(['b']); - }); - - it('markUsed updates last_used_at, only on success', () => { - const reg = MeshCentralRegistry.getInstance(); - reg.upsert(sampleMaterial()); - reg.markUsed('inst-uuid-1'); - expect(reg.getActive()?.lastUsedAt).toBeGreaterThan(0); - }); - - it('markRejected stores reason and timestamp', () => { - const reg = MeshCentralRegistry.getInstance(); - reg.upsert(sampleMaterial()); - reg.markRejected('inst-uuid-1', 'token_fingerprint_mismatch'); - const row = reg.getActive(); - expect(row?.lastRejectedAt).toBeGreaterThan(0); - expect(row?.lastRejectReason).toBe('token_fingerprint_mismatch'); - }); -}); diff --git a/backend/src/__tests__/mesh-instance-id-change-integration.test.ts b/backend/src/__tests__/mesh-instance-id-change-integration.test.ts deleted file mode 100644 index 5a8c72ac..00000000 --- a/backend/src/__tests__/mesh-instance-id-change-integration.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Central instance id rotation invariant. - * - * `MeshCentralRegistry.upsert` is the single source of truth for the - * locally-cached callback material. When central regenerates its - * `instance_id` (factory reset, DB swap, container rebuild without volume), - * the next bootstrap frame arrives with a different `centralInstanceId`. The - * registry MUST: - * - persist the new row, - * - emit a single `central-instance-changed` event so subscribers (loud - * log, dialer reset) can react. - * - * Failing this invariant means the peer would keep dialing back with the - * old JWT (rejected with `instance_mismatch`) without anyone noticing. - */ -import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; -import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; -import { MeshCentralRegistry } from '../services/MeshCentralRegistry'; -import { DatabaseService } from '../services/DatabaseService'; - -let tmpDir: string; -beforeAll(async () => { tmpDir = await setupTestDb(); }); -afterAll(() => cleanupTestDb(tmpDir)); - -describe('Central instance id change', () => { - beforeEach(() => { - MeshCentralRegistry.resetForTest(); - // Per-test cleanup of the mesh_centrals table. The DB lives once per - // file (beforeAll); resetting setupTestDb per test would invalidate - // the DatabaseService singleton's connection on Linux. - DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_centrals').run(); - }); - - it('drops the old row and accepts the new one with a central-instance-changed event', () => { - const reg = MeshCentralRegistry.getInstance(); - reg.upsert({ - centralInstanceId: 'old-uuid', - centralApiUrl: 'https://central.example.com', - callbackJwt: 'jwt-old', - jwtIssuedAt: 1, - jwtExpiresAt: 999999, - }); - const events: Array<{ previousInstanceId: string; newInstanceId: string }> = []; - reg.on('central-instance-changed', (e: { previousInstanceId: string; newInstanceId: string }) => events.push(e)); - - reg.upsert({ - centralInstanceId: 'new-uuid', - centralApiUrl: 'https://central.example.com', - callbackJwt: 'jwt-new', - jwtIssuedAt: 2, - jwtExpiresAt: 999999, - }); - - expect(events).toHaveLength(1); - expect(events[0].previousInstanceId).toBe('old-uuid'); - expect(events[0].newInstanceId).toBe('new-uuid'); - // getActive returns the most-recently-bootstrapped row. - expect(reg.getActive()?.centralInstanceId).toBe('new-uuid'); - expect(reg.getActive()?.callbackJwt).toBe('jwt-new'); - }); - - it('an upsert with the same instance id does NOT emit central-instance-changed', () => { - const reg = MeshCentralRegistry.getInstance(); - reg.upsert({ - centralInstanceId: 'stable-uuid', - centralApiUrl: 'https://central.example.com', - callbackJwt: 'jwt-1', - jwtIssuedAt: 1, - jwtExpiresAt: 999999, - }); - const events: Array = []; - reg.on('central-instance-changed', (e: unknown) => events.push(e)); - reg.upsert({ - centralInstanceId: 'stable-uuid', - centralApiUrl: 'https://central.example.com', - callbackJwt: 'jwt-2-rotated', - jwtIssuedAt: 2, - jwtExpiresAt: 999999, - }); - expect(events).toHaveLength(0); - // Same id, the JWT material is overwritten in place. - expect(reg.getActive()?.callbackJwt).toBe('jwt-2-rotated'); - }); -}); diff --git a/backend/src/__tests__/mesh-proxy-tunnel-dialer-handshake.test.ts b/backend/src/__tests__/mesh-proxy-tunnel-dialer-handshake.test.ts deleted file mode 100644 index be52fa48..00000000 --- a/backend/src/__tests__/mesh-proxy-tunnel-dialer-handshake.test.ts +++ /dev/null @@ -1,164 +0,0 @@ -/** - * `MeshProxyTunnelDialer.maybeSendBootstrap`: capability-gated - * `mesh_handshake` send. - * - * Central mints a signed `mesh_tunnel`-scoped JWT bound to the peer's - * `api_token` fingerprint and pushes it as the first text frame on the - * fresh WS, but only when the peer advertises the - * `mesh_proxy_callback_bootstrap` capability AND a canonical central - * origin (`SENCHO_PRIMARY_URL`) is configured. All other paths must - * remain silent (no frame) so the legacy peer-side first-frame state - * machine can fall straight through to TCP traffic. - */ -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; -import jwt from 'jsonwebtoken'; -import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; -import { MeshProxyTunnelDialer } from '../services/MeshProxyTunnelDialer'; -import { NodeRegistry } from '../services/NodeRegistry'; -import { OFFLINE_META } from '../services/CapabilityRegistry'; -import { DatabaseService } from '../services/DatabaseService'; - -let tmpDir: string; -const CENTRAL_INSTANCE_ID = 'central-instance-for-handshake-test'; - -beforeAll(async () => { - tmpDir = await setupTestDb(); - // Seed system_state.instance_id (LicenseService.initialize() is not - // invoked in the test harness; mesh handshake requires this value). - DatabaseService.getInstance().setSystemState('instance_id', CENTRAL_INSTANCE_ID); -}); - -afterAll(() => { - cleanupTestDb(tmpDir); -}); - -describe('MeshProxyTunnelDialer capability-gated handshake', () => { - let nodeSeq = 0; - - beforeEach(() => { - MeshProxyTunnelDialer.resetForTest(); - process.env.SENCHO_PRIMARY_URL = 'https://central.example.com'; - nodeSeq += 1; - }); - - afterEach(() => { - delete process.env.SENCHO_PRIMARY_URL; - vi.restoreAllMocks(); - }); - - function fakeWs(): { send: ReturnType; sentFrames: unknown[] } { - const sentFrames: unknown[] = []; - return { - send: vi.fn((data: string) => { sentFrames.push(JSON.parse(data)); }), - sentFrames, - }; - } - - function seedProxyNode(): number { - const db = DatabaseService.getInstance(); - const id = db.addNode({ - name: `peer-handshake-${nodeSeq}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, - type: 'remote', - mode: 'proxy', - api_url: 'https://peer.example.com', - api_token: 'peer-token-for-fp', - compose_dir: '/tmp', - is_default: false, - }); - db.setNodeMeshEnabled(id, true); - return id; - } - - type DialerPrivate = { maybeSendBootstrap: (nodeId: number, ws: unknown) => Promise }; - - it('sends mesh_handshake when peer advertises mesh_proxy_callback_bootstrap', async () => { - vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue({ - ...OFFLINE_META, - version: '0.79.0', - capabilities: ['stacks', 'mesh_proxy_callback_bootstrap'], - online: true, - startedAt: Date.now(), - }); - const id = seedProxyNode(); - const ws = fakeWs(); - await (MeshProxyTunnelDialer.getInstance() as unknown as DialerPrivate).maybeSendBootstrap(id, ws); - expect(ws.send).toHaveBeenCalledOnce(); - expect(ws.sentFrames[0]).toMatchObject({ t: 'mesh_handshake', v: 1 }); - }); - - it('skips mesh_handshake when peer lacks the capability', async () => { - vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue({ - ...OFFLINE_META, - version: '0.78.0', - capabilities: ['stacks'], - online: true, - startedAt: Date.now(), - }); - const id = seedProxyNode(); - const ws = fakeWs(); - await (MeshProxyTunnelDialer.getInstance() as unknown as DialerPrivate).maybeSendBootstrap(id, ws); - expect(ws.send).not.toHaveBeenCalled(); - }); - - it('skips mesh_handshake when meta fetch returns offline shape', async () => { - vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue({ - ...OFFLINE_META, - online: false, - }); - const id = seedProxyNode(); - const ws = fakeWs(); - await (MeshProxyTunnelDialer.getInstance() as unknown as DialerPrivate).maybeSendBootstrap(id, ws); - expect(ws.send).not.toHaveBeenCalled(); - }); - - it('skips mesh_handshake when SENCHO_PRIMARY_URL is unset (fail-safe)', async () => { - delete process.env.SENCHO_PRIMARY_URL; - vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue({ - ...OFFLINE_META, - version: '0.79.0', - capabilities: ['mesh_proxy_callback_bootstrap'], - online: true, - startedAt: Date.now(), - }); - const id = seedProxyNode(); - const ws = fakeWs(); - await (MeshProxyTunnelDialer.getInstance() as unknown as DialerPrivate).maybeSendBootstrap(id, ws); - expect(ws.send).not.toHaveBeenCalled(); - }); - - it('signs the JWT with auth_jwt_secret using HS256 and includes the required claims', async () => { - vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue({ - ...OFFLINE_META, - version: '0.79.0', - capabilities: ['mesh_proxy_callback_bootstrap'], - online: true, - startedAt: Date.now(), - }); - const id = seedProxyNode(); - const ws = fakeWs(); - await (MeshProxyTunnelDialer.getInstance() as unknown as DialerPrivate).maybeSendBootstrap(id, ws); - const frame = ws.sentFrames[0] as { - meshTunnelJwt: string; - centralApiUrl: string; - centralInstanceId: string; - peerNodeId: number; - jwtExpiresAt: number; - }; - expect(frame.centralApiUrl).toBe('https://central.example.com'); - expect(frame.centralInstanceId).toBe(CENTRAL_INSTANCE_ID); - expect(frame.peerNodeId).toBe(id); - expect(typeof frame.jwtExpiresAt).toBe('number'); - const decoded = jwt.decode(frame.meshTunnelJwt, { complete: true }); - expect(decoded?.header.alg).toBe('HS256'); - const payload = decoded?.payload as Record; - expect(payload).toMatchObject({ - sub: String(id), - iss: CENTRAL_INSTANCE_ID, - aud: 'https://central.example.com', - scope: 'mesh_tunnel', - kid: 'v1', - }); - expect(typeof payload.peer_token_fp).toBe('string'); - expect((payload.peer_token_fp as string).length).toBe(16); - }); -}); diff --git a/backend/src/__tests__/mesh-proxy-tunnel-dialer-no-idle-close.test.ts b/backend/src/__tests__/mesh-proxy-tunnel-dialer-no-idle-close.test.ts new file mode 100644 index 00000000..eb235d40 --- /dev/null +++ b/backend/src/__tests__/mesh-proxy-tunnel-dialer-no-idle-close.test.ts @@ -0,0 +1,61 @@ +/** + * `MeshProxyTunnelDialer` default-TTL contract. + * + * The bridge is now a persistent bidirectional control-plane channel + * (peer→central reverse traffic multiplexes over the same WS that + * central uses to send forward `tcp_open` frames). The default + * `SENCHO_MESH_PROXY_TUNNEL_IDLE_MS` is `0`, which disables the idle + * sweeper entirely, so the dialer never tears the bridge down for + * inactivity. The env override is retained as an escape hatch for the + * legacy stream-scoped behavior. + */ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { MeshProxyTunnelDialer } from '../services/MeshProxyTunnelDialer'; + +let prev: string | undefined; + +beforeEach(() => { + prev = process.env.SENCHO_MESH_PROXY_TUNNEL_IDLE_MS; + delete process.env.SENCHO_MESH_PROXY_TUNNEL_IDLE_MS; +}); + +afterEach(() => { + if (prev === undefined) delete process.env.SENCHO_MESH_PROXY_TUNNEL_IDLE_MS; + else process.env.SENCHO_MESH_PROXY_TUNNEL_IDLE_MS = prev; +}); + +describe('MeshProxyTunnelDialer idle-close default', () => { + it('defaults to no idle close (TTL 0) and never starts the sweep timer', () => { + const dialer = MeshProxyTunnelDialer.resetForTest() as unknown as { + idleTtlMs: number; + idleCheckTimer: NodeJS.Timeout | null; + }; + expect(dialer.idleTtlMs).toBe(0); + expect(dialer.idleCheckTimer).toBeNull(); + }); + + it('respects an explicit env override to re-enable the legacy stream-scoped TTL', () => { + process.env.SENCHO_MESH_PROXY_TUNNEL_IDLE_MS = '300000'; + const dialer = MeshProxyTunnelDialer.resetForTest() as unknown as { + idleTtlMs: number; + idleCheckTimer: NodeJS.Timeout | null; + }; + expect(dialer.idleTtlMs).toBe(300_000); + expect(dialer.idleCheckTimer).not.toBeNull(); + }); + + it('respects an explicit override of `0` (no idle close) without scheduling', () => { + process.env.SENCHO_MESH_PROXY_TUNNEL_IDLE_MS = '0'; + const dialer = MeshProxyTunnelDialer.resetForTest() as unknown as { + idleTtlMs: number; + idleCheckTimer: NodeJS.Timeout | null; + }; + expect(dialer.idleTtlMs).toBe(0); + expect(dialer.idleCheckTimer).toBeNull(); + }); + + it('exposes isDialing(nodeId) which is false when no dial is in flight', () => { + const dialer = MeshProxyTunnelDialer.resetForTest(); + expect(dialer.isDialing(42)).toBe(false); + }); +}); diff --git a/backend/src/__tests__/mesh-proxy-tunnel-first-frame.test.ts b/backend/src/__tests__/mesh-proxy-tunnel-first-frame.test.ts deleted file mode 100644 index 240b05c7..00000000 --- a/backend/src/__tests__/mesh-proxy-tunnel-first-frame.test.ts +++ /dev/null @@ -1,227 +0,0 @@ -/** - * `meshProxyTunnel.ts` first-frame state machine. - * - * The PEER side of a proxy-mode mesh tunnel optionally consumes a - * `mesh_handshake` JSON frame as the FIRST text frame from central, - * persists the bootstrap material via MeshCentralRegistry, then yields - * control to the existing TcpStreamSwitchboard. Out-of-order or - * malformed handshake frames close the WS with code 1008. - */ -import http from 'http'; -import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; -import WebSocket from 'ws'; -import type { AddressInfo } from 'net'; -import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; - -let tmpDir: string; -let handleMeshProxyTunnel: typeof import('../websocket/meshProxyTunnel').handleMeshProxyTunnel; -let MeshService: typeof import('../services/MeshService').MeshService; -let MeshCentralRegistry: typeof import('../services/MeshCentralRegistry').MeshCentralRegistry; -let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; - -interface ServerHandle { - server: http.Server; - port: number; - close: () => Promise; -} - -async function startServer(): Promise { - const server = http.createServer(); - server.on('upgrade', (req, socket, head) => { - const pathname = new URL(req.url ?? '/', 'http://localhost').pathname; - if (pathname === '/api/mesh/proxy-tunnel') { - void handleMeshProxyTunnel(req, socket, head); - } else { - socket.destroy(); - } - }); - await new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve())); - const port = (server.address() as AddressInfo).port; - return { - server, - port, - close: () => new Promise((resolve) => server.close(() => resolve())), - }; -} - -function dialTunnel(port: number, query: string = ''): Promise { - return new Promise((resolve, reject) => { - const ws = new WebSocket(`ws://127.0.0.1:${port}/api/mesh/proxy-tunnel${query}`); - ws.once('open', () => resolve(ws)); - ws.once('error', reject); - }); -} - -function makeHandshakeFrame(overrides: Partial<{ - centralInstanceId: string; - centralApiUrl: string; - meshTunnelJwt: string; - jwtExpiresAt: number; - peerNodeId: number; -}> = {}): string { - return JSON.stringify({ - t: 'mesh_handshake', - v: 1, - peerNodeId: overrides.peerNodeId ?? 7, - centralInstanceId: overrides.centralInstanceId ?? 'central-instance-abc', - centralApiUrl: overrides.centralApiUrl ?? 'https://central.example.test', - meshTunnelJwt: overrides.meshTunnelJwt ?? 'jwt.token.body', - jwtExpiresAt: overrides.jwtExpiresAt ?? Math.floor(Date.now() / 1000) + 3600, - }); -} - -beforeAll(async () => { - tmpDir = await setupTestDb(); - ({ handleMeshProxyTunnel } = await import('../websocket/meshProxyTunnel')); - ({ MeshService } = await import('../services/MeshService')); - ({ MeshCentralRegistry } = await import('../services/MeshCentralRegistry')); - ({ DatabaseService } = await import('../services/DatabaseService')); -}); - -afterAll(() => { - cleanupTestDb(tmpDir); -}); - -beforeEach(() => { - delete process.env.SENCHO_MODE; - MeshService.getInstance().setReverseDialer(null); - MeshService.getInstance().setProxyTunnelSelfCentralNodeId(null); - MeshCentralRegistry.resetForTest(); - DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_centrals').run(); -}); - -describe('meshProxyTunnel first-frame state machine', () => { - it('consumes a mesh_handshake first frame and persists material via MeshCentralRegistry', async () => { - const srv = await startServer(); - try { - const ws = await dialTunnel(srv.port, '?nodeId=7'); - await new Promise((r) => setTimeout(r, 20)); - - const expiresAt = Math.floor(Date.now() / 1000) + 7200; - ws.send(makeHandshakeFrame({ - centralInstanceId: 'central-instance-xyz', - centralApiUrl: 'https://central.example.test/', - meshTunnelJwt: 'callback.jwt.value', - jwtExpiresAt: expiresAt, - })); - await new Promise((r) => setTimeout(r, 30)); - - const row = MeshCentralRegistry.getInstance().getActive(); - expect(row).not.toBeNull(); - expect(row?.centralInstanceId).toBe('central-instance-xyz'); - // Trailing slash should be stripped to keep the persisted URL canonical. - expect(row?.centralApiUrl).toBe('https://central.example.test'); - expect(row?.callbackJwt).toBe('callback.jwt.value'); - expect(row?.jwtExpiresAt).toBe(expiresAt); - - ws.close(1000, 'test cleanup'); - await new Promise((r) => setTimeout(r, 30)); - } finally { - await srv.close(); - } - }); - - it('passes through if first frame is not a mesh_handshake (delegates to switchboard)', async () => { - const srv = await startServer(); - try { - const ws = await dialTunnel(srv.port, '?nodeId=7'); - await new Promise((r) => setTimeout(r, 20)); - - // Send a known non-handshake JSON frame (tcp_close for a non-existent - // stream is a no-op the switchboard accepts without side effects). - ws.send(JSON.stringify({ t: 'tcp_close', s: 999999 })); - await new Promise((r) => setTimeout(r, 30)); - - // No handshake material should have been persisted. - expect(MeshCentralRegistry.getInstance().getActive()).toBeNull(); - // WS should still be open (no protocol error close). - expect(ws.readyState).toBe(WebSocket.OPEN); - - ws.close(1000, 'test cleanup'); - await new Promise((r) => setTimeout(r, 30)); - } finally { - await srv.close(); - } - }); - - it('closes the WS as protocol error when mesh_handshake arrives after a non-handshake frame', async () => { - const srv = await startServer(); - try { - const ws = await dialTunnel(srv.port, '?nodeId=7'); - await new Promise((r) => setTimeout(r, 20)); - - // First a non-handshake frame, then the handshake. - ws.send(JSON.stringify({ t: 'tcp_close', s: 999999 })); - await new Promise((r) => setTimeout(r, 10)); - - const closeInfo = new Promise<{ code: number; reason: string }>((resolve) => { - ws.once('close', (code, reason) => resolve({ code, reason: reason.toString() })); - }); - ws.send(makeHandshakeFrame()); - const info = await closeInfo; - expect(info.code).toBe(1008); - expect(MeshCentralRegistry.getInstance().getActive()).toBeNull(); - } finally { - await srv.close(); - } - }); - - it('rejects malformed mesh_handshake (missing required fields) and closes WS as protocol error', async () => { - const srv = await startServer(); - try { - const ws = await dialTunnel(srv.port, '?nodeId=7'); - await new Promise((r) => setTimeout(r, 20)); - - const closeInfo = new Promise<{ code: number; reason: string }>((resolve) => { - ws.once('close', (code, reason) => resolve({ code, reason: reason.toString() })); - }); - // Recognized 't' discriminator but missing required fields. - ws.send(JSON.stringify({ t: 'mesh_handshake', v: 1 })); - const info = await closeInfo; - expect(info.code).toBe(1008); - expect(MeshCentralRegistry.getInstance().getActive()).toBeNull(); - } finally { - await srv.close(); - } - }); - - it('persists mesh_handshake material even when reverse-dialer CAS rejects the bridge', async () => { - // Simulate a peer-initiated callback bridge that already owns the - // reverseDialer slot. Central's Trigger-2 dial then arrives with a - // rotated mesh_handshake frame; the CAS swap must refuse the new - // bridge, but the bootstrap material has to land on the peer - // anyway so subsequent peer-initiated callbacks authenticate with - // the rotated JWT. - MeshService.getInstance().setReverseDialer({ - openMeshTcpStream: () => null, - }); - - const srv = await startServer(); - try { - const ws = await dialTunnel(srv.port, '?nodeId=7'); - - const expiresAt = Math.floor(Date.now() / 1000) + 7200; - const closeInfo = new Promise<{ code: number }>((resolve) => { - ws.once('close', (code) => resolve({ code })); - }); - ws.send(makeHandshakeFrame({ - centralInstanceId: 'rotation-central-id', - centralApiUrl: 'https://central.example.test', - meshTunnelJwt: 'rotation.jwt.value', - jwtExpiresAt: expiresAt, - })); - - const info = await closeInfo; - expect(info.code).toBe(1013); - - const row = MeshCentralRegistry.getInstance().getActive(); - expect(row).not.toBeNull(); - expect(row?.centralInstanceId).toBe('rotation-central-id'); - expect(row?.centralApiUrl).toBe('https://central.example.test'); - expect(row?.callbackJwt).toBe('rotation.jwt.value'); - expect(row?.jwtExpiresAt).toBe(expiresAt); - } finally { - await srv.close(); - } - }); -}); diff --git a/backend/src/__tests__/mesh-proxy-tunnel-from-peer.test.ts b/backend/src/__tests__/mesh-proxy-tunnel-from-peer.test.ts deleted file mode 100644 index e499a015..00000000 --- a/backend/src/__tests__/mesh-proxy-tunnel-from-peer.test.ts +++ /dev/null @@ -1,282 +0,0 @@ -/** - * `meshProxyTunnelFromPeer.ts`: central-side ingress for peer-initiated - * dial-back tunnels. Validates the chain of JWT claims and node-state - * preconditions that the peer's `mesh_tunnel` bootstrap token must satisfy - * before the upgrade succeeds and a proxy bridge is registered. - * - * The chain (in order): algorithm whitelist, signature, scope, audience - * (= SENCHO_PRIMARY_URL), issuer (= central instance_id), exp, iat clock - * skew (60s), node existence, mode==proxy, api_token fingerprint match. - * Every failure point returns HTTP 401 with a JSON `{reason}` body using - * a stable machine-readable code; the happy path constructs a - * `PilotTunnelBridge` and registers it via - * `PilotTunnelManager.replaceOrRegisterProxyBridge`. - */ -import http from 'http'; -import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; -import WebSocket from 'ws'; -import jwt from 'jsonwebtoken'; -import { createHash } from 'crypto'; -import type { AddressInfo } from 'net'; -import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; - -let tmpDir: string; -let handleMeshProxyTunnelFromPeerUpgrade: typeof import('../websocket/meshProxyTunnelFromPeer').handleMeshProxyTunnelFromPeerUpgrade; -let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; -let PilotTunnelManager: typeof import('../services/PilotTunnelManager').PilotTunnelManager; - -interface ServerHandle { - server: http.Server; - port: number; - close: () => Promise; -} - -async function startServer(): Promise { - const server = http.createServer(); - server.on('upgrade', (req, socket, head) => { - const pathname = new URL(req.url ?? '/', 'http://localhost').pathname; - if (pathname === '/api/mesh/proxy-tunnel-from-peer') { - handleMeshProxyTunnelFromPeerUpgrade(req, socket, head); - } else { - socket.destroy(); - } - }); - await new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve())); - const port = (server.address() as AddressInfo).port; - return { - server, - port, - close: () => new Promise((resolve) => server.close(() => resolve())), - }; -} - -interface UpgradeOutcome { - kind: 'open' | 'unexpected' | 'error'; - status?: number; - body?: string; - ws?: WebSocket; -} - -function attemptUpgrade(port: number, token: string): Promise { - return new Promise((resolve) => { - const ws = new WebSocket(`ws://127.0.0.1:${port}/api/mesh/proxy-tunnel-from-peer`, { - headers: { authorization: `Bearer ${token}` }, - }); - const timer = setTimeout(() => resolve({ kind: 'error' }), 3000); - ws.once('open', () => { - clearTimeout(timer); - resolve({ kind: 'open', ws }); - }); - ws.once('unexpected-response', (_req, res) => { - const chunks: Buffer[] = []; - res.on('data', (c: Buffer) => chunks.push(c)); - res.on('end', () => { - clearTimeout(timer); - resolve({ kind: 'unexpected', status: res.statusCode, body: Buffer.concat(chunks).toString('utf8') }); - }); - }); - ws.once('error', () => { - // 'unexpected-response' fires first; 'error' here is the - // post-handshake failure ws raises after the server destroys - // the socket. Already resolved above. - }); - }); -} - -function parseReason(body: string | undefined): string | null { - if (!body) return null; - try { - const parsed = JSON.parse(body) as { reason?: unknown }; - return typeof parsed.reason === 'string' ? parsed.reason : null; - } catch { - return null; - } -} - -const CANONICAL_ORIGIN = 'https://central.example.com'; -const INSTANCE_ID = 'test-central-instance'; -const PEER_API_TOKEN = 'peer-token-123'; - -let secret: string; -let peerNodeId: number; -let srv: ServerHandle; - -beforeAll(async () => { - tmpDir = await setupTestDb(); - ({ handleMeshProxyTunnelFromPeerUpgrade } = await import('../websocket/meshProxyTunnelFromPeer')); - ({ DatabaseService } = await import('../services/DatabaseService')); - ({ PilotTunnelManager } = await import('../services/PilotTunnelManager')); -}); - -beforeEach(async () => { - process.env.SENCHO_PRIMARY_URL = CANONICAL_ORIGIN; - const db = DatabaseService.getInstance(); - secret = db.getGlobalSettings().auth_jwt_secret; - db.setSystemState('instance_id', INSTANCE_ID); - // Unique peer per test to avoid PilotTunnelManager collisions across - // the 12 cases (one happy path actually registers a bridge). - peerNodeId = db.addNode({ - name: `peer-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, - type: 'remote', - mode: 'proxy', - api_url: 'https://peer.example.com', - api_token: PEER_API_TOKEN, - compose_dir: '/tmp', - is_default: false, - }); - db.setNodeMeshEnabled(peerNodeId, true); - if (!srv) srv = await startServer(); -}); - -afterAll(async () => { - if (srv) await srv.close(); - delete process.env.SENCHO_PRIMARY_URL; - cleanupTestDb(tmpDir); -}); - -function expectedFp(token: string = PEER_API_TOKEN): string { - return createHash('sha256').update(token).digest('hex').slice(0, 16); -} - -function makeJwt(overrides: Partial> = {}, signOpts: { alg?: jwt.Algorithm; secret?: string } = {}): string { - const payload: Record = { - sub: String(peerNodeId), - iss: INSTANCE_ID, - aud: CANONICAL_ORIGIN, - scope: 'mesh_tunnel', - iat: Math.floor(Date.now() / 1000), - exp: Math.floor(Date.now() / 1000) + 3600, - kid: 'v1', - peer_token_fp: expectedFp(), - ...overrides, - }; - return jwt.sign(payload, signOpts.secret ?? secret, { algorithm: signOpts.alg ?? 'HS256' }); -} - -describe('/api/mesh/proxy-tunnel-from-peer validation chain', () => { - it('rejects alg=none (algorithm_mismatch)', async () => { - // jsonwebtoken refuses to sign with alg=none unless explicitly - // enabled and given a null secret; we build the token manually so - // the test exercises the central's defence, not the library's. - const header = Buffer.from(JSON.stringify({ alg: 'none', typ: 'JWT' })).toString('base64url'); - const body = Buffer.from(JSON.stringify({ - sub: String(peerNodeId), iss: INSTANCE_ID, aud: CANONICAL_ORIGIN, - scope: 'mesh_tunnel', iat: Math.floor(Date.now() / 1000), - exp: Math.floor(Date.now() / 1000) + 3600, peer_token_fp: expectedFp(), - })).toString('base64url'); - const token = `${header}.${body}.`; - const outcome = await attemptUpgrade(srv.port, token); - expect(outcome.kind).toBe('unexpected'); - expect(outcome.status).toBe(401); - expect(parseReason(outcome.body)).toBe('algorithm_mismatch'); - }); - - it('rejects alg=RS256 (algorithm_mismatch)', async () => { - const { generateKeyPairSync } = await import('crypto'); - const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); - const token = jwt.sign({ - sub: String(peerNodeId), iss: INSTANCE_ID, aud: CANONICAL_ORIGIN, - scope: 'mesh_tunnel', exp: Math.floor(Date.now() / 1000) + 3600, - peer_token_fp: expectedFp(), - }, privateKey, { algorithm: 'RS256' }); - const outcome = await attemptUpgrade(srv.port, token); - expect(outcome.kind).toBe('unexpected'); - expect(outcome.status).toBe(401); - expect(parseReason(outcome.body)).toBe('algorithm_mismatch'); - }); - - it('rejects bad signature (signature_invalid)', async () => { - const token = makeJwt({}, { secret: 'wrong-secret-not-the-real-one' }); - const outcome = await attemptUpgrade(srv.port, token); - expect(outcome.kind).toBe('unexpected'); - expect(outcome.status).toBe(401); - expect(parseReason(outcome.body)).toBe('signature_invalid'); - }); - - it('rejects scope mismatch', async () => { - const token = makeJwt({ scope: 'pilot_tunnel' }); - const outcome = await attemptUpgrade(srv.port, token); - expect(outcome.kind).toBe('unexpected'); - expect(outcome.status).toBe(401); - expect(parseReason(outcome.body)).toBe('scope_mismatch'); - }); - - it('rejects audience mismatch', async () => { - const token = makeJwt({ aud: 'https://other.example.com' }); - const outcome = await attemptUpgrade(srv.port, token); - expect(outcome.kind).toBe('unexpected'); - expect(outcome.status).toBe(401); - expect(parseReason(outcome.body)).toBe('audience_mismatch'); - }); - - it('rejects issuer mismatch', async () => { - const token = makeJwt({ iss: 'wrong-instance-id' }); - const outcome = await attemptUpgrade(srv.port, token); - expect(outcome.kind).toBe('unexpected'); - expect(outcome.status).toBe(401); - expect(parseReason(outcome.body)).toBe('instance_mismatch'); - }); - - it('rejects expired token (stale)', async () => { - const token = makeJwt({ exp: Math.floor(Date.now() / 1000) - 10 }); - const outcome = await attemptUpgrade(srv.port, token); - expect(outcome.kind).toBe('unexpected'); - expect(outcome.status).toBe(401); - // jsonwebtoken throws on expired tokens before our exp check sees it, - // so the rejection surfaces as signature_invalid via the verify catch. - // The contract: stale tokens are rejected with a 401 and some - // deterministic reason code; accept either of the two equivalent - // failures since both convey "stale credential" to the operator. - const reason = parseReason(outcome.body); - expect(['stale', 'signature_invalid']).toContain(reason); - }); - - it('rejects clock-skewed token (clock_skew)', async () => { - const token = makeJwt({ iat: Math.floor(Date.now() / 1000) + 600 }); - const outcome = await attemptUpgrade(srv.port, token); - expect(outcome.kind).toBe('unexpected'); - expect(outcome.status).toBe(401); - expect(parseReason(outcome.body)).toBe('clock_skew'); - }); - - it('rejects missing node (node_deleted)', async () => { - const token = makeJwt({ sub: '999999' }); - const outcome = await attemptUpgrade(srv.port, token); - expect(outcome.kind).toBe('unexpected'); - expect(outcome.status).toBe(401); - expect(parseReason(outcome.body)).toBe('node_deleted'); - }); - - it('rejects mode mismatch', async () => { - DatabaseService.getInstance().updateNode(peerNodeId, { mode: 'pilot_agent' }); - const token = makeJwt(); - const outcome = await attemptUpgrade(srv.port, token); - expect(outcome.kind).toBe('unexpected'); - expect(outcome.status).toBe(401); - expect(parseReason(outcome.body)).toBe('mode_mismatch'); - }); - - it('rejects token fingerprint mismatch', async () => { - const token = makeJwt(); - // Rotate the api_token after minting; the JWT now carries the - // fingerprint of the old token, but getNode returns the new one. - DatabaseService.getInstance().updateNode(peerNodeId, { api_token: 'rotated-token-xyz' }); - const outcome = await attemptUpgrade(srv.port, token); - expect(outcome.kind).toBe('unexpected'); - expect(outcome.status).toBe(401); - expect(parseReason(outcome.body)).toBe('token_fingerprint_mismatch'); - }); - - it('accepts a fully valid token and registers a proxy bridge', async () => { - const token = makeJwt(); - const outcome = await attemptUpgrade(srv.port, token); - expect(outcome.kind).toBe('open'); - // Give the bridge.start() microtasks a moment to land and register. - await new Promise((r) => setTimeout(r, 50)); - const bridge = PilotTunnelManager.getInstance().getBridge(peerNodeId); - expect(bridge).not.toBeNull(); - try { outcome.ws?.close(1000, 'test cleanup'); } catch { /* ignore */ } - // Allow the manager's 'closed' handler to remove the bridge entry. - await new Promise((r) => setTimeout(r, 30)); - }); -}); diff --git a/backend/src/__tests__/mesh-service-proactive-bootstrap-fanout.test.ts b/backend/src/__tests__/mesh-service-proxy-tunnel-reconcile.test.ts similarity index 62% rename from backend/src/__tests__/mesh-service-proactive-bootstrap-fanout.test.ts rename to backend/src/__tests__/mesh-service-proxy-tunnel-reconcile.test.ts index 7568fdb2..77885066 100644 --- a/backend/src/__tests__/mesh-service-proactive-bootstrap-fanout.test.ts +++ b/backend/src/__tests__/mesh-service-proxy-tunnel-reconcile.test.ts @@ -1,15 +1,11 @@ /** - * `MeshService.proactiveBootstrapFanout` (Trigger 3). + * `MeshService.proactiveBridgeFanout` and the bridge-reconcile loop. * - * At central startup, after `MeshService.start()` finishes, iterate the - * mesh-enabled proxy-mode nodes that have at least one `mesh_stacks` row - * and call `MeshProxyTunnelDialer.ensureBridge(nodeId)` on each. This - * proactively re-establishes the central->peer bridges so the - * capability-gated handshake can mint and ship bootstrap material to any - * peer that just came online or just got upgraded to v0.79+. - * - * Concurrency 4, 250ms stagger, fire-and-forget. Failures do not abort - * the fan-out. + * Central proactively dials every mesh-enabled proxy-mode peer at startup + * and on every reconcile tick so peer→central reverse traffic always has a + * live forward WS to multiplex through. The fanout selects ALL such peers + * regardless of whether they have any `mesh_stacks` rows yet, because the + * bridge is a control-plane primitive, not stack-scoped. */ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; @@ -61,7 +57,13 @@ function clearNodesAndMeshStacks(): void { db.prepare('DELETE FROM nodes WHERE is_default = 0').run(); } -describe('MeshService.proactiveBootstrapFanout (Trigger 3)', () => { +function callFanout(): Promise { + return (MeshService.getInstance() as unknown as { + proactiveBridgeFanout: () => Promise; + }).proactiveBridgeFanout(); +} + +describe('MeshService.proactiveBridgeFanout', () => { beforeEach(() => { clearNodesAndMeshStacks(); MeshProxyTunnelDialer.resetForTest(); @@ -71,7 +73,7 @@ describe('MeshService.proactiveBootstrapFanout (Trigger 3)', () => { vi.restoreAllMocks(); }); - it('iterates only mesh-enabled proxy-mode nodes that have mesh_stacks rows', async () => { + it('iterates every mesh-enabled proxy-mode node regardless of mesh_stacks rows', async () => { const calls: number[] = []; vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge') .mockImplementation(async (id: number) => { @@ -79,22 +81,20 @@ describe('MeshService.proactiveBootstrapFanout (Trigger 3)', () => { return null; }); - // Eligible: mesh-enabled proxy node with a mesh_stacks row. - const eligible = seedProxyNode(true); - insertMeshStack(eligible, `stack-eligible-${uniqueSuffix()}`); + const withStack = seedProxyNode(true); + insertMeshStack(withStack, `stack-${uniqueSuffix()}`); - // Ineligible: mesh-enabled proxy node WITHOUT a mesh_stacks row. - seedProxyNode(true); + // Mesh-enabled proxy node WITHOUT any mesh_stacks row. The old + // SQL excluded these; this assertion locks in the new behavior. + const withoutStack = seedProxyNode(true); - // Ineligible: mesh-disabled proxy node WITH a mesh_stacks row. + // Mesh-disabled proxy node WITH a mesh_stacks row. Must still be skipped. const meshOff = seedProxyNode(false); insertMeshStack(meshOff, `stack-meshoff-${uniqueSuffix()}`); - await (MeshService.getInstance() as unknown as { - proactiveBootstrapFanout: () => Promise; - }).proactiveBootstrapFanout(); + await callFanout(); - expect(calls).toEqual([eligible]); + expect(calls.sort((a, b) => a - b)).toEqual([withStack, withoutStack].sort((a, b) => a - b)); }); it('throttles to concurrency 4', async () => { @@ -109,14 +109,9 @@ describe('MeshService.proactiveBootstrapFanout (Trigger 3)', () => { return null; }); - for (let i = 0; i < 12; i++) { - const id = seedProxyNode(true); - insertMeshStack(id, `stack-${i}-${uniqueSuffix()}`); - } + for (let i = 0; i < 12; i++) seedProxyNode(true); - await (MeshService.getInstance() as unknown as { - proactiveBootstrapFanout: () => Promise; - }).proactiveBootstrapFanout(); + await callFanout(); expect(maxInflight).toBeLessThanOrEqual(4); expect(maxInflight).toBeGreaterThan(0); @@ -124,11 +119,7 @@ describe('MeshService.proactiveBootstrapFanout (Trigger 3)', () => { it('failures do not abort the fanout', async () => { const ids: number[] = []; - for (let i = 0; i < 3; i++) { - const id = seedProxyNode(true); - insertMeshStack(id, `stack-fail-${i}-${uniqueSuffix()}`); - ids.push(id); - } + for (let i = 0; i < 3; i++) ids.push(seedProxyNode(true)); const failingId = ids[1]; const calls: number[] = []; @@ -139,10 +130,19 @@ describe('MeshService.proactiveBootstrapFanout (Trigger 3)', () => { return null; }); - await (MeshService.getInstance() as unknown as { - proactiveBootstrapFanout: () => Promise; - }).proactiveBootstrapFanout(); + await callFanout(); expect(calls.slice().sort((a, b) => a - b)).toEqual(ids.slice().sort((a, b) => a - b)); }); + + it('repeated fanout calls invoke ensureBridge each tick (reconcile semantics)', async () => { + const ensureSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge') + .mockResolvedValue(null); + + const id = seedProxyNode(true); + await callFanout(); + await callFanout(); + + expect(ensureSpy.mock.calls.filter(([n]) => n === id).length).toBe(2); + }); }); diff --git a/backend/src/__tests__/mesh-service.test.ts b/backend/src/__tests__/mesh-service.test.ts index 18fabc71..5fe56193 100644 --- a/backend/src/__tests__/mesh-service.test.ts +++ b/backend/src/__tests__/mesh-service.test.ts @@ -1021,10 +1021,9 @@ describe('MeshService.openCrossNode (BUG-4)', () => { }; } - // Install a stub reverseDialer so openCrossNode skips the peer-side - // bootstrap path (PeerToCentralMeshSessionDialer.ensureSession). The - // dispatch behavior under test relies on dialMeshTcpStream being called - // directly; the bootstrap kick would short-circuit before that mock fires. + // Install a stub reverseDialer so openCrossNode routes through the + // multiplex path rather than depending on a live MeshProxyTunnelDialer + // bridge being open against a real peer. const stubDialer = { openMeshTcpStream: vi.fn() }; beforeEach(() => { MeshService.getInstance().setReverseDialer(stubDialer); }); afterEach(() => { MeshService.getInstance().setReverseDialer(null); }); @@ -1092,16 +1091,13 @@ describe('MeshService.openCrossNode (BUG-4)', () => { }); }); -describe('MeshService.openCrossNode peer-recovery gating', () => { - // Regression cover for the v0.81.0 forward-direction break: on central - // (no mesh_centrals row, no reverseDialer because central is the bridge - // initiator side) the peer-recovery branch incorrectly fired, found no - // PeerToCentralMeshSessionDialer session, and destroyed the inbound - // socket with route.resolve.fail forward-from-peer no_session. The - // intent of the branch was peer cold-start; the guard - // `!this.reverseDialer` alone could not distinguish "I am central" from - // "I am a peer waiting for central to dial in", because both states - // present as reverseDialer===null. +describe('MeshService.openCrossNode without reverseDialer', () => { + // Forward + reverse mesh traffic now share the same central-initiated + // bridge. When openCrossNode runs without a reverseDialer installed it + // falls straight through to dialMeshTcpStream, which on central uses + // MeshProxyTunnelDialer.ensureBridge to reach a proxy peer. The + // previous peer-recovery branch (which depended on the now-removed + // mesh_centrals table) is gone. function makeFakeStream(streamId: number): MeshTcpStreamLike & EventEmitter { const ee = new EventEmitter() as MeshTcpStreamLike & EventEmitter & { destroyed: boolean }; ee.destroyed = false; @@ -1115,17 +1111,11 @@ describe('MeshService.openCrossNode peer-recovery gating', () => { return { destroy: vi.fn(), end: vi.fn(), on: vi.fn(), write: vi.fn() }; } - // No stub reverseDialer here: the whole point is to exercise the - // !reverseDialer code path. Reset the registry per test so the mesh_centrals - // row state is deterministic. - beforeEach(async () => { - const { MeshCentralRegistry } = await import('../services/MeshCentralRegistry'); - MeshCentralRegistry.resetForTest(); - DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_centrals').run(); + beforeEach(() => { MeshService.getInstance().setReverseDialer(null); }); - it('central (no mesh_centrals row) skips peer-recovery and calls dialMeshTcpStream', async () => { + it('central (no reverseDialer) falls through to dialMeshTcpStream without route.resolve.fail forward-from-peer', async () => { const svc = MeshService.getInstance(); const target: MeshTarget = { nodeId: 7, stack: 'audit-mesh-proxy', service: 'echo', @@ -1142,11 +1132,7 @@ describe('MeshService.openCrossNode peer-recovery gating', () => { .openCrossNode(target, fakeSrc); const events = svc.getActivity({ limit: 50 }); - const dispatch = events.find((e) => e.type === 'route.dispatch'); - expect(dispatch).toBeDefined(); - - // The bug surface: a route.resolve.fail with direction=forward-from-peer - // would mean central wrongly went through the peer-recovery branch. + expect(events.some((e) => e.type === 'route.dispatch')).toBe(true); const wrongFail = events.find((e) => e.type === 'route.resolve.fail' && (e.details as { direction?: string } | undefined)?.direction === 'forward-from-peer', @@ -1156,48 +1142,6 @@ describe('MeshService.openCrossNode peer-recovery gating', () => { expect(fakeSrc.destroy).not.toHaveBeenCalled(); fakeStream.emit('close'); }); - - it('proxy-peer (mesh_centrals row + no session) still emits route.resolve.fail forward-from-peer no_session', async () => { - const { MeshCentralRegistry } = await import('../services/MeshCentralRegistry'); - const { PeerToCentralMeshSessionDialer } = await import('../services/PeerToCentralMeshSessionDialer'); - MeshCentralRegistry.getInstance().upsert({ - centralInstanceId: 'central-uuid-test', - centralApiUrl: 'https://central.example.com', - callbackJwt: 'eyJhbGciOiJIUzI1NiJ9.fake.token', - jwtIssuedAt: Math.floor(Date.now() / 1000), - jwtExpiresAt: Math.floor(Date.now() / 1000) + 90 * 24 * 3600, - }); - const ensureSpy = vi.spyOn( - PeerToCentralMeshSessionDialer.getInstance(), - 'ensureSession', - ).mockResolvedValue(null); - - const svc = MeshService.getInstance(); - const target: MeshTarget = { - nodeId: 1, stack: 'audit-mesh-central', service: 'echo', - port: 9000, alias: 'echo.audit-mesh-central.Local.sencho', - }; - const dialSpy = vi.spyOn( - svc as unknown as { dialMeshTcpStream: (t: MeshTarget) => MeshTcpStreamLike | null }, - 'dialMeshTcpStream', - ); - - const fakeSrc = makeFakeSocket(); - await (svc as unknown as { openCrossNode: (t: MeshTarget, s: unknown) => Promise }) - .openCrossNode(target, fakeSrc); - - const events = svc.getActivity({ limit: 50 }); - const fail = events.find((e) => - e.type === 'route.resolve.fail' - && (e.details as { direction?: string; reason?: string } | undefined)?.direction === 'forward-from-peer' - && (e.details as { reason?: string } | undefined)?.reason === 'no_session', - ); - expect(fail).toBeDefined(); - expect(ensureSpy).toHaveBeenCalled(); - // Peer-recovery aborted dispatch before reaching dialMeshTcpStream. - expect(dialSpy).not.toHaveBeenCalled(); - expect(fakeSrc.destroy).toHaveBeenCalled(); - }); }); describe('MeshService pilot handleAccept dispatch', () => { diff --git a/backend/src/__tests__/mesh-setup-error-classification.test.ts b/backend/src/__tests__/mesh-setup-error-classification.test.ts index 1dea6cbb..3dec2f0f 100644 --- a/backend/src/__tests__/mesh-setup-error-classification.test.ts +++ b/backend/src/__tests__/mesh-setup-error-classification.test.ts @@ -5,13 +5,11 @@ import type { MeshActivityEvent, MeshDataPlaneStatus } from '../services/MeshSer let tmpDir: string; let MeshService: typeof import('../services/MeshService').MeshService; let DockerController: typeof import('../services/DockerController').default; -let capability: typeof import('../services/CapabilityRegistry'); beforeAll(async () => { tmpDir = await setupTestDb(); ({ MeshService } = await import('../services/MeshService')); ({ default: DockerController } = await import('../services/DockerController')); - capability = await import('../services/CapabilityRegistry'); }); afterAll(() => { @@ -43,7 +41,6 @@ beforeEach(() => { message: 'mesh data plane has not initialized yet', subnet: '', }; - capability.enableCapability('mesh_proxy_callback_bootstrap'); }); afterEach(() => { @@ -51,7 +48,6 @@ afterEach(() => { else process.env.SENCHO_MESH_SUBNET = prevSubnetEnv; if (prevHostnameEnv === undefined) delete process.env.HOSTNAME; else process.env.HOSTNAME = prevHostnameEnv; - capability.enableCapability('mesh_proxy_callback_bootstrap'); vi.restoreAllMocks(); }); @@ -94,7 +90,6 @@ describe('MeshService.setupMeshNetwork failure classification', () => { expect(status.ok).toBe(false); expect(status.reason).toBe('subnet_invalid'); expect(status.subnet).toBe('not-a-cidr'); - expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap'); const entry = lastDisable(svc); expect(entry?.level).toBe('error'); expect(entry?.details?.reason).toBe('subnet_invalid'); @@ -114,7 +109,6 @@ describe('MeshService.setupMeshNetwork failure classification', () => { expect(status.reason).toBe('subnet_overlap'); expect(status.subnet).toBe('10.42.0.0/24'); expect(status.message).toMatch(/overlap/i); - expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap'); const entry = lastDisable(svc); expect(entry?.level).toBe('error'); expect(entry?.details?.reason).toBe('subnet_overlap'); @@ -169,9 +163,6 @@ describe('MeshService.setupMeshNetwork failure classification', () => { const entry = lastDisable(svc); expect(entry?.level).toBe('warn'); expect(entry?.details?.reason).toBe('not_in_docker'); - // Capability is still stripped even in the warn-level not_in_docker - // case; pilot processes outside Docker must not advertise it. - expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap'); }); it('records the 404-on-inspect path as not_in_docker at level warn', async () => { @@ -187,11 +178,9 @@ describe('MeshService.setupMeshNetwork failure classification', () => { expect(lastDisable(svc)?.level).toBe('warn'); }); - it('records success as ok and re-enables the capability', async () => { + it('records success as ok', async () => { process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24'; process.env.HOSTNAME = 'sencho'; - // Force the capability off first to prove the success path flips it back on. - capability.disableCapability('mesh_proxy_callback_bootstrap'); mockDocker(); const svc = MeshService.getInstance(); await callSetup(svc); @@ -200,7 +189,6 @@ describe('MeshService.setupMeshNetwork failure classification', () => { expect(status.reason).toBe('ok'); expect(status.subnet).toBe('10.42.0.0/24'); expect(status.message).toBeNull(); - expect(capability.getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap'); }); it('preserves the legacy networkSetupError getter on failure', async () => { diff --git a/backend/src/__tests__/mesh-startup-primary-url-warning.test.ts b/backend/src/__tests__/mesh-startup-primary-url-warning.test.ts deleted file mode 100644 index f34e95bc..00000000 --- a/backend/src/__tests__/mesh-startup-primary-url-warning.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest'; -import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; -import { MeshService } from '../services/MeshService'; -import { DatabaseService } from '../services/DatabaseService'; - -function seedMeshedProxyNode(): void { - const db = DatabaseService.getInstance(); - const id = db.addNode({ - name: `p-${Date.now()}-${Math.random()}`, - type: 'remote', - mode: 'proxy', - api_url: 'https://p.example.com', - api_token: 't', - compose_dir: '/tmp', - is_default: false, - }); - db.setNodeMeshEnabled(id, true); -} - -let tmpDir: string; -beforeAll(async () => { tmpDir = await setupTestDb(); }); -afterAll(() => cleanupTestDb(tmpDir)); - -describe('SENCHO_PRIMARY_URL preflight warning', () => { - let originalPrimaryUrl: string | undefined; - - beforeEach(() => { - originalPrimaryUrl = process.env.SENCHO_PRIMARY_URL; - // The DB singleton persists across tests in the same process; wipe any - // non-default nodes seeded by a prior test so each case starts clean. - // setupTestDb itself runs once per file (beforeAll above) to avoid - // invalidating the DatabaseService singleton's open connection. - DatabaseService.getInstance().getDb().prepare('DELETE FROM nodes WHERE is_default = 0').run(); - }); - - afterEach(() => { - if (originalPrimaryUrl === undefined) { - delete process.env.SENCHO_PRIMARY_URL; - } else { - process.env.SENCHO_PRIMARY_URL = originalPrimaryUrl; - } - }); - - it('warns when SENCHO_PRIMARY_URL is unset and a mesh-enabled proxy node exists', () => { - delete process.env.SENCHO_PRIMARY_URL; - seedMeshedProxyNode(); - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); - const svc = MeshService.getInstance() as unknown as { maybeWarnUnsetPrimaryUrl: () => void }; - svc.maybeWarnUnsetPrimaryUrl(); - expect(warn.mock.calls.some(c => /SENCHO_PRIMARY_URL is unset/.test(String(c[0])))).toBe(true); - warn.mockRestore(); - }); - - it('does not warn when SENCHO_PRIMARY_URL is set', () => { - process.env.SENCHO_PRIMARY_URL = 'https://central.example.com'; - seedMeshedProxyNode(); - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); - const svc = MeshService.getInstance() as unknown as { maybeWarnUnsetPrimaryUrl: () => void }; - svc.maybeWarnUnsetPrimaryUrl(); - expect(warn.mock.calls.some(c => /SENCHO_PRIMARY_URL is unset/.test(String(c[0])))).toBe(false); - warn.mockRestore(); - }); - - it('does not warn when SENCHO_PRIMARY_URL is unset but no mesh-enabled proxy node exists', () => { - delete process.env.SENCHO_PRIMARY_URL; - // No seeding: zero mesh-enabled proxy nodes (only the default local node). - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); - const svc = MeshService.getInstance() as unknown as { maybeWarnUnsetPrimaryUrl: () => void }; - svc.maybeWarnUnsetPrimaryUrl(); - expect(warn.mock.calls.some(c => /SENCHO_PRIMARY_URL is unset/.test(String(c[0])))).toBe(false); - warn.mockRestore(); - }); -}); diff --git a/backend/src/__tests__/mesh-status-reverse-callback.test.ts b/backend/src/__tests__/mesh-status-reverse-callback.test.ts new file mode 100644 index 00000000..28e0e39e --- /dev/null +++ b/backend/src/__tests__/mesh-status-reverse-callback.test.ts @@ -0,0 +1,109 @@ +/** + * `MeshService.getStatus().reverseCallbackStatus` discriminator. + * + * Surfaces the per-node state of the peer→central reverse path: forward + * bridge open (`connected`), dial in flight (`connecting`), no bridge and + * not dialing (`unavailable`), or non-proxy node (`not_applicable`). The + * Routing tab consumes this to render a transient pill while central + * reconciles a dropped bridge. + */ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { MeshService } from '../services/MeshService'; +import { MeshProxyTunnelDialer } from '../services/MeshProxyTunnelDialer'; +import { DatabaseService } from '../services/DatabaseService'; + +let tmpDir: string; +beforeAll(async () => { tmpDir = await setupTestDb(); }); +afterAll(() => cleanupTestDb(tmpDir)); + +function uniqueSuffix(): string { + return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; +} + +function seedProxyNode(meshEnabled: boolean): number { + const db = DatabaseService.getInstance(); + const id = db.addNode({ + name: `peer-status-${uniqueSuffix()}`, + type: 'remote', + mode: 'proxy', + api_url: `https://peer-${uniqueSuffix()}.example.com`, + api_token: `tok-${uniqueSuffix()}`, + compose_dir: '/tmp', + is_default: false, + }); + if (meshEnabled) db.setNodeMeshEnabled(id, true); + return id; +} + +function seedPilotNode(): number { + const db = DatabaseService.getInstance(); + return db.addNode({ + name: `pilot-${uniqueSuffix()}`, + type: 'remote', + mode: 'pilot_agent', + api_url: `https://pilot-${uniqueSuffix()}.example.com`, + api_token: `tok-${uniqueSuffix()}`, + compose_dir: '/tmp', + is_default: false, + }); +} + +beforeEach(() => { + DatabaseService.getInstance().getDb().prepare('DELETE FROM nodes WHERE is_default = 0').run(); + MeshProxyTunnelDialer.resetForTest(); + vi.restoreAllMocks(); +}); + +describe('MeshService.getStatus reverseCallbackStatus', () => { + it('returns connected when the dialer has a bridge for a mesh-enabled proxy peer', async () => { + const id = seedProxyNode(true); + vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'hasBridge') + .mockImplementation((n: number) => n === id); + + const statuses = await MeshService.getInstance().getStatus(); + const entry = statuses.find((s) => s.nodeId === id); + expect(entry?.reverseCallbackStatus).toBe('connected'); + }); + + it('returns connecting when a dial is in flight for the peer', async () => { + const id = seedProxyNode(true); + vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'hasBridge').mockReturnValue(false); + vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'isDialing') + .mockImplementation((n: number) => n === id); + + const statuses = await MeshService.getInstance().getStatus(); + const entry = statuses.find((s) => s.nodeId === id); + expect(entry?.reverseCallbackStatus).toBe('connecting'); + }); + + it('returns unavailable when no bridge and no dial in flight', async () => { + const id = seedProxyNode(true); + vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'hasBridge').mockReturnValue(false); + vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'isDialing').mockReturnValue(false); + + const statuses = await MeshService.getInstance().getStatus(); + const entry = statuses.find((s) => s.nodeId === id); + expect(entry?.reverseCallbackStatus).toBe('unavailable'); + }); + + it('returns not_applicable for the local node', async () => { + const statuses = await MeshService.getInstance().getStatus(); + const local = statuses.find((s) => s.nodeName === 'Local' || s.nodeId === 1); + expect(local?.reverseCallbackStatus).toBe('not_applicable'); + }); + + it('returns not_applicable for pilot-mode peers', async () => { + const id = seedPilotNode(); + const statuses = await MeshService.getInstance().getStatus(); + const entry = statuses.find((s) => s.nodeId === id); + expect(entry?.reverseCallbackStatus).toBe('not_applicable'); + }); + + it('returns not_applicable for mesh-disabled proxy peers', async () => { + const id = seedProxyNode(false); + const statuses = await MeshService.getInstance().getStatus(); + const entry = statuses.find((s) => s.nodeId === id); + expect(entry?.reverseCallbackStatus).toBe('not_applicable'); + }); +}); diff --git a/backend/src/__tests__/mesh-token-rotation-integration.test.ts b/backend/src/__tests__/mesh-token-rotation-integration.test.ts deleted file mode 100644 index 217cf406..00000000 --- a/backend/src/__tests__/mesh-token-rotation-integration.test.ts +++ /dev/null @@ -1,197 +0,0 @@ -/** - * Triggers 1 + 2: proactive mesh bridge bootstrap on mesh-enable and on - * api_token rotation. - * - * Trigger 1 (mesh-enable): after `setNodeMeshEnabled(true)`, the service - * fire-and-forgets `MeshProxyTunnelDialer.ensureBridge(nodeId)` for any - * remote/proxy peer. The capability-gated handshake then ships - * `mesh_handshake` material to peers that just came online or just got - * upgraded to a build that advertises `mesh_proxy_callback_bootstrap`. - * - * Trigger 2 (api_token rotation): when the nodes router persists a new - * `api_token` for a mesh-enabled proxy peer, it closes the existing bridge - * with reason 'peer token rotated' and re-dials. The next ensureBridge mints - * a JWT whose fingerprint matches the freshly stored token; without this - * trigger the remote would reject the upgrade with token_fingerprint_mismatch - * until the next idle re-dial. - */ -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; -import request from 'supertest'; -import jwt from 'jsonwebtoken'; -import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb'; -import { MeshService } from '../services/MeshService'; -import { MeshProxyTunnelDialer } from '../services/MeshProxyTunnelDialer'; -import { DatabaseService } from '../services/DatabaseService'; - -let tmpDir: string; -let app: import('express').Express; -let authHeader: string; - -beforeAll(async () => { - tmpDir = await setupTestDb(); - ({ app } = await import('../index')); - const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' }); - authHeader = `Bearer ${token}`; -}); - -afterAll(() => { - cleanupTestDb(tmpDir); -}); - -function uniqueSuffix(): string { - return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; -} - -function seedProxyNode(meshEnabled: boolean): number { - const db = DatabaseService.getInstance(); - const id = db.addNode({ - name: `peer-rotate-${uniqueSuffix()}`, - type: 'remote', - mode: 'proxy', - api_url: `https://peer-${uniqueSuffix()}.example.com`, - api_token: `tok-${uniqueSuffix()}`, - compose_dir: '/tmp', - is_default: false, - }); - if (meshEnabled) { - db.setNodeMeshEnabled(id, true); - } - return id; -} - -function clearTestNodes(): void { - const db = DatabaseService.getInstance().getDb(); - db.prepare('DELETE FROM mesh_stacks').run(); - db.prepare('DELETE FROM nodes WHERE is_default = 0').run(); -} - -describe('Trigger 1: enableForNode triggers proactive bridge bootstrap', () => { - beforeEach(() => { - clearTestNodes(); - MeshProxyTunnelDialer.resetForTest(); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - it('calls ensureBridge for a remote proxy-mode node', async () => { - const ensureSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge') - .mockResolvedValue(null); - const nodeId = seedProxyNode(false); - - await MeshService.getInstance().enableForNode(nodeId); - - // Trigger is fire-and-forget; wait a microtask for the void promise. - await new Promise((r) => setImmediate(r)); - expect(ensureSpy).toHaveBeenCalledWith(nodeId); - }); - - it('skips local nodes', async () => { - const ensureSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge') - .mockResolvedValue(null); - const localId = DatabaseService.getInstance().getNodes()[0].id; - - await MeshService.getInstance().enableForNode(localId); - - await new Promise((r) => setImmediate(r)); - expect(ensureSpy).not.toHaveBeenCalled(); - }); - - it('swallows ensureBridge rejections without throwing', async () => { - vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge') - .mockRejectedValue(new Error('peer unreachable')); - const nodeId = seedProxyNode(false); - - await expect(MeshService.getInstance().enableForNode(nodeId)).resolves.toBeUndefined(); - await new Promise((r) => setImmediate(r)); - }); -}); - -describe('Trigger 2: api_token rotation forces re-bootstrap', () => { - beforeEach(() => { - clearTestNodes(); - MeshProxyTunnelDialer.resetForTest(); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - it('calls closeBridge then ensureBridge when rotation handler runs', async () => { - const closeSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'closeBridge'); - const ensureSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge') - .mockResolvedValue(null); - const nodeId = seedProxyNode(true); - - const res = await request(app) - .put(`/api/nodes/${nodeId}`) - .set('Authorization', authHeader) - .send({ api_token: 'rotated-token-abc' }); - - expect(res.status).toBe(200); - await new Promise((r) => setImmediate(r)); - - expect(closeSpy).toHaveBeenCalledWith(nodeId, 'peer token rotated'); - expect(ensureSpy).toHaveBeenCalledWith(nodeId); - const closeOrder = closeSpy.mock.invocationCallOrder[0]; - const ensureOrder = ensureSpy.mock.invocationCallOrder[0]; - expect(closeOrder).toBeLessThan(ensureOrder); - }); - - it('does not fire when api_token is not in the update payload', async () => { - const closeSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'closeBridge'); - const ensureSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge') - .mockResolvedValue(null); - const nodeId = seedProxyNode(true); - - const res = await request(app) - .put(`/api/nodes/${nodeId}`) - .set('Authorization', authHeader) - .send({ name: `renamed-${uniqueSuffix()}` }); - - expect(res.status).toBe(200); - await new Promise((r) => setImmediate(r)); - - expect(closeSpy).not.toHaveBeenCalled(); - expect(ensureSpy).not.toHaveBeenCalled(); - }); - - it('does not fire when mesh is disabled on the node', async () => { - const closeSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'closeBridge'); - const ensureSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge') - .mockResolvedValue(null); - const nodeId = seedProxyNode(false); - - const res = await request(app) - .put(`/api/nodes/${nodeId}`) - .set('Authorization', authHeader) - .send({ api_token: 'rotated-token-xyz' }); - - expect(res.status).toBe(200); - await new Promise((r) => setImmediate(r)); - - expect(closeSpy).not.toHaveBeenCalled(); - expect(ensureSpy).not.toHaveBeenCalled(); - }); - - it('does not fire when api_token in payload equals the current value', async () => { - const closeSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'closeBridge'); - const ensureSpy = vi.spyOn(MeshProxyTunnelDialer.getInstance(), 'ensureBridge') - .mockResolvedValue(null); - const nodeId = seedProxyNode(true); - const existingToken = DatabaseService.getInstance().getNode(nodeId)?.api_token; - expect(existingToken).toBeTruthy(); - - const res = await request(app) - .put(`/api/nodes/${nodeId}`) - .set('Authorization', authHeader) - .send({ name: `renamed-${uniqueSuffix()}`, api_token: existingToken }); - - expect(res.status).toBe(200); - await new Promise((r) => setImmediate(r)); - - expect(closeSpy).not.toHaveBeenCalled(); - expect(ensureSpy).not.toHaveBeenCalled(); - }); -}); diff --git a/backend/src/__tests__/mesh-version-skew-integration.test.ts b/backend/src/__tests__/mesh-version-skew-integration.test.ts deleted file mode 100644 index 07ce451b..00000000 --- a/backend/src/__tests__/mesh-version-skew-integration.test.ts +++ /dev/null @@ -1,112 +0,0 @@ -/** - * Version skew invariant: central must NOT send a `mesh_handshake` to a peer - * that lacks the `mesh_proxy_callback_bootstrap` capability. - * - * The capability gate keeps central rolling forward without breaking older - * peers: a peer that does not understand the bootstrap frame would otherwise - * see a stray JSON text frame before the TCP frames it expects. The dialer's - * `maybeSendBootstrap` must read the capability list from the remote's meta - * endpoint and silently skip the send when the bit is absent. - * - * This is complementary to `mesh-proxy-tunnel-dialer-handshake.test.ts`, - * which exercises the same code path with a finer-grained set of inputs; - * this file locks in the same invariant as an integration so a future - * refactor that moves the capability check to a different layer does not - * silently regress. - */ -import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest'; -import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; -import { NodeRegistry } from '../services/NodeRegistry'; -import { MeshProxyTunnelDialer } from '../services/MeshProxyTunnelDialer'; -import { OFFLINE_META } from '../services/CapabilityRegistry'; -import { MeshCentralRegistry } from '../services/MeshCentralRegistry'; -import { DatabaseService } from '../services/DatabaseService'; - -let tmpDir: string; -beforeAll(async () => { tmpDir = await setupTestDb(); }); -afterAll(() => cleanupTestDb(tmpDir)); - -describe('Version skew: peer without mesh_proxy_callback_bootstrap', () => { - beforeEach(() => { - // Per-test reset of dialer + registry singletons + env. The DB lives - // once per file (beforeAll above); resetting setupTestDb per test - // would invalidate the DatabaseService singleton on Linux. - MeshProxyTunnelDialer.resetForTest(); - MeshCentralRegistry.resetForTest(); - process.env.SENCHO_PRIMARY_URL = 'https://central.example.com'; - DatabaseService.getInstance().setSystemState('instance_id', 'test-central-instance'); - }); - afterEach(() => { - delete process.env.SENCHO_PRIMARY_URL; - vi.restoreAllMocks(); - }); - - type DialerPrivate = { - maybeSendBootstrap: (nodeId: number, ws: unknown) => Promise; - }; - - it('central does not send mesh_handshake when capability is absent', async () => { - vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue({ - ...OFFLINE_META, - version: '0.78.0', - capabilities: ['stacks'], - online: true, - startedAt: Date.now(), - }); - - // Seed a proxy node so maybeSendBootstrap has something to look up. - const db = DatabaseService.getInstance(); - const id = db.addNode({ - name: `n-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, - type: 'remote', - mode: 'proxy', - api_url: 'https://n', - api_token: 't', - compose_dir: '/tmp', - is_default: false, - }); - db.setNodeMeshEnabled(id, true); - - const sentFrames: unknown[] = []; - const fakeWs = { send: vi.fn((data: string) => sentFrames.push(JSON.parse(data))) }; - - await (MeshProxyTunnelDialer.getInstance() as unknown as DialerPrivate) - .maybeSendBootstrap(id, fakeWs); - - expect(fakeWs.send).not.toHaveBeenCalled(); - expect(sentFrames).toHaveLength(0); - // No persistence as a side effect either. - expect(MeshCentralRegistry.getInstance().getActive()).toBeNull(); - }); - - it('central does send mesh_handshake when the capability flips to advertised', async () => { - // Sanity: the same code path with the capability present must take - // the affirmative branch, so we know the skip above is gated on the - // capability and not on some unrelated precondition. - vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue({ - ...OFFLINE_META, - version: '0.79.0', - capabilities: ['stacks', 'mesh_proxy_callback_bootstrap'], - online: true, - startedAt: Date.now(), - }); - const db = DatabaseService.getInstance(); - const id = db.addNode({ - name: `n-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, - type: 'remote', - mode: 'proxy', - api_url: 'https://n', - api_token: 't', - compose_dir: '/tmp', - is_default: false, - }); - db.setNodeMeshEnabled(id, true); - - const sentFrames: unknown[] = []; - const fakeWs = { send: vi.fn((data: string) => sentFrames.push(JSON.parse(data))) }; - await (MeshProxyTunnelDialer.getInstance() as unknown as DialerPrivate) - .maybeSendBootstrap(id, fakeWs); - expect(fakeWs.send).toHaveBeenCalledOnce(); - expect(sentFrames[0]).toMatchObject({ t: 'mesh_handshake', v: 1 }); - }); -}); diff --git a/backend/src/__tests__/peer-to-central-mesh-session-dialer.test.ts b/backend/src/__tests__/peer-to-central-mesh-session-dialer.test.ts deleted file mode 100644 index 364abd05..00000000 --- a/backend/src/__tests__/peer-to-central-mesh-session-dialer.test.ts +++ /dev/null @@ -1,275 +0,0 @@ -/** - * `PeerToCentralMeshSessionDialer`: peer-side counterpart to the central - * ingress at `/api/mesh/proxy-tunnel-from-peer` (Task 9). Reads cached - * central material from `MeshCentralRegistry`, opens a WebSocket to - * central carrying the bootstrapped JWT in the Authorization header, and - * branches on the upgrade outcome: - * - * - successful open: marks the cache row "used", arms the bridge. - * - 401 with a terminal reason code (stale, signature_invalid, ...): - * clears the cache so we stop dialing on a bad credential. - * - 401 with a transient reason (clock_skew, mode_mismatch, ...): - * keeps the cache, marks rejected so the operator can see why. - * - 404 endpoint_not_found: keeps the cache and arms a longer backoff - * so we do not hammer an older central that has not yet shipped the - * peer ingress endpoint. - * - * The dial logic is also rate-limited per dialer process (5 attempts / - * 60s window) so a misbehaving central or a misconfigured peer cannot - * flood either side with handshake traffic. - */ -import http from 'http'; -import type { AddressInfo } from 'net'; -import { WebSocketServer, type WebSocket as WsClient } from 'ws'; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; -import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; - -let tmpDir: string; -let PeerToCentralMeshSessionDialer: typeof import('../services/PeerToCentralMeshSessionDialer').PeerToCentralMeshSessionDialer; -let MeshCentralRegistry: typeof import('../services/MeshCentralRegistry').MeshCentralRegistry; -let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; -let TcpStreamSwitchboardCtor: typeof import('../mesh/tcpStreamSwitchboard').TcpStreamSwitchboard; -let MeshService: typeof import('../services/MeshService').MeshService; - -interface RejectingServer { - server: http.Server; - url: string; - lastAuthHeader: string | null; - lastPath: string | null; - close: () => Promise; -} - -/** - * Spins up a minimal HTTP server that intercepts WebSocket upgrades and - * responds with a fixed status + JSON body, mimicking how the central - * ingress rejects bad credentials. Records the auth header and path so - * tests can assert the dialer is sending the right material. - */ -async function startRejectingServer(status: number, reason: string): Promise { - const handle: RejectingServer = { - server: http.createServer(), - url: '', - lastAuthHeader: null, - lastPath: null, - close: () => new Promise((resolve) => handle.server.close(() => resolve())), - }; - handle.server.on('upgrade', (req, socket) => { - handle.lastAuthHeader = (req.headers['authorization'] as string | undefined) ?? null; - handle.lastPath = req.url ?? null; - const body = JSON.stringify({ reason }); - const statusText = status === 404 ? 'Not Found' : 'Unauthorized'; - const head = [ - `HTTP/1.1 ${status} ${statusText}`, - 'Content-Type: application/json', - `Content-Length: ${Buffer.byteLength(body)}`, - 'Connection: close', - '', - body, - ].join('\r\n'); - try { socket.write(head); } catch { /* ignore */ } - try { socket.destroy(); } catch { /* ignore */ } - }); - await new Promise((resolve) => handle.server.listen(0, '127.0.0.1', () => resolve())); - const port = (handle.server.address() as AddressInfo).port; - handle.url = `http://127.0.0.1:${port}`; - return handle; -} - -beforeAll(async () => { - tmpDir = await setupTestDb(); - ({ PeerToCentralMeshSessionDialer } = await import('../services/PeerToCentralMeshSessionDialer')); - ({ MeshCentralRegistry } = await import('../services/MeshCentralRegistry')); - ({ DatabaseService } = await import('../services/DatabaseService')); - ({ TcpStreamSwitchboard: TcpStreamSwitchboardCtor } = await import('../mesh/tcpStreamSwitchboard')); - ({ MeshService } = await import('../services/MeshService')); -}); - -afterAll(() => { - cleanupTestDb(tmpDir); -}); - -beforeEach(() => { - MeshCentralRegistry.resetForTest(); - PeerToCentralMeshSessionDialer.resetForTest(); - DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_centrals').run(); -}); - -afterEach(() => { - PeerToCentralMeshSessionDialer.resetForTest(); - MeshCentralRegistry.resetForTest(); -}); - -describe('PeerToCentralMeshSessionDialer', () => { - it('returns null when no central material is cached', async () => { - const result = await PeerToCentralMeshSessionDialer.getInstance().ensureSession(); - expect(result).toBeNull(); - }); - - it('dials using the cached URL + JWT bearer (asserted by server-side recording)', async () => { - const srv = await startRejectingServer(401, 'clock_skew'); - try { - MeshCentralRegistry.getInstance().upsert({ - centralInstanceId: 'inst-a', - centralApiUrl: srv.url, - callbackJwt: 'fake.jwt.token', - jwtIssuedAt: 1, - jwtExpiresAt: Math.floor(Date.now() / 1000) + 3600, - }); - const result = await PeerToCentralMeshSessionDialer.getInstance().ensureSession(); - expect(result).toBeNull(); - expect(srv.lastPath).toBe('/api/mesh/proxy-tunnel-from-peer'); - expect(srv.lastAuthHeader).toBe('Bearer fake.jwt.token'); - } finally { - await srv.close(); - } - }); - - it('clears cache on 401 reason=stale', async () => { - const srv = await startRejectingServer(401, 'stale'); - try { - MeshCentralRegistry.getInstance().upsert({ - centralInstanceId: 'inst-b', - centralApiUrl: srv.url, - callbackJwt: 'jwt', - jwtIssuedAt: 1, - jwtExpiresAt: Math.floor(Date.now() / 1000) + 3600, - }); - await PeerToCentralMeshSessionDialer.getInstance().ensureSession(); - expect(MeshCentralRegistry.getInstance().getActive()).toBeNull(); - } finally { - await srv.close(); - } - }); - - it('keeps cache on 401 reason=clock_skew but records the rejection', async () => { - const srv = await startRejectingServer(401, 'clock_skew'); - try { - MeshCentralRegistry.getInstance().upsert({ - centralInstanceId: 'inst-c', - centralApiUrl: srv.url, - callbackJwt: 'jwt', - jwtIssuedAt: 1, - jwtExpiresAt: Math.floor(Date.now() / 1000) + 3600, - }); - await PeerToCentralMeshSessionDialer.getInstance().ensureSession(); - const row = MeshCentralRegistry.getInstance().getActive(); - expect(row).not.toBeNull(); - expect(row?.lastRejectReason).toBe('clock_skew'); - } finally { - await srv.close(); - } - }); - - it('keeps cache on HTTP 404 endpoint_not_found and rate-limits future dials', async () => { - const srv = await startRejectingServer(404, 'whatever'); - try { - MeshCentralRegistry.getInstance().upsert({ - centralInstanceId: 'inst-d', - centralApiUrl: srv.url, - callbackJwt: 'jwt', - jwtIssuedAt: 1, - jwtExpiresAt: Math.floor(Date.now() / 1000) + 3600, - }); - const dialer = PeerToCentralMeshSessionDialer.getInstance(); - await dialer.ensureSession(); - // Cache must survive a 404 (older central, transient infra). - expect(MeshCentralRegistry.getInstance().getActive()).not.toBeNull(); - // The second call must not even hit the server: the endpoint - // backoff blocks new dials for the configured window. - srv.lastAuthHeader = null; - const second = await dialer.ensureSession(); - expect(second).toBeNull(); - expect(srv.lastAuthHeader).toBeNull(); - } finally { - await srv.close(); - } - }); - - it('rate-limits dials after the configured attempt cap', async () => { - const srv = await startRejectingServer(401, 'clock_skew'); - try { - MeshCentralRegistry.getInstance().upsert({ - centralInstanceId: 'inst-e', - centralApiUrl: srv.url, - callbackJwt: 'jwt', - jwtIssuedAt: 1, - jwtExpiresAt: Math.floor(Date.now() / 1000) + 3600, - }); - const dialer = PeerToCentralMeshSessionDialer.getInstance(); - // 5 dials should land; the 6th must be suppressed by the limiter. - for (let i = 0; i < 5; i++) { - await dialer.ensureSession(); - } - srv.lastAuthHeader = null; - const sixth = await dialer.ensureSession(); - expect(sixth).toBeNull(); - expect(srv.lastAuthHeader).toBeNull(); - } finally { - await srv.close(); - } - }); - - it('marks the row used on successful WS open and installs a reverseDialer', async () => { - const wss = new WebSocketServer({ noServer: true }); - const srv = http.createServer(); - srv.on('upgrade', (req, socket, head) => { - if (req.url?.startsWith('/api/mesh/proxy-tunnel-from-peer')) { - wss.handleUpgrade(req, socket, head, () => { /* accept */ }); - } else { - socket.destroy(); - } - }); - await new Promise((resolve) => srv.listen(0, '127.0.0.1', () => resolve())); - const port = (srv.address() as AddressInfo).port; - const url = `http://127.0.0.1:${port}`; - // Start with a clean reverseDialer slot on the local MeshService so - // setReverseDialer's CAS swap succeeds inside attachSwitchboard. - MeshService.getInstance().setReverseDialer(null); - let switchboard: InstanceType | null = null; - try { - MeshCentralRegistry.getInstance().upsert({ - centralInstanceId: 'inst-success', - centralApiUrl: url, - callbackJwt: 'fake.jwt', - jwtIssuedAt: 1, - jwtExpiresAt: 9999999999, - }); - switchboard = await PeerToCentralMeshSessionDialer.getInstance().ensureSession(); - expect(switchboard).not.toBeNull(); - // The R1-A2 design puts a TcpStreamSwitchboard on the peer end of - // this WS, not a PilotTunnelBridge (which is central's role). A - // future regression that returns the wrong shape would surface - // here before the more abstract reverseDialer-installed check. - expect(switchboard).toBeInstanceOf(TcpStreamSwitchboardCtor); - expect(PeerToCentralMeshSessionDialer.getInstance().hasSession()).toBe(true); - // markUsed is called synchronously inside attachSwitchboard before - // ensureSession resolves, so the DB row reflects it immediately. - expect(MeshCentralRegistry.getInstance().getActive()?.lastUsedAt ?? 0).toBeGreaterThan(0); - // The R1-A2 wiring under test: MeshService.reverseDialer must be - // populated so MeshService.dialMeshTcpStream routes peer-side - // cross-fleet traffic through this callback bridge instead of - // falling through to PilotTunnelManager.ensureBridge(centralId), - // which has no record for central on a proxy peer. - const meshSvc = MeshService.getInstance() as unknown as { reverseDialer: unknown }; - expect(meshSvc.reverseDialer).not.toBeNull(); - } finally { - try { switchboard?.cleanup('test done'); } catch { /* ignore */ } - // The dialer owns the client-side WS and the switchboard doesn't - // close it on cleanup; tear it down explicitly so wss.close can - // resolve (it waits for all client connections to disconnect). - try { - const inst = PeerToCentralMeshSessionDialer.getInstance() as unknown as { currentWs: WsClient | null }; - inst.currentWs?.close(1000, 'test done'); - } catch { /* ignore */ } - MeshService.getInstance().setReverseDialer(null); - await new Promise((resolve) => wss.close(() => resolve())); - await new Promise((resolve) => srv.close(() => resolve())); - } - }); - - it('singleton returns the same instance', () => { - const a = PeerToCentralMeshSessionDialer.getInstance(); - const b = PeerToCentralMeshSessionDialer.getInstance(); - expect(a).toBe(b); - }); -}); diff --git a/backend/src/__tests__/pilot-metrics.test.ts b/backend/src/__tests__/pilot-metrics.test.ts index ca8b6738..59314ff9 100644 --- a/backend/src/__tests__/pilot-metrics.test.ts +++ b/backend/src/__tests__/pilot-metrics.test.ts @@ -66,17 +66,12 @@ describe('PilotMetrics persistence', () => { proxy_bridges_total: 7, proxy_dials_failed: 5, proxy_idle_closes: 2, - proxy_bridges_peer_initiated_total: 3, - mesh_central_bootstraps_total: 4, - mesh_callback_dials_failed_total: 0, - mesh_callback_auth_failures_total: 0, }); PilotMetrics.load(db); const snap = PilotMetrics.snapshot(); expect(snap.tunnels_total).toBe(11); expect(snap.proxy_dials_failed).toBe(5); expect(snap.proxy_bridges_total).toBe(7); - expect(snap.mesh_central_bootstraps_total).toBe(4); }); it('threshold flush: increments past the threshold trigger a single persist', () => { @@ -150,7 +145,7 @@ describe('PilotMetrics persistence', () => { expect(snap.proxy_dials_failed).toBe(9); expect(snap.proxy_bridges_total).toBe(0); expect(snap.tunnels_total).toBe(0); - expect(snap.mesh_callback_auth_failures_total).toBe(0); + expect(snap.proxy_idle_closes).toBe(0); }); it('stop() cancels a pending interval flush without persisting', () => { diff --git a/backend/src/__tests__/pilot-mode-regression-integration.test.ts b/backend/src/__tests__/pilot-mode-regression-integration.test.ts deleted file mode 100644 index c0f56363..00000000 --- a/backend/src/__tests__/pilot-mode-regression-integration.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Pilot-mode regression invariant: a pilot-agent tunnel always wins over a - * peer-initiated proxy-mode bridge for the same nodeId. - * - * The two registration paths share the `PilotTunnelManager.bridges` map. - * Without the `bridgeKinds` index, a peer-initiated dial-back could quietly - * replace a live pilot tunnel and break the agent's reverse-stream relay. - * `replaceOrRegisterProxyBridge` is the single point that has to refuse the - * replacement; this test locks that contract in. - * - * Mirrors the `injectBridgeForTest` pattern used elsewhere in the suite so - * we exercise the manager invariant without owning a real WebSocket. - */ -import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { EventEmitter } from 'events'; -import { PilotTunnelManager } from '../services/PilotTunnelManager'; -import type { PilotTunnelBridge } from '../services/PilotTunnelBridge'; - -function makeFakeBridge(): EventEmitter & { - close: ReturnType; - getActiveStreamCount: () => number; -} { - const ee = new EventEmitter() as EventEmitter & { - close: ReturnType; - getActiveStreamCount: () => number; - }; - ee.close = vi.fn(); - ee.getActiveStreamCount = () => 0; - return ee; -} - -describe('Pilot-mode regression: pilot tunnel wins over peer-initiated proxy bridge', () => { - beforeEach(() => { - PilotTunnelManager.resetForTest(); - }); - - it('refuses replaceOrRegisterProxyBridge when a pilot tunnel exists for the same nodeId', () => { - const mgr = PilotTunnelManager.getInstance(); - const pilot = makeFakeBridge(); - const proxy = makeFakeBridge(); - mgr.injectBridgeForTest(7, pilot as unknown as PilotTunnelBridge, 'pilot'); - - expect(() => mgr.replaceOrRegisterProxyBridge(7, proxy as unknown as PilotTunnelBridge)) - .toThrow(/pilot tunnel/); - - // Pilot bridge is still the resident bridge for nodeId 7. - expect(mgr.getBridge(7)).toBe(pilot); - // The pilot bridge must not be closed by the rejected replacement. - expect(pilot.close).not.toHaveBeenCalled(); - }); - - it('allows replaceOrRegisterProxyBridge to swap one proxy bridge for another', () => { - const mgr = PilotTunnelManager.getInstance(); - const oldProxy = makeFakeBridge(); - const newProxy = makeFakeBridge(); - mgr.injectBridgeForTest(9, oldProxy as unknown as PilotTunnelBridge, 'proxy'); - - mgr.replaceOrRegisterProxyBridge(9, newProxy as unknown as PilotTunnelBridge); - - // The new dial is the source of truth; old proxy is closed. - expect(oldProxy.close).toHaveBeenCalledOnce(); - expect(mgr.getBridge(9)).toBe(newProxy); - }); - - it('replaceOrRegisterProxyBridge on an empty slot just registers (no refuse, no close)', () => { - const mgr = PilotTunnelManager.getInstance(); - const proxy = makeFakeBridge(); - - mgr.replaceOrRegisterProxyBridge(11, proxy as unknown as PilotTunnelBridge); - - expect(mgr.getBridge(11)).toBe(proxy); - expect(proxy.close).not.toHaveBeenCalled(); - }); -}); diff --git a/backend/src/__tests__/pilot-tunnel-manager-replace.test.ts b/backend/src/__tests__/pilot-tunnel-manager-replace.test.ts deleted file mode 100644 index 0cea829c..00000000 --- a/backend/src/__tests__/pilot-tunnel-manager-replace.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -/** - * PilotTunnelManager.replaceOrRegisterProxyBridge: peer-initiated proxy - * bridges (Phase R1) need to be able to supersede a previous proxy bridge - * for the same nodeId, but must never shadow a live pilot tunnel (which - * always wins the bridge slot). - */ -import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { EventEmitter } from 'events'; -import { PilotTunnelManager } from '../services/PilotTunnelManager'; -import type { PilotTunnelBridge } from '../services/PilotTunnelBridge'; - -type FakeBridge = PilotTunnelBridge & { - close: ReturnType void>>; - getActiveStreamCount: ReturnType number>>; -}; - -function makeFakeBridge(): FakeBridge { - const ee = new EventEmitter() as unknown as FakeBridge; - ee.close = vi.fn<(code?: number, reason?: string) => void>(); - ee.getActiveStreamCount = vi.fn<() => number>().mockReturnValue(0); - return ee; -} - -describe('PilotTunnelManager.replaceOrRegisterProxyBridge', () => { - beforeEach(() => PilotTunnelManager.resetForTest?.()); - - it('registers when no existing bridge', () => { - const mgr = PilotTunnelManager.getInstance(); - const bridge = makeFakeBridge(); - mgr.replaceOrRegisterProxyBridge(42, bridge); - expect(mgr.getBridge(42)).toBe(bridge); - }); - - it('replaces when existing bridge is a proxy bridge (closes the old one)', () => { - const mgr = PilotTunnelManager.getInstance(); - const oldBridge = makeFakeBridge(); - const newBridge = makeFakeBridge(); - mgr.injectBridgeForTest(42, oldBridge, 'proxy'); - mgr.replaceOrRegisterProxyBridge(42, newBridge); - expect(oldBridge.close).toHaveBeenCalledWith(1000, 'replaced-by-newer-proxy'); - expect(mgr.getBridge(42)).toBe(newBridge); - }); - - it('throws when existing bridge is a pilot tunnel (preserves the pilot tunnel)', () => { - const mgr = PilotTunnelManager.getInstance(); - const pilotBridge = makeFakeBridge(); - const proxyBridge = makeFakeBridge(); - mgr.injectBridgeForTest(42, pilotBridge, 'pilot'); - expect(() => mgr.replaceOrRegisterProxyBridge(42, proxyBridge)) - .toThrow(/pilot tunnel.*proxy bridge refused/); - expect(mgr.getBridge(42)).toBe(pilotBridge); - expect(pilotBridge.close).not.toHaveBeenCalled(); - }); -}); diff --git a/backend/src/__tests__/system-pilot-tunnels-central-callback.test.ts b/backend/src/__tests__/system-pilot-tunnels-central-callback.test.ts deleted file mode 100644 index 46207bdc..00000000 --- a/backend/src/__tests__/system-pilot-tunnels-central-callback.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Tests for the `centralCallback` diag block returned by - * GET /api/system/pilot-tunnels. The block reads from MeshCentralRegistry - * (cached central material + last-used/last-rejected timestamps) and from - * PeerToCentralMeshSessionDialer (live bridge presence). Counters for the - * peer-callback path live in PilotMetrics and are exposed via the existing - * `counters` field of the same response, so this block only adds the - * cached-row diagnostics that PilotMetrics does not track. - */ -import { describe, it, expect, beforeAll, beforeEach, afterAll } from 'vitest'; -import request from 'supertest'; -import type { Express } from 'express'; -import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; -import { MeshCentralRegistry } from '../services/MeshCentralRegistry'; -import { PeerToCentralMeshSessionDialer } from '../services/PeerToCentralMeshSessionDialer'; -import { DatabaseService } from '../services/DatabaseService'; - -let tmpDir: string; -let app: Express; -let cookie: string; - -beforeAll(async () => { - tmpDir = await setupTestDb(); - ({ app } = await import('../index')); - cookie = await loginAsTestAdmin(app); -}); - -afterAll(() => cleanupTestDb(tmpDir)); - -beforeEach(() => { - MeshCentralRegistry.resetForTest(); - PeerToCentralMeshSessionDialer.resetForTest(); - DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_centrals').run(); -}); - -describe('/api/system/pilot-tunnels centralCallback diag', () => { - it('returns null centralCallback fields when no central material is cached', async () => { - const res = await request(app).get('/api/system/pilot-tunnels').set('Cookie', cookie); - expect(res.status).toBe(200); - expect(res.body.centralCallback).toEqual({ - bridgeOpen: false, - lastBootstrapAt: null, - lastDialOkAt: null, - lastDialFailAt: null, - lastDialFailReason: null, - }); - }); - - it('returns populated centralCallback fields when material is present', async () => { - MeshCentralRegistry.getInstance().upsert({ - centralInstanceId: 'inst-1', - centralApiUrl: 'https://central.example.com', - callbackJwt: 'jwt-value', - jwtIssuedAt: 1, - jwtExpiresAt: 9999999999, - }); - MeshCentralRegistry.getInstance().markUsed('inst-1'); - - const res = await request(app).get('/api/system/pilot-tunnels').set('Cookie', cookie); - expect(res.status).toBe(200); - expect(res.body.centralCallback.bridgeOpen).toBe(false); - expect(res.body.centralCallback.lastBootstrapAt).toBeGreaterThan(0); - expect(res.body.centralCallback.lastDialOkAt).toBeGreaterThan(0); - expect(res.body.centralCallback.lastDialFailAt).toBeNull(); - expect(res.body.centralCallback.lastDialFailReason).toBeNull(); - }); - - it('surfaces last reject reason when central marks a dial failed', async () => { - MeshCentralRegistry.getInstance().upsert({ - centralInstanceId: 'inst-2', - centralApiUrl: 'https://central.example.com', - callbackJwt: 'jwt-value', - jwtIssuedAt: 1, - jwtExpiresAt: 9999999999, - }); - MeshCentralRegistry.getInstance().markRejected('inst-2', 'signature_invalid'); - - const res = await request(app).get('/api/system/pilot-tunnels').set('Cookie', cookie); - expect(res.status).toBe(200); - expect(res.body.centralCallback.lastDialFailAt).toBeGreaterThan(0); - expect(res.body.centralCallback.lastDialFailReason).toBe('signature_invalid'); - }); -}); diff --git a/backend/src/routes/metrics.ts b/backend/src/routes/metrics.ts index f61a5b5b..f820dce6 100644 --- a/backend/src/routes/metrics.ts +++ b/backend/src/routes/metrics.ts @@ -6,8 +6,6 @@ import { DatabaseService } from '../services/DatabaseService'; import { CacheService } from '../services/CacheService'; import { NodeRegistry } from '../services/NodeRegistry'; import { PilotTunnelManager } from '../services/PilotTunnelManager'; -import { MeshCentralRegistry } from '../services/MeshCentralRegistry'; -import { PeerToCentralMeshSessionDialer } from '../services/PeerToCentralMeshSessionDialer'; import { authMiddleware } from '../middleware/auth'; import { requireAdmin } from '../middleware/tierGates'; import { STATS_CACHE_TTL_MS, SYSTEM_STATS_CACHE_TTL_MS } from '../helpers/constants'; @@ -263,21 +261,7 @@ metricsRouter.get('/system/pilot-tunnels', authMiddleware, async (req: Request, if (!requireAdmin(req, res)) return; try { const snapshot = PilotTunnelManager.getInstance().getMetricsSnapshot(); - // centralCallback exposes the peer-side view of the symmetric mesh - // callback path (cached central material plus the live bridge presence). - // PilotMetrics already tracks the attempt/success counters via - // `counters.mesh_central_bootstraps_total` and friends; this block adds - // the per-instance last-success / last-failure diagnostics that live in - // MeshCentralRegistry rather than PilotMetrics. - const cached = MeshCentralRegistry.getInstance().getActive(); - const centralCallback = { - bridgeOpen: PeerToCentralMeshSessionDialer.getInstance().hasSession(), - lastBootstrapAt: cached?.lastBootstrapAt ?? null, - lastDialOkAt: cached?.lastUsedAt ?? null, - lastDialFailAt: cached?.lastRejectedAt ?? null, - lastDialFailReason: cached?.lastRejectReason ?? null, - }; - res.json({ ...snapshot, centralCallback }); + res.json(snapshot); } catch (error) { console.error('Failed to fetch pilot tunnel metrics:', error); res.status(500).json({ error: 'Failed to fetch pilot tunnel metrics' }); diff --git a/backend/src/services/CapabilityRegistry.ts b/backend/src/services/CapabilityRegistry.ts index 27d71f2e..7d0df575 100644 --- a/backend/src/services/CapabilityRegistry.ts +++ b/backend/src/services/CapabilityRegistry.ts @@ -34,7 +34,6 @@ export const CAPABILITIES = [ 'registries', 'self-update', 'vulnerability-scanning', - 'mesh_proxy_callback_bootstrap', ] as const; export type Capability = (typeof CAPABILITIES)[number]; diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 2f99b6e3..9c1e2ab6 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -1489,23 +1489,13 @@ export class DatabaseService { } catch (e) { console.warn('[DatabaseService] mesh_stacks migration:', (e as Error).message); } - try { - this.db.prepare(` - CREATE TABLE IF NOT EXISTS mesh_centrals ( - central_instance_id TEXT PRIMARY KEY, - central_api_url TEXT NOT NULL, - callback_jwt TEXT NOT NULL, - jwt_issued_at INTEGER NOT NULL, - jwt_expires_at INTEGER NOT NULL, - last_bootstrap_at INTEGER NOT NULL, - last_used_at INTEGER, - last_rejected_at INTEGER, - last_reject_reason TEXT - ) - `).run(); - } catch (e) { - console.warn('[DatabaseService] Could not create mesh_centrals:', (e as Error).message); - } + // mesh_centrals was the peer-side cache of the reverse-callback JWT + // (central → peer bootstrap material). Peer→central traffic now + // multiplexes over the existing forward WS via `tcp_open_reverse`, + // so the table is no longer written or read. Drop it on every boot; + // idempotent. + try { this.db.prepare('DROP TABLE IF EXISTS mesh_centrals').run(); } + catch (e) { console.warn('[DatabaseService] Could not drop mesh_centrals:', (e as Error).message); } this.tryAddColumn('nodes', 'mesh_enabled', 'INTEGER NOT NULL DEFAULT 0'); } diff --git a/backend/src/services/MeshCentralRegistry.ts b/backend/src/services/MeshCentralRegistry.ts deleted file mode 100644 index ab55b264..00000000 --- a/backend/src/services/MeshCentralRegistry.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { EventEmitter } from 'events'; -import { DatabaseService } from './DatabaseService'; - -export interface MeshCentralMaterial { - centralInstanceId: string; - centralApiUrl: string; - callbackJwt: string; - jwtIssuedAt: number; - jwtExpiresAt: number; -} - -export interface MeshCentralRow extends MeshCentralMaterial { - lastBootstrapAt: number; - lastUsedAt: number | null; - lastRejectedAt: number | null; - lastRejectReason: string | null; -} - -export class MeshCentralRegistry extends EventEmitter { - private static instance: MeshCentralRegistry | null = null; - private warnedMultiRow = false; - - private constructor() { super(); } - - public static getInstance(): MeshCentralRegistry { - if (!this.instance) this.instance = new MeshCentralRegistry(); - return this.instance; - } - - public static resetForTest(): void { - this.instance = null; - } - - public upsert(material: MeshCentralMaterial): void { - const db = DatabaseService.getInstance().getDb(); - const prior = this.getActive(); - db.prepare(` - INSERT INTO mesh_centrals ( - central_instance_id, central_api_url, callback_jwt, - jwt_issued_at, jwt_expires_at, last_bootstrap_at, - last_used_at, last_rejected_at, last_reject_reason - ) VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, NULL) - ON CONFLICT(central_instance_id) DO UPDATE SET - central_api_url = excluded.central_api_url, - callback_jwt = excluded.callback_jwt, - jwt_issued_at = excluded.jwt_issued_at, - jwt_expires_at = excluded.jwt_expires_at, - last_bootstrap_at = excluded.last_bootstrap_at - `).run( - material.centralInstanceId, - material.centralApiUrl, - material.callbackJwt, - material.jwtIssuedAt, - material.jwtExpiresAt, - Date.now(), - ); - - const isInstanceChange = prior && prior.centralInstanceId !== material.centralInstanceId; - if (isInstanceChange) { - this.emit('central-instance-changed', { - previousInstanceId: prior!.centralInstanceId, - newInstanceId: material.centralInstanceId, - }); - } - this.emit('central-bootstrap', { centralInstanceId: material.centralInstanceId }); - } - - public getActive(): MeshCentralRow | null { - const db = DatabaseService.getInstance().getDb(); - const rows = db.prepare(` - SELECT * FROM mesh_centrals ORDER BY last_bootstrap_at DESC - `).all() as Array<{ - central_instance_id: string; - central_api_url: string; - callback_jwt: string; - jwt_issued_at: number; - jwt_expires_at: number; - last_bootstrap_at: number; - last_used_at: number | null; - last_rejected_at: number | null; - last_reject_reason: string | null; - }>; - if (rows.length === 0) return null; - if (rows.length > 1 && !this.warnedMultiRow) { - console.warn(`[MeshCentralRegistry] multiple central rows present (${rows.length}), multi-central unsupported in v0.79`); - this.warnedMultiRow = true; - } - const r = rows[0]; - return { - centralInstanceId: r.central_instance_id, - centralApiUrl: r.central_api_url, - callbackJwt: r.callback_jwt, - jwtIssuedAt: r.jwt_issued_at, - jwtExpiresAt: r.jwt_expires_at, - lastBootstrapAt: r.last_bootstrap_at, - lastUsedAt: r.last_used_at, - lastRejectedAt: r.last_rejected_at, - lastRejectReason: r.last_reject_reason, - }; - } - - public clearForInstance(centralInstanceId: string): void { - DatabaseService.getInstance().getDb() - .prepare(`DELETE FROM mesh_centrals WHERE central_instance_id = ?`) - .run(centralInstanceId); - } - - public markUsed(centralInstanceId: string): void { - DatabaseService.getInstance().getDb() - .prepare(`UPDATE mesh_centrals SET last_used_at = ? WHERE central_instance_id = ?`) - .run(Date.now(), centralInstanceId); - } - - public markRejected(centralInstanceId: string, reason: string): void { - DatabaseService.getInstance().getDb() - .prepare(`UPDATE mesh_centrals SET last_rejected_at = ?, last_reject_reason = ? WHERE central_instance_id = ?`) - .run(Date.now(), reason, centralInstanceId); - } -} diff --git a/backend/src/services/MeshProxyTunnelDialer.ts b/backend/src/services/MeshProxyTunnelDialer.ts index 8e49063c..079a0e7f 100644 --- a/backend/src/services/MeshProxyTunnelDialer.ts +++ b/backend/src/services/MeshProxyTunnelDialer.ts @@ -1,12 +1,9 @@ import { EventEmitter } from 'events'; -import { createHash } from 'crypto'; -import jwt from 'jsonwebtoken'; import WebSocket from 'ws'; import { MAX_FRAME_SIZE_BYTES } from '../pilot/protocol'; import { PilotTunnelBridge, type MeshTunnelHandle } from './PilotTunnelBridge'; import { PilotTunnelManager } from './PilotTunnelManager'; import { NodeRegistry } from './NodeRegistry'; -import { DatabaseService } from './DatabaseService'; import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog'; import { httpUrlToWs } from '../utils/wsUrl'; import { isDebugEnabled } from '../utils/debug'; @@ -29,15 +26,16 @@ import type { MeshActivityType } from './MeshService'; * Lifecycle: * - `ensureBridge(nodeId)` dials if not connected; concurrent callers * dedupe through an in-flight promise map. - * - A periodic check tears down bridges that have seen no active streams - * for `SENCHO_MESH_PROXY_TUNNEL_IDLE_MS` (default 5 minutes). Setting - * the env var to `0` disables idle close (tunnel persists until the - * remote drops it or central shuts down). + * - The bridge is a persistent bidirectional control-plane channel: + * central → peer `tcp_open` frames and peer → central `tcp_open_reverse` + * frames flow over the same WS. The default idle TTL is 0 (no idle + * close); `SENCHO_MESH_PROXY_TUNNEL_IDLE_MS` overrides the default for + * operators who still want stream-scoped tunnels. * - Recent failures are cached for 60 seconds so a misconfigured remote * does not cause a continuous redial storm; `MeshService.getStatus` * consults the cache to surface a `reachableReason` to the UI. */ -const DEFAULT_IDLE_TTL_MS = 5 * 60 * 1000; +const DEFAULT_IDLE_TTL_MS = 0; const IDLE_CHECK_INTERVAL_MS = 60 * 1000; const HANDSHAKE_TIMEOUT_MS = 15_000; const FAILURE_CACHE_TTL_MS = 60 * 1000; @@ -148,6 +146,11 @@ export class MeshProxyTunnelDialer extends EventEmitter { return this.bridges.get(nodeId) ?? null; } + /** True when a dial for this node is currently in flight. */ + public isDialing(nodeId: number): boolean { + return this.inflight.has(nodeId); + } + public getRecentFailure(nodeId: number): DialFailure | null { const entry = this.recentFailures.get(nodeId); if (!entry) return null; @@ -262,18 +265,6 @@ export class MeshProxyTunnelDialer extends EventEmitter { return null; } - // Capability-gated mesh_handshake: pushes a signed `mesh_tunnel` JWT - // and the central origin so the peer can dial central back for - // reverse cross-fleet TCP streams (R2). Skipped silently when the - // peer lacks the capability or `SENCHO_PRIMARY_URL` is unset so - // legacy peers keep their forward-only TCP path untouched. - await this.maybeSendBootstrap(nodeId, ws); - - if (this.stopped) { - try { ws.close(1001, 'dialer shutdown'); } catch { /* ignore */ } - return null; - } - const bridge = new PilotTunnelBridge(nodeId, ws); try { await bridge.start(); @@ -451,47 +442,6 @@ export class MeshProxyTunnelDialer extends EventEmitter { this.redialTimers.set(nodeId, timer); } - /** - * If the peer advertises `mesh_proxy_callback_bootstrap` and central has - * a canonical origin (`SENCHO_PRIMARY_URL`), mint a `mesh_tunnel`-scoped - * JWT bound to the peer's `api_token` fingerprint and push it as the - * first text frame on the freshly-opened WS. The peer's - * `meshProxyTunnel` first-frame state machine persists the bootstrap - * material before yielding to TCP traffic. Every other path is a silent - * skip so legacy peers continue to receive raw TCP frames immediately. - */ - private async maybeSendBootstrap(nodeId: number, ws: WebSocket): Promise { - const canonicalOrigin = (process.env.SENCHO_PRIMARY_URL ?? '').replace(/\/+$/, ''); - if (!canonicalOrigin) return; - - let meta; - try { meta = await NodeRegistry.getInstance().fetchMetaForNode(nodeId); } - catch { return; } - if (!meta?.online) return; - if (!meta.capabilities?.includes('mesh_proxy_callback_bootstrap')) return; - - const db = DatabaseService.getInstance(); - const node = db.getNode(nodeId); - if (!node || node.type !== 'remote' || node.mode !== 'proxy' || !node.api_token) return; - - const authSecret = db.getGlobalSettings().auth_jwt_secret; - const centralInstanceId = db.getSystemState('instance_id'); - if (!authSecret || !centralInstanceId) return; - - const built = buildHandshakeFrame(nodeId, node.api_token, canonicalOrigin, centralInstanceId, authSecret); - try { - ws.send(JSON.stringify(built.frame)); - void this.logActivity(nodeId, 'open.ok', { - bootstrapSent: true, - centralApiUrl: redactSensitiveText(canonicalOrigin), - kid: 'v1', - jwtExpiresAt: built.expSec, - }); - } catch (err) { - console.warn(`[MeshProxyDialer] mesh_handshake send failed for node ${nodeId}: ${(err as Error).message}`.replace(/[\n\r]/g, '')); - } - } - private async logActivity( nodeId: number, event: ProxyTunnelEvent, @@ -517,58 +467,6 @@ export class MeshProxyTunnelDialer extends EventEmitter { } } -interface HandshakeFrame { - t: 'mesh_handshake'; - v: 1; - peerNodeId: number; - centralInstanceId: string; - centralApiUrl: string; - meshTunnelJwt: string; - jwtExpiresAt: number; -} - -/** - * Build the mesh_handshake frame and the associated JWT. Pure helper so the - * dialer's `maybeSendBootstrap` stays under the 30-line ceiling. - */ -function buildHandshakeFrame( - nodeId: number, - apiToken: string, - canonicalOrigin: string, - centralInstanceId: string, - authSecret: string, -): { frame: HandshakeFrame; expSec: number } { - const peerTokenFp = createHash('sha256').update(apiToken).digest('hex').slice(0, 16); - const nowSec = Math.floor(Date.now() / 1000); - const expSec = nowSec + 365 * 24 * 3600; - const meshTunnelJwt = jwt.sign( - { - sub: String(nodeId), - iss: centralInstanceId, - aud: canonicalOrigin, - scope: 'mesh_tunnel', - iat: nowSec, - exp: expSec, - kid: 'v1', - peer_token_fp: peerTokenFp, - }, - authSecret, - { algorithm: 'HS256' }, - ); - return { - frame: { - t: 'mesh_handshake', - v: 1, - peerNodeId: nodeId, - centralInstanceId, - centralApiUrl: canonicalOrigin, - meshTunnelJwt, - jwtExpiresAt: expSec, - }, - expSec, - }; -} - function classifyDialError(err: unknown): { code: DialFailureCode; message: string } { const message = sanitizeForLog((err as Error).message || String(err)); const httpStatus = (err as Error & { httpStatus?: number }).httpStatus; diff --git a/backend/src/services/MeshService.ts b/backend/src/services/MeshService.ts index 63dfb2ec..f40db049 100644 --- a/backend/src/services/MeshService.ts +++ b/backend/src/services/MeshService.ts @@ -4,7 +4,7 @@ import fs from 'fs/promises'; import { EventEmitter } from 'events'; import * as YAML from 'yaml'; import { ComposeService } from './ComposeService'; -import { DatabaseService } from './DatabaseService'; +import { DatabaseService, type NodeMode } from './DatabaseService'; import DockerController from './DockerController'; import { FileSystemService } from './FileSystemService'; import { LicenseService } from './LicenseService'; @@ -14,7 +14,6 @@ import { NodeRegistry } from './NodeRegistry'; import { PilotTunnelManager } from './PilotTunnelManager'; import { MeshProxyTunnelDialer, type DialFailureCode } from './MeshProxyTunnelDialer'; import { generateOverrideYaml, MeshAlias, SENCHO_MESH_NETWORK } from './MeshComposeOverride'; -import { disableCapability, enableCapability } from './CapabilityRegistry'; import { lookupContainerIp } from '../mesh/containerLookup'; import { sanitizeForLog } from '../utils/safeLog'; import { isPathWithinBase, isValidStackName } from '../utils/validation'; @@ -97,7 +96,7 @@ export type MeshActivityType = | 'forwarder.listen' | 'forwarder.unlisten' | 'forwarder.error' | 'proxy-tunnel.open.ok' | 'proxy-tunnel.open.fail' | 'proxy-tunnel.close' | 'mesh.proxy_tunnel.identify' - | 'mesh_handshake.received'; + | 'mesh.reconcile.fail'; export interface MeshActivityEvent { ts: number; @@ -148,13 +147,28 @@ export interface MeshRegenSummary { * - `pilot`: a remote with a pilot agent. Live-tunnel state is captured * separately in `pilotConnected`. * - `proxy`: a remote that central reaches via the long-lived api_token. - * Mesh tunnels are opened on demand by `MeshProxyTunnelDialer`; the - * operator sees no badge while the configuration is sound. + * Central maintains a persistent bidirectional WS to each mesh-enabled + * proxy peer, reconciled periodically; the operator sees no badge while + * the bridge is up. * - `unreachable`: configuration or runtime problem keeps mesh traffic * from flowing. `reachableReason` carries an actionable hint. */ export type MeshReachableMode = 'local' | 'pilot' | 'proxy' | 'unreachable'; +/** + * State of the peer→central reverse path. The forward WS to a proxy-mode + * peer is bidirectional; peer→central traffic flows over the same WS via + * `tcp_open_reverse`. This discriminator surfaces whether that bridge is + * currently usable so the Routing tab can show a transient pill while the + * dialer is reconnecting. + * - `connected`: forward WS is open; peer can dispatch reverse streams. + * - `connecting`: dial in flight; transient. + * - `unavailable`: no bridge and no dial in flight (peer just rebooted, or + * last dial cached a failure). + * - `not_applicable`: not a proxy-mode peer, or mesh disabled on this node. + */ +export type MeshReverseCallbackStatus = 'connected' | 'connecting' | 'unavailable' | 'not_applicable'; + export interface MeshNodeStatus { nodeId: number; nodeName: string; @@ -174,6 +188,8 @@ export interface MeshNodeStatus { reachableMode: MeshReachableMode; /** Short, operator-facing reason when `reachableMode === 'unreachable'`. Null otherwise. */ reachableReason: string | null; + /** Peer→central reverse path state. `not_applicable` for non-proxy peers. */ + reverseCallbackStatus: MeshReverseCallbackStatus; optedInStacks: string[]; activeStreamCount: number; } @@ -245,6 +261,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { private activity: MeshActivityEvent[] = []; private activeStreams = new Map(); private aliasRefreshTimer?: NodeJS.Timeout; + private bridgeReconcileTimer?: NodeJS.Timeout; private routeErrorMap = new Map(); private routeLatencyMap = new Map(); private activityListeners = new Set<(e: MeshActivityEvent) => void>(); @@ -335,8 +352,6 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { this.selfCentralNodeId = this.resolveSelfCentralNodeId(); - this.maybeWarnUnsetPrimaryUrl(); - await this.setupMeshNetwork(); try { await this.refreshAliasCache(); @@ -355,10 +370,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { }); } await this.regenerateAllOverrides(); - // Trigger 3: proactively re-establish central->peer bridges so any - // peer that is online and capable receives bootstrap material on - // startup. Fire-and-forget so start() does not block on remote I/O. - void this.proactiveBootstrapFanout(); + // Proactively dial every mesh-enabled proxy peer so the forward WS + // (which also carries peer→central reverse traffic) is up before any + // user request hits it. Fire-and-forget so start() does not block on + // remote I/O. + void this.proactiveBridgeFanout(); + this.startBridgeReconcileLoop(); this.aliasRefreshTimer = setInterval(() => { void (async () => { try { @@ -388,49 +405,25 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { clearInterval(this.aliasRefreshTimer); this.aliasRefreshTimer = undefined; } + this.stopBridgeReconcileLoop(); await this.forwarder.shutdown(); } /** - * Operator-facing preflight: if `SENCHO_PRIMARY_URL` is unset at - * startup and the central has at least one mesh-enabled proxy-mode - * node, warn that the Task 8 capability-gated handshake will fall - * back to an inferred origin for the callback bootstrap. The matching - * fail-safe in `MeshProxyTunnelDialer` silently skips the bootstrap - * when the env is unset; this warn makes the consequence visible. + * Walk every mesh-enabled proxy-mode peer and call + * `MeshProxyTunnelDialer.ensureBridge(nodeId)` on each. The bridge is + * the persistent bidirectional control-plane channel: central→peer + * `tcp_open` and peer→central `tcp_open_reverse` both flow over the + * same WS. Bounded concurrency 4 with a 250 ms stagger; failures are + * logged and never abort the fan-out. Called both at startup and on + * every reconcile tick. `ensureBridge` short-circuits on already-open + * bridges, so steady-state cost is one Map lookup per peer. */ - private maybeWarnUnsetPrimaryUrl(): void { - if (process.env.SENCHO_PRIMARY_URL) return; - const row = DatabaseService.getInstance().getDb().prepare(` - SELECT COUNT(*) as n FROM nodes - WHERE type='remote' AND mode='proxy' AND mesh_enabled=1 - `).get() as { n: number }; - if (row.n > 0) { - console.warn( - '[Mesh] SENCHO_PRIMARY_URL is unset; mesh callback bootstrap will use ' + - 'inferred origin. Set SENCHO_PRIMARY_URL to make peer-initiated mesh ' + - 'callbacks robust against reverse-proxy edge cases.' - ); - } - } - - /** - * Trigger 3: at central startup, walk every mesh-enabled proxy-mode - * peer that already has at least one `mesh_stacks` row and call - * `MeshProxyTunnelDialer.ensureBridge(nodeId)`. The dialer's - * capability-gated handshake then mints and ships bootstrap material - * to peers that just came online or just got upgraded to a build that - * advertises `mesh_proxy_callback_bootstrap`. Bounded concurrency 4 - * with a 250ms stagger; failures are logged and never abort the - * fan-out. - */ - private async proactiveBootstrapFanout(): Promise { + private async proactiveBridgeFanout(): Promise { const rows = DatabaseService.getInstance().getDb().prepare(` - SELECT DISTINCT n.id AS id - FROM nodes n - INNER JOIN mesh_stacks ms ON ms.node_id = n.id - WHERE n.type = 'remote' AND n.mode = 'proxy' AND n.mesh_enabled = 1 - ORDER BY n.id + SELECT id FROM nodes + WHERE type = 'remote' AND mode = 'proxy' AND mesh_enabled = 1 + ORDER BY id `).all() as Array<{ id: number }>; const queue = rows.map((r) => r.id); @@ -445,8 +438,8 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { this.logActivity({ source: 'mesh', level: 'warn', type: 'proxy-tunnel.open.fail', nodeId, - message: `boot proactive bootstrap failed: ${sanitizeForLog((err as Error).message)}`, - details: { trigger: 'startup_fanout' }, + message: `proxy-tunnel reconcile dial failed: ${sanitizeForLog((err as Error).message)}`, + details: { trigger: 'reconcile' }, }); } await new Promise((r) => setTimeout(r, 250)); @@ -456,6 +449,35 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { await Promise.all(Array.from({ length: workerCount }, () => worker())); } + /** + * Schedule the proxy-tunnel reconcile tick. Interval is overridable via + * `SENCHO_MESH_RECONCILE_INTERVAL_MS` (default 60_000 ms) so an operator + * can tune the peer-reboot detection window. Idempotent: a second call + * is a no-op while the timer is live. + */ + private startBridgeReconcileLoop(): void { + if (this.bridgeReconcileTimer) return; + const raw = process.env.SENCHO_MESH_RECONCILE_INTERVAL_MS; + const parsed = raw === undefined ? Number.NaN : Number(raw); + const intervalMs = Number.isFinite(parsed) && parsed >= 1000 ? parsed : 60_000; + this.bridgeReconcileTimer = setInterval(() => { + void this.proactiveBridgeFanout().catch((err) => { + this.logActivity({ + source: 'mesh', level: 'error', type: 'mesh.reconcile.fail', + message: `bridge reconcile threw: ${sanitizeForLog((err as Error).message)}`, + }); + }); + }, intervalMs); + this.bridgeReconcileTimer.unref?.(); + } + + private stopBridgeReconcileLoop(): void { + if (this.bridgeReconcileTimer) { + clearInterval(this.bridgeReconcileTimer); + this.bridgeReconcileTimer = undefined; + } + } + public getSenchoIp(): string | null { return this.senchoIp; } @@ -481,7 +503,6 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { /** * Single recording path for every mesh-setup failure. Keeps the legacy * `networkSetupError` string in sync, sets the typed `dataPlaneStatus`, - * strips `mesh_proxy_callback_bootstrap` from advertised capabilities, * and emits a `mesh.disable` activity entry. Callers pass `level: 'warn'` * for expected conditions (`not_in_docker` in dev mode) and `'error'` for * real failures. @@ -500,7 +521,6 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { this.networkSetupError = message; this.dataPlaneStatus = { ok: false, reason, message, subnet }; this.senchoIp = null; - disableCapability('mesh_proxy_callback_bootstrap'); this.logActivity({ source: 'mesh', level, @@ -580,7 +600,6 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { this.networkSetupError = null; this.dataPlaneStatus = { ok: true, reason: 'ok', message: null, subnet }; - enableCapability('mesh_proxy_callback_bootstrap'); } /** @@ -856,15 +875,14 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { source: 'mesh', level: 'info', type: 'mesh.enable', nodeId, message: `mesh enabled on node ${nodeId}`, }); - // Trigger 1: proactive bootstrap. If the peer is a proxy-mode remote, - // dial the mesh callback bridge immediately so the capability-gated - // handshake can ship `mesh_handshake` material without waiting for - // the next forwarder dial. Fire-and-forget; failures are logged by - // the dialer. + // When mesh is enabled on a proxy peer, dial the persistent bridge + // immediately so the next forward (or peer-initiated reverse) + // request has the WS already up. Fire-and-forget; failures are + // logged by the dialer. const node = DatabaseService.getInstance().getNode(nodeId); if (node && node.type === 'remote' && node.mode === 'proxy') { void MeshProxyTunnelDialer.getInstance().ensureBridge(nodeId).catch((err) => { - console.warn(`[Mesh] proactive bootstrap on mesh-enable failed for node ${nodeId}: ${(err as Error).message}`); + console.warn(`[Mesh] proxy-tunnel dial on mesh-enable failed for node ${nodeId}: ${(err as Error).message}`); }); } } @@ -1812,59 +1830,23 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { message: `cross-node dispatch to ${target.alias} on node ${target.nodeId}`, }); - // Peer-side recovery: when this Sencho is acting as a proxy-mode peer - // (mesh_centrals has a row from a prior bootstrap) and the - // reverseDialer is not currently installed, the inbound bridge from - // central is either cold-start (peer just rebooted) or has been torn - // down (idle close, central restart). Kick the symmetric-dial path - // so the peer re-opens its callback WS to central before attempting - // the cross-node dispatch. - // - // Central instances never have a mesh_centrals row (central is not a - // peer of itself), so this branch is correctly skipped on central. - // Central falls straight through to dialMeshTcpStream which uses its - // own PilotTunnelManager + MeshProxyTunnelDialer to reach the target - // peer. Without this gate, central enters the branch on every - // forward dispatch, finds no session, and destroys the inbound - // socket with route.resolve.fail forward-from-peer no_session. - if (!this.reverseDialer) { - const { MeshCentralRegistry } = await import('./MeshCentralRegistry'); - const isProxyPeer = MeshCentralRegistry.getInstance().getActive() !== null; - if (isProxyPeer) { - try { - const { PeerToCentralMeshSessionDialer } = await import('./PeerToCentralMeshSessionDialer'); - const session = await PeerToCentralMeshSessionDialer.getInstance().ensureSession(); - if (!session) { - this.logActivity({ - source: 'mesh', level: 'warn', type: 'route.resolve.fail', - nodeId: target.nodeId, alias: target.alias, - message: `peer cross-node dispatch failed: no central callback session available`, - details: { direction: 'forward-from-peer', reason: 'no_session' }, - }); - try { src.destroy(); } catch { /* ignore */ } - return; - } - } catch (err) { - this.logActivity({ - source: 'mesh', level: 'warn', type: 'route.resolve.fail', - nodeId: target.nodeId, alias: target.alias, - message: `peer cross-node dispatch failed: bootstrap threw`, - details: { direction: 'forward-from-peer', reason: 'bootstrap_threw' }, - }); - try { src.destroy(); } catch { /* ignore */ } - return; - } - } - } - const tcpStream = await this.dialMeshTcpStream(target); if (!tcpStream) { + // Three failure shapes on this path: + // - central with reverseDialer somehow installed (unusual; relay edge case) + // - central without reverseDialer (normal): ensureBridge could not reach the target peer + // - peer side: target nodeId is central's id, which the peer's NodeRegistry does + // not know as a remote proxy target, so ensureBridge returns null. The actionable + // condition is "central has not dialed the bridge yet"; tell the operator. + const targetIsLocalKnownRemote = NodeRegistry.getInstance().getNode(target.nodeId)?.type === 'remote'; + const message = this.reverseDialer + ? `cannot open reverse mesh stream to node ${target.nodeId}` + : targetIsLocalKnownRemote + ? `no mesh tunnel reachable for node ${target.nodeId}` + : `peer cross-node dispatch deferred: waiting for central to dial the reverse bridge`; this.logActivity({ source: 'pilot', level: 'error', type: 'tunnel.fail', - nodeId: target.nodeId, alias: target.alias, - message: this.reverseDialer - ? `cannot open reverse mesh stream to node ${target.nodeId}` - : `no mesh tunnel reachable for node ${target.nodeId}`, + nodeId: target.nodeId, alias: target.alias, message, }); try { src.destroy(); } catch { /* ignore */ } return; @@ -2148,12 +2130,14 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { const localNodeId = NodeRegistry.getInstance().getDefaultNodeId(); const localListening = this.isLocalForwarderActive(); const ptm = PilotTunnelManager.getInstance(); + const dialer = MeshProxyTunnelDialer.getInstance(); return nodes.map((node) => { const reach = this.computeReachable(node, localNodeId); + const meshEnabled = db.getNodeMeshEnabled(node.id); return { nodeId: node.id, nodeName: node.name, - enabled: db.getNodeMeshEnabled(node.id), + enabled: meshEnabled, localForwarderListening: node.id === localNodeId ? localListening : null, // `pilotConnected` stays at its original meaning: a pilot // tunnel is currently registered for this node. The @@ -2163,12 +2147,26 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { pilotConnected: node.type !== 'remote' || ptm.hasActiveTunnel(node.id), reachableMode: reach.mode, reachableReason: reach.reason, + reverseCallbackStatus: this.computeReverseCallbackStatus(node, meshEnabled, dialer), optedInStacks: db.listMeshStacks(node.id).map((s) => s.stack_name), activeStreamCount: this.activeStreams.size, }; }); } + private computeReverseCallbackStatus( + node: { id: number; type: 'local' | 'remote'; mode: NodeMode }, + meshEnabled: boolean, + dialer: MeshProxyTunnelDialer, + ): MeshReverseCallbackStatus { + if (node.type !== 'remote' || node.mode !== 'proxy' || !meshEnabled) { + return 'not_applicable'; + } + if (dialer.hasBridge(node.id)) return 'connected'; + if (dialer.isDialing(node.id)) return 'connecting'; + return 'unavailable'; + } + /** True when the local Sencho's forwarder is started and bound to at least one alias port. */ private isLocalForwarderActive(): boolean { return this.started && this.forwarder.getListenerPorts().length > 0; diff --git a/backend/src/services/PeerToCentralMeshSessionDialer.ts b/backend/src/services/PeerToCentralMeshSessionDialer.ts deleted file mode 100644 index 5a489456..00000000 --- a/backend/src/services/PeerToCentralMeshSessionDialer.ts +++ /dev/null @@ -1,276 +0,0 @@ -/** - * PeerToCentralMeshSessionDialer: peer-side counterpart to the central - * ingress at `/api/mesh/proxy-tunnel-from-peer`. Reads cached central - * material from `MeshCentralRegistry`, opens a WebSocket to central - * carrying the bootstrapped JWT in the Authorization header, and on a - * successful upgrade installs a local `PilotTunnelBridge` whose loopback - * URL the local proxy can target as if central had dialed us. - * - * Failure handling branches on the central's machine-readable reason - * code returned in the 401 body: terminal codes (stale, - * signature_invalid, fingerprint, node_deleted, instance_mismatch, - * audience_mismatch) clear the cache so we stop dialing with a known-bad - * credential. Transient codes (clock_skew, mode_mismatch) keep the cache - * but record the rejection so an operator can see why we stalled. - * HTTP 404 (older central without the peer ingress endpoint) keeps the - * cache and arms a longer backoff window so we do not hammer the remote. - * - * Process-local rate limit: at most 5 dial attempts per 60s window. - * Concurrency: a single in-flight promise is shared across callers so - * parallel `ensureSession()` calls coalesce into one dial. - */ -import { EventEmitter } from 'events'; -import WebSocket from 'ws'; -import { - MAX_FRAME_SIZE_BYTES, - decodeBinaryFrame, - decodeJsonFrame, - wsDataToBuffer, - wsDataToString, -} from '../pilot/protocol'; -import { MeshCentralRegistry } from './MeshCentralRegistry'; -import { - attachTcpStreamSwitchboard, - resolveByComposeLabels, - type TcpStreamSwitchboard, - type ReverseTcpStreamHandle, -} from '../mesh/tcpStreamSwitchboard'; -import { PilotMetrics } from './PilotMetrics'; -import { httpUrlToWs } from '../utils/wsUrl'; -import { sanitizeForLog } from '../utils/safeLog'; - -interface SwitchboardReverseDialer { - openMeshTcpStream(target: { nodeId: number; stack: string; service: string; port: number }): ReverseTcpStreamHandle | null; -} - -const HANDSHAKE_TIMEOUT_MS = 15_000; -const RATE_LIMIT_WINDOW_MS = 60_000; -const RATE_LIMIT_MAX = 5; -const ENDPOINT_UNAVAILABLE_BACKOFF_MS = 5 * 60_000; -const CALLBACK_PATH = '/api/mesh/proxy-tunnel-from-peer'; - -/** - * Reasons that indicate the cached central material is permanently bad - * (wrong secret, wrong fingerprint, node removed on central, etc.) and - * should be evicted so the next mesh-enable cycle remints fresh - * material. `endpoint_not_found` is explicitly NOT in this set because - * a 404 typically means "central is older than the peer ingress", - * which is recoverable by waiting. - */ -const REJECT_REASONS_CLEAR = new Set([ - 'stale', - 'token_fingerprint_mismatch', - 'node_deleted', - 'instance_mismatch', - 'signature_invalid', - 'audience_mismatch', - 'unknown', -]); - -interface DialError extends Error { - reason?: string; - httpStatus?: number; -} - -export class PeerToCentralMeshSessionDialer extends EventEmitter { - private static instance: PeerToCentralMeshSessionDialer | null = null; - private currentSession: TcpStreamSwitchboard | null = null; - private currentWs: WebSocket | null = null; - private inflight: Promise | null = null; - private recentDials: number[] = []; - private endpointUnavailableUntil = 0; - - private constructor() { super(); } - - public static getInstance(): PeerToCentralMeshSessionDialer { - if (!this.instance) this.instance = new PeerToCentralMeshSessionDialer(); - return this.instance; - } - - public static resetForTest(): void { - if (this.instance) { - try { this.instance.currentSession?.cleanup('test reset'); } catch { /* ignore */ } - try { this.instance.currentWs?.close(1000, 'test reset'); } catch { /* ignore */ } - } - this.instance = null; - } - - public hasSession(): boolean { - return this.currentSession !== null; - } - - public async ensureSession(): Promise { - if (this.currentSession) return this.currentSession; - if (Date.now() < this.endpointUnavailableUntil) return null; - if (this.isRateLimited()) return null; - if (this.inflight) return this.inflight; - this.inflight = this.dial().finally(() => { this.inflight = null; }); - return this.inflight; - } - - private isRateLimited(): boolean { - const now = Date.now(); - this.recentDials = this.recentDials.filter((ts) => now - ts < RATE_LIMIT_WINDOW_MS); - return this.recentDials.length >= RATE_LIMIT_MAX; - } - - private async dial(): Promise { - const material = MeshCentralRegistry.getInstance().getActive(); - if (!material) return null; - this.recentDials.push(Date.now()); - const wsUrl = httpUrlToWs(material.centralApiUrl) + CALLBACK_PATH; - const ws = new WebSocket(wsUrl, { - headers: { Authorization: `Bearer ${material.callbackJwt}` }, - handshakeTimeout: HANDSHAKE_TIMEOUT_MS, - maxPayload: MAX_FRAME_SIZE_BYTES, - }); - try { - await this.awaitOpen(ws); - } catch (err) { - ws.on('error', () => { /* swallow tail */ }); - try { ws.close(); } catch { /* ignore */ } - this.handleDialFailure(err, material.centralInstanceId); - return null; - } - return this.attachSwitchboard(ws, material.centralInstanceId); - } - - /** - * Wire the peer-initiated callback WS into the local MeshService. The - * R1-A2 design puts the peer end of the bridge in TcpStreamSwitchboard - * mode (peer multiplexes streams; central side runs PilotTunnelBridge). - * Without this wiring the WS opens cleanly but MeshService.reverseDialer - * stays null, so MeshService.dialMeshTcpStream falls through to - * PilotTunnelManager.ensureBridge(centralNodeId), which has no record - * for central on a proxy-mode peer (peers do not enroll central) and - * fails with proxy-tunnel.open.fail reason=no_target. End state matches - * the v0.78.1 reverse-direction failure even though the callback bridge - * is alive. - * - * Wiring symmetric to the central-initiated handler at - * `meshProxyTunnel.ts:115-163`: - * - attachTcpStreamSwitchboard with the same compose-label resolver - * - SwitchboardReverseDialer that delegates to switchboard.openReverseStream - * - setReverseDialer(localDialer, null) with CAS so a concurrent - * central-initiated tunnel does not get silently overwritten - * - ws.on('message') dispatches JSON/binary frames to the switchboard - * - ws.on('close'/'error') tears down switchboard + clears reverseDialer - */ - private async attachSwitchboard(ws: WebSocket, instanceId: string): Promise { - let switchboard: TcpStreamSwitchboard; - try { - switchboard = attachTcpStreamSwitchboard({ - ws, - resolveTarget: resolveByComposeLabels, - logLabel: 'MeshCallback', - }); - } catch (err) { - try { ws.close(1011, 'switchboard attach failed'); } catch { /* ignore */ } - PilotMetrics.increment('mesh_callback_dials_failed_total'); - console.warn(`[PeerToCentralMeshSessionDialer] attach failed: ${sanitizeForLog((err as Error).message)}`); - return null; - } - - const { MeshService } = await import('./MeshService'); - const meshService = MeshService.getInstance(); - const localDialer: SwitchboardReverseDialer = { - openMeshTcpStream(target) { - return switchboard.openReverseStream(target); - }, - }; - const installed = meshService.setReverseDialer(localDialer, null); - if (!installed) { - console.warn('[PeerToCentralMeshSessionDialer] reverse dialer already installed; rejecting concurrent callback bridge'); - switchboard.cleanup('reverse dialer already installed'); - try { ws.close(1013, 'reverse dialer already installed'); } catch { /* ignore */ } - PilotMetrics.increment('mesh_callback_dials_failed_total'); - return null; - } - - const onMessage = (data: unknown, isBinary: boolean): void => { - try { - if (isBinary) { - const buf = wsDataToBuffer(data); - if (!buf) return; - switchboard.handleBinaryFrame(decodeBinaryFrame(buf)); - return; - } - const text = wsDataToString(data); - if (text == null) return; - switchboard.handleJsonFrame(decodeJsonFrame(text)); - } catch (err) { - console.warn(`[PeerToCentralMeshSessionDialer] malformed frame: ${sanitizeForLog((err as Error).message)}`); - } - }; - - let tornDown = false; - const teardown = (): void => { - if (tornDown) return; - tornDown = true; - ws.off('message', onMessage); - try { switchboard.cleanup('mesh callback bridge closed'); } catch { /* ignore */ } - meshService.setReverseDialer(null, localDialer); - if (this.currentSession === switchboard) this.currentSession = null; - if (this.currentWs === ws) this.currentWs = null; - }; - - ws.on('message', onMessage); - ws.once('close', teardown); - ws.once('error', (err) => { - console.warn(`[PeerToCentralMeshSessionDialer] ws error: ${sanitizeForLog(err.message)}`); - teardown(); - }); - - this.currentSession = switchboard; - this.currentWs = ws; - PilotMetrics.increment('mesh_central_bootstraps_total'); - MeshCentralRegistry.getInstance().markUsed(instanceId); - return switchboard; - } - - private awaitOpen(ws: WebSocket): Promise { - return new Promise((resolve, reject) => { - const cleanup = (): void => { - ws.removeAllListeners('open'); - ws.removeAllListeners('error'); - ws.removeAllListeners('unexpected-response'); - }; - ws.once('open', () => { cleanup(); resolve(); }); - ws.once('error', (err) => { cleanup(); reject(err); }); - ws.once('unexpected-response', (_req, res) => { - cleanup(); - let body = ''; - res.on('data', (chunk: Buffer) => { body += chunk.toString('utf8'); }); - res.on('end', () => reject(this.buildUpgradeError(res.statusCode, body))); - }); - }); - } - - private buildUpgradeError(statusCode: number | undefined, body: string): DialError { - let reason = 'unknown'; - try { - const parsed = JSON.parse(body) as { reason?: unknown }; - if (typeof parsed.reason === 'string') reason = parsed.reason; - } catch { /* keep unknown */ } - if (statusCode === 404) reason = 'endpoint_not_found'; - const err = new Error(`upgrade rejected: HTTP ${statusCode ?? 'unknown'} reason=${reason}`) as DialError; - err.reason = reason; - err.httpStatus = statusCode; - return err; - } - - private handleDialFailure(err: unknown, instanceId: string): void { - const reason = (err as DialError).reason ?? 'unknown'; - PilotMetrics.increment('mesh_callback_dials_failed_total'); - if (REJECT_REASONS_CLEAR.has(reason)) { - MeshCentralRegistry.getInstance().clearForInstance(instanceId); - PilotMetrics.increment('mesh_callback_auth_failures_total'); - } else { - MeshCentralRegistry.getInstance().markRejected(instanceId, reason); - if (reason === 'endpoint_not_found') { - this.endpointUnavailableUntil = Date.now() + ENDPOINT_UNAVAILABLE_BACKOFF_MS; - } - } - console.warn(`[PeerToCentralMeshSessionDialer] dial failed: ${sanitizeForLog(`reason=${reason}`)}`); - } -} diff --git a/backend/src/services/PilotMetrics.ts b/backend/src/services/PilotMetrics.ts index 118881af..f70e3163 100644 --- a/backend/src/services/PilotMetrics.ts +++ b/backend/src/services/PilotMetrics.ts @@ -28,16 +28,8 @@ export interface Counters { proxy_bridges_total: number; /** Failed proxy-tunnel dial attempts (all reasons). Pair with `proxy_bridges_total` for an attempt/success ratio; pair with `proxy_idle_closes` for retention. */ proxy_dials_failed: number; - /** Proxy-tunnel teardowns initiated by the dialer's idle sweep (zero active streams for the configured TTL). */ + /** Proxy-tunnel teardowns initiated by the dialer's idle sweep (zero active streams for the configured TTL). Effectively 0 by default since the dialer no longer idle-closes the bridge. */ proxy_idle_closes: number; - /** Proxy bridges registered via the peer-initiated dial-back path (`/api/mesh/proxy-tunnel-from-peer`). Disjoint from `proxy_bridges_total`, which counts central-initiated dials. */ - proxy_bridges_peer_initiated_total: number; - /** Successful peer-to-central dial-back sessions (counted on WS open + bridge start). Disjoint from `proxy_bridges_peer_initiated_total`, which is incremented by the central-side ingress at register time. Useful for operators investigating asymmetric counts (peer thinks it dialed, central never registered). */ - mesh_central_bootstraps_total: number; - /** Peer-to-central dial-back attempts that did not complete a WS open. Covers transport errors, upgrade rejections, and bridge.start failures. */ - mesh_callback_dials_failed_total: number; - /** Subset of `mesh_callback_dials_failed_total` where central responded 401 with a terminal reason code (stale, signature_invalid, ...) that caused the peer to clear its cached central material. */ - mesh_callback_auth_failures_total: number; } const ZERO_COUNTERS: Counters = { @@ -49,10 +41,6 @@ const ZERO_COUNTERS: Counters = { proxy_bridges_total: 0, proxy_dials_failed: 0, proxy_idle_closes: 0, - proxy_bridges_peer_initiated_total: 0, - mesh_central_bootstraps_total: 0, - mesh_callback_dials_failed_total: 0, - mesh_callback_auth_failures_total: 0, }; export const PILOT_METRICS_FLUSH_INTERVAL_MS = 1_000; diff --git a/backend/src/services/PilotTunnelManager.ts b/backend/src/services/PilotTunnelManager.ts index d2d2dab0..36d89b37 100644 --- a/backend/src/services/PilotTunnelManager.ts +++ b/backend/src/services/PilotTunnelManager.ts @@ -36,11 +36,10 @@ export class PilotTunnelCapacityError extends Error { /** * Discriminator for the two flavors of bridge that share the `bridges` * map: `'pilot'` is set by `registerTunnel` (agent-initiated long-lived - * tunnel); `'proxy'` is set by `registerProxyBridge` / - * `replaceOrRegisterProxyBridge` (central- or peer-initiated short-lived - * proxy bridge). Used by the rejection-message formatter and by - * `replaceOrRegisterProxyBridge` so a peer-initiated dial can supersede a - * previous proxy bridge but never shadow a live pilot tunnel. + * tunnel); `'proxy'` is set by `registerProxyBridge` (central-initiated + * persistent bridge dialed by `MeshProxyTunnelDialer`). Used by the + * rejection-message formatter so a pilot tunnel and a proxy bridge for the + * same nodeId cannot coexist silently. */ export type BridgeKind = 'pilot' | 'proxy'; @@ -282,46 +281,12 @@ export class PilotTunnelManager extends EventEmitter { this.emit('proxy-bridge-up', nodeId); } - /** - * Register a peer-initiated proxy bridge for an existing remote. If a proxy - * bridge already exists for this nodeId, close it (the new dial is the source - * of truth) and replace. If a pilot tunnel exists, refuse: pilot tunnels - * always win over peer-initiated proxy bridges. - */ - public replaceOrRegisterProxyBridge(nodeId: number, bridge: PilotTunnelBridge): void { - const existingKind = this.bridgeKinds.get(nodeId); - if (existingKind === 'pilot') { - throw new Error(this.formatBridgeConflict(nodeId, existingKind)); - } - if (existingKind === 'proxy') { - const old = this.bridges.get(nodeId); - this.bridges.delete(nodeId); - this.bridgeKinds.delete(nodeId); - try { old?.close(1000, 'replaced-by-newer-proxy'); } catch { /* best-effort cleanup */ } - } - if (this.bridges.size >= PILOT_TUNNEL_HARD_LIMIT) { - PilotMetrics.increment('tunnels_rejected_capacity'); - throw new PilotTunnelCapacityError(PILOT_TUNNEL_HARD_LIMIT); - } - bridge.once('closed', () => { - if (this.bridges.get(nodeId) === bridge) { - this.bridges.delete(nodeId); - this.bridgeKinds.delete(nodeId); - this.emit('proxy-bridge-down', nodeId, 'remote_closed'); - } - }); - this.bridges.set(nodeId, bridge); - this.bridgeKinds.set(nodeId, 'proxy'); - } - /** * Single source of truth for the "slot already held" rejection - * message thrown by `registerProxyBridge` and - * `replaceOrRegisterProxyBridge`. Picks the bridge-kind subject and - * the rejection tail so a stale call site cannot drift from the - * other. `undefined` collapses to the pilot branch defensively; - * production code always sets `bridgeKinds` whenever `bridges` is - * set, so this is unreachable in practice. + * message thrown by `registerProxyBridge`. Picks the bridge-kind + * subject and the rejection tail. `undefined` collapses to the pilot + * branch defensively; production code always sets `bridgeKinds` + * whenever `bridges` is set, so this is unreachable in practice. */ private formatBridgeConflict(nodeId: number, existingKind: BridgeKind | undefined): string { return existingKind === 'proxy' diff --git a/backend/src/websocket/meshProxyTunnel.ts b/backend/src/websocket/meshProxyTunnel.ts index 53f626c7..9a06cf3a 100644 --- a/backend/src/websocket/meshProxyTunnel.ts +++ b/backend/src/websocket/meshProxyTunnel.ts @@ -11,44 +11,11 @@ import { import { sanitizeForLog } from '../utils/safeLog'; import { isDebugEnabled } from '../utils/debug'; import { rejectUpgrade as reject } from './reject'; -import { MeshCentralRegistry, type MeshCentralMaterial } from '../services/MeshCentralRegistry'; - -/** - * Bootstrap phase for the first-frame state machine. - * - * Central MAY send a `mesh_handshake` JSON frame as the FIRST text frame - * after upgrade. If it arrives we persist the callback material via - * MeshCentralRegistry and transition to `consumed`. Any frame other than - * the handshake (or no frame at all) moves us to `past`, after which a - * later handshake is a protocol error. - */ -type BootstrapPhase = 'awaiting' | 'consumed' | 'past'; - -interface MeshHandshakeFrame { - t: 'mesh_handshake'; - v: number; - peerNodeId: number; - centralInstanceId: string; - centralApiUrl: string; - meshTunnelJwt: string; - jwtExpiresAt: number; -} - -function isMeshHandshakeFrame(parsed: unknown): parsed is MeshHandshakeFrame { - return typeof parsed === 'object' && parsed !== null - && (parsed as { t?: unknown }).t === 'mesh_handshake' - && typeof (parsed as { v?: unknown }).v === 'number' - && typeof (parsed as { peerNodeId?: unknown }).peerNodeId === 'number' - && typeof (parsed as { centralInstanceId?: unknown }).centralInstanceId === 'string' - && typeof (parsed as { centralApiUrl?: unknown }).centralApiUrl === 'string' - && typeof (parsed as { meshTunnelJwt?: unknown }).meshTunnelJwt === 'string' - && typeof (parsed as { jwtExpiresAt?: unknown }).jwtExpiresAt === 'number'; -} /** * Mesh proxy-tunnel ingress. * - * The remote side of a Phase C proxy-mode mesh tunnel. Central dials + * The peer side of a proxy-mode mesh tunnel. Central dials * `WSS /api/mesh/proxy-tunnel` using the fleet credential as a * Bearer header (either the `node_proxy` JWT generated by the remote's * Settings → Nodes → Generate Token flow, or a `full-admin` api_token @@ -65,21 +32,12 @@ function isMeshHandshakeFrame(parsed: unknown): parsed is MeshHandshakeFrame { * the reverse-dialer on the local MeshService so meshed containers on * this Sencho can dial cross-node aliases via `tcp_open_reverse` over * the same WS. On disconnect the reverse-dialer registration is - * cleared. + * cleared. Central maintains the bridge persistently (no idle close; + * reconciled on a timer), so peer→central traffic always has a live + * channel in steady state. */ const wss = new WebSocketServer({ noServer: true, maxPayload: MAX_FRAME_SIZE_BYTES }); -/** - * Upper bound on the pre-CAS first-frame wait inside attachSwitchboard. - * Sized to cover localhost-loopback RTT plus a slim margin for the - * central-side ms-scale gap between `ws.send(mesh_handshake)` and the - * registerProxyBridge throw + ws.close that follows on a refused dial. - * On a dial that carries no bootstrap, this is pure added latency - * before the reverse-dialer install; 30ms is well below operator - * perception and well above realistic localhost RTT. - */ -const FIRST_FRAME_WAIT_MS = 30; - interface SwitchboardReverseDialer { openMeshTcpStream(target: { nodeId: number; stack: string; service: string; port: number }): ReverseTcpStreamHandle | null; } @@ -126,74 +84,18 @@ export async function handleMeshProxyTunnel(req: IncomingMessage, socket: Duplex async function attachSwitchboard(ws: WebSocket, peerNodeId: number | null): Promise { let switchboard: TcpStreamSwitchboard | null = null; let meshServiceCleanup: (() => void) | null = null; - let phase: BootstrapPhase = 'awaiting'; const onMessage = (data: unknown, isBinary: boolean): void => { if (!switchboard) return; try { if (isBinary) { - if (phase === 'awaiting') phase = 'past'; const buf = wsDataToBuffer(data); if (!buf) return; switchboard.handleBinaryFrame(decodeBinaryFrame(buf)); return; } const text = wsDataToString(data); - if (text == null) { - if (phase === 'awaiting') phase = 'past'; - return; - } - let parsedForBootstrap: unknown = null; - let parseSucceeded = false; - try { - parsedForBootstrap = JSON.parse(text); - parseSucceeded = true; - } catch { - // Not valid JSON; defer to decodeJsonFrame's stricter error path below. - } - - if (parseSucceeded && typeof parsedForBootstrap === 'object' - && parsedForBootstrap !== null - && (parsedForBootstrap as { t?: unknown }).t === 'mesh_handshake') { - if (phase !== 'awaiting') { - ws.close(1008, 'mesh_handshake out of order'); - return; - } - if (!isMeshHandshakeFrame(parsedForBootstrap)) { - ws.close(1008, 'malformed mesh_handshake'); - return; - } - const material: MeshCentralMaterial = { - centralInstanceId: parsedForBootstrap.centralInstanceId, - centralApiUrl: parsedForBootstrap.centralApiUrl.replace(/\/+$/, ''), - callbackJwt: parsedForBootstrap.meshTunnelJwt, - jwtIssuedAt: Math.floor(Date.now() / 1000), - jwtExpiresAt: parsedForBootstrap.jwtExpiresAt, - }; - try { - MeshCentralRegistry.getInstance().upsert(material); - phase = 'consumed'; - const centralInstanceId = parsedForBootstrap.centralInstanceId; - const jwtExpiresAt = parsedForBootstrap.jwtExpiresAt; - void (async () => { - try { - const { MeshService: MeshSvc } = await import('../services/MeshService'); - MeshSvc.getInstance().logActivity({ - source: 'mesh', level: 'info', - type: 'mesh_handshake.received', - message: `mesh callback bootstrap received from central ${centralInstanceId}`, - details: { centralInstanceId, jwtExpiresAt }, - }); - } catch { /* best-effort log; bootstrap success path must not depend on it */ } - })(); - } catch (err) { - console.warn(`[meshProxyTunnel] mesh_handshake persist failed: ${sanitizeForLog((err as Error).message)}`); - try { ws.close(1011, 'bootstrap persist failed'); } catch { /* ignore */ } - } - return; - } - - if (phase === 'awaiting') phase = 'past'; + if (text == null) return; switchboard.handleJsonFrame(decodeJsonFrame(text)); } catch (err) { if (isDebugEnabled()) { @@ -221,11 +123,6 @@ async function attachSwitchboard(ws: WebSocket, peerNodeId: number | null): Prom logLabel: 'MeshProxy', }); - // Wire listeners synchronously so any first frame from central - // reaches onMessage even when the reverse-dialer CAS swap below - // refuses this bridge. Bootstrap material identifies central for - // the next peer-initiated callback and must persist independent - // of whether this particular WS becomes the active bridge. ws.on('message', onMessage); ws.once('close', teardown); ws.once('error', (err) => { @@ -251,23 +148,7 @@ async function attachSwitchboard(ws: WebSocket, peerNodeId: number | null): Prom const installed = meshService.setReverseDialer(localDialer, null); if (!installed) { console.warn('[MeshProxy] reverse dialer already installed; rejecting concurrent tunnel'); - // Hold the WS open briefly so any in-flight first frame can - // land in onMessage before we send the close. Central - // optionally sends a mesh_handshake as the first frame on - // Trigger 1 / Trigger 2 dials; that bootstrap material must - // persist via MeshCentralRegistry regardless of whether this - // WS becomes the active bridge. The wait resolves as soon as - // any 'message' arrives (the listener attached above runs - // synchronously on the event and updates the registry); the - // timeout bounds the close-latency overhead for refused - // dials that carry no bootstrap. - await new Promise((resolve) => { - const onFirst = (): void => { clearTimeout(t); resolve(); }; - const t = setTimeout(() => { ws.off('message', onFirst); resolve(); }, FIRST_FRAME_WAIT_MS); - ws.once('message', onFirst); - }); try { ws.close(1013, 'reverse dialer already installed'); } catch { /* ignore */ } - // teardown fires on the 'close' event and runs switchboard.cleanup. return; } // Install central's view of this peer's nodeId so handleAccept @@ -276,11 +157,6 @@ async function attachSwitchboard(ws: WebSocket, peerNodeId: number | null): Prom // // The install persists across bridge lifecycles: the peer's identity // in central's namespace is stable for the enrollment, not per-WS. - // Null-clearing on teardown caused dispatch to fall back to - // getDefaultNodeId() after idle close, which collides with central's - // own nodeId for Local and misdispatches cross-fleet traffic to the - // same-node path. A subsequent install with a different nodeId - // (e.g. re-enrollment) is detected by the setter's overwrite-warn. if (peerNodeId !== null) { meshService.setProxyTunnelSelfCentralNodeId(peerNodeId); } diff --git a/backend/src/websocket/meshProxyTunnelFromPeer.ts b/backend/src/websocket/meshProxyTunnelFromPeer.ts deleted file mode 100644 index 51d7d574..00000000 --- a/backend/src/websocket/meshProxyTunnelFromPeer.ts +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Central-side ingress for peer-initiated proxy-mode mesh tunnels. - * - * After the bootstrap exchange in `meshProxyTunnel.ts` (Task 7/8), a peer - * with the central's `mesh_tunnel` JWT can dial *back* to central at this - * endpoint. The validation chain enforces every claim the bootstrap mint - * produced and confirms the peer's stored fingerprint still matches its - * current api_token. Each failure path returns HTTP 401 with a stable - * machine-readable `reason` code in the JSON body, so the dialer side - * (PeerToCentralMeshSessionDialer, Task 10) can act on the reason without - * scraping prose. - * - * Auth runs MANUALLY in this handler. Express middleware does not run on - * WebSocket upgrades, and the credential here is a Bearer JWT minted by - * the central itself, not a user session. - */ -import type { IncomingMessage } from 'http'; -import type { Duplex } from 'stream'; -import jwt, { type Algorithm, type JwtPayload } from 'jsonwebtoken'; -import { createHash } from 'crypto'; -import { WebSocketServer } from 'ws'; -import { DatabaseService } from '../services/DatabaseService'; -import { PilotTunnelBridge } from '../services/PilotTunnelBridge'; -import { PilotTunnelManager } from '../services/PilotTunnelManager'; -import { PilotMetrics } from '../services/PilotMetrics'; -import { sanitizeForLog } from '../utils/safeLog'; - -const wss = new WebSocketServer({ noServer: true }); -const CLOCK_SKEW_SEC = 60; - -type RejectReason = - | 'algorithm_mismatch' | 'signature_invalid' - | 'scope_mismatch' | 'audience_mismatch' | 'instance_mismatch' - | 'stale' | 'clock_skew' | 'node_deleted' | 'mode_mismatch' - | 'token_fingerprint_mismatch' | 'malformed'; - -function rejectUpgrade(socket: Duplex, reason: RejectReason): void { - const body = JSON.stringify({ reason }); - const headers = [ - 'HTTP/1.1 401 Unauthorized', - 'Content-Type: application/json', - `Content-Length: ${Buffer.byteLength(body)}`, - 'Connection: close', - '', - body, - ].join('\r\n'); - try { socket.write(headers); } catch { /* socket already gone */ } - try { socket.destroy(); } catch { /* socket already gone */ } -} - -function extractBearer(req: IncomingMessage): string | null { - const h = req.headers['authorization']; - if (typeof h !== 'string' || !h.startsWith('Bearer ')) return null; - return h.slice('Bearer '.length).trim() || null; -} - -export function handleMeshProxyTunnelFromPeerUpgrade( - req: IncomingMessage, - socket: Duplex, - head: Buffer, -): void { - const token = extractBearer(req); - if (!token) { rejectUpgrade(socket, 'malformed'); return; } - - let header: { alg?: string; kid?: string } = {}; - try { - const decoded = jwt.decode(token, { complete: true }); - header = (decoded?.header ?? {}) as typeof header; - } catch { rejectUpgrade(socket, 'malformed'); return; } - if (header.alg !== 'HS256') { rejectUpgrade(socket, 'algorithm_mismatch'); return; } - - const settings = DatabaseService.getInstance().getGlobalSettings(); - const secret = settings.auth_jwt_secret; - if (!secret) { rejectUpgrade(socket, 'malformed'); return; } - - let payload: JwtPayload; - try { - payload = jwt.verify(token, secret, { algorithms: ['HS256' as Algorithm] }) as JwtPayload; - } catch { rejectUpgrade(socket, 'signature_invalid'); return; } - - if (payload.scope !== 'mesh_tunnel') { rejectUpgrade(socket, 'scope_mismatch'); return; } - - // SENCHO_PRIMARY_URL must be set on central for peer-initiated dial-back - // to operate. The Task 8 bootstrap-mint side already skips when it's - // unset; this is the matching fail-safe on the verify side. Reject all - // peer dials with audience_mismatch when central has no canonical - // origin to validate against. - const canonicalOrigin = (process.env.SENCHO_PRIMARY_URL ?? '').replace(/\/+$/, ''); - if (!canonicalOrigin) { rejectUpgrade(socket, 'audience_mismatch'); return; } - if (payload.aud !== canonicalOrigin) { rejectUpgrade(socket, 'audience_mismatch'); return; } - - // instance_id is operational state, not user-defined config, so it - // lives in system_state rather than global_settings. - const instanceId = DatabaseService.getInstance().getSystemState('instance_id'); - if (payload.iss !== instanceId) { rejectUpgrade(socket, 'instance_mismatch'); return; } - - const nowSec = Math.floor(Date.now() / 1000); - if (typeof payload.exp !== 'number' || payload.exp <= nowSec) { - rejectUpgrade(socket, 'stale'); return; - } - if (typeof payload.iat !== 'number' || payload.iat > nowSec + CLOCK_SKEW_SEC) { - rejectUpgrade(socket, 'clock_skew'); return; - } - - const peerNodeId = Number(payload.sub); - if (!Number.isInteger(peerNodeId) || peerNodeId <= 0) { - rejectUpgrade(socket, 'malformed'); return; - } - const node = DatabaseService.getInstance().getNode(peerNodeId); - if (!node) { rejectUpgrade(socket, 'node_deleted'); return; } - if (node.type !== 'remote' || node.mode !== 'proxy') { - rejectUpgrade(socket, 'mode_mismatch'); return; - } - - if (!node.api_token) { rejectUpgrade(socket, 'token_fingerprint_mismatch'); return; } - const expectedFp = createHash('sha256').update(node.api_token).digest('hex').slice(0, 16); - const claimedFp = payload['peer_token_fp']; - if (typeof claimedFp !== 'string' || claimedFp !== expectedFp) { - rejectUpgrade(socket, 'token_fingerprint_mismatch'); return; - } - - wss.handleUpgrade(req, socket, head, (ws) => { - try { - const bridge = new PilotTunnelBridge(peerNodeId, ws); - bridge.start().then(() => { - try { - PilotTunnelManager.getInstance().replaceOrRegisterProxyBridge(peerNodeId, bridge); - PilotMetrics.increment('proxy_bridges_peer_initiated_total'); - } catch (err) { - try { bridge.close(1013, 'manager rejected'); } catch { /* ignore */ } - console.warn(`[meshProxyTunnelFromPeer] register failed: ${sanitizeForLog((err as Error).message)}`); - } - }).catch((err) => { - try { bridge.close(1011, 'bridge start failed'); } catch { /* ignore */ } - console.warn(`[meshProxyTunnelFromPeer] bridge start failed: ${sanitizeForLog((err as Error).message)}`); - }); - } catch (err) { - console.warn(`[meshProxyTunnelFromPeer] bridge construct failed: ${sanitizeForLog((err as Error).message)}`); - try { ws.close(1011, 'bridge init failed'); } catch { /* ignore */ } - } - }); -} diff --git a/backend/src/websocket/upgradeHandler.ts b/backend/src/websocket/upgradeHandler.ts index 67f09119..220e3abd 100644 --- a/backend/src/websocket/upgradeHandler.ts +++ b/backend/src/websocket/upgradeHandler.ts @@ -8,7 +8,6 @@ import { NodeRegistry } from '../services/NodeRegistry'; import { COOKIE_NAME } from '../helpers/constants'; import { handlePilotTunnel } from './pilotTunnel'; import { handleMeshProxyTunnel } from './meshProxyTunnel'; -import { handleMeshProxyTunnelFromPeerUpgrade } from './meshProxyTunnelFromPeer'; import { handleNotificationsWs } from './notifications'; import { handleRemoteForwarder } from './remoteForwarder'; import { handleLogsWs } from './logs'; @@ -33,15 +32,14 @@ function parseCookies(req: IncomingMessage): Record { * * Dispatch order (first match wins): * 1. `/api/pilot/tunnel` -> handlePilotTunnel (own auth, own wss) - * 2. `/api/mesh/proxy-tunnel-from-peer` -> handleMeshProxyTunnelFromPeerUpgrade (own JWT chain, central-side dial-back) - * 3. shared cookie/Bearer auth + JWT verify (rejects unauthenticated) - * 4. API token scope gate (read-only / deploy-only restricted to logs + notifications) - * 5. `/api/mesh/proxy-tunnel` -> handleMeshProxyTunnel (machine-to-machine: node_proxy or full-admin api_token) - * 6. `/ws/notifications` local -> handleNotificationsWs - * 7. remote nodeId path -> handleRemoteForwarder - * 8. `/api/stacks/:name/logs` -> handleLogsWs - * 9. `/api/system/host-console` -> handleHostConsoleWs - * 10. fallback -> handleGenericWs (`/ws` exec + stats) + * 2. shared cookie/Bearer auth + JWT verify (rejects unauthenticated) + * 3. API token scope gate (read-only / deploy-only restricted to logs + notifications) + * 4. `/api/mesh/proxy-tunnel` -> handleMeshProxyTunnel (machine-to-machine: node_proxy or full-admin api_token; bidirectional bridge for both forward and reverse mesh traffic) + * 5. `/ws/notifications` local -> handleNotificationsWs + * 6. remote nodeId path -> handleRemoteForwarder + * 7. `/api/stacks/:name/logs` -> handleLogsWs + * 8. `/api/system/host-console` -> handleHostConsoleWs + * 9. fallback -> handleGenericWs (`/ws` exec + stats) */ export function attachUpgrade( server: http.Server, @@ -52,22 +50,16 @@ export function attachUpgrade( attachGenericConnectionHandlers(wss); server.on('upgrade', async (req, socket, head) => { - // Pilot-agent tunnel ingress: machine credentials, no cookies. - // Mesh proxy-tunnel-from-peer ingress: peer-initiated dial-back over - // a `mesh_tunnel`-scoped JWT (minted earlier during the bootstrap - // exchange). Both handlers run their own auth before the shared - // cookie/Bearer pipeline because their credentials are not user - // sessions and would fail the shared user-existence check. + // Pilot-agent tunnel ingress: machine credentials, no cookies. Runs its + // own auth before the shared cookie/Bearer pipeline because the + // credential is not a user session and would fail the shared + // user-existence check. try { const reqUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`); if (reqUrl.pathname === '/api/pilot/tunnel') { await handlePilotTunnel(req, socket, head, pilotTunnelWss); return; } - if (reqUrl.pathname === '/api/mesh/proxy-tunnel-from-peer') { - handleMeshProxyTunnelFromPeerUpgrade(req, socket, head); - return; - } } catch { // URL parse error falls through and will be rejected below. } diff --git a/docs/features/sencho-mesh.mdx b/docs/features/sencho-mesh.mdx index d4d3dec4..bfea30d9 100644 --- a/docs/features/sencho-mesh.mdx +++ b/docs/features/sencho-mesh.mdx @@ -71,19 +71,16 @@ services: Sencho's static IP on the network is ` + 2` (so `10.42.0.2` for the example above). Operators can configure each node independently; the override generator pushes alias hostnames that resolve to whichever IP the deploying node uses locally. -## Bidirectional secure callback +## Bidirectional traffic -Proxy-mode mesh nodes can re-establish their secure mesh callback to central -when traffic starts from that node. Central is the hub; the callback is signed -with a `mesh_tunnel`-scoped credential bound to the node's API token. - - - **Prerequisite:** set `SENCHO_PRIMARY_URL` on central to its canonical public - origin. This value is used as the audience for the mesh callback credential - and as the callback URL on the node side. Without it, central infers an origin - from inbound request headers; the inference works in standard installs but is - not the intended deploy shape for fleets where reverse-proxy headers may differ. - +Mesh traffic flows in both directions over a single persistent WebSocket +between central and each proxy-mode peer. Central dials the bridge to every +mesh-enabled peer on startup and re-dials any dropped bridge on its +reconcile tick (every 60 seconds by default). When a container on the peer +needs to reach a service on central, the request multiplexes over that same +bridge: no extra inbound listener on central, no public URL required on +the central side. This is the path that makes "call a service on any node +by hostname" work in NAT'd homelab topologies out of the box. ## Test upstream diff --git a/frontend/src/components/fleet/MeshStackTopologySheet.test.tsx b/frontend/src/components/fleet/MeshStackTopologySheet.test.tsx index 3bb4b1cb..3ebb2582 100644 --- a/frontend/src/components/fleet/MeshStackTopologySheet.test.tsx +++ b/frontend/src/components/fleet/MeshStackTopologySheet.test.tsx @@ -21,6 +21,7 @@ function makeNode(over: Partial & Pick )} + {status.reverseCallbackStatus === 'connecting' && ( + + reconnecting + + )} + {status.reverseCallbackStatus === 'unavailable' && ( + + reverse unavailable + + )}
& Pick