From 946db9df5251d4b437dec3847fce4f32ddc84283 Mon Sep 17 00:00:00 2001 From: Anso Date: Thu, 14 May 2026 23:26:41 -0400 Subject: [PATCH] fix(mesh): accept node_proxy at the proxy-tunnel WS upgrade (#1050) The mesh proxy-tunnel WS handler was gated on full-admin api_token scope only, so node_proxy JWTs (the credential the Add Remote Node dialog tells operators to generate via Settings -> Nodes -> Generate Token) were silently rejected with HTTP 403. Operators followed the dialog's instructions, enrolled the node cleanly, saw reachableMode='proxy' with no negative badge, opted a stack into mesh, watched the redeploy complete, and only then discovered no bytes flow. node_proxy already authorizes every /api/* surface on the remote (deploy, exec, host console, filesystem), which is a strict superset of what the mesh proxy-tunnel does. Gating mesh more strictly was theatre, not security, and it created a UX trap with no in-product signal pointing at the scope mismatch. Change the upgrade dispatcher to accept any machine-to-machine credential: node_proxy JWT or full-admin api_token. Session cookies and restricted api_token scopes (read-only, deploy-only) remain rejected through the same paths as before. Adds positive/negative coverage in upgrade-order.test.ts for all four credential shapes so the gate stays pinned against future re-tightening. Updates user-facing copy in the mesh docs and Settings -> API Tokens description so the documented path matches the actual code. If we ever introduce a demoted node_proxy variant (audit-only, read-only fleet member), the right move is differentiated node_proxy scope claims inside the tunnel JWT (tracker F-A3), not requiring a separate token type for mesh. --- backend/src/__tests__/upgrade-order.test.ts | 76 +++++++++++++++++++- backend/src/services/MeshService.ts | 2 +- backend/src/websocket/meshProxyTunnel.ts | 15 ++-- backend/src/websocket/upgradeHandler.ts | 13 ++-- docs/features/sencho-mesh.mdx | 4 +- frontend/src/components/settings/registry.ts | 2 +- 6 files changed, 96 insertions(+), 16 deletions(-) diff --git a/backend/src/__tests__/upgrade-order.test.ts b/backend/src/__tests__/upgrade-order.test.ts index 21f8db49..c76abc9b 100644 --- a/backend/src/__tests__/upgrade-order.test.ts +++ b/backend/src/__tests__/upgrade-order.test.ts @@ -11,6 +11,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import WebSocket from 'ws'; import jwt from 'jsonwebtoken'; +import crypto from 'crypto'; import type { AddressInfo } from 'net'; import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb'; @@ -50,9 +51,10 @@ describe('WebSocket upgrade dispatch order', () => { cleanupTestDb(tmpDir); }); - function connect(pathAndQuery: string, opts: { cookie?: string } = {}): WebSocket { + function connect(pathAndQuery: string, opts: { cookie?: string; bearer?: string } = {}): WebSocket { const headers: Record = {}; if (opts.cookie) headers['cookie'] = opts.cookie; + if (opts.bearer) headers['authorization'] = `Bearer ${opts.bearer}`; return new WebSocket(`ws://127.0.0.1:${port}${pathAndQuery}`, { headers }); } @@ -143,6 +145,78 @@ describe('WebSocket upgrade dispatch order', () => { try { ws.terminate(); } catch { /* ignore */ } }); + describe('/api/mesh/proxy-tunnel scope gating', () => { + // The mesh proxy-tunnel ingress is machine-to-machine. The dispatch + // ladder must accept the credentials that fleet enrollment produces + // (node_proxy JWTs) AND the full-admin api_token scope, while rejecting + // session cookies and restricted api_token scopes. These cases pin the + // scope contract so a future re-tightening cannot silently regress to + // the "full-admin only" behaviour that trapped operators following the + // Add Remote Node dialog's Node Token instructions. + + it('accepts a node_proxy Bearer (fleet enrollment token) at the upgrade and reaches the handler', async () => { + const nodeProxyToken = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); + const ws = connect('/api/mesh/proxy-tunnel', { bearer: nodeProxyToken }); + const outcome = await waitForOutcome(ws); + // Pilot-mode rejection (404) or normal open both indicate the upgrade + // passed the scope gate and was handed to handleMeshProxyTunnel. + // The pre-fix behaviour was unequivocal: HTTP 403 from the dispatcher. + expect(outcome.kind).not.toBe('unexpected'); + if (outcome.kind === 'unexpected') { + expect(outcome.status).not.toBe(403); + } + try { ws.terminate(); } catch { /* ignore */ } + }); + + it('accepts a full-admin api_token at the upgrade and reaches the handler', async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + const rawToken = jwt.sign({ scope: 'api_token', jti: crypto.randomUUID() }, TEST_JWT_SECRET, { expiresIn: '1h' }); + const tokenHash = crypto.createHash('sha256').update(rawToken).digest('hex'); + const adminId = DatabaseService.getInstance().getUserByUsername(TEST_USERNAME)!.id; + DatabaseService.getInstance().addApiToken({ + token_hash: tokenHash, + name: `mesh-scope-gate-${Date.now()}`, + scope: 'full-admin', + user_id: adminId, + created_at: Date.now(), + expires_at: null, + }); + const ws = connect('/api/mesh/proxy-tunnel', { bearer: rawToken }); + const outcome = await waitForOutcome(ws); + expect(outcome.kind).not.toBe('unexpected'); + if (outcome.kind === 'unexpected') { + expect(outcome.status).not.toBe(403); + } + try { ws.terminate(); } catch { /* ignore */ } + }); + + it('rejects a read-only api_token at the upgrade with HTTP 403', async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + const rawToken = jwt.sign({ scope: 'api_token', jti: crypto.randomUUID() }, TEST_JWT_SECRET, { expiresIn: '1h' }); + const tokenHash = crypto.createHash('sha256').update(rawToken).digest('hex'); + const adminId = DatabaseService.getInstance().getUserByUsername(TEST_USERNAME)!.id; + DatabaseService.getInstance().addApiToken({ + token_hash: tokenHash, + name: `mesh-scope-readonly-${Date.now()}`, + scope: 'read-only', + user_id: adminId, + created_at: Date.now(), + expires_at: null, + }); + const ws = connect('/api/mesh/proxy-tunnel', { bearer: rawToken }); + const outcome = await waitForOutcome(ws); + expect(outcome.kind).toBe('unexpected'); + if (outcome.kind === 'unexpected') expect(outcome.status).toBe(403); + }); + + it('rejects a session cookie at the upgrade with HTTP 403 (mesh is not a UI surface)', async () => { + const ws = connect('/api/mesh/proxy-tunnel', { cookie: sessionCookie }); + const outcome = await waitForOutcome(ws); + expect(outcome.kind).toBe('unexpected'); + if (outcome.kind === 'unexpected') expect(outcome.status).toBe(403); + }); + }); + it('dispatches /api/pilot/tunnel to the pilot handler (rejects non-pilot bearer before path-based dispatch)', async () => { // A plain session cookie is a valid *user* JWT but not a pilot JWT. The // pilot handler runs first (before the shared cookie/Bearer auth) and diff --git a/backend/src/services/MeshService.ts b/backend/src/services/MeshService.ts index c9e4b466..b9dcc313 100644 --- a/backend/src/services/MeshService.ts +++ b/backend/src/services/MeshService.ts @@ -27,7 +27,7 @@ const SLOW_PROBE_THRESHOLD_MS = 500; const DEFAULT_MESH_SUBNET = '172.30.0.0/24'; const REACHABLE_REASON: Record = { - auth_failed: 'api token rejected (scope must be full-admin)', + auth_failed: 'api token rejected by remote', endpoint_not_found: 'remote does not support proxy mesh', tls_failed: 'TLS handshake failed', no_target: 'proxy target missing', diff --git a/backend/src/websocket/meshProxyTunnel.ts b/backend/src/websocket/meshProxyTunnel.ts index e595347c..0abbf8e2 100644 --- a/backend/src/websocket/meshProxyTunnel.ts +++ b/backend/src/websocket/meshProxyTunnel.ts @@ -16,14 +16,17 @@ import { rejectUpgrade as reject } from './reject'; * Mesh proxy-tunnel ingress. * * The remote side of a Phase C proxy-mode mesh tunnel. Central dials - * `WSS /api/mesh/proxy-tunnel` using the long-lived `api_token` - * as a Bearer credential; this handler upgrades the connection and wires - * the shared `TcpStreamSwitchboard` to handle `tcp_open` / `tcp_open_ack` - * / `tcp_open_reverse` / `tcp_close` and `TcpData` frames. + * `WSS /api/mesh/proxy-tunnel` using the fleet credential as a + * Bearer header (either the `node_proxy` JWT generated by the remote's + * Settings → Nodes → Generate Token flow, or a `full-admin` api_token + * for non-fleet integrations); this handler upgrades the connection and + * wires the shared `TcpStreamSwitchboard` to handle `tcp_open` / + * `tcp_open_ack` / `tcp_open_reverse` / `tcp_close` and `TcpData` frames. * * Auth + scope gating happens in `upgradeHandler.ts` before this handler - * runs (require `full-admin` api_token scope). The handler itself trusts - * the upgrade; the WS credential is the only trust boundary. + * runs: it accepts node_proxy and full-admin api_token, and rejects + * session cookies plus restricted api_token scopes. The handler itself + * trusts the upgrade; the WS credential is the only trust boundary. * * Bidirectional: when the tunnel opens, the handler registers itself as * the reverse-dialer on the local MeshService so meshed containers on diff --git a/backend/src/websocket/upgradeHandler.ts b/backend/src/websocket/upgradeHandler.ts index a32cbcb4..0374950e 100644 --- a/backend/src/websocket/upgradeHandler.ts +++ b/backend/src/websocket/upgradeHandler.ts @@ -33,7 +33,7 @@ function parseCookies(req: IncomingMessage): Record { * 1. `/api/pilot/tunnel` -> handlePilotTunnel (own auth, own wss) * 2. shared cookie/Bearer auth + JWT verify (rejects unauthenticated) * 3. API token scope gate (read-only / deploy-only restricted to logs + notifications) - * 4. `/api/mesh/proxy-tunnel` -> handleMeshProxyTunnel (requires full-admin api_token scope) + * 4. `/api/mesh/proxy-tunnel` -> handleMeshProxyTunnel (machine-to-machine: node_proxy or full-admin api_token) * 5. `/ws/notifications` local -> handleNotificationsWs * 6. remote nodeId path -> handleRemoteForwarder * 7. `/api/stacks/:name/logs` -> handleLogsWs @@ -123,11 +123,14 @@ export function attachUpgrade( } // Mesh proxy-tunnel ingress: a sibling Sencho is dialing this node - // to carry mesh TCP traffic. Require an api_token Bearer with the - // full-admin scope; mesh manipulates traffic and must not be - // reachable under a session cookie or a node_proxy JWT. + // to carry mesh TCP traffic. Accept any machine-to-machine credential: + // node_proxy JWT (the token enrolled nodes carry) or a full-admin + // api_token. Session cookies fall through to a 403 here because their + // decoded scope is undefined (isProxyToken=false, wsApiTokenScope=null). + // Restricted api_token scopes (read-only, deploy-only) are blocked + // earlier by the scope gate above before this branch is reached. if (pathname === '/api/mesh/proxy-tunnel') { - if (wsApiTokenScope !== 'full-admin') { + if (!isProxyToken && wsApiTokenScope !== 'full-admin') { return reject(socket, 403, 'Forbidden'); } await handleMeshProxyTunnel(req, socket, head); diff --git a/docs/features/sencho-mesh.mdx b/docs/features/sencho-mesh.mdx index bd4ac3c4..55e5b9e3 100644 --- a/docs/features/sencho-mesh.mdx +++ b/docs/features/sencho-mesh.mdx @@ -48,7 +48,7 @@ Four guarantees: 1. **Only opted-in services are reachable.** A stack reaches another stack's services through the mesh only if both stacks have explicitly opted in. The Pilot agent on the target node refuses any request for a non-opted service. 2. **Aliases are not internet-reachable.** Sencho listens on an internal Docker network; nothing about the mesh exposes new ports beyond the host's existing firewall posture. -3. **Traffic is encrypted in transit.** Cross-node bytes ride an authenticated WSS tunnel between Sencho instances. Distributed API nodes additionally require an API token with the **full-admin** scope; tokens with narrower scopes cannot carry mesh traffic. +3. **Traffic is encrypted in transit.** Cross-node bytes ride an authenticated WSS tunnel between Sencho instances. The Node Token generated during fleet enrollment is the credential that authorizes the mesh tunnel; restricted API token scopes (read-only, deploy-only) cannot carry mesh traffic. 4. **Tier-gated and audit-logged.** Only Admiral users can configure the mesh. Enable, disable, opt-in, and opt-out events write durable rows to the audit log with the actor's identity. What the mesh does **not** do: @@ -142,7 +142,7 @@ A few things are deliberately out of scope for the first release: Central could not open a mesh tunnel to this node. The badge tooltip shows the specific reason. While a node is in this state the **mesh toggle and Add stack to mesh action on the node card are disabled** so a redeploy is not triggered against an unreachable target. Common causes: - - `api token rejected (scope must be full-admin)` — the API token configured for this node is restricted. Open the remote Sencho, generate a token with the **full-admin** scope, and update the node's credentials in **Settings → Nodes**. + - `api token rejected by remote` — the credential central uses to dial this node is not accepted. Open the remote Sencho, generate a fresh Node Token via **Settings → Nodes → Generate Token**, and paste it back into the node's credentials in central's **Settings → Nodes**. - `remote does not support proxy mesh` — the remote Sencho is on a version that predates proxy-mode mesh. Update the remote and the badge clears on the next refresh. - `TLS handshake failed` — the remote serves a certificate Node's default trust store does not accept. Use a certificate issued by a trusted authority on the remote. - `api_url not set` or `api token missing` — the node was added without credentials. Edit the node in **Settings → Nodes** and supply the URL and token. diff --git a/frontend/src/components/settings/registry.ts b/frontend/src/components/settings/registry.ts index 351ec42d..864b3257 100644 --- a/frontend/src/components/settings/registry.ts +++ b/frontend/src/components/settings/registry.ts @@ -87,7 +87,7 @@ export const SETTINGS_ITEMS: readonly SettingsItemMeta[] = [ id: 'api-tokens', group: 'identity', label: 'API Tokens', - description: 'Long-lived bearer tokens for CI, scripts, and remote nodes.', + description: 'Long-lived bearer tokens for CI and scripts.', keywords: ['bearer', 'automation', 'ci', 'scripts', 'scopes'], tier: 'admiral', scope: 'global',