feat: pilot agent outbound-mode for remote nodes (#667)

* feat: pilot agent outbound-mode for remote nodes

Adds a second mode for managing remote nodes: the agent dials an outbound
WebSocket tunnel to the primary, so the remote host no longer needs an
inbound port, a reachable URL, or its own TLS certificate. Works behind
NAT, residential routers, and corporate firewalls.

The primary multiplexes HTTP and WebSocket requests over a single tunnel
via a hybrid JSON + binary frame protocol, bridged through a per-tunnel
loopback server so existing proxy and upgrade handlers route pilot-mode
nodes identically to proxy-mode ones.

Enrollment uses a single-use 15-minute pilot_enroll JWT exchanged for a
long-lived pilot_tunnel credential on first connect. Proxy mode continues
to work unchanged and both modes are supported side-by-side.

* test(e2e): switch to proxy mode before asserting api_url field

Remote nodes default to Pilot Agent mode, which hides the api_url input.
The SSRF-validation tests need proxy mode, so the helper now selects
Distributed API Proxy after picking Remote type before asserting the
field is visible.

* fix(e2e): wire Combobox id prop so node-mode selector resolves

The Combobox trigger button had no id, leaving its Label orphaned and
making getByRole name-based lookups fail. Adding id to the primitive,
passing id="node-mode" from NodeManager, and updating the E2E helper to
use #node-mode fixes both the a11y regression and the CI timeout.
This commit is contained in:
Anso
2026-04-17 20:31:43 -04:00
committed by GitHub
parent 0a536ae653
commit 8e7a567f69
15 changed files with 1812 additions and 27 deletions
+351
View File
@@ -0,0 +1,351 @@
import fs from 'fs';
import path from 'path';
import http from 'http';
import WebSocket from 'ws';
import { getSenchoVersion } from '../services/CapabilityRegistry';
import {
BinaryFrameType,
PROTOCOL_VERSION,
decodeBinaryFrame,
decodeJsonFrame,
encodeBinaryFrame,
encodeJsonFrame,
wsDataToBuffer,
wsDataToString,
} from './protocol';
const RECONNECT_MIN_MS = 1_000;
const RECONNECT_MAX_MS = 60_000;
const PING_INTERVAL_MS = 30_000;
const TOKEN_PATH = path.join(process.env.DATA_DIR || '/app/data', 'pilot.jwt');
/**
* Pilot agent: dials the primary via outbound WebSocket and tunnels every
* inbound frame to the agent's own loopback HTTP server (the fully-booted
* Sencho app). Because the tunnel is the only ingress, the agent needs no
* open port, no TLS certificate, and no reachable address.
*/
export function startPilotAgent(loopbackPort: number): void {
const primaryUrl = process.env.SENCHO_PRIMARY_URL;
if (!primaryUrl) {
console.error('[Pilot] SENCHO_PRIMARY_URL is required when SENCHO_MODE=pilot');
process.exit(1);
}
const enrollToken = process.env.SENCHO_ENROLL_TOKEN;
const persistedToken = readPersistedToken();
if (!enrollToken && !persistedToken) {
console.error('[Pilot] SENCHO_ENROLL_TOKEN is required on first boot');
process.exit(1);
}
const agent = new PilotAgent({
primaryUrl,
loopbackPort,
initialToken: persistedToken || enrollToken!,
enrolling: !persistedToken,
});
agent.start();
}
interface AgentOptions {
primaryUrl: string;
loopbackPort: number;
initialToken: string;
enrolling: boolean;
}
class PilotAgent {
private readonly options: AgentOptions;
private token: string;
private backoff = RECONNECT_MIN_MS;
private ws: WebSocket | null = null;
private pingTimer?: NodeJS.Timeout;
private reconnectTimer?: NodeJS.Timeout;
private readonly httpStreams = new Map<number, { req: http.ClientRequest }>();
private readonly wsStreams = new Map<number, WebSocket>();
private shuttingDown = false;
private readonly agentVersion: string;
constructor(options: AgentOptions) {
this.options = options;
this.token = options.initialToken;
this.agentVersion = getSenchoVersion() || '0.0.0';
}
public start(): void {
this.connect();
process.on('SIGTERM', () => this.shutdown());
process.on('SIGINT', () => this.shutdown());
}
private shutdown(): void {
this.shuttingDown = true;
if (this.pingTimer) clearInterval(this.pingTimer);
if (this.reconnectTimer) { clearTimeout(this.reconnectTimer); this.reconnectTimer = undefined; }
try { this.ws?.close(1000, 'agent shutdown'); } catch { /* ignore */ }
}
private connect(): void {
if (this.shuttingDown) return;
const wsUrl = this.options.primaryUrl.replace(/^http/, 'ws').replace(/\/$/, '') + '/api/pilot/tunnel';
const ws = new WebSocket(wsUrl, {
headers: {
Authorization: `Bearer ${this.token}`,
'x-sencho-agent-version': this.agentVersion,
},
handshakeTimeout: 15_000,
});
this.ws = ws;
ws.on('open', () => {
console.log('[Pilot] Tunnel connected to', this.options.primaryUrl);
this.backoff = RECONNECT_MIN_MS;
try {
ws.send(encodeJsonFrame({
t: 'hello',
version: PROTOCOL_VERSION,
role: 'agent',
agentVersion: this.agentVersion,
}));
} catch (err) {
console.error('[Pilot] Failed to send hello:', (err as Error).message);
}
this.pingTimer = setInterval(() => {
if (ws.readyState === WebSocket.OPEN) {
try { ws.ping(); } catch { /* surfaced via error */ }
}
}, PING_INTERVAL_MS);
});
ws.on('message', (data, isBinary) => this.handleFrame(data, isBinary));
ws.on('close', (code, reason) => {
console.log('[Pilot] Tunnel closed:', code, reason?.toString?.() ?? '');
this.cleanupAfterDisconnect();
this.scheduleReconnect();
});
ws.on('error', (err) => {
console.warn('[Pilot] Tunnel error:', err.message);
// 'close' will follow; reconnect is scheduled there.
});
}
private cleanupAfterDisconnect(): void {
if (this.pingTimer) { clearInterval(this.pingTimer); this.pingTimer = undefined; }
for (const [, entry] of this.httpStreams) {
try { entry.req.destroy(); } catch { /* ignore */ }
}
this.httpStreams.clear();
for (const [, ws] of this.wsStreams) {
try { ws.close(1006, 'tunnel closed'); } catch { /* ignore */ }
}
this.wsStreams.clear();
}
private scheduleReconnect(): void {
if (this.shuttingDown) return;
const jitter = Math.floor(Math.random() * 500);
const delay = this.backoff + jitter;
this.reconnectTimer = setTimeout(() => {
this.reconnectTimer = undefined;
this.connect();
}, delay);
this.backoff = Math.min(this.backoff * 2, RECONNECT_MAX_MS);
}
private handleFrame(data: unknown, isBinary: boolean): void {
try {
if (isBinary) {
const buf = wsDataToBuffer(data);
if (!buf) return;
this.handleBinaryFrame(decodeBinaryFrame(buf));
} else {
const text = wsDataToString(data);
if (text == null) return;
const frame = decodeJsonFrame(text);
this.handleJsonFrame(frame);
}
} catch (err) {
console.warn('[Pilot] Malformed frame from primary:', (err as Error).message);
}
}
private handleJsonFrame(frame: ReturnType<typeof decodeJsonFrame>): void {
const ws = this.ws;
if (!ws) return;
switch (frame.t) {
case 'hello': {
if (frame.version !== PROTOCOL_VERSION) {
console.error(`[Pilot] Protocol version ${frame.version} from primary is incompatible with agent (${PROTOCOL_VERSION}); exiting.`);
this.shuttingDown = true;
try { ws.close(1002, 'incompatible version'); } catch { /* ignore */ }
process.exit(1);
}
break;
}
case 'ctrl': {
if (frame.op === 'enroll_ack' && frame.payload && typeof frame.payload.token === 'string') {
this.token = frame.payload.token;
persistToken(this.token);
console.log('[Pilot] Enrollment complete; long-lived token persisted.');
}
break;
}
case 'http_req': this.onHttpReq(frame); break;
case 'http_req_end': this.onHttpReqEnd(frame.s); break;
case 'ws_open': this.onWsOpen(frame); break;
case 'ws_msg_text': this.onWsMsgText(frame.s, frame.data); break;
case 'ws_close': this.onWsClose(frame.s, frame.code, frame.reason); break;
default:
// Other frame types are primary-bound only; agent ignores.
break;
}
}
private handleBinaryFrame(frame: ReturnType<typeof decodeBinaryFrame>): void {
switch (frame.type) {
case BinaryFrameType.HttpReqBody: {
const entry = this.httpStreams.get(frame.streamId);
if (!entry) return;
try { entry.req.write(frame.payload); } catch { /* ignore */ }
break;
}
case BinaryFrameType.WsMessageBinary: {
const ws = this.wsStreams.get(frame.streamId);
if (!ws) return;
try { ws.send(frame.payload, { binary: true }); } catch { /* ignore */ }
break;
}
default:
break;
}
}
// --- HTTP dispatch (tunnel -> loopback) ---
private onHttpReq(frame: Extract<ReturnType<typeof decodeJsonFrame>, { t: 'http_req' }>): void {
const ws = this.ws;
if (!ws) return;
const req = http.request({
host: '127.0.0.1',
port: this.options.loopbackPort,
method: frame.method,
path: frame.path,
headers: { ...frame.headers, host: `127.0.0.1:${this.options.loopbackPort}` },
}, (res) => {
const outHeaders: Record<string, string> = {};
for (const [k, v] of Object.entries(res.headers)) {
if (typeof v === 'string') outHeaders[k] = v;
else if (Array.isArray(v)) outHeaders[k] = v.join(', ');
}
try {
ws.send(encodeJsonFrame({
t: 'http_res',
s: frame.s,
status: res.statusCode || 200,
headers: outHeaders,
}));
} catch { /* ignore */ }
res.on('data', (chunk: Buffer) => {
try { ws.send(encodeBinaryFrame(BinaryFrameType.HttpResBody, frame.s, chunk), { binary: true }); } catch { /* ignore */ }
});
res.on('end', () => {
try { ws.send(encodeJsonFrame({ t: 'http_res_end', s: frame.s })); } catch { /* ignore */ }
this.httpStreams.delete(frame.s);
});
res.on('error', () => {
try { ws.send(encodeJsonFrame({ t: 'http_err', s: frame.s, code: 'bad_response', message: 'upstream error' })); } catch { /* ignore */ }
this.httpStreams.delete(frame.s);
});
});
req.on('error', (err) => {
try {
ws.send(encodeJsonFrame({
t: 'http_err',
s: frame.s,
code: 'agent_error',
message: err.message || 'agent request failed',
}));
} catch { /* ignore */ }
this.httpStreams.delete(frame.s);
});
this.httpStreams.set(frame.s, { req });
}
private onHttpReqEnd(streamId: number): void {
const entry = this.httpStreams.get(streamId);
if (!entry) return;
try { entry.req.end(); } catch { /* ignore */ }
}
// --- WebSocket dispatch (tunnel -> loopback) ---
private onWsOpen(frame: Extract<ReturnType<typeof decodeJsonFrame>, { t: 'ws_open' }>): void {
const ws = this.ws;
if (!ws) return;
const target = `ws://127.0.0.1:${this.options.loopbackPort}${frame.path}`;
const client = new WebSocket(target, {
headers: { ...frame.headers, host: `127.0.0.1:${this.options.loopbackPort}` },
});
client.on('open', () => {
try { ws.send(encodeJsonFrame({ t: 'ws_accept', s: frame.s, headers: {} })); } catch { /* ignore */ }
this.wsStreams.set(frame.s, client);
});
client.on('message', (data, isBinary) => {
if (isBinary) {
try { ws.send(encodeBinaryFrame(BinaryFrameType.WsMessageBinary, frame.s, wsDataToBuffer(data) ?? Buffer.alloc(0)), { binary: true }); } catch { /* ignore */ }
} else {
try { ws.send(encodeJsonFrame({ t: 'ws_msg_text', s: frame.s, data: wsDataToString(data) ?? '' })); } catch { /* ignore */ }
}
});
client.on('close', (code, reason) => {
try { ws.send(encodeJsonFrame({ t: 'ws_close', s: frame.s, code, reason: reason?.toString?.() })); } catch { /* ignore */ }
this.wsStreams.delete(frame.s);
});
client.on('error', () => {
try { ws.send(encodeJsonFrame({ t: 'ws_reject', s: frame.s, status: 502, message: 'agent websocket failed' })); } catch { /* ignore */ }
this.wsStreams.delete(frame.s);
});
}
private onWsMsgText(streamId: number, data: string): void {
const ws = this.wsStreams.get(streamId);
if (!ws) return;
try { ws.send(data); } catch { /* ignore */ }
}
private onWsClose(streamId: number, code: number, reason?: string): void {
const ws = this.wsStreams.get(streamId);
if (!ws) return;
try { ws.close(code, reason); } catch { /* ignore */ }
this.wsStreams.delete(streamId);
}
}
function readPersistedToken(): string | null {
try {
if (fs.existsSync(TOKEN_PATH)) {
return fs.readFileSync(TOKEN_PATH, 'utf8').trim() || null;
}
} catch { /* ignore */ }
return null;
}
function persistToken(token: string): void {
try {
const dir = path.dirname(TOKEN_PATH);
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(TOKEN_PATH, token, { mode: 0o600 });
} catch (err) {
console.warn('[Pilot] Failed to persist tunnel token:', (err as Error).message);
}
}
+213
View File
@@ -0,0 +1,213 @@
/**
* Pilot Tunnel wire protocol.
*
* A single WebSocket between primary and pilot agent carries many multiplexed
* HTTP requests and nested WebSocket streams. Each stream is identified by a
* monotonically increasing streamId allocated by the primary (the originator
* of every request).
*
* Wire format is hybrid:
* - Text frames: JSON envelopes for metadata (open/close/headers/control)
* - Binary frames: raw payload bytes with a 5-byte prefix
* [ 1 byte: BinaryFrameType ][ 4 bytes: streamId (big-endian) ][ bytes... ]
*
* JSON is inspectable and low-overhead for small control messages; binary
* avoids base64 bloat on body chunks and WS message payloads.
*/
export const PROTOCOL_VERSION = 1;
// --- Binary frame types (first byte of a binary WS frame) ---
export enum BinaryFrameType {
HttpReqBody = 0x01,
HttpResBody = 0x02,
WsMessageBinary = 0x03,
}
// --- JSON envelope types ---
export type JsonFrame =
| HelloFrame
| HttpReqFrame
| HttpReqEndFrame
| HttpResFrame
| HttpResEndFrame
| HttpErrorFrame
| WsOpenFrame
| WsAcceptFrame
| WsRejectFrame
| WsMessageTextFrame
| WsCloseFrame
| ControlFrame;
export interface HelloFrame {
t: 'hello';
version: number;
role: 'primary' | 'agent';
agentVersion?: string;
}
export interface HttpReqFrame {
t: 'http_req';
s: number;
method: string;
path: string;
headers: Record<string, string>;
}
export interface HttpReqEndFrame {
t: 'http_req_end';
s: number;
}
export interface HttpResFrame {
t: 'http_res';
s: number;
status: number;
headers: Record<string, string>;
}
export interface HttpResEndFrame {
t: 'http_res_end';
s: number;
}
export interface HttpErrorFrame {
t: 'http_err';
s: number;
code: 'timeout' | 'tunnel_down' | 'bad_response' | 'agent_error';
message: string;
}
export interface WsOpenFrame {
t: 'ws_open';
s: number;
path: string;
headers: Record<string, string>;
}
export interface WsAcceptFrame {
t: 'ws_accept';
s: number;
headers: Record<string, string>;
}
export interface WsRejectFrame {
t: 'ws_reject';
s: number;
status: number;
message: string;
}
export interface WsMessageTextFrame {
t: 'ws_msg_text';
s: number;
data: string;
}
export interface WsCloseFrame {
t: 'ws_close';
s: number;
code: number;
reason?: string;
}
export interface ControlFrame {
t: 'ctrl';
op: 'enroll_ack' | 'node_info' | 'ping' | 'pong';
payload?: Record<string, unknown>;
}
// --- Serialize / parse ---
export function encodeJsonFrame(frame: JsonFrame): string {
return JSON.stringify(frame);
}
export function decodeJsonFrame(raw: string): JsonFrame {
const parsed = JSON.parse(raw);
if (!parsed || typeof parsed !== 'object' || typeof parsed.t !== 'string') {
throw new Error('invalid frame: missing type discriminator');
}
return parsed as JsonFrame;
}
/**
* Build a binary frame: [type][streamId BE][payload].
* Returns a fresh Buffer owned by the caller.
*/
export function encodeBinaryFrame(type: BinaryFrameType, streamId: number, payload: Buffer): Buffer {
if (!Number.isInteger(streamId) || streamId < 0 || streamId > 0xffffffff) {
throw new Error(`invalid streamId: ${streamId}`);
}
const out = Buffer.allocUnsafe(5 + payload.length);
out.writeUInt8(type, 0);
out.writeUInt32BE(streamId, 1);
payload.copy(out, 5);
return out;
}
export interface DecodedBinaryFrame {
type: BinaryFrameType;
streamId: number;
payload: Buffer;
}
export function decodeBinaryFrame(buf: Buffer): DecodedBinaryFrame {
if (buf.length < 5) {
throw new Error(`binary frame too short: ${buf.length} bytes`);
}
const type = buf.readUInt8(0) as BinaryFrameType;
if (type !== BinaryFrameType.HttpReqBody &&
type !== BinaryFrameType.HttpResBody &&
type !== BinaryFrameType.WsMessageBinary) {
throw new Error(`unknown binary frame type: ${type}`);
}
const streamId = buf.readUInt32BE(1);
const payload = buf.subarray(5);
return { type, streamId, payload };
}
// --- WS payload normalization ---
export function wsDataToBuffer(data: unknown): Buffer | null {
if (Buffer.isBuffer(data)) return data;
if (data instanceof ArrayBuffer) return Buffer.from(data);
if (Array.isArray(data)) return Buffer.concat(data.map((d) => Buffer.isBuffer(d) ? d : Buffer.from(d as ArrayBuffer)));
return null;
}
export function wsDataToString(data: unknown): string | null {
if (typeof data === 'string') return data;
const buf = wsDataToBuffer(data);
return buf ? buf.toString('utf8') : null;
}
// --- Close codes ---
export const PilotCloseCode = {
Replaced: 4000,
EnrollmentRegenerated: 4001,
ProtocolError: 1002,
} as const;
// --- Stream id allocation ---
/**
* Monotonic stream id generator. Primary is the sole allocator.
* Wraps at 2^31 (well above practical per-tunnel concurrency).
*/
export class StreamIdAllocator {
private next: number;
constructor(start = 1) {
this.next = start;
}
allocate(): number {
const id = this.next;
this.next = this.next >= 0x7fffffff ? 1 : this.next + 1;
return id;
}
}