mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-07-28 04:38:59 +00:00
feat: pilot agent outbound-mode for remote nodes (#667)
* feat: pilot agent outbound-mode for remote nodes Adds a second mode for managing remote nodes: the agent dials an outbound WebSocket tunnel to the primary, so the remote host no longer needs an inbound port, a reachable URL, or its own TLS certificate. Works behind NAT, residential routers, and corporate firewalls. The primary multiplexes HTTP and WebSocket requests over a single tunnel via a hybrid JSON + binary frame protocol, bridged through a per-tunnel loopback server so existing proxy and upgrade handlers route pilot-mode nodes identically to proxy-mode ones. Enrollment uses a single-use 15-minute pilot_enroll JWT exchanged for a long-lived pilot_tunnel credential on first connect. Proxy mode continues to work unchanged and both modes are supported side-by-side. * test(e2e): switch to proxy mode before asserting api_url field Remote nodes default to Pilot Agent mode, which hides the api_url input. The SSRF-validation tests need proxy mode, so the helper now selects Distributed API Proxy after picking Remote type before asserting the field is visible. * fix(e2e): wire Combobox id prop so node-mode selector resolves The Combobox trigger button had no id, leaving its Label orphaned and making getByRole name-based lookups fail. Adding id to the primitive, passing id="node-mode" from NodeManager, and updating the E2E helper to use #node-mode fixes both the a11y regression and the CI timeout.
This commit is contained in:
Anso
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* Tests for the pilot tunnel wire protocol.
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
BinaryFrameType,
|
||||
PROTOCOL_VERSION,
|
||||
StreamIdAllocator,
|
||||
decodeBinaryFrame,
|
||||
decodeJsonFrame,
|
||||
encodeBinaryFrame,
|
||||
encodeJsonFrame,
|
||||
} from '../pilot/protocol';
|
||||
|
||||
describe('JSON frame roundtrip', () => {
|
||||
it('roundtrips a hello frame', () => {
|
||||
const frame = encodeJsonFrame({ t: 'hello', version: PROTOCOL_VERSION, role: 'agent', agentVersion: '0.1.0' });
|
||||
const decoded = decodeJsonFrame(frame);
|
||||
expect(decoded).toEqual({ t: 'hello', version: PROTOCOL_VERSION, role: 'agent', agentVersion: '0.1.0' });
|
||||
});
|
||||
|
||||
it('roundtrips an http_req frame', () => {
|
||||
const raw = encodeJsonFrame({
|
||||
t: 'http_req',
|
||||
s: 7,
|
||||
method: 'POST',
|
||||
path: '/api/stacks',
|
||||
headers: { 'content-type': 'application/json', 'x-sencho-tier': 'paid' },
|
||||
});
|
||||
const decoded = decodeJsonFrame(raw);
|
||||
expect(decoded.t).toBe('http_req');
|
||||
if (decoded.t !== 'http_req') throw new Error('narrowing');
|
||||
expect(decoded.s).toBe(7);
|
||||
expect(decoded.method).toBe('POST');
|
||||
expect(decoded.headers['x-sencho-tier']).toBe('paid');
|
||||
});
|
||||
|
||||
it('roundtrips ws lifecycle frames', () => {
|
||||
const open = decodeJsonFrame(encodeJsonFrame({ t: 'ws_open', s: 3, path: '/ws', headers: {} }));
|
||||
expect(open.t).toBe('ws_open');
|
||||
const accept = decodeJsonFrame(encodeJsonFrame({ t: 'ws_accept', s: 3, headers: {} }));
|
||||
expect(accept.t).toBe('ws_accept');
|
||||
const close = decodeJsonFrame(encodeJsonFrame({ t: 'ws_close', s: 3, code: 1000, reason: 'done' }));
|
||||
expect(close.t).toBe('ws_close');
|
||||
});
|
||||
|
||||
it('rejects malformed JSON', () => {
|
||||
expect(() => decodeJsonFrame('not json')).toThrow();
|
||||
});
|
||||
|
||||
it('rejects missing type discriminator', () => {
|
||||
expect(() => decodeJsonFrame('{"foo":1}')).toThrow(/type discriminator/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Binary frame roundtrip', () => {
|
||||
it('roundtrips an http request body chunk', () => {
|
||||
const payload = Buffer.from('hello world');
|
||||
const encoded = encodeBinaryFrame(BinaryFrameType.HttpReqBody, 42, payload);
|
||||
const decoded = decodeBinaryFrame(encoded);
|
||||
expect(decoded.type).toBe(BinaryFrameType.HttpReqBody);
|
||||
expect(decoded.streamId).toBe(42);
|
||||
expect(decoded.payload.equals(payload)).toBe(true);
|
||||
});
|
||||
|
||||
it('roundtrips a ws binary message', () => {
|
||||
const payload = Buffer.from([0xde, 0xad, 0xbe, 0xef]);
|
||||
const encoded = encodeBinaryFrame(BinaryFrameType.WsMessageBinary, 1, payload);
|
||||
const decoded = decodeBinaryFrame(encoded);
|
||||
expect(decoded.type).toBe(BinaryFrameType.WsMessageBinary);
|
||||
expect(decoded.streamId).toBe(1);
|
||||
expect(decoded.payload.equals(payload)).toBe(true);
|
||||
});
|
||||
|
||||
it('roundtrips an empty payload', () => {
|
||||
const encoded = encodeBinaryFrame(BinaryFrameType.HttpResBody, 99, Buffer.alloc(0));
|
||||
const decoded = decodeBinaryFrame(encoded);
|
||||
expect(decoded.payload.length).toBe(0);
|
||||
expect(decoded.streamId).toBe(99);
|
||||
});
|
||||
|
||||
it('handles max streamId', () => {
|
||||
const encoded = encodeBinaryFrame(BinaryFrameType.HttpReqBody, 0xffffffff, Buffer.from([0]));
|
||||
const decoded = decodeBinaryFrame(encoded);
|
||||
expect(decoded.streamId).toBe(0xffffffff);
|
||||
});
|
||||
|
||||
it('rejects negative streamId', () => {
|
||||
expect(() => encodeBinaryFrame(BinaryFrameType.HttpReqBody, -1, Buffer.alloc(0))).toThrow(/streamId/);
|
||||
});
|
||||
|
||||
it('rejects streamId above uint32 max', () => {
|
||||
expect(() => encodeBinaryFrame(BinaryFrameType.HttpReqBody, 0x100000000, Buffer.alloc(0))).toThrow(/streamId/);
|
||||
});
|
||||
|
||||
it('rejects frames shorter than the 5-byte header', () => {
|
||||
expect(() => decodeBinaryFrame(Buffer.from([0x01, 0x00]))).toThrow(/too short/);
|
||||
});
|
||||
|
||||
it('rejects unknown binary frame types', () => {
|
||||
const buf = Buffer.alloc(5);
|
||||
buf.writeUInt8(0x99, 0);
|
||||
expect(() => decodeBinaryFrame(buf)).toThrow(/unknown binary frame type/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('StreamIdAllocator', () => {
|
||||
it('allocates monotonically starting at 1', () => {
|
||||
const alloc = new StreamIdAllocator();
|
||||
expect(alloc.allocate()).toBe(1);
|
||||
expect(alloc.allocate()).toBe(2);
|
||||
expect(alloc.allocate()).toBe(3);
|
||||
});
|
||||
|
||||
it('wraps before overflowing int32', () => {
|
||||
const alloc = new StreamIdAllocator();
|
||||
(alloc as unknown as { next: number }).next = 0x7fffffff;
|
||||
expect(alloc.allocate()).toBe(0x7fffffff);
|
||||
expect(alloc.allocate()).toBe(1);
|
||||
});
|
||||
});
|
||||
+215
-9
@@ -28,6 +28,8 @@ import { ImageUpdateService } from './services/ImageUpdateService';
|
||||
import { templateService } from './services/TemplateService';
|
||||
import { ErrorParser } from './utils/ErrorParser';
|
||||
import { NodeRegistry } from './services/NodeRegistry';
|
||||
import { PilotTunnelManager } from './services/PilotTunnelManager';
|
||||
import { encodeJsonFrame as encodePilotJsonFrame, PROTOCOL_VERSION as PILOT_PROTOCOL_VERSION, PilotCloseCode } from './pilot/protocol';
|
||||
import { FleetSyncService } from './services/FleetSyncService';
|
||||
import { LicenseService, type LicenseTier, type LicenseVariant, isLicenseTier, isLicenseVariant, normalizeTier, normalizeVariant, PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from './services/LicenseService';
|
||||
import { WebhookService } from './services/WebhookService';
|
||||
@@ -503,7 +505,11 @@ const authMiddleware = async (req: Request, res: Response, next: NextFunction):
|
||||
|
||||
// Node proxy tokens: Sencho-to-Sencho communication, not user sessions.
|
||||
// Handle before user resolution since proxy tokens have no username.
|
||||
if (decoded.scope === 'node_proxy') {
|
||||
// pilot_tunnel scope is the equivalent credential for pilot-agent-mode
|
||||
// nodes; it arrives on requests the primary forwarded through a tunnel
|
||||
// after the primary itself re-signed/trusted them. Same tier-header trust
|
||||
// rules apply.
|
||||
if (decoded.scope === 'node_proxy' || decoded.scope === 'pilot_tunnel') {
|
||||
req.user = { username: 'node-proxy', role: 'admin', userId: 0 };
|
||||
|
||||
// Distributed License Enforcement: trust tier headers only from authenticated node proxy requests.
|
||||
@@ -3600,6 +3606,11 @@ const server = http.createServer(app);
|
||||
// WebSocket server with authentication
|
||||
const wss = new WebSocketServer({ noServer: true });
|
||||
|
||||
// Dedicated WS server for accepting inbound pilot-agent tunnels. Agents dial
|
||||
// /api/pilot/tunnel; the handshake verifies a pilot_enroll or pilot_tunnel
|
||||
// JWT, then hands the socket off to PilotTunnelManager.
|
||||
const pilotTunnelWss = new WebSocketServer({ noServer: true });
|
||||
|
||||
let terminalWs: WebSocket | null = null;
|
||||
|
||||
// Notification push - set of authenticated browser clients subscribed to real-time alerts
|
||||
@@ -3614,8 +3625,110 @@ NotificationService.getInstance().setBroadcaster((notification) => {
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Handle a pilot-agent tunnel upgrade. Accepts either:
|
||||
* - pilot_enroll (15m, one-time): consume the enrollment row, mint a
|
||||
* long-lived pilot_tunnel token, send it back in a ctrl enroll_ack frame.
|
||||
* - pilot_tunnel (365d): accept the socket directly.
|
||||
* In both cases, the accepted WebSocket is handed to PilotTunnelManager.
|
||||
*/
|
||||
async function handlePilotTunnelUpgrade(
|
||||
req: import('http').IncomingMessage,
|
||||
socket: import('stream').Duplex,
|
||||
head: Buffer,
|
||||
): Promise<void> {
|
||||
const reject = (status: number, message: string) => {
|
||||
try { socket.write(`HTTP/1.1 ${status} ${message}\r\n\r\n`); } catch { /* ignore */ }
|
||||
try { socket.destroy(); } catch { /* ignore */ }
|
||||
};
|
||||
|
||||
const authHeader = req.headers['authorization'];
|
||||
const header = Array.isArray(authHeader) ? authHeader[0] : authHeader;
|
||||
const token = header?.startsWith('Bearer ') ? header.slice(7) : null;
|
||||
if (!token) return reject(401, 'Unauthorized');
|
||||
|
||||
const db = DatabaseService.getInstance();
|
||||
const jwtSecret = db.getGlobalSettings().auth_jwt_secret;
|
||||
if (!jwtSecret) return reject(500, 'Internal Server Error');
|
||||
|
||||
let decoded: { scope?: string; nodeId?: number; enrollNonce?: string };
|
||||
try {
|
||||
decoded = jwt.verify(token, jwtSecret) as typeof decoded;
|
||||
} catch {
|
||||
return reject(401, 'Unauthorized');
|
||||
}
|
||||
|
||||
if (decoded.scope !== 'pilot_enroll' && decoded.scope !== 'pilot_tunnel') {
|
||||
return reject(403, 'Forbidden');
|
||||
}
|
||||
if (typeof decoded.nodeId !== 'number') return reject(400, 'Bad Request');
|
||||
|
||||
const node = db.getNode(decoded.nodeId);
|
||||
if (!node || node.type !== 'remote' || node.mode !== 'pilot_agent') {
|
||||
return reject(404, 'Not Found');
|
||||
}
|
||||
|
||||
let mintedTunnelToken: string | null = null;
|
||||
if (decoded.scope === 'pilot_enroll') {
|
||||
const tokenHash = crypto.createHash('sha256').update(token).digest('hex');
|
||||
const row = db.consumePilotEnrollment(tokenHash);
|
||||
if (!row || row.node_id !== decoded.nodeId) {
|
||||
return reject(401, 'Unauthorized');
|
||||
}
|
||||
mintedTunnelToken = jwt.sign(
|
||||
{ scope: 'pilot_tunnel', nodeId: decoded.nodeId },
|
||||
jwtSecret,
|
||||
{ expiresIn: '365d' },
|
||||
);
|
||||
}
|
||||
|
||||
const agentVersionHeader = req.headers['x-sencho-agent-version'];
|
||||
const agentVersion = Array.isArray(agentVersionHeader) ? agentVersionHeader[0] : agentVersionHeader;
|
||||
|
||||
pilotTunnelWss.handleUpgrade(req, socket, head, async (ws) => {
|
||||
try {
|
||||
ws.send(encodePilotJsonFrame({
|
||||
t: 'hello',
|
||||
version: PILOT_PROTOCOL_VERSION,
|
||||
role: 'primary',
|
||||
}));
|
||||
if (mintedTunnelToken) {
|
||||
ws.send(encodePilotJsonFrame({
|
||||
t: 'ctrl',
|
||||
op: 'enroll_ack',
|
||||
payload: { token: mintedTunnelToken, nodeId: decoded.nodeId },
|
||||
}));
|
||||
}
|
||||
} catch {
|
||||
try { ws.close(1011, 'hello failed'); } catch { /* ignore */ }
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await PilotTunnelManager.getInstance().registerTunnel(decoded.nodeId!, ws, agentVersion);
|
||||
} catch (err) {
|
||||
console.error('[Pilot] Failed to register tunnel:', (err as Error).message);
|
||||
try { ws.close(1011, 'registration failed'); } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Handle WebSocket upgrade with JWT authentication
|
||||
server.on('upgrade', async (req, socket, head) => {
|
||||
// Pilot-agent tunnel ingress: agents dial /api/pilot/tunnel and present
|
||||
// either a short-lived pilot_enroll token (consumed once) or a long-lived
|
||||
// pilot_tunnel token. Handled independently of user/session auth because
|
||||
// these are machine credentials and carry no cookies.
|
||||
try {
|
||||
const reqUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||
if (reqUrl.pathname === '/api/pilot/tunnel') {
|
||||
await handlePilotTunnelUpgrade(req, socket, head);
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// URL parse error falls through to the normal flow, which will reject.
|
||||
}
|
||||
|
||||
// Parse cookies from the upgrade request
|
||||
const cookieHeader = req.headers.cookie || '';
|
||||
const cookies = Object.fromEntries(
|
||||
@@ -8123,7 +8236,7 @@ app.post('/api/nodes', async (req: Request, res: Response) => {
|
||||
}
|
||||
if (!requirePermission(req, res, 'node:manage')) return;
|
||||
try {
|
||||
const { name, type, compose_dir, is_default, api_url, api_token } = req.body;
|
||||
const { name, type, compose_dir, is_default, api_url, api_token, mode } = req.body;
|
||||
|
||||
if (!name || typeof name !== 'string') {
|
||||
return res.status(400).json({ error: 'Node name is required' });
|
||||
@@ -8131,9 +8244,12 @@ app.post('/api/nodes', async (req: Request, res: Response) => {
|
||||
if (!type || !['local', 'remote'].includes(type)) {
|
||||
return res.status(400).json({ error: 'Node type must be "local" or "remote"' });
|
||||
}
|
||||
if (type === 'remote') {
|
||||
|
||||
const resolvedMode: 'proxy' | 'pilot_agent' = type === 'remote' && mode === 'pilot_agent' ? 'pilot_agent' : 'proxy';
|
||||
|
||||
if (type === 'remote' && resolvedMode === 'proxy') {
|
||||
if (!api_url || typeof api_url !== 'string') {
|
||||
return res.status(400).json({ error: 'API URL is required for remote nodes' });
|
||||
return res.status(400).json({ error: 'API URL is required for proxy-mode remote nodes' });
|
||||
}
|
||||
const urlCheck = isValidRemoteUrl(api_url);
|
||||
if (!urlCheck.valid) {
|
||||
@@ -8146,18 +8262,25 @@ app.post('/api/nodes', async (req: Request, res: Response) => {
|
||||
type,
|
||||
compose_dir: compose_dir || '/app/compose',
|
||||
is_default: is_default || false,
|
||||
api_url: api_url || '',
|
||||
api_token: api_token || '',
|
||||
api_url: resolvedMode === 'pilot_agent' ? '' : (api_url || ''),
|
||||
api_token: resolvedMode === 'pilot_agent' ? '' : (api_token || ''),
|
||||
mode: resolvedMode,
|
||||
});
|
||||
|
||||
// Notify subscribers (e.g. DockerEventManager) so a new local node gets
|
||||
// its event stream spun up immediately, not on next restart.
|
||||
NodeRegistry.getInstance().notifyNodeAdded(id);
|
||||
|
||||
const isPlainHttp = type === 'remote' && api_url && api_url.startsWith('http://');
|
||||
let enrollment: ReturnType<typeof mintPilotEnrollment> | null = null;
|
||||
if (resolvedMode === 'pilot_agent') {
|
||||
enrollment = mintPilotEnrollment(id, req);
|
||||
}
|
||||
|
||||
const isPlainHttp = resolvedMode === 'proxy' && type === 'remote' && api_url && api_url.startsWith('http://');
|
||||
res.json({
|
||||
success: true,
|
||||
id,
|
||||
...(enrollment && { enrollment }),
|
||||
...(isPlainHttp && {
|
||||
warning: 'This node uses plain HTTP. Use HTTPS or a VPN for connections over the public internet.'
|
||||
})
|
||||
@@ -8171,6 +8294,77 @@ app.post('/api/nodes', async (req: Request, res: Response) => {
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Mint a fresh 15-minute, single-use enrollment token for a pilot-mode node.
|
||||
* Stores a sha256 hash in pilot_enrollments so the token itself is never
|
||||
* recoverable from the DB. The returned `dockerRun` string is a copy-paste
|
||||
* starter command the admin runs on the remote host.
|
||||
*/
|
||||
function mintPilotEnrollment(nodeId: number, req: Request): { token: string; expiresAt: number; dockerRun: string } {
|
||||
const db = DatabaseService.getInstance();
|
||||
const jwtSecret = db.getGlobalSettings().auth_jwt_secret;
|
||||
if (!jwtSecret) throw new Error('JWT secret not configured');
|
||||
|
||||
const ttlSeconds = 15 * 60;
|
||||
const expiresAt = Date.now() + ttlSeconds * 1000;
|
||||
const enrollNonce = crypto.randomUUID();
|
||||
const token = jwt.sign(
|
||||
{ scope: 'pilot_enroll', nodeId, enrollNonce },
|
||||
jwtSecret,
|
||||
{ expiresIn: ttlSeconds },
|
||||
);
|
||||
const tokenHash = crypto.createHash('sha256').update(token).digest('hex');
|
||||
db.createPilotEnrollment(nodeId, tokenHash, expiresAt);
|
||||
|
||||
const forwardedProto = req.headers['x-forwarded-proto'];
|
||||
const protoHeader = Array.isArray(forwardedProto) ? forwardedProto[0] : forwardedProto;
|
||||
const protocol = protoHeader || req.protocol || 'http';
|
||||
const host = req.get('host') || 'localhost:3000';
|
||||
const primaryUrl = `${protocol}://${host}`;
|
||||
|
||||
const dockerRun =
|
||||
`docker run -d --restart=unless-stopped --name sencho-agent ` +
|
||||
`-v /var/run/docker.sock:/var/run/docker.sock ` +
|
||||
`-v sencho-agent-data:/app/data ` +
|
||||
`-v /opt/docker/sencho:/app/compose ` +
|
||||
`-e SENCHO_MODE=pilot ` +
|
||||
`-e SENCHO_PRIMARY_URL=${primaryUrl} ` +
|
||||
`-e SENCHO_ENROLL_TOKEN=${token} ` +
|
||||
`saelix/sencho:latest`;
|
||||
|
||||
return { token, expiresAt, dockerRun };
|
||||
}
|
||||
|
||||
// Regenerate an enrollment token for an existing pilot-mode node. Used when
|
||||
// the first token expired before the agent was started, or when the agent
|
||||
// container was lost and needs to re-enroll from scratch.
|
||||
app.post('/api/nodes/:id/pilot/enroll', async (req: Request, res: Response) => {
|
||||
if (req.apiTokenScope) {
|
||||
res.status(403).json({ error: 'API tokens cannot manage nodes.', code: 'SCOPE_DENIED' });
|
||||
return;
|
||||
}
|
||||
const nodeIdStr = req.params.id as string;
|
||||
if (!requirePermission(req, res, 'node:manage', 'node', nodeIdStr)) return;
|
||||
try {
|
||||
const nodeId = parseInt(nodeIdStr, 10);
|
||||
if (!Number.isFinite(nodeId)) {
|
||||
return res.status(400).json({ error: 'Invalid node id' });
|
||||
}
|
||||
const node = DatabaseService.getInstance().getNode(nodeId);
|
||||
if (!node) return res.status(404).json({ error: 'Node not found' });
|
||||
if (node.type !== 'remote' || node.mode !== 'pilot_agent') {
|
||||
return res.status(400).json({ error: 'Enrollment only applies to pilot-agent nodes' });
|
||||
}
|
||||
// Close any existing tunnel so the re-enrolling agent cleanly replaces it.
|
||||
PilotTunnelManager.getInstance().closeTunnel(nodeId, PilotCloseCode.EnrollmentRegenerated, 'enrollment regenerated');
|
||||
const enrollment = mintPilotEnrollment(nodeId, req);
|
||||
res.json({ success: true, enrollment });
|
||||
} catch (error: any) {
|
||||
console.error('Failed to regenerate pilot enrollment:', error);
|
||||
res.status(500).json({ error: error.message || 'Failed to regenerate enrollment' });
|
||||
}
|
||||
});
|
||||
|
||||
// Update a node
|
||||
app.put('/api/nodes/:id', async (req: Request, res: Response) => {
|
||||
if (req.apiTokenScope) {
|
||||
@@ -8376,8 +8570,20 @@ async function startServer() {
|
||||
}, MFA_REPLAY_PURGE_INTERVAL_MS);
|
||||
mfaReplayPurgeTimer.unref();
|
||||
|
||||
server.listen(PORT, () => {
|
||||
console.log(`Server running on port ${PORT}`);
|
||||
// Pilot-agent mode: bind only to loopback so no external port is exposed.
|
||||
// All traffic is demultiplexed from the primary via the pilot tunnel.
|
||||
const isPilotAgent = process.env.SENCHO_MODE === 'pilot';
|
||||
const listenHost = isPilotAgent ? '127.0.0.1' : undefined;
|
||||
|
||||
server.listen(PORT, listenHost, () => {
|
||||
console.log(`Server running on ${listenHost || '0.0.0.0'}:${PORT}${isPilotAgent ? ' (pilot-agent mode)' : ''}`);
|
||||
if (isPilotAgent) {
|
||||
// Start the outbound tunnel client once the local HTTP server is ready
|
||||
// to accept loopback traffic from the tunnel.
|
||||
import('./pilot/agent').then((m) => m.startPilotAgent(PORT)).catch((err) => {
|
||||
console.error('[Pilot] Agent startup failed:', err);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,351 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import http from 'http';
|
||||
import WebSocket from 'ws';
|
||||
import { getSenchoVersion } from '../services/CapabilityRegistry';
|
||||
import {
|
||||
BinaryFrameType,
|
||||
PROTOCOL_VERSION,
|
||||
decodeBinaryFrame,
|
||||
decodeJsonFrame,
|
||||
encodeBinaryFrame,
|
||||
encodeJsonFrame,
|
||||
wsDataToBuffer,
|
||||
wsDataToString,
|
||||
} from './protocol';
|
||||
|
||||
const RECONNECT_MIN_MS = 1_000;
|
||||
const RECONNECT_MAX_MS = 60_000;
|
||||
const PING_INTERVAL_MS = 30_000;
|
||||
const TOKEN_PATH = path.join(process.env.DATA_DIR || '/app/data', 'pilot.jwt');
|
||||
|
||||
/**
|
||||
* Pilot agent: dials the primary via outbound WebSocket and tunnels every
|
||||
* inbound frame to the agent's own loopback HTTP server (the fully-booted
|
||||
* Sencho app). Because the tunnel is the only ingress, the agent needs no
|
||||
* open port, no TLS certificate, and no reachable address.
|
||||
*/
|
||||
export function startPilotAgent(loopbackPort: number): void {
|
||||
const primaryUrl = process.env.SENCHO_PRIMARY_URL;
|
||||
if (!primaryUrl) {
|
||||
console.error('[Pilot] SENCHO_PRIMARY_URL is required when SENCHO_MODE=pilot');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const enrollToken = process.env.SENCHO_ENROLL_TOKEN;
|
||||
const persistedToken = readPersistedToken();
|
||||
|
||||
if (!enrollToken && !persistedToken) {
|
||||
console.error('[Pilot] SENCHO_ENROLL_TOKEN is required on first boot');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const agent = new PilotAgent({
|
||||
primaryUrl,
|
||||
loopbackPort,
|
||||
initialToken: persistedToken || enrollToken!,
|
||||
enrolling: !persistedToken,
|
||||
});
|
||||
agent.start();
|
||||
}
|
||||
|
||||
interface AgentOptions {
|
||||
primaryUrl: string;
|
||||
loopbackPort: number;
|
||||
initialToken: string;
|
||||
enrolling: boolean;
|
||||
}
|
||||
|
||||
class PilotAgent {
|
||||
private readonly options: AgentOptions;
|
||||
private token: string;
|
||||
private backoff = RECONNECT_MIN_MS;
|
||||
private ws: WebSocket | null = null;
|
||||
private pingTimer?: NodeJS.Timeout;
|
||||
private reconnectTimer?: NodeJS.Timeout;
|
||||
private readonly httpStreams = new Map<number, { req: http.ClientRequest }>();
|
||||
private readonly wsStreams = new Map<number, WebSocket>();
|
||||
private shuttingDown = false;
|
||||
private readonly agentVersion: string;
|
||||
|
||||
constructor(options: AgentOptions) {
|
||||
this.options = options;
|
||||
this.token = options.initialToken;
|
||||
this.agentVersion = getSenchoVersion() || '0.0.0';
|
||||
}
|
||||
|
||||
public start(): void {
|
||||
this.connect();
|
||||
process.on('SIGTERM', () => this.shutdown());
|
||||
process.on('SIGINT', () => this.shutdown());
|
||||
}
|
||||
|
||||
private shutdown(): void {
|
||||
this.shuttingDown = true;
|
||||
if (this.pingTimer) clearInterval(this.pingTimer);
|
||||
if (this.reconnectTimer) { clearTimeout(this.reconnectTimer); this.reconnectTimer = undefined; }
|
||||
try { this.ws?.close(1000, 'agent shutdown'); } catch { /* ignore */ }
|
||||
}
|
||||
|
||||
private connect(): void {
|
||||
if (this.shuttingDown) return;
|
||||
|
||||
const wsUrl = this.options.primaryUrl.replace(/^http/, 'ws').replace(/\/$/, '') + '/api/pilot/tunnel';
|
||||
const ws = new WebSocket(wsUrl, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.token}`,
|
||||
'x-sencho-agent-version': this.agentVersion,
|
||||
},
|
||||
handshakeTimeout: 15_000,
|
||||
});
|
||||
this.ws = ws;
|
||||
|
||||
ws.on('open', () => {
|
||||
console.log('[Pilot] Tunnel connected to', this.options.primaryUrl);
|
||||
this.backoff = RECONNECT_MIN_MS;
|
||||
try {
|
||||
ws.send(encodeJsonFrame({
|
||||
t: 'hello',
|
||||
version: PROTOCOL_VERSION,
|
||||
role: 'agent',
|
||||
agentVersion: this.agentVersion,
|
||||
}));
|
||||
} catch (err) {
|
||||
console.error('[Pilot] Failed to send hello:', (err as Error).message);
|
||||
}
|
||||
this.pingTimer = setInterval(() => {
|
||||
if (ws.readyState === WebSocket.OPEN) {
|
||||
try { ws.ping(); } catch { /* surfaced via error */ }
|
||||
}
|
||||
}, PING_INTERVAL_MS);
|
||||
});
|
||||
|
||||
ws.on('message', (data, isBinary) => this.handleFrame(data, isBinary));
|
||||
ws.on('close', (code, reason) => {
|
||||
console.log('[Pilot] Tunnel closed:', code, reason?.toString?.() ?? '');
|
||||
this.cleanupAfterDisconnect();
|
||||
this.scheduleReconnect();
|
||||
});
|
||||
ws.on('error', (err) => {
|
||||
console.warn('[Pilot] Tunnel error:', err.message);
|
||||
// 'close' will follow; reconnect is scheduled there.
|
||||
});
|
||||
}
|
||||
|
||||
private cleanupAfterDisconnect(): void {
|
||||
if (this.pingTimer) { clearInterval(this.pingTimer); this.pingTimer = undefined; }
|
||||
for (const [, entry] of this.httpStreams) {
|
||||
try { entry.req.destroy(); } catch { /* ignore */ }
|
||||
}
|
||||
this.httpStreams.clear();
|
||||
for (const [, ws] of this.wsStreams) {
|
||||
try { ws.close(1006, 'tunnel closed'); } catch { /* ignore */ }
|
||||
}
|
||||
this.wsStreams.clear();
|
||||
}
|
||||
|
||||
private scheduleReconnect(): void {
|
||||
if (this.shuttingDown) return;
|
||||
const jitter = Math.floor(Math.random() * 500);
|
||||
const delay = this.backoff + jitter;
|
||||
this.reconnectTimer = setTimeout(() => {
|
||||
this.reconnectTimer = undefined;
|
||||
this.connect();
|
||||
}, delay);
|
||||
this.backoff = Math.min(this.backoff * 2, RECONNECT_MAX_MS);
|
||||
}
|
||||
|
||||
private handleFrame(data: unknown, isBinary: boolean): void {
|
||||
try {
|
||||
if (isBinary) {
|
||||
const buf = wsDataToBuffer(data);
|
||||
if (!buf) return;
|
||||
this.handleBinaryFrame(decodeBinaryFrame(buf));
|
||||
} else {
|
||||
const text = wsDataToString(data);
|
||||
if (text == null) return;
|
||||
const frame = decodeJsonFrame(text);
|
||||
this.handleJsonFrame(frame);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('[Pilot] Malformed frame from primary:', (err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
private handleJsonFrame(frame: ReturnType<typeof decodeJsonFrame>): void {
|
||||
const ws = this.ws;
|
||||
if (!ws) return;
|
||||
switch (frame.t) {
|
||||
case 'hello': {
|
||||
if (frame.version !== PROTOCOL_VERSION) {
|
||||
console.error(`[Pilot] Protocol version ${frame.version} from primary is incompatible with agent (${PROTOCOL_VERSION}); exiting.`);
|
||||
this.shuttingDown = true;
|
||||
try { ws.close(1002, 'incompatible version'); } catch { /* ignore */ }
|
||||
process.exit(1);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'ctrl': {
|
||||
if (frame.op === 'enroll_ack' && frame.payload && typeof frame.payload.token === 'string') {
|
||||
this.token = frame.payload.token;
|
||||
persistToken(this.token);
|
||||
console.log('[Pilot] Enrollment complete; long-lived token persisted.');
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'http_req': this.onHttpReq(frame); break;
|
||||
case 'http_req_end': this.onHttpReqEnd(frame.s); break;
|
||||
case 'ws_open': this.onWsOpen(frame); break;
|
||||
case 'ws_msg_text': this.onWsMsgText(frame.s, frame.data); break;
|
||||
case 'ws_close': this.onWsClose(frame.s, frame.code, frame.reason); break;
|
||||
default:
|
||||
// Other frame types are primary-bound only; agent ignores.
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private handleBinaryFrame(frame: ReturnType<typeof decodeBinaryFrame>): void {
|
||||
switch (frame.type) {
|
||||
case BinaryFrameType.HttpReqBody: {
|
||||
const entry = this.httpStreams.get(frame.streamId);
|
||||
if (!entry) return;
|
||||
try { entry.req.write(frame.payload); } catch { /* ignore */ }
|
||||
break;
|
||||
}
|
||||
case BinaryFrameType.WsMessageBinary: {
|
||||
const ws = this.wsStreams.get(frame.streamId);
|
||||
if (!ws) return;
|
||||
try { ws.send(frame.payload, { binary: true }); } catch { /* ignore */ }
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// --- HTTP dispatch (tunnel -> loopback) ---
|
||||
|
||||
private onHttpReq(frame: Extract<ReturnType<typeof decodeJsonFrame>, { t: 'http_req' }>): void {
|
||||
const ws = this.ws;
|
||||
if (!ws) return;
|
||||
|
||||
const req = http.request({
|
||||
host: '127.0.0.1',
|
||||
port: this.options.loopbackPort,
|
||||
method: frame.method,
|
||||
path: frame.path,
|
||||
headers: { ...frame.headers, host: `127.0.0.1:${this.options.loopbackPort}` },
|
||||
}, (res) => {
|
||||
const outHeaders: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(res.headers)) {
|
||||
if (typeof v === 'string') outHeaders[k] = v;
|
||||
else if (Array.isArray(v)) outHeaders[k] = v.join(', ');
|
||||
}
|
||||
try {
|
||||
ws.send(encodeJsonFrame({
|
||||
t: 'http_res',
|
||||
s: frame.s,
|
||||
status: res.statusCode || 200,
|
||||
headers: outHeaders,
|
||||
}));
|
||||
} catch { /* ignore */ }
|
||||
|
||||
res.on('data', (chunk: Buffer) => {
|
||||
try { ws.send(encodeBinaryFrame(BinaryFrameType.HttpResBody, frame.s, chunk), { binary: true }); } catch { /* ignore */ }
|
||||
});
|
||||
res.on('end', () => {
|
||||
try { ws.send(encodeJsonFrame({ t: 'http_res_end', s: frame.s })); } catch { /* ignore */ }
|
||||
this.httpStreams.delete(frame.s);
|
||||
});
|
||||
res.on('error', () => {
|
||||
try { ws.send(encodeJsonFrame({ t: 'http_err', s: frame.s, code: 'bad_response', message: 'upstream error' })); } catch { /* ignore */ }
|
||||
this.httpStreams.delete(frame.s);
|
||||
});
|
||||
});
|
||||
|
||||
req.on('error', (err) => {
|
||||
try {
|
||||
ws.send(encodeJsonFrame({
|
||||
t: 'http_err',
|
||||
s: frame.s,
|
||||
code: 'agent_error',
|
||||
message: err.message || 'agent request failed',
|
||||
}));
|
||||
} catch { /* ignore */ }
|
||||
this.httpStreams.delete(frame.s);
|
||||
});
|
||||
|
||||
this.httpStreams.set(frame.s, { req });
|
||||
}
|
||||
|
||||
private onHttpReqEnd(streamId: number): void {
|
||||
const entry = this.httpStreams.get(streamId);
|
||||
if (!entry) return;
|
||||
try { entry.req.end(); } catch { /* ignore */ }
|
||||
}
|
||||
|
||||
// --- WebSocket dispatch (tunnel -> loopback) ---
|
||||
|
||||
private onWsOpen(frame: Extract<ReturnType<typeof decodeJsonFrame>, { t: 'ws_open' }>): void {
|
||||
const ws = this.ws;
|
||||
if (!ws) return;
|
||||
|
||||
const target = `ws://127.0.0.1:${this.options.loopbackPort}${frame.path}`;
|
||||
const client = new WebSocket(target, {
|
||||
headers: { ...frame.headers, host: `127.0.0.1:${this.options.loopbackPort}` },
|
||||
});
|
||||
|
||||
client.on('open', () => {
|
||||
try { ws.send(encodeJsonFrame({ t: 'ws_accept', s: frame.s, headers: {} })); } catch { /* ignore */ }
|
||||
this.wsStreams.set(frame.s, client);
|
||||
});
|
||||
client.on('message', (data, isBinary) => {
|
||||
if (isBinary) {
|
||||
try { ws.send(encodeBinaryFrame(BinaryFrameType.WsMessageBinary, frame.s, wsDataToBuffer(data) ?? Buffer.alloc(0)), { binary: true }); } catch { /* ignore */ }
|
||||
} else {
|
||||
try { ws.send(encodeJsonFrame({ t: 'ws_msg_text', s: frame.s, data: wsDataToString(data) ?? '' })); } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
client.on('close', (code, reason) => {
|
||||
try { ws.send(encodeJsonFrame({ t: 'ws_close', s: frame.s, code, reason: reason?.toString?.() })); } catch { /* ignore */ }
|
||||
this.wsStreams.delete(frame.s);
|
||||
});
|
||||
client.on('error', () => {
|
||||
try { ws.send(encodeJsonFrame({ t: 'ws_reject', s: frame.s, status: 502, message: 'agent websocket failed' })); } catch { /* ignore */ }
|
||||
this.wsStreams.delete(frame.s);
|
||||
});
|
||||
}
|
||||
|
||||
private onWsMsgText(streamId: number, data: string): void {
|
||||
const ws = this.wsStreams.get(streamId);
|
||||
if (!ws) return;
|
||||
try { ws.send(data); } catch { /* ignore */ }
|
||||
}
|
||||
|
||||
private onWsClose(streamId: number, code: number, reason?: string): void {
|
||||
const ws = this.wsStreams.get(streamId);
|
||||
if (!ws) return;
|
||||
try { ws.close(code, reason); } catch { /* ignore */ }
|
||||
this.wsStreams.delete(streamId);
|
||||
}
|
||||
}
|
||||
|
||||
function readPersistedToken(): string | null {
|
||||
try {
|
||||
if (fs.existsSync(TOKEN_PATH)) {
|
||||
return fs.readFileSync(TOKEN_PATH, 'utf8').trim() || null;
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
return null;
|
||||
}
|
||||
|
||||
function persistToken(token: string): void {
|
||||
try {
|
||||
const dir = path.dirname(TOKEN_PATH);
|
||||
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(TOKEN_PATH, token, { mode: 0o600 });
|
||||
} catch (err) {
|
||||
console.warn('[Pilot] Failed to persist tunnel token:', (err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
/**
|
||||
* Pilot Tunnel wire protocol.
|
||||
*
|
||||
* A single WebSocket between primary and pilot agent carries many multiplexed
|
||||
* HTTP requests and nested WebSocket streams. Each stream is identified by a
|
||||
* monotonically increasing streamId allocated by the primary (the originator
|
||||
* of every request).
|
||||
*
|
||||
* Wire format is hybrid:
|
||||
* - Text frames: JSON envelopes for metadata (open/close/headers/control)
|
||||
* - Binary frames: raw payload bytes with a 5-byte prefix
|
||||
* [ 1 byte: BinaryFrameType ][ 4 bytes: streamId (big-endian) ][ bytes... ]
|
||||
*
|
||||
* JSON is inspectable and low-overhead for small control messages; binary
|
||||
* avoids base64 bloat on body chunks and WS message payloads.
|
||||
*/
|
||||
|
||||
export const PROTOCOL_VERSION = 1;
|
||||
|
||||
// --- Binary frame types (first byte of a binary WS frame) ---
|
||||
|
||||
export enum BinaryFrameType {
|
||||
HttpReqBody = 0x01,
|
||||
HttpResBody = 0x02,
|
||||
WsMessageBinary = 0x03,
|
||||
}
|
||||
|
||||
// --- JSON envelope types ---
|
||||
|
||||
export type JsonFrame =
|
||||
| HelloFrame
|
||||
| HttpReqFrame
|
||||
| HttpReqEndFrame
|
||||
| HttpResFrame
|
||||
| HttpResEndFrame
|
||||
| HttpErrorFrame
|
||||
| WsOpenFrame
|
||||
| WsAcceptFrame
|
||||
| WsRejectFrame
|
||||
| WsMessageTextFrame
|
||||
| WsCloseFrame
|
||||
| ControlFrame;
|
||||
|
||||
export interface HelloFrame {
|
||||
t: 'hello';
|
||||
version: number;
|
||||
role: 'primary' | 'agent';
|
||||
agentVersion?: string;
|
||||
}
|
||||
|
||||
export interface HttpReqFrame {
|
||||
t: 'http_req';
|
||||
s: number;
|
||||
method: string;
|
||||
path: string;
|
||||
headers: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface HttpReqEndFrame {
|
||||
t: 'http_req_end';
|
||||
s: number;
|
||||
}
|
||||
|
||||
export interface HttpResFrame {
|
||||
t: 'http_res';
|
||||
s: number;
|
||||
status: number;
|
||||
headers: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface HttpResEndFrame {
|
||||
t: 'http_res_end';
|
||||
s: number;
|
||||
}
|
||||
|
||||
export interface HttpErrorFrame {
|
||||
t: 'http_err';
|
||||
s: number;
|
||||
code: 'timeout' | 'tunnel_down' | 'bad_response' | 'agent_error';
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface WsOpenFrame {
|
||||
t: 'ws_open';
|
||||
s: number;
|
||||
path: string;
|
||||
headers: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface WsAcceptFrame {
|
||||
t: 'ws_accept';
|
||||
s: number;
|
||||
headers: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface WsRejectFrame {
|
||||
t: 'ws_reject';
|
||||
s: number;
|
||||
status: number;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface WsMessageTextFrame {
|
||||
t: 'ws_msg_text';
|
||||
s: number;
|
||||
data: string;
|
||||
}
|
||||
|
||||
export interface WsCloseFrame {
|
||||
t: 'ws_close';
|
||||
s: number;
|
||||
code: number;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export interface ControlFrame {
|
||||
t: 'ctrl';
|
||||
op: 'enroll_ack' | 'node_info' | 'ping' | 'pong';
|
||||
payload?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
// --- Serialize / parse ---
|
||||
|
||||
export function encodeJsonFrame(frame: JsonFrame): string {
|
||||
return JSON.stringify(frame);
|
||||
}
|
||||
|
||||
export function decodeJsonFrame(raw: string): JsonFrame {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (!parsed || typeof parsed !== 'object' || typeof parsed.t !== 'string') {
|
||||
throw new Error('invalid frame: missing type discriminator');
|
||||
}
|
||||
return parsed as JsonFrame;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a binary frame: [type][streamId BE][payload].
|
||||
* Returns a fresh Buffer owned by the caller.
|
||||
*/
|
||||
export function encodeBinaryFrame(type: BinaryFrameType, streamId: number, payload: Buffer): Buffer {
|
||||
if (!Number.isInteger(streamId) || streamId < 0 || streamId > 0xffffffff) {
|
||||
throw new Error(`invalid streamId: ${streamId}`);
|
||||
}
|
||||
const out = Buffer.allocUnsafe(5 + payload.length);
|
||||
out.writeUInt8(type, 0);
|
||||
out.writeUInt32BE(streamId, 1);
|
||||
payload.copy(out, 5);
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface DecodedBinaryFrame {
|
||||
type: BinaryFrameType;
|
||||
streamId: number;
|
||||
payload: Buffer;
|
||||
}
|
||||
|
||||
export function decodeBinaryFrame(buf: Buffer): DecodedBinaryFrame {
|
||||
if (buf.length < 5) {
|
||||
throw new Error(`binary frame too short: ${buf.length} bytes`);
|
||||
}
|
||||
const type = buf.readUInt8(0) as BinaryFrameType;
|
||||
if (type !== BinaryFrameType.HttpReqBody &&
|
||||
type !== BinaryFrameType.HttpResBody &&
|
||||
type !== BinaryFrameType.WsMessageBinary) {
|
||||
throw new Error(`unknown binary frame type: ${type}`);
|
||||
}
|
||||
const streamId = buf.readUInt32BE(1);
|
||||
const payload = buf.subarray(5);
|
||||
return { type, streamId, payload };
|
||||
}
|
||||
|
||||
// --- WS payload normalization ---
|
||||
|
||||
export function wsDataToBuffer(data: unknown): Buffer | null {
|
||||
if (Buffer.isBuffer(data)) return data;
|
||||
if (data instanceof ArrayBuffer) return Buffer.from(data);
|
||||
if (Array.isArray(data)) return Buffer.concat(data.map((d) => Buffer.isBuffer(d) ? d : Buffer.from(d as ArrayBuffer)));
|
||||
return null;
|
||||
}
|
||||
|
||||
export function wsDataToString(data: unknown): string | null {
|
||||
if (typeof data === 'string') return data;
|
||||
const buf = wsDataToBuffer(data);
|
||||
return buf ? buf.toString('utf8') : null;
|
||||
}
|
||||
|
||||
// --- Close codes ---
|
||||
|
||||
export const PilotCloseCode = {
|
||||
Replaced: 4000,
|
||||
EnrollmentRegenerated: 4001,
|
||||
ProtocolError: 1002,
|
||||
} as const;
|
||||
|
||||
// --- Stream id allocation ---
|
||||
|
||||
/**
|
||||
* Monotonic stream id generator. Primary is the sole allocator.
|
||||
* Wraps at 2^31 (well above practical per-tunnel concurrency).
|
||||
*/
|
||||
export class StreamIdAllocator {
|
||||
private next: number;
|
||||
|
||||
constructor(start = 1) {
|
||||
this.next = start;
|
||||
}
|
||||
|
||||
allocate(): number {
|
||||
const id = this.next;
|
||||
this.next = this.next >= 0x7fffffff ? 1 : this.next + 1;
|
||||
return id;
|
||||
}
|
||||
}
|
||||
@@ -26,16 +26,28 @@ export interface StackAlert {
|
||||
last_fired_at?: number;
|
||||
}
|
||||
|
||||
export type NodeMode = 'proxy' | 'pilot_agent';
|
||||
|
||||
export interface Node {
|
||||
id: number;
|
||||
name: string;
|
||||
type: 'local' | 'remote';
|
||||
mode: NodeMode;
|
||||
compose_dir: string;
|
||||
is_default: boolean;
|
||||
status: 'online' | 'offline' | 'unknown';
|
||||
created_at: number;
|
||||
api_url?: string;
|
||||
api_token?: string;
|
||||
pilot_last_seen?: number | null;
|
||||
pilot_agent_version?: string | null;
|
||||
}
|
||||
|
||||
export interface PilotEnrollment {
|
||||
node_id: number;
|
||||
token_hash: string;
|
||||
expires_at: number;
|
||||
used_at: number | null;
|
||||
}
|
||||
|
||||
export interface Label {
|
||||
@@ -483,6 +495,14 @@ export class DatabaseService {
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pilot_enrollments (
|
||||
node_id INTEGER PRIMARY KEY,
|
||||
token_hash TEXT NOT NULL,
|
||||
expires_at INTEGER NOT NULL,
|
||||
used_at INTEGER,
|
||||
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS system_state (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
@@ -798,6 +818,11 @@ export class DatabaseService {
|
||||
maybeAddCol('nodes', 'api_url', "TEXT DEFAULT ''");
|
||||
maybeAddCol('nodes', 'api_token', "TEXT DEFAULT ''");
|
||||
|
||||
// Pilot Agent outbound-mode columns
|
||||
maybeAddCol('nodes', 'mode', "TEXT NOT NULL DEFAULT 'proxy'");
|
||||
maybeAddCol('nodes', 'pilot_last_seen', 'INTEGER');
|
||||
maybeAddCol('nodes', 'pilot_agent_version', 'TEXT');
|
||||
|
||||
// Scheduled operations migrations
|
||||
maybeAddCol('scheduled_task_runs', 'triggered_by', "TEXT NOT NULL DEFAULT 'scheduler'");
|
||||
maybeAddCol('scheduled_tasks', 'prune_targets', 'TEXT DEFAULT NULL');
|
||||
@@ -1278,36 +1303,42 @@ export class DatabaseService {
|
||||
return {
|
||||
...row,
|
||||
is_default: row.is_default === 1,
|
||||
mode: (row.mode === 'pilot_agent' ? 'pilot_agent' : 'proxy') as NodeMode,
|
||||
api_token: row.api_token ? crypto.decrypt(row.api_token) : '',
|
||||
pilot_last_seen: row.pilot_last_seen ?? null,
|
||||
pilot_agent_version: row.pilot_agent_version ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
private static readonly NODE_COLUMNS =
|
||||
'id, name, type, compose_dir, is_default, status, created_at, api_url, api_token, mode, pilot_last_seen, pilot_agent_version';
|
||||
|
||||
public getNodes(): Node[] {
|
||||
const stmt = this.db.prepare('SELECT id, name, type, compose_dir, is_default, status, created_at, api_url, api_token FROM nodes ORDER BY is_default DESC, name ASC');
|
||||
const stmt = this.db.prepare(`SELECT ${DatabaseService.NODE_COLUMNS} FROM nodes ORDER BY is_default DESC, name ASC`);
|
||||
return stmt.all().map((row: any) => this.decryptNodeRow(row));
|
||||
}
|
||||
|
||||
public getNode(id: number): Node | undefined {
|
||||
const stmt = this.db.prepare('SELECT id, name, type, compose_dir, is_default, status, created_at, api_url, api_token FROM nodes WHERE id = ?');
|
||||
const stmt = this.db.prepare(`SELECT ${DatabaseService.NODE_COLUMNS} FROM nodes WHERE id = ?`);
|
||||
const row = stmt.get(id) as any;
|
||||
if (!row) return undefined;
|
||||
return this.decryptNodeRow(row);
|
||||
}
|
||||
|
||||
public getDefaultNode(): Node | undefined {
|
||||
const stmt = this.db.prepare('SELECT id, name, type, compose_dir, is_default, status, created_at, api_url, api_token FROM nodes WHERE is_default = 1 LIMIT 1');
|
||||
const stmt = this.db.prepare(`SELECT ${DatabaseService.NODE_COLUMNS} FROM nodes WHERE is_default = 1 LIMIT 1`);
|
||||
const row = stmt.get() as any;
|
||||
if (!row) return undefined;
|
||||
return this.decryptNodeRow(row);
|
||||
}
|
||||
|
||||
public addNode(node: Omit<Node, 'id' | 'status' | 'created_at'>): number {
|
||||
public addNode(node: Omit<Node, 'id' | 'status' | 'created_at' | 'mode'> & { mode?: NodeMode }): number {
|
||||
if (node.is_default) {
|
||||
this.db.prepare('UPDATE nodes SET is_default = 0').run();
|
||||
}
|
||||
const crypto = CryptoService.getInstance();
|
||||
const stmt = this.db.prepare(
|
||||
'INSERT INTO nodes (name, type, compose_dir, is_default, status, created_at, api_url, api_token) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
|
||||
'INSERT INTO nodes (name, type, compose_dir, is_default, status, created_at, api_url, api_token, mode) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)'
|
||||
);
|
||||
const result = stmt.run(
|
||||
node.name,
|
||||
@@ -1317,7 +1348,8 @@ export class DatabaseService {
|
||||
'unknown',
|
||||
Date.now(),
|
||||
node.api_url || '',
|
||||
node.api_token ? crypto.encrypt(node.api_token) : ''
|
||||
node.api_token ? crypto.encrypt(node.api_token) : '',
|
||||
node.mode || 'proxy'
|
||||
);
|
||||
return result.lastInsertRowid as number;
|
||||
}
|
||||
@@ -1343,6 +1375,9 @@ export class DatabaseService {
|
||||
fields.push('api_token = ?');
|
||||
values.push(updates.api_token ? CryptoService.getInstance().encrypt(updates.api_token) : '');
|
||||
}
|
||||
if (updates.mode !== undefined) { fields.push('mode = ?'); values.push(updates.mode); }
|
||||
if (updates.pilot_last_seen !== undefined) { fields.push('pilot_last_seen = ?'); values.push(updates.pilot_last_seen); }
|
||||
if (updates.pilot_agent_version !== undefined) { fields.push('pilot_agent_version = ?'); values.push(updates.pilot_agent_version); }
|
||||
|
||||
if (fields.length === 0) return;
|
||||
|
||||
@@ -1370,6 +1405,43 @@ export class DatabaseService {
|
||||
this.db.prepare('UPDATE nodes SET status = ? WHERE id = ?').run(status, id);
|
||||
}
|
||||
|
||||
// --- Pilot enrollments ---
|
||||
|
||||
public getPilotEnrollment(nodeId: number): PilotEnrollment | undefined {
|
||||
const row = this.db.prepare(
|
||||
'SELECT node_id, token_hash, expires_at, used_at FROM pilot_enrollments WHERE node_id = ?'
|
||||
).get(nodeId) as PilotEnrollment | undefined;
|
||||
return row;
|
||||
}
|
||||
|
||||
public createPilotEnrollment(nodeId: number, tokenHash: string, expiresAt: number): void {
|
||||
this.db.prepare(
|
||||
`INSERT INTO pilot_enrollments (node_id, token_hash, expires_at, used_at)
|
||||
VALUES (?, ?, ?, NULL)
|
||||
ON CONFLICT(node_id) DO UPDATE SET
|
||||
token_hash = excluded.token_hash,
|
||||
expires_at = excluded.expires_at,
|
||||
used_at = NULL`
|
||||
).run(nodeId, tokenHash, expiresAt);
|
||||
}
|
||||
|
||||
public consumePilotEnrollment(tokenHash: string): PilotEnrollment | undefined {
|
||||
const now = Date.now();
|
||||
return this.db.transaction(() => {
|
||||
const row = this.db.prepare(
|
||||
`SELECT node_id, token_hash, expires_at, used_at FROM pilot_enrollments
|
||||
WHERE token_hash = ? AND used_at IS NULL AND expires_at > ?`
|
||||
).get(tokenHash, now) as PilotEnrollment | undefined;
|
||||
if (!row) return undefined;
|
||||
this.db.prepare('UPDATE pilot_enrollments SET used_at = ? WHERE node_id = ?').run(now, row.node_id);
|
||||
return { ...row, used_at: now };
|
||||
})();
|
||||
}
|
||||
|
||||
public deletePilotEnrollment(nodeId: number): void {
|
||||
this.db.prepare('DELETE FROM pilot_enrollments WHERE node_id = ?').run(nodeId);
|
||||
}
|
||||
|
||||
// --- Stack Update Status ---
|
||||
|
||||
public upsertStackUpdateStatus(nodeId: number, stackName: string, hasUpdate: boolean, checkedAt: number): void {
|
||||
|
||||
@@ -3,6 +3,7 @@ import axios from 'axios';
|
||||
import { EventEmitter } from 'events';
|
||||
import { DatabaseService, Node } from './DatabaseService';
|
||||
import { fetchRemoteMeta } from './CapabilityRegistry';
|
||||
import { PilotTunnelManager } from './PilotTunnelManager';
|
||||
|
||||
/**
|
||||
* NodeRegistry: Manages connections for multiple nodes.
|
||||
@@ -99,12 +100,22 @@ export class NodeRegistry extends EventEmitter {
|
||||
/**
|
||||
* Get the HTTP proxy target for a remote node.
|
||||
* Returns { apiUrl, apiToken } for use by the HTTP proxy middleware.
|
||||
*
|
||||
* Pilot-agent nodes resolve to the loopback URL of their active tunnel
|
||||
* bridge; the bridge strips the bearer token and re-authenticates
|
||||
* implicitly via the pre-verified tunnel socket.
|
||||
*/
|
||||
public getProxyTarget(nodeId: number): { apiUrl: string; apiToken: string } | null {
|
||||
const node = DatabaseService.getInstance().getNode(nodeId);
|
||||
if (!node || node.type !== 'remote' || !node.api_url || !node.api_token) {
|
||||
return null;
|
||||
if (!node || node.type !== 'remote') return null;
|
||||
|
||||
if (node.mode === 'pilot_agent') {
|
||||
const loopbackUrl = PilotTunnelManager.getInstance().getLoopbackUrl(nodeId);
|
||||
if (!loopbackUrl) return null;
|
||||
return { apiUrl: loopbackUrl, apiToken: '' };
|
||||
}
|
||||
|
||||
if (!node.api_url || !node.api_token) return null;
|
||||
return { apiUrl: node.api_url, apiToken: node.api_token };
|
||||
}
|
||||
|
||||
@@ -122,12 +133,48 @@ export class NodeRegistry extends EventEmitter {
|
||||
}
|
||||
|
||||
if (node.type === 'remote') {
|
||||
if (node.mode === 'pilot_agent') {
|
||||
return this.testPilotConnection(node);
|
||||
}
|
||||
return this.testRemoteConnection(node);
|
||||
}
|
||||
|
||||
return this.testLocalConnection(nodeId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether a pilot-agent node has an active tunnel. Does not call
|
||||
* any endpoint; tunnel liveness is tracked in-process by
|
||||
* PilotTunnelManager and via the JWT handshake that originally accepted
|
||||
* the agent.
|
||||
*/
|
||||
private async testPilotConnection(node: Node): Promise<{ success: boolean; error?: string; info?: any }> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const active = PilotTunnelManager.getInstance().hasActiveTunnel(node.id);
|
||||
if (!active) {
|
||||
db.updateNodeStatus(node.id, 'offline');
|
||||
return { success: false, error: 'Pilot agent is not connected. Start the agent container or regenerate the enrollment token.' };
|
||||
}
|
||||
db.updateNodeStatus(node.id, 'online');
|
||||
return {
|
||||
success: true,
|
||||
info: {
|
||||
name: node.name,
|
||||
serverVersion: 'Pilot Agent',
|
||||
senchoVersion: node.pilot_agent_version ?? null,
|
||||
capabilities: [],
|
||||
os: 'Remote (tunnel)',
|
||||
architecture: 'Remote',
|
||||
containers: '-',
|
||||
containersRunning: '-',
|
||||
images: '-',
|
||||
memTotal: 0,
|
||||
cpus: '-',
|
||||
pilotLastSeen: node.pilot_last_seen ?? null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private async testLocalConnection(nodeId: number): Promise<{ success: boolean; error?: string; info?: any }> {
|
||||
const db = DatabaseService.getInstance();
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,385 @@
|
||||
import http, { IncomingMessage, Server as HttpServer, ServerResponse } from 'http';
|
||||
import { Socket } from 'net';
|
||||
import { EventEmitter } from 'events';
|
||||
import { WebSocket, WebSocketServer } from 'ws';
|
||||
import {
|
||||
BinaryFrameType,
|
||||
DecodedBinaryFrame,
|
||||
StreamIdAllocator,
|
||||
decodeBinaryFrame,
|
||||
decodeJsonFrame,
|
||||
encodeBinaryFrame,
|
||||
encodeJsonFrame,
|
||||
wsDataToBuffer,
|
||||
wsDataToString,
|
||||
} from '../pilot/protocol';
|
||||
|
||||
const BUFFER_HIGH_WATER_MARK = 4 * 1024 * 1024;
|
||||
const PING_INTERVAL_MS = 30_000;
|
||||
|
||||
interface HttpStreamState {
|
||||
kind: 'http';
|
||||
res: ServerResponse;
|
||||
headersWritten: boolean;
|
||||
}
|
||||
|
||||
interface WsStreamState {
|
||||
kind: 'ws';
|
||||
rawSocket?: Socket;
|
||||
rawHead?: Buffer;
|
||||
upgradeRequest: IncomingMessage;
|
||||
clientWs?: WebSocket;
|
||||
}
|
||||
|
||||
type StreamState = HttpStreamState | WsStreamState;
|
||||
|
||||
/**
|
||||
* Per-tunnel bridge: hosts a loopback HTTP server that demuxes requests into
|
||||
* wire frames sent over the pilot WebSocket, and remuxes response frames back
|
||||
* to the loopback caller.
|
||||
*
|
||||
* The primary's existing http-proxy-middleware setup treats the loopback URL
|
||||
* as just another remote target, so HTTP and WebSocket proxy logic, header
|
||||
* stripping/injection, and license-tier propagation all work unchanged.
|
||||
*/
|
||||
export class PilotTunnelBridge extends EventEmitter {
|
||||
private readonly tunnelWs: WebSocket;
|
||||
private readonly loopback: HttpServer;
|
||||
private readonly wsUpgradeServer: WebSocketServer;
|
||||
private readonly streamIds = new StreamIdAllocator();
|
||||
private readonly streams = new Map<number, StreamState>();
|
||||
private readonly connectedAt = Date.now();
|
||||
private loopbackUrl = '';
|
||||
private pingTimer?: NodeJS.Timeout;
|
||||
private closed = false;
|
||||
|
||||
constructor(_nodeId: number, tunnelWs: WebSocket) {
|
||||
super();
|
||||
this.tunnelWs = tunnelWs;
|
||||
this.loopback = http.createServer();
|
||||
this.wsUpgradeServer = new WebSocketServer({ noServer: true });
|
||||
|
||||
this.loopback.on('request', (req, res) => this.handleLoopbackRequest(req, res));
|
||||
this.loopback.on('upgrade', (req, socket, head) => this.handleLoopbackUpgrade(req, socket as Socket, head));
|
||||
this.loopback.on('clientError', (_err, socket) => {
|
||||
try { socket.destroy(); } catch { /* ignore */ }
|
||||
});
|
||||
|
||||
this.tunnelWs.on('message', (data, isBinary) => this.handleTunnelMessage(data, isBinary));
|
||||
this.tunnelWs.on('close', () => this.onTunnelClose());
|
||||
this.tunnelWs.on('error', () => this.onTunnelClose());
|
||||
}
|
||||
|
||||
public async start(): Promise<void> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onError = (err: Error) => reject(err);
|
||||
this.loopback.once('error', onError);
|
||||
this.loopback.listen(0, '127.0.0.1', () => {
|
||||
const addr = this.loopback.address();
|
||||
if (!addr || typeof addr === 'string') {
|
||||
reject(new Error('loopback server returned unexpected address'));
|
||||
return;
|
||||
}
|
||||
this.loopbackUrl = `http://127.0.0.1:${addr.port}`;
|
||||
this.loopback.removeListener('error', onError);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
this.pingTimer = setInterval(() => {
|
||||
if (this.tunnelWs.readyState === WebSocket.OPEN) {
|
||||
try { this.tunnelWs.ping(); } catch { /* surfaced via 'error' */ }
|
||||
}
|
||||
}, PING_INTERVAL_MS);
|
||||
}
|
||||
|
||||
public getLoopbackUrl(): string { return this.loopbackUrl; }
|
||||
public getConnectedAt(): number { return this.connectedAt; }
|
||||
|
||||
public close(code = 1000, reason = 'closed by primary'): void {
|
||||
if (this.closed) return;
|
||||
this.closed = true;
|
||||
if (this.pingTimer) { clearInterval(this.pingTimer); this.pingTimer = undefined; }
|
||||
|
||||
for (const [, state] of this.streams) this.teardownStream(state);
|
||||
this.streams.clear();
|
||||
|
||||
try { this.tunnelWs.close(code, reason); } catch { /* ignore */ }
|
||||
try { this.loopback.close(); } catch { /* ignore */ }
|
||||
try { this.wsUpgradeServer.close(); } catch { /* ignore */ }
|
||||
this.emit('closed');
|
||||
}
|
||||
|
||||
// --- Loopback HTTP ingress ---
|
||||
|
||||
private handleLoopbackRequest(req: IncomingMessage, res: ServerResponse): void {
|
||||
if (this.closed || this.tunnelWs.readyState !== WebSocket.OPEN) {
|
||||
res.statusCode = 502;
|
||||
res.end('pilot tunnel not ready');
|
||||
return;
|
||||
}
|
||||
|
||||
const streamId = this.streamIds.allocate();
|
||||
this.streams.set(streamId, { kind: 'http', res, headersWritten: false });
|
||||
|
||||
const headers: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(req.headers)) {
|
||||
if (typeof v === 'string') headers[k] = v;
|
||||
else if (Array.isArray(v)) headers[k] = v.join(', ');
|
||||
}
|
||||
|
||||
this.sendJson({
|
||||
t: 'http_req',
|
||||
s: streamId,
|
||||
method: req.method || 'GET',
|
||||
path: req.url || '/',
|
||||
headers,
|
||||
});
|
||||
|
||||
req.on('data', (chunk: Buffer) => {
|
||||
if (!this.streams.has(streamId)) return;
|
||||
this.sendBinary(BinaryFrameType.HttpReqBody, streamId, chunk);
|
||||
if (this.tunnelWs.bufferedAmount > BUFFER_HIGH_WATER_MARK) req.pause();
|
||||
});
|
||||
req.on('end', () => {
|
||||
if (!this.streams.has(streamId)) return;
|
||||
this.sendJson({ t: 'http_req_end', s: streamId });
|
||||
});
|
||||
req.on('error', () => {
|
||||
const s = this.streams.get(streamId);
|
||||
if (s) this.teardownStream(s);
|
||||
this.streams.delete(streamId);
|
||||
});
|
||||
|
||||
res.on('close', () => {
|
||||
// Client disconnected before response finished.
|
||||
if (this.streams.has(streamId)) {
|
||||
this.streams.delete(streamId);
|
||||
this.sendJson({ t: 'http_err', s: streamId, code: 'tunnel_down', message: 'client aborted' });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private handleLoopbackUpgrade(req: IncomingMessage, socket: Socket, head: Buffer): void {
|
||||
if (this.closed || this.tunnelWs.readyState !== WebSocket.OPEN) {
|
||||
socket.write('HTTP/1.1 502 Bad Gateway\r\n\r\n');
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
|
||||
const streamId = this.streamIds.allocate();
|
||||
this.streams.set(streamId, {
|
||||
kind: 'ws',
|
||||
rawSocket: socket,
|
||||
rawHead: head,
|
||||
upgradeRequest: req,
|
||||
});
|
||||
|
||||
const headers: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(req.headers)) {
|
||||
if (typeof v === 'string') headers[k] = v;
|
||||
else if (Array.isArray(v)) headers[k] = v.join(', ');
|
||||
}
|
||||
|
||||
this.sendJson({
|
||||
t: 'ws_open',
|
||||
s: streamId,
|
||||
path: req.url || '/',
|
||||
headers,
|
||||
});
|
||||
|
||||
socket.on('error', () => {
|
||||
const s = this.streams.get(streamId);
|
||||
if (s) this.teardownStream(s);
|
||||
this.streams.delete(streamId);
|
||||
});
|
||||
socket.on('close', () => {
|
||||
if (this.streams.has(streamId)) {
|
||||
this.sendJson({ t: 'ws_close', s: streamId, code: 1006, reason: 'client closed' });
|
||||
this.streams.delete(streamId);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// --- Tunnel ingress (frames from agent) ---
|
||||
|
||||
private handleTunnelMessage(data: unknown, isBinary: boolean): void {
|
||||
if (this.closed) return;
|
||||
try {
|
||||
if (isBinary) {
|
||||
const buf = wsDataToBuffer(data);
|
||||
if (!buf) return;
|
||||
this.handleBinaryFrame(decodeBinaryFrame(buf));
|
||||
} else {
|
||||
const text = wsDataToString(data);
|
||||
if (text == null) return;
|
||||
const frame = decodeJsonFrame(text);
|
||||
this.handleJsonFrame(frame);
|
||||
}
|
||||
} catch {
|
||||
// Malformed frame: kill the tunnel to force re-sync.
|
||||
this.close(1002, 'protocol error');
|
||||
}
|
||||
}
|
||||
|
||||
private handleJsonFrame(frame: ReturnType<typeof decodeJsonFrame>): void {
|
||||
switch (frame.t) {
|
||||
case 'http_res': {
|
||||
const s = this.streams.get(frame.s);
|
||||
if (!s || s.kind !== 'http') return;
|
||||
if (!s.headersWritten) {
|
||||
try {
|
||||
s.res.writeHead(frame.status, frame.headers);
|
||||
} catch { /* headers already sent or invalid */ }
|
||||
s.headersWritten = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'http_res_end': {
|
||||
const s = this.streams.get(frame.s);
|
||||
if (!s || s.kind !== 'http') return;
|
||||
try { s.res.end(); } catch { /* ignore */ }
|
||||
this.streams.delete(frame.s);
|
||||
break;
|
||||
}
|
||||
case 'http_err': {
|
||||
const s = this.streams.get(frame.s);
|
||||
if (!s) return;
|
||||
if (s.kind === 'http' && !s.headersWritten) {
|
||||
try {
|
||||
s.res.writeHead(502, { 'content-type': 'text/plain' });
|
||||
s.res.end(`pilot tunnel error: ${frame.code} ${frame.message}`);
|
||||
} catch { /* ignore */ }
|
||||
} else {
|
||||
this.teardownStream(s);
|
||||
}
|
||||
this.streams.delete(frame.s);
|
||||
break;
|
||||
}
|
||||
case 'ws_accept': {
|
||||
const s = this.streams.get(frame.s);
|
||||
if (!s || s.kind !== 'ws' || !s.rawSocket || !s.rawHead) return;
|
||||
this.wsUpgradeServer.handleUpgrade(s.upgradeRequest, s.rawSocket, s.rawHead, (ws) => {
|
||||
s.clientWs = ws;
|
||||
s.rawSocket = undefined;
|
||||
s.rawHead = undefined;
|
||||
ws.on('message', (msg, isBin) => {
|
||||
if (isBin) {
|
||||
this.sendBinary(BinaryFrameType.WsMessageBinary, frame.s, wsDataToBuffer(msg) ?? Buffer.alloc(0));
|
||||
} else {
|
||||
this.sendJson({ t: 'ws_msg_text', s: frame.s, data: wsDataToString(msg) ?? '' });
|
||||
}
|
||||
});
|
||||
ws.on('close', (code, reason) => {
|
||||
if (this.streams.has(frame.s)) {
|
||||
this.sendJson({ t: 'ws_close', s: frame.s, code, reason: reason?.toString?.() });
|
||||
this.streams.delete(frame.s);
|
||||
}
|
||||
});
|
||||
ws.on('error', () => {
|
||||
if (this.streams.has(frame.s)) this.streams.delete(frame.s);
|
||||
});
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 'ws_reject': {
|
||||
const s = this.streams.get(frame.s);
|
||||
if (!s || s.kind !== 'ws' || !s.rawSocket) return;
|
||||
try {
|
||||
s.rawSocket.write(`HTTP/1.1 ${frame.status} ${frame.message}\r\n\r\n`);
|
||||
s.rawSocket.destroy();
|
||||
} catch { /* ignore */ }
|
||||
this.streams.delete(frame.s);
|
||||
break;
|
||||
}
|
||||
case 'ws_msg_text': {
|
||||
const s = this.streams.get(frame.s);
|
||||
if (!s || s.kind !== 'ws' || !s.clientWs) return;
|
||||
try { s.clientWs.send(frame.data); } catch { /* ignore */ }
|
||||
break;
|
||||
}
|
||||
case 'ws_close': {
|
||||
const s = this.streams.get(frame.s);
|
||||
if (!s || s.kind !== 'ws') return;
|
||||
if (s.clientWs) {
|
||||
try { s.clientWs.close(frame.code, frame.reason); } catch { /* ignore */ }
|
||||
} else if (s.rawSocket) {
|
||||
try { s.rawSocket.destroy(); } catch { /* ignore */ }
|
||||
}
|
||||
this.streams.delete(frame.s);
|
||||
break;
|
||||
}
|
||||
case 'ctrl': {
|
||||
// Primary-side bridge does not act on control ops today; the
|
||||
// upgrade handler consumes enroll_ack before registerTunnel is
|
||||
// called, and ping/pong are handled by the WS layer.
|
||||
break;
|
||||
}
|
||||
default:
|
||||
// Ignore unknown JSON frame types for forward compatibility.
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private handleBinaryFrame(frame: DecodedBinaryFrame): void {
|
||||
const s = this.streams.get(frame.streamId);
|
||||
if (!s) return;
|
||||
switch (frame.type) {
|
||||
case BinaryFrameType.HttpResBody: {
|
||||
if (s.kind !== 'http') return;
|
||||
if (!s.headersWritten) {
|
||||
// Agent sent body before headers; synthesize 200 so we don't drop data.
|
||||
try { s.res.writeHead(200); } catch { /* ignore */ }
|
||||
s.headersWritten = true;
|
||||
}
|
||||
try { s.res.write(frame.payload); } catch { /* ignore */ }
|
||||
break;
|
||||
}
|
||||
case BinaryFrameType.WsMessageBinary: {
|
||||
if (s.kind !== 'ws' || !s.clientWs) return;
|
||||
try { s.clientWs.send(frame.payload, { binary: true }); } catch { /* ignore */ }
|
||||
break;
|
||||
}
|
||||
case BinaryFrameType.HttpReqBody:
|
||||
// Agent never originates request bodies; ignore for defense-in-depth.
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private onTunnelClose(): void {
|
||||
if (this.closed) return;
|
||||
this.close(1006, 'tunnel closed');
|
||||
}
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
private sendJson(frame: Parameters<typeof encodeJsonFrame>[0]): void {
|
||||
if (this.tunnelWs.readyState !== WebSocket.OPEN) return;
|
||||
try { this.tunnelWs.send(encodeJsonFrame(frame)); } catch { /* ignore */ }
|
||||
}
|
||||
|
||||
private sendBinary(type: BinaryFrameType, streamId: number, payload: Buffer): void {
|
||||
if (this.tunnelWs.readyState !== WebSocket.OPEN) return;
|
||||
try { this.tunnelWs.send(encodeBinaryFrame(type, streamId, payload), { binary: true }); } catch { /* ignore */ }
|
||||
}
|
||||
|
||||
private teardownStream(state: StreamState): void {
|
||||
if (state.kind === 'http') {
|
||||
try {
|
||||
if (!state.headersWritten) {
|
||||
state.res.writeHead(502, { 'content-type': 'text/plain' });
|
||||
state.res.end('pilot tunnel closed');
|
||||
} else {
|
||||
state.res.end();
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
} else {
|
||||
if (state.clientWs) {
|
||||
try { state.clientWs.close(1011, 'tunnel closed'); } catch { /* ignore */ }
|
||||
} else if (state.rawSocket) {
|
||||
try { state.rawSocket.destroy(); } catch { /* ignore */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
import { EventEmitter } from 'events';
|
||||
import WebSocket from 'ws';
|
||||
import { PilotTunnelBridge } from './PilotTunnelBridge';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import { PilotCloseCode } from '../pilot/protocol';
|
||||
|
||||
/**
|
||||
* PilotTunnelManager: singleton registry of active pilot tunnels.
|
||||
*
|
||||
* Each enrolled pilot-agent node holds one outbound WebSocket to the primary.
|
||||
* For every such tunnel we spin up a local loopback HTTP server that demuxes
|
||||
* requests into frames. Remote-proxy code paths (http-proxy-middleware and the
|
||||
* WebSocket upgrade handler) can then treat pilot nodes identically to standard
|
||||
* proxy nodes by pointing at the loopback URL.
|
||||
*
|
||||
* Events:
|
||||
* - 'tunnel-up' (nodeId: number) after a tunnel is accepted
|
||||
* - 'tunnel-down' (nodeId: number) after a tunnel closes (for any reason)
|
||||
*/
|
||||
export class PilotTunnelManager extends EventEmitter {
|
||||
private static instance: PilotTunnelManager;
|
||||
private bridges: Map<number, PilotTunnelBridge> = new Map();
|
||||
|
||||
private constructor() {
|
||||
super();
|
||||
this.setMaxListeners(50);
|
||||
}
|
||||
|
||||
public static getInstance(): PilotTunnelManager {
|
||||
if (!PilotTunnelManager.instance) {
|
||||
PilotTunnelManager.instance = new PilotTunnelManager();
|
||||
}
|
||||
return PilotTunnelManager.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Accept a newly handshaked pilot tunnel. Replaces any prior tunnel for the
|
||||
* same node (split-brain prevention): the previous bridge is closed
|
||||
* before the new one is installed.
|
||||
*
|
||||
* Resolves once the loopback HTTP server is listening.
|
||||
*/
|
||||
public async registerTunnel(nodeId: number, ws: WebSocket, agentVersion?: string): Promise<void> {
|
||||
const existing = this.bridges.get(nodeId);
|
||||
if (existing) {
|
||||
existing.close(PilotCloseCode.Replaced, 'replaced by newer tunnel');
|
||||
this.bridges.delete(nodeId);
|
||||
}
|
||||
|
||||
const bridge = new PilotTunnelBridge(nodeId, ws);
|
||||
bridge.once('closed', () => {
|
||||
if (this.bridges.get(nodeId) === bridge) {
|
||||
this.bridges.delete(nodeId);
|
||||
DatabaseService.getInstance().updateNodeStatus(nodeId, 'offline');
|
||||
this.emit('tunnel-down', nodeId);
|
||||
}
|
||||
});
|
||||
await bridge.start();
|
||||
|
||||
this.bridges.set(nodeId, bridge);
|
||||
const db = DatabaseService.getInstance();
|
||||
db.updateNodeStatus(nodeId, 'online');
|
||||
db.updateNode(nodeId, {
|
||||
pilot_last_seen: Date.now(),
|
||||
pilot_agent_version: agentVersion ?? null,
|
||||
});
|
||||
this.emit('tunnel-up', nodeId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the loopback base URL (http://127.0.0.1:PORT) for a node's active
|
||||
* tunnel, or null if no tunnel is currently registered.
|
||||
*/
|
||||
public getLoopbackUrl(nodeId: number): string | null {
|
||||
const bridge = this.bridges.get(nodeId);
|
||||
return bridge ? bridge.getLoopbackUrl() : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* True if a tunnel for this node is registered and healthy.
|
||||
*/
|
||||
public hasActiveTunnel(nodeId: number): boolean {
|
||||
return this.bridges.has(nodeId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Force-close a tunnel (e.g., on node deletion).
|
||||
*/
|
||||
public closeTunnel(nodeId: number, code = 1000, reason = 'closed by primary'): void {
|
||||
const bridge = this.bridges.get(nodeId);
|
||||
if (!bridge) return;
|
||||
bridge.close(code, reason);
|
||||
this.bridges.delete(nodeId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Snapshot of currently active tunnels.
|
||||
*/
|
||||
public listActive(): Array<{ nodeId: number; loopbackUrl: string; connectedAt: number }> {
|
||||
return Array.from(this.bridges.entries()).map(([nodeId, bridge]) => ({
|
||||
nodeId,
|
||||
loopbackUrl: bridge.getLoopbackUrl(),
|
||||
connectedAt: bridge.getConnectedAt(),
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Record an application-level heartbeat from the agent.
|
||||
*/
|
||||
public touch(nodeId: number): void {
|
||||
if (!this.bridges.has(nodeId)) return;
|
||||
DatabaseService.getInstance().updateNode(nodeId, { pilot_last_seen: Date.now() });
|
||||
}
|
||||
}
|
||||
@@ -102,6 +102,7 @@
|
||||
"features/global-observability",
|
||||
"features/host-console",
|
||||
"features/multi-node",
|
||||
"features/pilot-agent",
|
||||
"features/fleet-view",
|
||||
"features/fleet-sync",
|
||||
"features/remote-updates",
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
---
|
||||
title: Pilot Agent
|
||||
description: Add remote nodes behind NAT, residential networks, or corporate firewalls without exposing any inbound port.
|
||||
---
|
||||
|
||||
Pilot Agent mode connects a remote server to your primary Sencho instance through a single outbound WebSocket tunnel. Every request the primary sends to that node, HTTP or WebSocket, rides through this tunnel. The remote host never opens an inbound port and never needs a TLS certificate.
|
||||
|
||||
## When to use Pilot Agent
|
||||
|
||||
Pick Pilot Agent when the remote host:
|
||||
- Sits behind NAT or a residential router.
|
||||
- Lives on a corporate network that blocks inbound connections.
|
||||
- Roams between networks (laptops, mobile hotspots, rotating cloud IPs).
|
||||
- Would otherwise need a reverse proxy, a dynamic DNS entry, or a self-signed TLS certificate just to be reachable.
|
||||
|
||||
Pick **Distributed API Proxy** (documented in [Multi-Node Management](/features/multi-node)) when the remote host already has a stable, reachable URL, for example a VPS with a public IP or a LAN server on a home network. Both modes are supported side-by-side, one per node.
|
||||
|
||||
## How it works
|
||||
|
||||
The agent runs inside a second container on the remote host, using the same `saelix/sencho:latest` image. Setting `SENCHO_MODE=pilot` plus a primary URL and a one-time enrollment token puts the container into agent mode. On boot it dials the primary at `wss://<primary>/api/pilot/tunnel` and holds that connection open. For every tunneled request the primary demultiplexes frames to an internal loopback server, which re-issues the request locally on the agent host against its Docker socket. License tier, role checks, and all other authorization continue to flow from the primary, exactly like proxy mode.
|
||||
|
||||
Only outbound HTTPS from the remote to the primary is required. Nothing else is exposed.
|
||||
|
||||
## Enrollment walkthrough
|
||||
|
||||
### 1. Add the node on the primary
|
||||
|
||||
On your primary instance open **Settings → Nodes** and click **Add Node**.
|
||||
|
||||
- **Type:** Remote
|
||||
- **Mode:** Pilot Agent (the default for remote nodes)
|
||||
- **Name:** any label, for example `homelab-nuc`
|
||||
- **Compose Directory:** the folder on the remote host where stack folders will live
|
||||
|
||||
Click **Add Node**. The form is replaced by an enrollment dialog with a one-line `docker run` command.
|
||||
|
||||
<Frame>
|
||||
<img src="/images/pilot-agent/enrollment-dialog.png" alt="Pilot Agent enrollment dialog with docker run command" />
|
||||
</Frame>
|
||||
|
||||
### 2. Run the command on the remote host
|
||||
|
||||
Copy the command and paste it on the remote host. It looks like:
|
||||
|
||||
```bash
|
||||
docker run -d --restart=unless-stopped --name sencho-agent \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-v sencho-agent-data:/app/data \
|
||||
-v /opt/docker/sencho:/app/compose \
|
||||
-e SENCHO_MODE=pilot \
|
||||
-e SENCHO_PRIMARY_URL=https://sencho.example.com \
|
||||
-e SENCHO_ENROLL_TOKEN=<short-lived token> \
|
||||
saelix/sencho:latest
|
||||
```
|
||||
|
||||
The enrollment token is single-use and expires after 15 minutes. On first connect the agent exchanges it for a long-lived tunnel credential, persisted inside the container volume at `/app/data/pilot.jwt`. Subsequent restarts reconnect automatically without needing a new token.
|
||||
|
||||
### 3. Confirm the node is online
|
||||
|
||||
The node flips to **Online** in the primary within a few seconds of the agent container starting. The **Endpoint** column shows `tunnel` alongside the time the primary last saw a frame from the agent.
|
||||
|
||||
Open the node in the sidebar switcher and use it exactly like any other node: deploy stacks, tail logs, open container terminals, watch host stats.
|
||||
|
||||
## Day-to-day operation
|
||||
|
||||
Once enrolled, a Pilot Agent node is indistinguishable from a proxy-mode node in the UI. The same switcher, the same stacks page, the same editor, the same host console, the same dashboard.
|
||||
|
||||
The agent reconnects automatically if the tunnel drops (transient network blip, primary restart, remote host reboot). It applies an exponential backoff between retries starting at 1 second and capping at 60 seconds, so reconnect traffic stays bounded even during long outages.
|
||||
|
||||
## Regenerating enrollment
|
||||
|
||||
If the agent container is destroyed before its first successful connect, or the enrollment token expires before you paste it on the remote, open the node's edit dialog on the primary and click **Regenerate enrollment token**. A fresh 15-minute token is issued and the previous tunnel (if any) is disconnected so the new agent replaces it cleanly.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
<AccordionGroup>
|
||||
<Accordion title="Node stays Offline after starting the agent container">
|
||||
Check the agent container's logs for the first `[Pilot]` line. It prints the primary URL it is dialing and the reason for any failed connect. Common causes: the primary URL is wrong or unreachable from the remote host, the enrollment token expired, or the primary is behind a reverse proxy that strips WebSocket upgrades.
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Tunnel keeps disconnecting and reconnecting">
|
||||
Usually caused by an HTTP proxy or load balancer in front of the primary with a short idle timeout. The tunnel sends a ping every 30 seconds, so raise the proxy's WebSocket idle timeout above that (90 seconds is a safe floor). If you terminate TLS on the proxy, make sure WebSocket upgrade passthrough is enabled.
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Enrollment token expired before I ran the docker command">
|
||||
Open **Settings → Nodes** on the primary, click the pencil icon on the pending node, then **Regenerate enrollment token**. The dialog shows a fresh command with a new token.
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="I ran the enrollment command on the wrong host">
|
||||
Stop and remove the agent container on the wrong host (`docker stop sencho-agent && docker rm sencho-agent`), then regenerate the enrollment token on the primary and run the command on the correct host.
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Primary was restored from backup and the agent will not reconnect">
|
||||
The agent's persisted tunnel credential is signed with the primary's JWT secret. If that secret is rotated or the primary is rebuilt from scratch, existing tunnels stop verifying. Regenerate enrollment for the affected node and restart the agent container so it consumes the fresh token.
|
||||
</Accordion>
|
||||
</AccordionGroup>
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 49 KiB |
@@ -31,6 +31,9 @@ test.describe('Node management', () => {
|
||||
// #node-type is a Radix UI combobox - click to open, then pick the option.
|
||||
await page.locator('#node-type').click();
|
||||
await page.getByRole('option', { name: /remote/i }).click();
|
||||
// Remote nodes default to Pilot Agent mode; switch to Proxy so the api_url field renders.
|
||||
await page.locator('#node-mode').click();
|
||||
await page.getByRole('button', { name: /distributed api proxy/i }).click();
|
||||
// Confirm the API URL field is now visible before proceeding
|
||||
await expect(page.locator('#node-api-url')).toBeVisible({ timeout: 3_000 });
|
||||
return true;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useState, useEffect, useCallback, useMemo } from 'react';
|
||||
import { useNodes } from '@/context/NodeContext';
|
||||
import type { Node } from '@/context/NodeContext';
|
||||
import type { Node, NodeMode } from '@/context/NodeContext';
|
||||
import { apiFetch } from '@/lib/api';
|
||||
import { toast } from '@/components/ui/toast-store';
|
||||
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter, DialogTrigger } from './ui/dialog';
|
||||
@@ -13,7 +13,8 @@ import { Separator } from './ui/separator';
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from './ui/table';
|
||||
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from './ui/tooltip';
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from './ui/select';
|
||||
import { Plus, Trash2, Wifi, WifiOff, Star, Pencil, Server, Monitor, Globe, Copy, KeyRound, Check, AlertTriangle, Calendar, RefreshCw } from 'lucide-react';
|
||||
import { Combobox } from './ui/combobox';
|
||||
import { Plus, Trash2, Wifi, WifiOff, Star, Pencil, Server, Monitor, Globe, Copy, KeyRound, Check, AlertTriangle, Calendar, RefreshCw, Terminal } from 'lucide-react';
|
||||
|
||||
interface NodeSchedulingSummary {
|
||||
active_tasks: number;
|
||||
@@ -41,15 +42,23 @@ function formatRelativeTime(timestamp: number): string {
|
||||
interface NodeFormData {
|
||||
name: string;
|
||||
type: 'local' | 'remote';
|
||||
mode: NodeMode;
|
||||
api_url: string;
|
||||
api_token: string;
|
||||
compose_dir: string;
|
||||
is_default: boolean;
|
||||
}
|
||||
|
||||
interface PilotEnrollment {
|
||||
token: string;
|
||||
expiresAt: number;
|
||||
dockerRun: string;
|
||||
}
|
||||
|
||||
const defaultFormData: NodeFormData = {
|
||||
name: '',
|
||||
type: 'remote',
|
||||
mode: 'pilot_agent',
|
||||
api_url: '',
|
||||
api_token: '',
|
||||
compose_dir: '/app/compose',
|
||||
@@ -72,6 +81,10 @@ export function NodeManager() {
|
||||
const [generatingToken, setGeneratingToken] = useState(false);
|
||||
const [tokenCopied, setTokenCopied] = useState(false);
|
||||
|
||||
// Pilot enrollment state (shown after creating a pilot-agent node)
|
||||
const [activeEnrollment, setActiveEnrollment] = useState<{ nodeId: number; nodeName: string; enrollment: PilotEnrollment } | null>(null);
|
||||
const [enrollmentCopied, setEnrollmentCopied] = useState(false);
|
||||
|
||||
// Per-node scheduling summary
|
||||
const [nodeSummary, setNodeSummary] = useState<Record<number, NodeSchedulingSummary>>({});
|
||||
|
||||
@@ -100,13 +113,23 @@ export function NodeManager() {
|
||||
const err = await res.json();
|
||||
throw new Error(err.error || 'Failed to create node');
|
||||
}
|
||||
const { id: newNodeId } = await res.json();
|
||||
const body = await res.json();
|
||||
const newNodeId: number | undefined = body.id;
|
||||
const enrollment: PilotEnrollment | undefined = body.enrollment;
|
||||
toast.success(`Node "${formData.name}" created successfully`);
|
||||
setCreateOpen(false);
|
||||
setFormData(defaultFormData);
|
||||
|
||||
// Auto-test the new node connection immediately
|
||||
if (newNodeId && formData.type === 'remote') {
|
||||
const isPilot = formData.type === 'remote' && formData.mode === 'pilot_agent';
|
||||
if (isPilot && newNodeId && enrollment) {
|
||||
// Keep the dialog open so the admin can copy the docker run command.
|
||||
setActiveEnrollment({ nodeId: newNodeId, nodeName: formData.name, enrollment });
|
||||
} else {
|
||||
setCreateOpen(false);
|
||||
setFormData(defaultFormData);
|
||||
}
|
||||
|
||||
// Auto-test the new node connection immediately (proxy mode only;
|
||||
// pilot agents flip online asynchronously once the container connects).
|
||||
if (newNodeId && formData.type === 'remote' && formData.mode === 'proxy') {
|
||||
setTesting(newNodeId);
|
||||
try {
|
||||
const testRes = await apiFetch(`/nodes/${newNodeId}/test`, { method: 'POST' });
|
||||
@@ -130,6 +153,34 @@ export function NodeManager() {
|
||||
}
|
||||
};
|
||||
|
||||
const regenerateEnrollment = async (node: Node) => {
|
||||
try {
|
||||
const res = await apiFetch(`/nodes/${node.id}/pilot/enroll`, { method: 'POST', localOnly: true });
|
||||
if (!res.ok) {
|
||||
const err = await res.json();
|
||||
throw new Error(err.error || 'Failed to regenerate enrollment');
|
||||
}
|
||||
const { enrollment } = (await res.json()) as { enrollment: PilotEnrollment };
|
||||
setActiveEnrollment({ nodeId: node.id, nodeName: node.name, enrollment });
|
||||
toast.success('New enrollment token generated');
|
||||
} catch (error) {
|
||||
toast.error((error as Error).message || 'Failed to regenerate enrollment');
|
||||
}
|
||||
};
|
||||
|
||||
const copyEnrollment = async () => {
|
||||
if (!activeEnrollment) return;
|
||||
const text = activeEnrollment.enrollment.dockerRun;
|
||||
try {
|
||||
await navigator.clipboard.writeText(text);
|
||||
setEnrollmentCopied(true);
|
||||
toast.success('Command copied to clipboard');
|
||||
setTimeout(() => setEnrollmentCopied(false), 2000);
|
||||
} catch {
|
||||
toast.error('Could not copy automatically - please select and copy the command manually.');
|
||||
}
|
||||
};
|
||||
|
||||
const handleEdit = async () => {
|
||||
if (!editingNodeId) return;
|
||||
try {
|
||||
@@ -155,6 +206,7 @@ export function NodeManager() {
|
||||
setFormData({
|
||||
name: node.name,
|
||||
type: node.type,
|
||||
mode: (node.mode === 'pilot_agent' ? 'pilot_agent' : 'proxy'),
|
||||
api_url: node.api_url || '',
|
||||
api_token: node.api_token || '',
|
||||
compose_dir: node.compose_dir,
|
||||
@@ -302,6 +354,24 @@ export function NodeManager() {
|
||||
</div>
|
||||
|
||||
{formData.type === 'remote' && (
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="node-mode">Mode</Label>
|
||||
<Combobox
|
||||
id="node-mode"
|
||||
value={formData.mode}
|
||||
onValueChange={(val) => setFormData({ ...formData, mode: val as NodeMode, api_url: '', api_token: '' })}
|
||||
options={[
|
||||
{ value: 'pilot_agent', label: 'Pilot Agent - outbound tunnel from remote host' },
|
||||
{ value: 'proxy', label: 'Distributed API Proxy - primary dials the remote' },
|
||||
]}
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Pilot Agent requires only outbound HTTPS from the remote host. Distributed API Proxy requires the remote host to expose an inbound port.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{formData.type === 'remote' && formData.mode === 'proxy' && (
|
||||
<>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="node-api-url">Sencho API URL</Label>
|
||||
@@ -394,7 +464,10 @@ export function NodeManager() {
|
||||
<Button variant="outline" onClick={() => setCreateOpen(false)}>Cancel</Button>
|
||||
<Button
|
||||
onClick={handleCreate}
|
||||
disabled={!formData.name || (formData.type === 'remote' && (!formData.api_url || !formData.api_token))}
|
||||
disabled={
|
||||
!formData.name ||
|
||||
(formData.type === 'remote' && formData.mode === 'proxy' && (!formData.api_url || !formData.api_token))
|
||||
}
|
||||
>
|
||||
Add Node
|
||||
</Button>
|
||||
@@ -446,6 +519,7 @@ export function NodeManager() {
|
||||
<TableHead className="w-10"></TableHead>
|
||||
<TableHead>Name</TableHead>
|
||||
<TableHead>Type</TableHead>
|
||||
<TableHead>Mode</TableHead>
|
||||
<TableHead>Endpoint</TableHead>
|
||||
<TableHead>Status</TableHead>
|
||||
<TableHead>Schedules</TableHead>
|
||||
@@ -477,8 +551,29 @@ export function NodeManager() {
|
||||
<TableCell>
|
||||
<Badge variant="outline">{node.type === 'local' ? 'Local' : 'Remote'}</Badge>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
{node.type === 'local' ? (
|
||||
<span className="text-muted-foreground text-sm">-</span>
|
||||
) : node.mode === 'pilot_agent' ? (
|
||||
<Badge variant="outline" className="gap-1 text-xs">
|
||||
<Terminal className="w-3 h-3" strokeWidth={1.5} />
|
||||
Pilot Agent
|
||||
</Badge>
|
||||
) : (
|
||||
<Badge variant="outline" className="gap-1 text-xs">
|
||||
<Globe className="w-3 h-3" strokeWidth={1.5} />
|
||||
Proxy
|
||||
</Badge>
|
||||
)}
|
||||
</TableCell>
|
||||
<TableCell className="text-muted-foreground text-sm font-mono">
|
||||
{node.type === 'local' ? 'docker.sock' : (node.api_url || '-')}
|
||||
{node.type === 'local'
|
||||
? 'docker.sock'
|
||||
: node.mode === 'pilot_agent'
|
||||
? (node.pilot_last_seen
|
||||
? `tunnel (seen ${formatRelativeTime(node.pilot_last_seen + 60_000)} ago)`
|
||||
: 'tunnel (waiting)')
|
||||
: (node.api_url || '-')}
|
||||
</TableCell>
|
||||
<TableCell>{getStatusBadge(node.status)}</TableCell>
|
||||
<TableCell>
|
||||
@@ -643,11 +738,33 @@ export function NodeManager() {
|
||||
<DialogTitle>Edit Node</DialogTitle>
|
||||
</DialogHeader>
|
||||
{renderFormFields()}
|
||||
{formData.type === 'remote' && formData.mode === 'pilot_agent' && editingNodeId !== null && (
|
||||
<div className="rounded-md border border-card-border bg-card/50 p-3 space-y-2">
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Re-enroll the agent if the container was lost or the enrollment token expired. The previous tunnel is disconnected automatically.
|
||||
</p>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => {
|
||||
const node = nodes.find((n) => n.id === editingNodeId);
|
||||
if (node) regenerateEnrollment(node);
|
||||
}}
|
||||
className="gap-1"
|
||||
>
|
||||
<RefreshCw className="w-3.5 h-3.5" strokeWidth={1.5} />
|
||||
Regenerate enrollment token
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
<DialogFooter>
|
||||
<Button variant="outline" onClick={() => { setEditOpen(false); setEditingNodeId(null); }}>Cancel</Button>
|
||||
<Button
|
||||
onClick={handleEdit}
|
||||
disabled={!formData.name || (formData.type === 'remote' && !formData.api_url)}
|
||||
disabled={
|
||||
!formData.name ||
|
||||
(formData.type === 'remote' && formData.mode === 'proxy' && !formData.api_url)
|
||||
}
|
||||
>
|
||||
Save Changes
|
||||
</Button>
|
||||
@@ -655,6 +772,57 @@ export function NodeManager() {
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
|
||||
{/* Pilot enrollment dialog (create + regenerate flows both open this) */}
|
||||
<Dialog
|
||||
open={activeEnrollment !== null}
|
||||
onOpenChange={(open) => {
|
||||
if (!open) {
|
||||
setActiveEnrollment(null);
|
||||
setEnrollmentCopied(false);
|
||||
setCreateOpen(false);
|
||||
setFormData(defaultFormData);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<DialogContent className="max-w-2xl">
|
||||
<DialogHeader className="pr-8">
|
||||
<DialogTitle>Enroll the pilot agent</DialogTitle>
|
||||
</DialogHeader>
|
||||
{activeEnrollment && (
|
||||
<div className="space-y-3 py-2">
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Run this command on <strong>{activeEnrollment.nodeName}</strong> to start the pilot agent. The token below is valid for 15 minutes and can only be used once.
|
||||
</p>
|
||||
<div className="rounded-md border border-card-border bg-muted/50 p-3">
|
||||
<pre className="text-xs font-mono whitespace-pre-wrap break-all text-foreground/90">{activeEnrollment.enrollment.dockerRun}</pre>
|
||||
</div>
|
||||
<div className="flex items-center justify-between">
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Expires <span className="font-mono tabular-nums">{formatRelativeTime(activeEnrollment.enrollment.expiresAt)}</span> from now.
|
||||
</p>
|
||||
<Button size="sm" variant="outline" className="gap-1" onClick={copyEnrollment}>
|
||||
{enrollmentCopied ? <Check className="w-3.5 h-3.5 text-success" /> : <Copy className="w-3.5 h-3.5" />}
|
||||
{enrollmentCopied ? 'Copied' : 'Copy command'}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<DialogFooter>
|
||||
<Button
|
||||
onClick={() => {
|
||||
setActiveEnrollment(null);
|
||||
setEnrollmentCopied(false);
|
||||
setCreateOpen(false);
|
||||
setEditOpen(false);
|
||||
setFormData(defaultFormData);
|
||||
}}
|
||||
>
|
||||
Done
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
|
||||
{/* Delete Confirmation */}
|
||||
<AlertDialog open={deleteOpen} onOpenChange={setDeleteOpen}>
|
||||
<AlertDialogContent>
|
||||
|
||||
@@ -17,6 +17,7 @@ interface ComboboxProps {
|
||||
emptyText?: string
|
||||
disabled?: boolean
|
||||
className?: string
|
||||
id?: string
|
||||
}
|
||||
|
||||
export function Combobox({
|
||||
@@ -28,6 +29,7 @@ export function Combobox({
|
||||
emptyText = "No results found.",
|
||||
disabled = false,
|
||||
className,
|
||||
id,
|
||||
}: ComboboxProps) {
|
||||
const [open, setOpen] = React.useState(false)
|
||||
const [search, setSearch] = React.useState("")
|
||||
@@ -95,6 +97,7 @@ export function Combobox({
|
||||
<button
|
||||
type="button"
|
||||
role="combobox"
|
||||
id={id}
|
||||
aria-expanded={false}
|
||||
disabled={disabled}
|
||||
onClick={() => { if (!disabled) setOpen(true) }}
|
||||
|
||||
@@ -2,16 +2,21 @@ import React, { createContext, useContext, useState, useEffect, useCallback, use
|
||||
import { apiFetch } from '@/lib/api';
|
||||
import type { Capability } from '@/lib/capabilities';
|
||||
|
||||
export type NodeMode = 'proxy' | 'pilot_agent';
|
||||
|
||||
export interface Node {
|
||||
id: number;
|
||||
name: string;
|
||||
type: 'local' | 'remote';
|
||||
mode?: NodeMode;
|
||||
compose_dir: string;
|
||||
is_default: boolean;
|
||||
status: 'online' | 'offline' | 'unknown';
|
||||
created_at: number;
|
||||
api_url?: string;
|
||||
api_token?: string;
|
||||
pilot_last_seen?: number | null;
|
||||
pilot_agent_version?: string | null;
|
||||
}
|
||||
|
||||
export interface NodeMeta {
|
||||
|
||||
Reference in New Issue
Block a user