gsadmin/sencho
1
0
Fork 0
mirror of https://github.com/Studio-Saelix/sencho.git synced 2026-08-09 02:12:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: fall back to enroll token when pilot tunnel JWT is rejected (#1566)

Browse Source 
* fix: fall back to enroll token when pilot tunnel JWT is rejected

On HTTP 401/404 upgrade rejection, delete stale pilot.jwt and retry with SENCHO_ENROLL_TOKEN.

* test: import pilot agent module after DATA_DIR is set in fallback test

The auth-fallback test statically imported pilot/agent, which freezes its
pilot.jwt path from DATA_DIR at module load, before setupTestDb redirects
DATA_DIR to a writable temp dir. On the Linux CI runner the path resolved
to a non-writable /app/data, so persistToken silently failed and the
round-trip assertion read null. Import the module dynamically in beforeAll
after setupTestDb, matching the sibling unit test.

* fix: remove unexpected-response listener that blocked pilot reconnect

The ws library skips abortHandshake when an unexpected-response listener
exists, so error and close never fire and the agent hangs in CONNECTING.
Detect auth rejection via the abortHandshake error message instead; the
close handler already performs enroll-token fallback and reconnect.
This commit is contained in:
Anso
2026-07-05 01:03:09 -04:00
committed by GitHub
parent fdbc1b1ebb
commit c677b8bb66
6 changed files with 368 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/pilot/agent.ts
+57 -4
View File
@@ -61,6 +61,7 @@ export function startPilotAgent(loopbackPort: number): void {
primaryUrl,
loopbackPort,
initialToken: persistedToken || enrollToken!,
enrollToken: enrollToken ?? null,
enrolling: !persistedToken,
});
// Register the agent as MeshService's reverse dialer so outbound
@@ -80,13 +81,18 @@ interface AgentOptions {
primaryUrl: string;
loopbackPort: number;
initialToken: string;
enrollToken: string | null;
enrolling: boolean;
}
export class PilotAgent {
private readonly options: AgentOptions;
private token: string;
/** Fallback credential from SENCHO_ENROLL_TOKEN when the persisted token is rejected. */
private readonly enrollToken: string | null;
private backoff = RECONNECT_MIN_MS;
/** Set when the WS upgrade is rejected with 401 or 404 before the handshake completes. */
private upgradeRejected = false;
private ws: WebSocket | null = null;
private pingTimer?: NodeJS.Timeout;
private reconnectTimer?: NodeJS.Timeout;
@@ -113,6 +119,7 @@ export class PilotAgent {
constructor(options: AgentOptions) {
this.options = options;
this.token = options.initialToken;
this.enrollToken = options.enrollToken;
this.agentVersion = getSenchoVersion() || '0.0.0';
this.customCa = readPilotCaBundle();
}
@@ -170,6 +177,8 @@ export class PilotAgent {
private connect(): void {
if (this.shuttingDown) return;
this.upgradeRejected = false;
const wsUrl = httpUrlToWs(this.options.primaryUrl) + '/api/pilot/tunnel';
const ws = new WebSocket(wsUrl, {
headers: {
@@ -187,6 +196,22 @@ export class PilotAgent {
...(this.customCa ? { ca: this.customCa } : {}),
});
this.ws = ws;
let opened = false;
let lastHandshakeError: string | null = null;
// Do NOT register an 'unexpected-response' listener here. The ws library
// skips abortHandshake (and therefore never emits 'error' or 'close')
// when a listener exists for that event. Auth rejection is detected via
// the error message abortHandshake emits: "Unexpected server response: N".
ws.on('error', (err) => {
console.warn('[Pilot] Tunnel error:', err.message);
lastHandshakeError = err.message;
if (/Unexpected server response: (401|404)/.test(err.message)) {
this.upgradeRejected = true;
}
// 'close' will follow; reconnect is scheduled there.
});
this.switchboard = attachTcpStreamSwitchboard({
ws,
resolveTarget: resolveByComposeLabels,
@@ -195,6 +220,7 @@ export class PilotAgent {
});
ws.on('open', () => {
opened = true;
// Backoff intentionally NOT reset here: a TCP-level connect that
// immediately fails the protocol handshake (incompatible version,
// bad token consumed at upgrade) would otherwise reset the
@@ -222,12 +248,19 @@ export class PilotAgent {
ws.on('close', (code, reason) => {
console.log('[Pilot] Tunnel closed:', code, reason?.toString?.() ?? '');
this.cleanupAfterDisconnect();
const authRejected = this.upgradeRejected
|| (!opened && lastHandshakeError != null && /Unexpected server response: (401|404)/.test(lastHandshakeError));
if (authRejected && this.enrollToken && this.token !== this.enrollToken) {
clearPersistedToken();
console.log('[Pilot] Persisted token rejected; falling back to enroll token');
this.token = this.enrollToken;
this.backoff = RECONNECT_MIN_MS;
}
this.upgradeRejected = false;
this.scheduleReconnect();
});
ws.on('error', (err) => {
console.warn('[Pilot] Tunnel error:', err.message);
// 'close' will follow; reconnect is scheduled there.
});
}
private cleanupAfterDisconnect(): void {
@@ -612,3 +645,23 @@ export function persistToken(token: string): void {
}
}
/**
* Remove the persisted long-lived tunnel token from disk. Used when the
* control instance rejects the stored credential at upgrade so the agent
* can fall back to SENCHO_ENROLL_TOKEN without re-poisoning on restart.
* ENOENT is normal when no file was written yet.
*
* Exposed for unit tests.
*/
export function clearPersistedToken(): void {
try {
fs.unlinkSync(TOKEN_PATH);
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') return;
console.warn(
`[Pilot] Failed to remove persisted tunnel token at ${sanitizeForLog(TOKEN_PATH)} (${sanitizeForLog(code ?? 'unknown')}: ${sanitizeForLog((err as Error).message)})`,
);
}
}