fix(pilot): preserve tunnel JWT across upgrade 401 reconnects (#1818)

Stop deleting pilot.jwt on any WebSocket 401 and falling back to an already-consumed enroll token. Attempt enroll recovery only on upgrade 401/404 when SENCHO_ENROLL_TOKEN is still fresh, restore the in-memory tunnel credential if that attempt fails, and surface a clear re-enrollment prompt instead of looping silently.

Closes #1817

backend/src/__tests__/pilot-agent-auth-fallback-unit.test.ts

@@ -2,6 +2,7 @@
  * Unit tests for PilotAgent auth fallback event handling using a stub WebSocket.
  */
 import { describe, it, expect, vi, beforeEach, afterEach, beforeAll, afterAll } from 'vitest';
+import jwt from 'jsonwebtoken';
 import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
 
 const { wsInstances, mockAttachSwitchboard } = vi.hoisted(() => ({
@@ -9,6 +10,8 @@ const { wsInstances, mockAttachSwitchboard } = vi.hoisted(() => ({
         emit: (event: string, ...args: unknown[]) => boolean;
         readyState: number;
         close: ReturnType<typeof vi.fn>;
+        terminate: ReturnType<typeof vi.fn>;
+        on: (event: string, listener: (...args: unknown[]) => void) => unknown;
     }>,
     mockAttachSwitchboard: vi.fn(() => ({
         handleJsonFrame: vi.fn(() => false),
@@ -30,9 +33,10 @@ vi.mock('ws', () => {
     class MockWebSocket extends EventEmitter {
         readyState = 0;
         close = vi.fn();
+        terminate = vi.fn();
         constructor(..._args: unknown[]) {
             super();
-            wsInstances.push(this);
+            wsInstances.push(this as never);
         }
     }
     return { default: MockWebSocket };
@@ -43,87 +47,294 @@ let PilotAgent: typeof import('../pilot/agent').PilotAgent;
 let readPersistedToken: typeof import('../pilot/agent').readPersistedToken;
 let persistToken: typeof import('../pilot/agent').persistToken;
 let clearPersistedToken: typeof import('../pilot/agent').clearPersistedToken;
+let isFreshEnrollToken: typeof import('../pilot/agent').isFreshEnrollToken;
 
 beforeAll(async () => {
     tmpDir = await setupTestDb();
-    ({ PilotAgent, readPersistedToken, persistToken, clearPersistedToken } = await import('../pilot/agent'));
+    ({
+        PilotAgent,
+        readPersistedToken,
+        persistToken,
+        clearPersistedToken,
+        isFreshEnrollToken,
+    } = await import('../pilot/agent'));
 });
 
 afterAll(() => {
     cleanupTestDb(tmpDir);
 });
 
+function mintFreshEnroll(): string {
+    return jwt.sign({ scope: 'pilot_enroll', nodeId: 1 }, 'unit-test-secret', { expiresIn: '15m' });
+}
+
+function mintExpiredEnroll(): string {
+    // Explicit exp: negative expiresIn is unreliable in jsonwebtoken.
+    return jwt.sign(
+        { scope: 'pilot_enroll', nodeId: 1, exp: Math.floor(Date.now() / 1000) - 120 },
+        'unit-test-secret',
+    );
+}
+
+function emitUpgradeReject(
+    ws: (typeof wsInstances)[number],
+    status: number,
+    reason?: string,
+): void {
+    const headers: Record<string, string> = {};
+    if (reason) headers['x-sencho-pilot-reject'] = reason;
+    ws.emit('unexpected-response', {}, {
+        statusCode: status,
+        headers,
+        resume: vi.fn(),
+    });
+}
+
+describe('isFreshEnrollToken', () => {
+    it('accepts an unexpired pilot_enroll JWT', () => {
+        expect(isFreshEnrollToken(mintFreshEnroll())).toBe(true);
+    });
+
+    it('rejects an expired pilot_enroll JWT', () => {
+        expect(isFreshEnrollToken(mintExpiredEnroll())).toBe(false);
+    });
+
+    it('rejects a non-JWT string', () => {
+        expect(isFreshEnrollToken('not-a-jwt')).toBe(false);
+    });
+});
+
 describe('PilotAgent auth fallback (stub WebSocket)', () => {
     beforeEach(() => {
         wsInstances.length = 0;
         vi.clearAllMocks();
         vi.spyOn(console, 'log').mockImplementation(() => { /* swallow */ });
         vi.spyOn(console, 'warn').mockImplementation(() => { /* swallow */ });
+        vi.useFakeTimers();
     });
 
     afterEach(() => {
+        vi.runOnlyPendingTimers();
+        vi.useRealTimers();
         vi.restoreAllMocks();
         clearPersistedToken();
     });
 
-    it('swaps to the enroll token and clears pilot.jwt after HTTP 401 on upgrade', () => {
+    it('swaps to a fresh enroll token after HTTP 401 and leaves pilot.jwt on disk', () => {
         persistToken('stale-on-disk');
+        const enroll = mintFreshEnroll();
         const agent = new PilotAgent({
             primaryUrl: 'http://primary.invalid',
             loopbackPort: 1,
             initialToken: 'stale-token',
-            enrollToken: 'fresh-enroll-token',
+            enrollToken: enroll,
             enrolling: false,
         });
 
         (agent as unknown as { connect: () => void }).connect();
 
         const firstWs = wsInstances[0]!;
-        // Real ws (no unexpected-response listener): abortHandshake emits error then close.
-        firstWs.emit('error', new Error('Unexpected server response: 401'));
-        firstWs.emit('close', 1006, Buffer.from(''));
+        emitUpgradeReject(firstWs, 401, 'invalid_token');
 
-        expect(readPersistedToken()).toBeNull();
-        expect((agent as unknown as { token: string }).token).toBe('fresh-enroll-token');
+        expect(readPersistedToken()).toBe('stale-on-disk');
+        expect((agent as unknown as { token: string }).token).toBe(enroll);
         // scheduleReconnect doubles backoff after scheduling the imminent retry.
         expect((agent as unknown as { backoff: number }).backoff).toBe(2_000);
     });
 
     it('swaps to the enroll token after HTTP 404 on upgrade', () => {
         persistToken('stale-on-disk');
+        const enroll = mintFreshEnroll();
         const agent = new PilotAgent({
             primaryUrl: 'http://primary.invalid',
             loopbackPort: 1,
             initialToken: 'stale-token',
-            enrollToken: 'fresh-enroll-token',
+            enrollToken: enroll,
             enrolling: false,
         });
 
         (agent as unknown as { connect: () => void }).connect();
+        emitUpgradeReject(wsInstances[0]!, 404, 'unknown_node');
 
-        const firstWs = wsInstances[0]!;
-        firstWs.emit('error', new Error('Unexpected server response: 404'));
-        firstWs.emit('close', 1006, Buffer.from(''));
-
-        expect(readPersistedToken()).toBeNull();
-        expect((agent as unknown as { token: string }).token).toBe('fresh-enroll-token');
+        expect(readPersistedToken()).toBe('stale-on-disk');
+        expect((agent as unknown as { token: string }).token).toBe(enroll);
     });
 
-    it('does not swap when already connecting with the enroll token', () => {
+    it('does not swap when the enroll token is expired', () => {
+        persistToken('stale-on-disk');
         const agent = new PilotAgent({
             primaryUrl: 'http://primary.invalid',
             loopbackPort: 1,
-            initialToken: 'only-enroll-token',
-            enrollToken: 'only-enroll-token',
+            initialToken: 'stale-token',
+            enrollToken: mintExpiredEnroll(),
+            enrolling: false,
+        });
+
+        (agent as unknown as { connect: () => void }).connect();
+        emitUpgradeReject(wsInstances[0]!, 401, 'invalid_token');
+
+        expect(readPersistedToken()).toBe('stale-on-disk');
+        expect((agent as unknown as { token: string }).token).toBe('stale-token');
+    });
+
+    it('does not swap on a clean close even with a fresh enroll token', () => {
+        persistToken('good-on-disk');
+        const enroll = mintFreshEnroll();
+        const agent = new PilotAgent({
+            primaryUrl: 'http://primary.invalid',
+            loopbackPort: 1,
+            initialToken: 'good-token',
+            enrollToken: enroll,
+            enrolling: false,
+        });
+
+        (agent as unknown as { connect: () => void }).connect();
+        wsInstances[0]!.emit('close', 1000, Buffer.from(''));
+
+        expect(readPersistedToken()).toBe('good-on-disk');
+        expect((agent as unknown as { token: string }).token).toBe('good-token');
+    });
+
+    it('does not swap when already connecting with the enroll token', () => {
+        const enroll = mintFreshEnroll();
+        const agent = new PilotAgent({
+            primaryUrl: 'http://primary.invalid',
+            loopbackPort: 1,
+            initialToken: enroll,
+            enrollToken: enroll,
             enrolling: true,
         });
 
         (agent as unknown as { connect: () => void }).connect();
+        emitUpgradeReject(wsInstances[0]!, 401, 'enrollment_used');
 
+        // Failed enroll dial with no prior tunnelToken leaves token as enroll.
+        expect((agent as unknown as { token: string }).token).toBe(enroll);
+    });
+
+    it('restores the in-memory tunnel token when enroll fallback also 401s', () => {
+        const enroll = mintFreshEnroll();
+        const agent = new PilotAgent({
+            primaryUrl: 'http://primary.invalid',
+            loopbackPort: 1,
+            initialToken: 'tunnel-in-memory',
+            enrollToken: enroll,
+            enrolling: false,
+        });
+        // Simulate persist failure: no file on disk.
+        clearPersistedToken();
+
+        (agent as unknown as { connect: () => void }).connect();
+        emitUpgradeReject(wsInstances[0]!, 401);
+        expect((agent as unknown as { token: string }).token).toBe(enroll);
+
+        // Advance into the reconnect attempt with the enroll token.
+        vi.runOnlyPendingTimers();
+        expect(wsInstances.length).toBe(2);
+        emitUpgradeReject(wsInstances[1]!, 401, 'enrollment_used');
+
+        expect((agent as unknown as { token: string }).token).toBe('tunnel-in-memory');
+        expect(readPersistedToken()).toBeNull();
+    });
+
+    it('restores the tunnel token when the enroll dial drops without a 401/404', () => {
+        const enroll = mintFreshEnroll();
+        const agent = new PilotAgent({
+            primaryUrl: 'http://primary.invalid',
+            loopbackPort: 1,
+            initialToken: 'tunnel-in-memory',
+            enrollToken: enroll,
+            enrolling: false,
+        });
+
+        (agent as unknown as { connect: () => void }).connect();
+        emitUpgradeReject(wsInstances[0]!, 401);
+        expect((agent as unknown as { token: string }).token).toBe(enroll);
+
+        vi.runOnlyPendingTimers();
+        // Clean close / network drop while dialing enroll (no rejectInfo).
+        wsInstances[1]!.emit('close', 1006, Buffer.from(''));
+
+        expect((agent as unknown as { token: string }).token).toBe('tunnel-in-memory');
+    });
+
+    it('still swaps when the reject reason is enrollment_used (header is diagnostic only)', () => {
+        persistToken('stale-on-disk');
+        const enroll = mintFreshEnroll();
+        const agent = new PilotAgent({
+            primaryUrl: 'http://primary.invalid',
+            loopbackPort: 1,
+            initialToken: 'stale-token',
+            enrollToken: enroll,
+            enrolling: false,
+        });
+
+        (agent as unknown as { connect: () => void }).connect();
+        emitUpgradeReject(wsInstances[0]!, 401, 'enrollment_used');
+
+        expect(readPersistedToken()).toBe('stale-on-disk');
+        expect((agent as unknown as { token: string }).token).toBe(enroll);
+    });
+
+    it('does not swap on hub 403', () => {
+        persistToken('good-on-disk');
+        const enroll = mintFreshEnroll();
+        const agent = new PilotAgent({
+            primaryUrl: 'http://primary.invalid',
+            loopbackPort: 1,
+            initialToken: 'good-token',
+            enrollToken: enroll,
+            enrolling: false,
+        });
+
+        (agent as unknown as { connect: () => void }).connect();
+        emitUpgradeReject(wsInstances[0]!, 403, 'bad_scope');
+
+        expect(readPersistedToken()).toBe('good-on-disk');
+        expect((agent as unknown as { token: string }).token).toBe('good-token');
+    });
+
+    it('does not double-schedule reconnect when a stale socket closes after the next connect', () => {
+        const agent = new PilotAgent({
+            primaryUrl: 'http://primary.invalid',
+            loopbackPort: 1,
+            initialToken: 'tunnel-token',
+            enrollToken: mintFreshEnroll(),
+            enrolling: false,
+        });
+
+        (agent as unknown as { connect: () => void }).connect();
         const firstWs = wsInstances[0]!;
-        firstWs.emit('error', new Error('Unexpected server response: 401'));
-        firstWs.emit('close', 1006, Buffer.from(''));
+        emitUpgradeReject(firstWs, 401);
+        const afterFirst = wsInstances.length;
 
-        expect((agent as unknown as { token: string }).token).toBe('only-enroll-token');
+        vi.runOnlyPendingTimers();
+        expect(wsInstances.length).toBe(afterFirst + 1);
+
+        // Late close from the first socket must be a no-op for reconnect.
+        firstWs.emit('close', 1006, Buffer.from(''));
+        const beforeAdvance = wsInstances.length;
+        vi.runOnlyPendingTimers();
+        expect(wsInstances.length).toBe(beforeAdvance);
+    });
+
+    it('keeps the tunnel token on opaque 401 when enroll is fresh (fallback still swaps in memory)', () => {
+        persistToken('stale-on-disk');
+        const enroll = mintFreshEnroll();
+        const agent = new PilotAgent({
+            primaryUrl: 'http://primary.invalid',
+            loopbackPort: 1,
+            initialToken: 'stale-token',
+            enrollToken: enroll,
+            enrolling: false,
+        });
+
+        (agent as unknown as { connect: () => void }).connect();
+        // No X-Sencho-Pilot-Reject header (proxy / older hub).
+        emitUpgradeReject(wsInstances[0]!, 401);
+
+        expect(readPersistedToken()).toBe('stale-on-disk');
+        expect((agent as unknown as { token: string }).token).toBe(enroll);
+        expect((agent as unknown as { backoff: number }).backoff).toBe(2_000);
     });
 });

backend/src/__tests__/pilot-agent-auth-fallback.test.ts

@@ -2,12 +2,12 @@
  * Regression tests for pilot agent auth fallback when a persisted tunnel
  * token is rejected at WebSocket upgrade (401 invalid JWT, 404 unknown node).
  *
- * Without fallback the agent reconnects forever with the same stale
- * pilot.jwt credential even when SENCHO_ENROLL_TOKEN carries a fresh
- * enrollment JWT.
+ * Preserves pilot.jwt until a successful enroll_ack; enroll fallback requires
+ * upgrade 401/404 plus a still-fresh SENCHO_ENROLL_TOKEN.
  */
-import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest';
+import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
 import http from 'http';
+import crypto from 'crypto';
 import jwt from 'jsonwebtoken';
 import { WebSocketServer } from 'ws';
 import { setupTestDb, cleanupTestDb, TEST_JWT_SECRET } from './helpers/setupTestDb';
@@ -18,11 +18,10 @@ import { WebSocket } from 'ws';
 let tmpDir: string;
 let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
 
-// agent.ts freezes its pilot.jwt path from DATA_DIR at module load, so it must
-// be imported only after setupTestDb() points DATA_DIR at the writable tmp dir.
 let readPersistedToken: typeof import('../pilot/agent').readPersistedToken;
 let persistToken: typeof import('../pilot/agent').persistToken;
 let clearPersistedToken: typeof import('../pilot/agent').clearPersistedToken;
+let PilotAgent: typeof import('../pilot/agent').PilotAgent;
 
 let server: http.Server;
 let port: number;
@@ -33,7 +32,7 @@ let nodeId: number;
 beforeAll(async () => {
     tmpDir = await setupTestDb();
     ({ DatabaseService } = await import('../services/DatabaseService'));
-    ({ readPersistedToken, persistToken, clearPersistedToken } = await import('../pilot/agent'));
+    ({ readPersistedToken, persistToken, clearPersistedToken, PilotAgent } = await import('../pilot/agent'));
 
     server = http.createServer();
     mainWss = new WebSocketServer({ noServer: true });
@@ -78,6 +77,7 @@ afterAll(async () => {
 afterEach(() => {
     PilotTunnelManager.getInstance().closeTunnel(nodeId);
     clearPersistedToken();
+    vi.useRealTimers();
 });
 
 function mintStaleTunnelTokenWrongSecret(): string {
@@ -96,6 +96,68 @@ function mintStaleTunnelTokenWrongNode(): string {
     );
 }
 
+function mintFreshEnrollForNode(): string {
+    const expiresAt = Date.now() + 15 * 60 * 1000;
+    const token = jwt.sign(
+        { scope: 'pilot_enroll', nodeId, enrollNonce: crypto.randomUUID() },
+        TEST_JWT_SECRET,
+        { expiresIn: '15m' },
+    );
+    const tokenHash = crypto.createHash('sha256').update(token).digest('hex');
+    DatabaseService.getInstance().createPilotEnrollment(nodeId, tokenHash, expiresAt);
+    return token;
+}
+
+function mintExpiredEnroll(): string {
+    return jwt.sign(
+        {
+            scope: 'pilot_enroll',
+            nodeId,
+            enrollNonce: crypto.randomUUID(),
+            exp: Math.floor(Date.now() / 1000) - 120,
+        },
+        TEST_JWT_SECRET,
+    );
+}
+
+function stopAgent(agent: InstanceType<typeof PilotAgent>): void {
+    (agent as unknown as { shuttingDown: boolean }).shuttingDown = true;
+    const timer = (agent as unknown as { reconnectTimer?: NodeJS.Timeout }).reconnectTimer;
+    if (timer) clearTimeout(timer);
+    (agent as unknown as { reconnectTimer?: NodeJS.Timeout }).reconnectTimer = undefined;
+    try { (agent as unknown as { ws: WebSocket | null }).ws?.terminate(); } catch { /* ignore */ }
+}
+
+function enrollmentRowUsed(): boolean {
+    const row = DatabaseService.getInstance().getDb()
+        .prepare('SELECT used_at FROM pilot_enrollments WHERE node_id = ?')
+        .get(nodeId) as { used_at: number | null } | undefined;
+    return row?.used_at != null;
+}
+
+function waitForToken(
+    agent: InstanceType<typeof PilotAgent>,
+    predicate: (token: string) => boolean,
+    timeoutMs = 3_000,
+): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const started = Date.now();
+        const tick = () => {
+            const token = (agent as unknown as { token: string }).token;
+            if (predicate(token)) {
+                resolve(token);
+                return;
+            }
+            if (Date.now() - started > timeoutMs) {
+                reject(new Error(`timed out waiting for token change; still ${token.slice(0, 24)}…`));
+                return;
+            }
+            setTimeout(tick, 25);
+        };
+        tick();
+    });
+}
+
 describe('clearPersistedToken', () => {
     it('removes an existing pilot.jwt file', () => {
         persistToken('stale-token');
@@ -112,7 +174,7 @@ describe('clearPersistedToken', () => {
 });
 
 describe('pilot tunnel upgrade rejection (in-process integration)', () => {
-    it('rejects a stale tunnel JWT signed with the wrong secret at upgrade', async () => {
+    it('rejects a stale tunnel JWT signed with the wrong secret at upgrade with reject header', async () => {
         const staleToken = mintStaleTunnelTokenWrongSecret();
         const ws = new WebSocket(`ws://127.0.0.1:${port}/api/pilot/tunnel`, {
             headers: {
@@ -120,14 +182,19 @@ describe('pilot tunnel upgrade rejection (in-process integration)', () => {
                 'x-sencho-agent-version': 'auth-fallback-test/1.0',
             },
         });
-        const result = await new Promise<{ status?: number }>((resolve) => {
+        const result = await new Promise<{ status?: number; reason?: string }>((resolve) => {
             ws.on('unexpected-response', (_req, res) => {
-                resolve({ status: res.statusCode });
+                const reasonHeader = res.headers['x-sencho-pilot-reject'];
+                resolve({
+                    status: res.statusCode,
+                    reason: Array.isArray(reasonHeader) ? reasonHeader[0] : reasonHeader,
+                });
                 res.destroy();
             });
             ws.on('error', () => { /* close follows */ });
         });
         expect(result.status).toBe(401);
+        expect(result.reason).toBe('invalid_token');
     });
 
     it('rejects a tunnel JWT for an unknown node with HTTP 404 at upgrade', async () => {
@@ -138,13 +205,173 @@ describe('pilot tunnel upgrade rejection (in-process integration)', () => {
                 'x-sencho-agent-version': 'auth-fallback-test/1.0',
             },
         });
-        const result = await new Promise<{ status?: number }>((resolve) => {
+        const result = await new Promise<{ status?: number; reason?: string }>((resolve) => {
             ws.on('unexpected-response', (_req, res) => {
-                resolve({ status: res.statusCode });
+                const reasonHeader = res.headers['x-sencho-pilot-reject'];
+                resolve({
+                    status: res.statusCode,
+                    reason: Array.isArray(reasonHeader) ? reasonHeader[0] : reasonHeader,
+                });
                 res.destroy();
             });
             ws.on('error', () => { /* close follows */ });
         });
         expect(result.status).toBe(404);
+        expect(result.reason).toBe('unknown_node');
+    });
+});
+
+describe('PilotAgent reconnect recovery (real hub)', () => {
+    it('swaps to a fresh enroll token on hub 401 and leaves pilot.jwt unchanged until enroll_ack', async () => {
+        const diskToken = mintStaleTunnelTokenWrongSecret();
+        persistToken(diskToken);
+        const enroll = mintFreshEnrollForNode();
+
+        const agent = new PilotAgent({
+            primaryUrl: `http://127.0.0.1:${port}`,
+            loopbackPort: 1,
+            initialToken: diskToken,
+            enrollToken: enroll,
+            enrolling: false,
+        });
+
+        (agent as unknown as { connect: () => void }).connect();
+        await waitForToken(agent, (t) => t === enroll);
+
+        expect(readPersistedToken()).toBe(diskToken);
+        expect(enrollmentRowUsed()).toBe(false);
+
+        // Next reconnect should complete enrollment and overwrite disk.
+        await waitForToken(agent, (t) => t !== enroll && t !== diskToken, 5_000);
+        // Allow persistToken + any in-flight sibling dial to settle.
+        await new Promise((r) => setTimeout(r, 100));
+        const finalToken = (agent as unknown as { token: string }).token;
+        expect(finalToken).not.toBe(diskToken);
+        expect(finalToken).not.toBe(enroll);
+        expect(readPersistedToken()).toBe(finalToken);
+        expect(enrollmentRowUsed()).toBe(true);
+
+        stopAgent(agent);
+    });
+
+    it('keeps pilot.jwt and dial token when enroll is expired', async () => {
+        const diskToken = mintStaleTunnelTokenWrongSecret();
+        persistToken(diskToken);
+        const agent = new PilotAgent({
+            primaryUrl: `http://127.0.0.1:${port}`,
+            loopbackPort: 1,
+            initialToken: diskToken,
+            enrollToken: mintExpiredEnroll(),
+            enrolling: false,
+        });
+
+        (agent as unknown as { connect: () => void }).connect();
+        await new Promise((r) => setTimeout(r, 400));
+
+        expect(readPersistedToken()).toBe(diskToken);
+        expect((agent as unknown as { token: string }).token).toBe(diskToken);
+
+        stopAgent(agent);
+    });
+
+    it('does not consume enrollment or swap on a post-handshake capacity close (1013)', async () => {
+        const goodTunnel = jwt.sign(
+            { scope: 'pilot_tunnel', nodeId },
+            TEST_JWT_SECRET,
+            { expiresIn: '365d' },
+        );
+        persistToken(goodTunnel);
+        const enroll = mintFreshEnrollForNode();
+
+        const agent = new PilotAgent({
+            primaryUrl: `http://127.0.0.1:${port}`,
+            loopbackPort: 1,
+            initialToken: goodTunnel,
+            enrollToken: enroll,
+            enrolling: false,
+        });
+
+        const up = new Promise<void>((resolve) => {
+            PilotTunnelManager.getInstance().once('tunnel-up', () => resolve());
+        });
+        (agent as unknown as { connect: () => void }).connect();
+        await up;
+
+        // Force a post-handshake close that mimics capacity (1013).
+        PilotTunnelManager.getInstance().closeTunnel(nodeId);
+        await new Promise((r) => setTimeout(r, 300));
+
+        expect((agent as unknown as { token: string }).token).toBe(goodTunnel);
+        expect(readPersistedToken()).toBe(goodTunnel);
+        expect(enrollmentRowUsed()).toBe(false);
+
+        stopAgent(agent);
+    });
+
+    it('schedules exactly one reconnect per rejected upgrade', async () => {
+        const diskToken = mintStaleTunnelTokenWrongSecret();
+        persistToken(diskToken);
+
+        const agent = new PilotAgent({
+            primaryUrl: `http://127.0.0.1:${port}`,
+            loopbackPort: 1,
+            initialToken: diskToken,
+            enrollToken: mintExpiredEnroll(),
+            enrolling: false,
+        });
+
+        const connectSpy = vi.spyOn(agent as unknown as { connect: () => void }, 'connect');
+        (agent as unknown as { connect: () => void }).connect();
+        // Initial call + wait for one scheduled reconnect.
+        await new Promise((r) => setTimeout(r, 1_600));
+
+        // connect() was invoked once by us and once by scheduleReconnect.
+        expect(connectSpy.mock.calls.length).toBe(2);
+
+        stopAgent(agent);
+        connectSpy.mockRestore();
+    });
+});
+
+describe('PilotAgent opaque proxy 401 (no Sencho reject header)', () => {
+    it('still attempts enroll fallback when a fresh enroll token is present', async () => {
+        const opaque = http.createServer((_req, res) => {
+            res.writeHead(401, { 'Content-Type': 'text/plain' });
+            res.end('Unauthorized');
+        });
+        const opaquePort = await new Promise<number>((resolve, reject) => {
+            opaque.listen(0, '127.0.0.1', () => {
+                const addr = opaque.address();
+                if (!addr || typeof addr === 'string') {
+                    reject(new Error('listen failed'));
+                    return;
+                }
+                resolve(addr.port);
+            });
+        });
+
+        const diskToken = 'opaque-disk-token';
+        persistToken(diskToken);
+        const enroll = jwt.sign(
+            { scope: 'pilot_enroll', nodeId: 1, enrollNonce: crypto.randomUUID() },
+            'irrelevant',
+            { expiresIn: '15m' },
+        );
+
+        const agent = new PilotAgent({
+            primaryUrl: `http://127.0.0.1:${opaquePort}`,
+            loopbackPort: 1,
+            initialToken: diskToken,
+            enrollToken: enroll,
+            enrolling: false,
+        });
+
+        (agent as unknown as { connect: () => void }).connect();
+        await waitForToken(agent, (t) => t === enroll);
+
+        expect(readPersistedToken()).toBe(diskToken);
+
+        stopAgent(agent);
+        await new Promise<void>((resolve) => opaque.close(() => resolve()));
     });
 });

backend/src/__tests__/reject-upgrade.test.ts

@@ -0,0 +1,46 @@
+/**
+ * Byte-level contract for rejectUpgrade: omitted headers stay historically
+ * identical; optional headers are written before the terminating blank line.
+ */
+import { describe, it, expect, vi } from 'vitest';
+import type { Duplex } from 'stream';
+import { rejectUpgrade } from '../websocket/reject';
+
+function captureWrite(): { socket: Duplex; chunks: string[] } {
+  const chunks: string[] = [];
+  const socket = {
+    write: vi.fn((data: string | Buffer) => {
+      chunks.push(typeof data === 'string' ? data : data.toString('utf8'));
+      return true;
+    }),
+    destroy: vi.fn(),
+  } as unknown as Duplex;
+  return { socket, chunks };
+}
+
+describe('rejectUpgrade', () => {
+  it('emits the historical byte sequence when headers are omitted', () => {
+    const { socket, chunks } = captureWrite();
+    rejectUpgrade(socket, 401, 'Unauthorized');
+    expect(chunks.join('')).toBe('HTTP/1.1 401 Unauthorized\r\n\r\n');
+    expect(socket.destroy).toHaveBeenCalledOnce();
+  });
+
+  it('writes optional headers before the terminating blank line', () => {
+    const { socket, chunks } = captureWrite();
+    rejectUpgrade(socket, 401, 'Unauthorized', {
+      'X-Sencho-Pilot-Reject': 'invalid_token',
+      Connection: 'close',
+    });
+    const body = chunks.join('');
+    expect(body.startsWith('HTTP/1.1 401 Unauthorized\r\n')).toBe(true);
+    expect(body).toContain('X-Sencho-Pilot-Reject: invalid_token\r\n');
+    expect(body).toContain('Connection: close\r\n');
+    expect(body.endsWith('\r\n\r\n')).toBe(true);
+    const blankIndex = body.lastIndexOf('\r\n\r\n');
+    const headerBlock = body.slice(0, blankIndex);
+    expect(headerBlock).toContain('X-Sencho-Pilot-Reject: invalid_token');
+    expect(headerBlock).toContain('Connection: close');
+    expect(socket.destroy).toHaveBeenCalledOnce();
+  });
+});