diff --git a/backend/src/__tests__/pilot-agent-auth-fallback-unit.test.ts b/backend/src/__tests__/pilot-agent-auth-fallback-unit.test.ts new file mode 100644 index 00000000..00e826a5 --- /dev/null +++ b/backend/src/__tests__/pilot-agent-auth-fallback-unit.test.ts @@ -0,0 +1,129 @@ +/** + * Unit tests for PilotAgent auth fallback event handling using a stub WebSocket. + */ +import { describe, it, expect, vi, beforeEach, afterEach, beforeAll, afterAll } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; + +const { wsInstances, mockAttachSwitchboard } = vi.hoisted(() => ({ + wsInstances: [] as Array<{ + emit: (event: string, ...args: unknown[]) => boolean; + readyState: number; + close: ReturnType; + }>, + mockAttachSwitchboard: vi.fn(() => ({ + handleJsonFrame: vi.fn(() => false), + handleBinaryFrame: vi.fn(() => false), + cleanup: vi.fn(), + tcpStreamCount: vi.fn(() => 0), + openReverseStream: vi.fn(() => null), + })), +})); + +vi.mock('../mesh/tcpStreamSwitchboard', () => ({ + attachTcpStreamSwitchboard: mockAttachSwitchboard, + resolveByComposeLabels: vi.fn(), +})); + +vi.mock('ws', () => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { EventEmitter } = require('events') as typeof import('events'); + class MockWebSocket extends EventEmitter { + readyState = 0; + close = vi.fn(); + constructor(..._args: unknown[]) { + super(); + wsInstances.push(this); + } + } + return { default: MockWebSocket }; +}); + +let tmpDir: string; +let PilotAgent: typeof import('../pilot/agent').PilotAgent; +let readPersistedToken: typeof import('../pilot/agent').readPersistedToken; +let persistToken: typeof import('../pilot/agent').persistToken; +let clearPersistedToken: typeof import('../pilot/agent').clearPersistedToken; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ PilotAgent, readPersistedToken, persistToken, clearPersistedToken } = await import('../pilot/agent')); +}); + +afterAll(() => { + cleanupTestDb(tmpDir); +}); + +describe('PilotAgent auth fallback (stub WebSocket)', () => { + beforeEach(() => { + wsInstances.length = 0; + vi.clearAllMocks(); + vi.spyOn(console, 'log').mockImplementation(() => { /* swallow */ }); + vi.spyOn(console, 'warn').mockImplementation(() => { /* swallow */ }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + clearPersistedToken(); + }); + + it('swaps to the enroll token and clears pilot.jwt after HTTP 401 on upgrade', () => { + persistToken('stale-on-disk'); + const agent = new PilotAgent({ + primaryUrl: 'http://primary.invalid', + loopbackPort: 1, + initialToken: 'stale-token', + enrollToken: 'fresh-enroll-token', + enrolling: false, + }); + + (agent as unknown as { connect: () => void }).connect(); + + const firstWs = wsInstances[0]!; + // Real ws (no unexpected-response listener): abortHandshake emits error then close. + firstWs.emit('error', new Error('Unexpected server response: 401')); + firstWs.emit('close', 1006, Buffer.from('')); + + expect(readPersistedToken()).toBeNull(); + expect((agent as unknown as { token: string }).token).toBe('fresh-enroll-token'); + // scheduleReconnect doubles backoff after scheduling the imminent retry. + expect((agent as unknown as { backoff: number }).backoff).toBe(2_000); + }); + + it('swaps to the enroll token after HTTP 404 on upgrade', () => { + persistToken('stale-on-disk'); + const agent = new PilotAgent({ + primaryUrl: 'http://primary.invalid', + loopbackPort: 1, + initialToken: 'stale-token', + enrollToken: 'fresh-enroll-token', + enrolling: false, + }); + + (agent as unknown as { connect: () => void }).connect(); + + const firstWs = wsInstances[0]!; + firstWs.emit('error', new Error('Unexpected server response: 404')); + firstWs.emit('close', 1006, Buffer.from('')); + + expect(readPersistedToken()).toBeNull(); + expect((agent as unknown as { token: string }).token).toBe('fresh-enroll-token'); + }); + + it('does not swap when already connecting with the enroll token', () => { + const agent = new PilotAgent({ + primaryUrl: 'http://primary.invalid', + loopbackPort: 1, + initialToken: 'only-enroll-token', + enrollToken: 'only-enroll-token', + enrolling: true, + }); + + (agent as unknown as { connect: () => void }).connect(); + + const firstWs = wsInstances[0]!; + firstWs.emit('error', new Error('Unexpected server response: 401')); + firstWs.emit('close', 1006, Buffer.from('')); + + expect((agent as unknown as { token: string }).token).toBe('only-enroll-token'); + }); +}); diff --git a/backend/src/__tests__/pilot-agent-auth-fallback.test.ts b/backend/src/__tests__/pilot-agent-auth-fallback.test.ts new file mode 100644 index 00000000..ab1a0e69 --- /dev/null +++ b/backend/src/__tests__/pilot-agent-auth-fallback.test.ts @@ -0,0 +1,150 @@ +/** + * Regression tests for pilot agent auth fallback when a persisted tunnel + * token is rejected at WebSocket upgrade (401 invalid JWT, 404 unknown node). + * + * Without fallback the agent reconnects forever with the same stale + * pilot.jwt credential even when SENCHO_ENROLL_TOKEN carries a fresh + * enrollment JWT. + */ +import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest'; +import http from 'http'; +import jwt from 'jsonwebtoken'; +import { WebSocketServer } from 'ws'; +import { setupTestDb, cleanupTestDb, TEST_JWT_SECRET } from './helpers/setupTestDb'; +import { attachUpgrade } from '../websocket/upgradeHandler'; +import { PilotTunnelManager } from '../services/PilotTunnelManager'; +import { WebSocket } from 'ws'; + +let tmpDir: string; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; + +// agent.ts freezes its pilot.jwt path from DATA_DIR at module load, so it must +// be imported only after setupTestDb() points DATA_DIR at the writable tmp dir. +let readPersistedToken: typeof import('../pilot/agent').readPersistedToken; +let persistToken: typeof import('../pilot/agent').persistToken; +let clearPersistedToken: typeof import('../pilot/agent').clearPersistedToken; + +let server: http.Server; +let port: number; +let pilotTunnelWss: WebSocketServer; +let mainWss: WebSocketServer; +let nodeId: number; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); + ({ readPersistedToken, persistToken, clearPersistedToken } = await import('../pilot/agent')); + + server = http.createServer(); + mainWss = new WebSocketServer({ noServer: true }); + pilotTunnelWss = new WebSocketServer({ noServer: true }); + attachUpgrade(server, { wss: mainWss, pilotTunnelWss }); + + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + const addr = server.address(); + if (!addr || typeof addr === 'string') { + reject(new Error('listen returned unexpected address')); + return; + } + port = addr.port; + resolve(); + }); + }); + + nodeId = DatabaseService.getInstance().addNode({ + name: `pilot-auth-fallback-${Date.now()}`, + type: 'remote', + mode: 'pilot_agent', + compose_dir: '/tmp/x', + is_default: false, + api_url: '', + api_token: '', + }); +}); + +afterAll(async () => { + const mgr = PilotTunnelManager.getInstance(); + mgr.closeTunnel(nodeId); + mgr.removeAllListeners('tunnel-up'); + mgr.removeAllListeners('tunnel-down'); + pilotTunnelWss.close(); + mainWss.close(); + await new Promise((resolve) => server.close(() => resolve())); + cleanupTestDb(tmpDir); +}); + +afterEach(() => { + PilotTunnelManager.getInstance().closeTunnel(nodeId); + clearPersistedToken(); +}); + +function mintStaleTunnelTokenWrongSecret(): string { + return jwt.sign( + { scope: 'pilot_tunnel', nodeId }, + 'wrong-secret-not-the-control-instance', + { expiresIn: '365d' }, + ); +} + +function mintStaleTunnelTokenWrongNode(): string { + return jwt.sign( + { scope: 'pilot_tunnel', nodeId: 99_999_999 }, + TEST_JWT_SECRET, + { expiresIn: '365d' }, + ); +} + +describe('clearPersistedToken', () => { + it('removes an existing pilot.jwt file', () => { + persistToken('stale-token'); + expect(readPersistedToken()).toBe('stale-token'); + clearPersistedToken(); + expect(readPersistedToken()).toBeNull(); + }); + + it('does not throw when the file is already absent', () => { + clearPersistedToken(); + expect(() => clearPersistedToken()).not.toThrow(); + expect(readPersistedToken()).toBeNull(); + }); +}); + +describe('pilot tunnel upgrade rejection (in-process integration)', () => { + it('rejects a stale tunnel JWT signed with the wrong secret at upgrade', async () => { + const staleToken = mintStaleTunnelTokenWrongSecret(); + const ws = new WebSocket(`ws://127.0.0.1:${port}/api/pilot/tunnel`, { + headers: { + Authorization: `Bearer ${staleToken}`, + 'x-sencho-agent-version': 'auth-fallback-test/1.0', + }, + }); + const result = await new Promise<{ status?: number }>((resolve) => { + ws.on('unexpected-response', (_req, res) => { + resolve({ status: res.statusCode }); + res.destroy(); + }); + ws.on('error', () => { /* close follows */ }); + }); + expect(result.status).toBe(401); + }); + + it('rejects a tunnel JWT for an unknown node with HTTP 404 at upgrade', async () => { + const staleToken = mintStaleTunnelTokenWrongNode(); + const ws = new WebSocket(`ws://127.0.0.1:${port}/api/pilot/tunnel`, { + headers: { + Authorization: `Bearer ${staleToken}`, + 'x-sencho-agent-version': 'auth-fallback-test/1.0', + }, + }); + const result = await new Promise<{ status?: number }>((resolve) => { + ws.on('unexpected-response', (_req, res) => { + resolve({ status: res.statusCode }); + res.destroy(); + }); + ws.on('error', () => { /* close follows */ }); + }); + expect(result.status).toBe(404); + }); +}); diff --git a/backend/src/__tests__/pilot-agent-fs-errors.test.ts b/backend/src/__tests__/pilot-agent-fs-errors.test.ts index b832f9bd..172eb147 100644 --- a/backend/src/__tests__/pilot-agent-fs-errors.test.ts +++ b/backend/src/__tests__/pilot-agent-fs-errors.test.ts @@ -19,11 +19,12 @@ */ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -const { mockReadFileSync, mockWriteFileSync, mockExistsSync, mockMkdirSync } = vi.hoisted(() => ({ +const { mockReadFileSync, mockWriteFileSync, mockExistsSync, mockMkdirSync, mockUnlinkSync } = vi.hoisted(() => ({ mockReadFileSync: vi.fn(), mockWriteFileSync: vi.fn(), mockExistsSync: vi.fn(), mockMkdirSync: vi.fn(), + mockUnlinkSync: vi.fn(), })); vi.mock('fs', () => { @@ -32,13 +33,14 @@ vi.mock('fs', () => { writeFileSync: mockWriteFileSync, existsSync: mockExistsSync, mkdirSync: mockMkdirSync, + unlinkSync: mockUnlinkSync, }; return { ...mock, default: mock }; }); // agent.ts is imported AFTER vi.mock so the mock is in place when the // module's top-level fs import resolves. -import { readPersistedToken, persistToken } from '../pilot/agent'; +import { readPersistedToken, persistToken, clearPersistedToken } from '../pilot/agent'; let errorSpy: ReturnType; let warnSpy: ReturnType; @@ -167,3 +169,25 @@ describe('persistToken', () => { expect(() => persistToken('test-token')).not.toThrow(); }); }); + +describe('clearPersistedToken', () => { + it('unlinks the token file on the happy path', () => { + clearPersistedToken(); + expect(mockUnlinkSync).toHaveBeenCalledWith(expect.stringContaining('pilot.jwt')); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it('does not warn on ENOENT (file already absent)', () => { + mockUnlinkSync.mockImplementationOnce(() => { throw fsError('ENOENT', 'no such file'); }); + clearPersistedToken(); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it('warns on EACCES (read-only volume)', () => { + mockUnlinkSync.mockImplementationOnce(() => { throw fsError('EACCES', 'permission denied'); }); + clearPersistedToken(); + expect(warnSpy).toHaveBeenCalledOnce(); + expect(String(warnSpy.mock.calls[0][0])).toContain('Failed to remove persisted tunnel token'); + expect(String(warnSpy.mock.calls[0][0])).toContain('EACCES'); + }); +}); diff --git a/backend/src/__tests__/pilot-agent-loopback-auth.test.ts b/backend/src/__tests__/pilot-agent-loopback-auth.test.ts index 0b88120e..ed0ea08e 100644 --- a/backend/src/__tests__/pilot-agent-loopback-auth.test.ts +++ b/backend/src/__tests__/pilot-agent-loopback-auth.test.ts @@ -33,6 +33,7 @@ describe('PilotAgent loopback auth injection', () => { primaryUrl: 'http://primary.invalid', loopbackPort: 1, initialToken: 'irrelevant-for-this-test', + enrollToken: null, enrolling: false, }); mintHeader = (agent as unknown as { getLoopbackAuthHeader: () => string | null }).getLoopbackAuthHeader.bind(agent); diff --git a/backend/src/pilot/agent.ts b/backend/src/pilot/agent.ts index 4419ab4a..58c30a6a 100644 --- a/backend/src/pilot/agent.ts +++ b/backend/src/pilot/agent.ts @@ -61,6 +61,7 @@ export function startPilotAgent(loopbackPort: number): void { primaryUrl, loopbackPort, initialToken: persistedToken || enrollToken!, + enrollToken: enrollToken ?? null, enrolling: !persistedToken, }); // Register the agent as MeshService's reverse dialer so outbound @@ -80,13 +81,18 @@ interface AgentOptions { primaryUrl: string; loopbackPort: number; initialToken: string; + enrollToken: string | null; enrolling: boolean; } export class PilotAgent { private readonly options: AgentOptions; private token: string; + /** Fallback credential from SENCHO_ENROLL_TOKEN when the persisted token is rejected. */ + private readonly enrollToken: string | null; private backoff = RECONNECT_MIN_MS; + /** Set when the WS upgrade is rejected with 401 or 404 before the handshake completes. */ + private upgradeRejected = false; private ws: WebSocket | null = null; private pingTimer?: NodeJS.Timeout; private reconnectTimer?: NodeJS.Timeout; @@ -113,6 +119,7 @@ export class PilotAgent { constructor(options: AgentOptions) { this.options = options; this.token = options.initialToken; + this.enrollToken = options.enrollToken; this.agentVersion = getSenchoVersion() || '0.0.0'; this.customCa = readPilotCaBundle(); } @@ -170,6 +177,8 @@ export class PilotAgent { private connect(): void { if (this.shuttingDown) return; + this.upgradeRejected = false; + const wsUrl = httpUrlToWs(this.options.primaryUrl) + '/api/pilot/tunnel'; const ws = new WebSocket(wsUrl, { headers: { @@ -187,6 +196,22 @@ export class PilotAgent { ...(this.customCa ? { ca: this.customCa } : {}), }); this.ws = ws; + let opened = false; + let lastHandshakeError: string | null = null; + + // Do NOT register an 'unexpected-response' listener here. The ws library + // skips abortHandshake (and therefore never emits 'error' or 'close') + // when a listener exists for that event. Auth rejection is detected via + // the error message abortHandshake emits: "Unexpected server response: N". + ws.on('error', (err) => { + console.warn('[Pilot] Tunnel error:', err.message); + lastHandshakeError = err.message; + if (/Unexpected server response: (401|404)/.test(err.message)) { + this.upgradeRejected = true; + } + // 'close' will follow; reconnect is scheduled there. + }); + this.switchboard = attachTcpStreamSwitchboard({ ws, resolveTarget: resolveByComposeLabels, @@ -195,6 +220,7 @@ export class PilotAgent { }); ws.on('open', () => { + opened = true; // Backoff intentionally NOT reset here: a TCP-level connect that // immediately fails the protocol handshake (incompatible version, // bad token consumed at upgrade) would otherwise reset the @@ -222,12 +248,19 @@ export class PilotAgent { ws.on('close', (code, reason) => { console.log('[Pilot] Tunnel closed:', code, reason?.toString?.() ?? ''); this.cleanupAfterDisconnect(); + + const authRejected = this.upgradeRejected + || (!opened && lastHandshakeError != null && /Unexpected server response: (401|404)/.test(lastHandshakeError)); + if (authRejected && this.enrollToken && this.token !== this.enrollToken) { + clearPersistedToken(); + console.log('[Pilot] Persisted token rejected; falling back to enroll token'); + this.token = this.enrollToken; + this.backoff = RECONNECT_MIN_MS; + } + this.upgradeRejected = false; + this.scheduleReconnect(); }); - ws.on('error', (err) => { - console.warn('[Pilot] Tunnel error:', err.message); - // 'close' will follow; reconnect is scheduled there. - }); } private cleanupAfterDisconnect(): void { @@ -612,3 +645,23 @@ export function persistToken(token: string): void { } } +/** + * Remove the persisted long-lived tunnel token from disk. Used when the + * control instance rejects the stored credential at upgrade so the agent + * can fall back to SENCHO_ENROLL_TOKEN without re-poisoning on restart. + * ENOENT is normal when no file was written yet. + * + * Exposed for unit tests. + */ +export function clearPersistedToken(): void { + try { + fs.unlinkSync(TOKEN_PATH); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') return; + console.warn( + `[Pilot] Failed to remove persisted tunnel token at ${sanitizeForLog(TOKEN_PATH)} (${sanitizeForLog(code ?? 'unknown')}: ${sanitizeForLog((err as Error).message)})`, + ); + } +} + diff --git a/docs/features/pilot-agent.mdx b/docs/features/pilot-agent.mdx index 3dbe1ced..45f729b7 100644 --- a/docs/features/pilot-agent.mdx +++ b/docs/features/pilot-agent.mdx @@ -151,7 +151,7 @@ Deploying through Compose also lets the control instance push over-the-air updat The agent boots, reads its configuration, and dials `wss:///api/pilot/tunnel` carrying the enrollment token in an `Authorization: Bearer` header. -The control instance verifies the token, marks the enrollment slot consumed, and replies with a `hello` frame followed by a control frame that carries a **long-lived tunnel JWT** (365-day expiry). The agent writes that token to its data volume at `/app/data/pilot.jwt`. The original enrollment token is now useless: the agent does not need it again, and the control instance refuses to accept it a second time. +The control instance verifies the token, marks the enrollment slot consumed, and replies with a `hello` frame followed by a control frame that carries a **long-lived tunnel JWT** (365-day expiry). The agent writes that token to its data volume at `/app/data/pilot.jwt`. The control instance will not accept the same enrollment token again, but leave `SENCHO_ENROLL_TOKEN` in the compose file: the agent uses it as a fallback if the persisted tunnel JWT is ever rejected at upgrade (for example after a secret rotation), clearing the stale file and re-enrolling automatically. The tunnel is now active. The Endpoint column in the Nodes table flips from `tunnel (waiting)` to `tunnel (seen Xs ago)` and the node's status badge turns Online. @@ -205,7 +205,7 @@ The agent dials `wss://`. Certificate validation is on by default against the sy ### What rotating the JWT secret does -The tunnel JWTs are signed with the control instance's `auth_jwt_secret`. If that secret rotates (because the control instance was rebuilt from scratch, restored into a different environment, or manually rotated), every existing tunnel JWT stops verifying. The agents will reconnect, fail authentication, and back off. Re-enroll each affected node (regenerate the enrollment token, then redeploy the agent on the remote with the refreshed compose file) to issue a new tunnel JWT signed by the current secret. +The tunnel JWTs are signed with the control instance's `auth_jwt_secret`. If that secret rotates (because the control instance was rebuilt from scratch, restored into a different environment, or manually rotated), every existing tunnel JWT stops verifying. Regenerate enrollment for each affected node, update the agent compose file with the fresh token, and restart the agent containers. When `SENCHO_ENROLL_TOKEN` is present in the running container, the agent clears the stale `pilot.jwt` and re-enrolls on its own; you do not need to delete the data volume by hand. ## Self-signed control-instance TLS certificates @@ -261,7 +261,7 @@ These environment variables are read by the **agent** container at boot. |---|---|---|---| | `SENCHO_MODE` | Yes | none | Must be `pilot`. Putting any other value here causes the container to start as a normal Sencho instance, not as an agent. | | `SENCHO_PRIMARY_URL` | Yes | none | The base URL of your control instance (e.g. `https://sencho.example.com`). The agent appends `/api/pilot/tunnel` and dials `wss://`. | -| `SENCHO_ENROLL_TOKEN` | First boot only | none | The 15-minute enrollment token issued by the control instance. Ignored on subsequent boots once `pilot.jwt` is on disk. | +| `SENCHO_ENROLL_TOKEN` | First boot; keep for recovery | none | The 15-minute enrollment token issued by the control instance. During normal operation the agent dials with `pilot.jwt` instead. If that persisted token is rejected at upgrade (for example after a secret rotation or node re-registration), the agent falls back to this value automatically when it is still present in the container environment, clears the stale file, and re-enrolls. Update it in compose after regenerating enrollment on the control instance. | | `SENCHO_PILOT_CA_FILE` | Optional | unset | Absolute path inside the container to a PEM bundle. Use when your control instance's TLS chain is rooted in a private CA. | | `DATA_DIR` | Optional | `/app/data` | Where the persisted `pilot.jwt` is stored. Override only if you are mounting a different volume layout. | | `COMPOSE_DIR` | Optional | `/app/compose` | Root directory where compose stack folders live. Enrollment sets it to the absolute path selected for the node and mounts that path identically on the host and in the agent. | @@ -281,7 +281,7 @@ These are the boundaries operators should know about before designing a fleet ar - **No mode conversion.** The Edit dialog shows a Mode field for an enrolled node, but switching a node between Pilot Agent and Distributed API Proxy after enrollment leaves the credentials and connection state inconsistent. To change modes, delete the node and re-create it in the desired mode. - **No audit log entries for enrollment lifecycle.** Node creation, enrollment regeneration, and node deletion do not write to the audit log today. This is on the roadmap. - **`pilot.jwt` is not cleaned up on node deletion.** When you delete a node from the control instance, the agent's persisted token stays on the remote's data volume. The agent will fail to reconnect on next restart, but the file persists. If you are repurposing the host, tear the agent down with `docker compose -f compose.yaml down -v` to remove the `sencho-agent-data` volume. -- **JWT-secret rotation invalidates every tunnel.** Rebuilding the control instance from scratch or rotating `auth_jwt_secret` requires re-enrolling every agent. There is no out-of-band re-issuance flow. +- **JWT-secret rotation invalidates existing tunnel JWTs.** Rebuilding the control instance from scratch or rotating `auth_jwt_secret` requires a fresh enrollment token on each agent. When that token is still in the container environment, the agent re-enrolls automatically without manual deletion of `pilot.jwt`. - **One tunnel per node.** Splitting a node's load across multiple control instances or running multiple agent containers against the same control instance for the same node is not supported. - **Mesh and pilot share the per-tunnel stream pool.** A node that runs heavy Sencho Mesh traffic counts those streams against the same 1024-stream cap as HTTP and WebSocket traffic. - **The agent has no UI of its own.** All operation flows through the control instance. The agent's container logs (`docker logs sencho-agent`) are the only direct visibility into agent-side behaviour. @@ -320,7 +320,7 @@ The generic node-connectivity issues (a node showing Offline, a pilot agent stuc - The persisted tunnel credential is signed with the control instance's `auth_jwt_secret`. If that secret was not part of the backup (or has been rotated for any other reason), existing tunnels stop verifying. Regenerate enrollment for each affected node from Settings → Nodes and restart the agent containers so they consume the fresh tokens. There is no global re-issuance command. + The persisted tunnel credential is signed with the control instance's `auth_jwt_secret`. If that secret was not part of the backup (or has been rotated for any other reason), existing tunnels stop verifying. Regenerate enrollment for each affected node from Settings → Nodes, update the agent compose file with the fresh `SENCHO_ENROLL_TOKEN`, and restart the agent containers. The agent removes the stale `pilot.jwt` and completes enrollment automatically when the new token is in the container environment.