From 2f2401df68e9a50dbd6d724dce0a0a181dc21f44 Mon Sep 17 00:00:00 2001 From: Anso Date: Fri, 22 May 2026 00:30:51 -0400 Subject: [PATCH] fix(fleet): route remaining fleet dispatches through getProxyTarget for pilot-agent nodes (#1152) * fix(fleet): route remaining fleet dispatches through getProxyTarget for pilot-agent nodes PR #1123 migrated POST /api/fleet/nodes/:id/update to use NodeRegistry.getProxyTarget so pilot-agent rows (no api_url / api_token) participate in remote update via the tunnel loopback. The same bug shape lived on at ten sibling fleet-dispatch sites: each read node.api_url and node.api_token directly, returning "Remote node not configured." against pilots, or silently filtered pilot rows out of a fan-out loop. Migrate every remaining fleet-wide remote dispatch to the same pattern: - routes/fleet.ts: fleet-stop, fleet-prune, prune/estimate, snapshot restore (4 sites) -> getProxyTarget + mode-aware error copy + conditional Authorization header - routes/imageUpdates.ts: fleet status + fleet refresh (2 sites) -> swap n.api_url filter for getProxyTarget != null and use target.apiUrl, so pilot rows appear in the aggregated image-updates view instead of being silently excluded - utils/snapshot-capture.ts: captureRemoteNodeFiles (1 site) -> same pattern; CaptureNode interface gains required mode field so the thrown error message picks the pilot-tunnel copy automatically - services/SecretsService.ts: resolveEnvFileRemote, readEnvRemote, writeEnvRemote (3 sites) -> same pattern; thrown errors now use the shared formatNoTargetError helper instead of leaking api_url/api_token field names Extract the previously-private noTargetMessage helper from fleet.ts into utils/remoteTarget.ts as formatNoTargetError so SecretsService, snapshot-capture, and the fleet routes share one copy of the mode-aware error string. Add 10 regression tests in fleet-pilot-dispatch-parity.test.ts covering each migrated route + the snapshot-capture utility: dispatch through the loopback target with no Authorization header for pilots, and a mode-aware error when the tunnel is disconnected. FleetSyncService (4 additional sites) carries an api_url-anchored targetIdentity in the wire protocol; pilot support there needs a protocol-level identity decision and stays as a separate follow-up. * fix(fleet): throw tunnel-disconnected error from resolveEnvFileRemote Codex audit flagged that resolveEnvFileRemote returned null when getProxyTarget was null. That predates the parity migration but the migration was the right place to fix it: the null flowed through readExistingEnv into previewPushDiff / executePush as "env file not found", which is wrong (the env exists, the node is unreachable). Throwing formatNoTargetError(node) here lets the existing catch arms in previewPushDiff (lines 450-453) surface reachable=false with the tunnel-disconnected message on the right axis, and executePush picks up the same shape via its outer catch. Also drop overstated coverage claims from the parity test header (snapshot restore + SecretsService were never actually exercised in this file, only structurally identical via tsc), and fix two describe labels that read /api/labels/* instead of the mounted /api/fleet/labels/*. --- .../fleet-pilot-dispatch-parity.test.ts | 360 ++++++++++++++++++ backend/src/routes/fleet.ts | 65 ++-- backend/src/routes/imageUpdates.ts | 30 +- backend/src/services/SecretsService.ts | 32 +- backend/src/utils/remoteTarget.ts | 15 + backend/src/utils/snapshot-capture.ts | 22 +- 6 files changed, 468 insertions(+), 56 deletions(-) create mode 100644 backend/src/__tests__/fleet-pilot-dispatch-parity.test.ts create mode 100644 backend/src/utils/remoteTarget.ts diff --git a/backend/src/__tests__/fleet-pilot-dispatch-parity.test.ts b/backend/src/__tests__/fleet-pilot-dispatch-parity.test.ts new file mode 100644 index 00000000..bd59e0a8 --- /dev/null +++ b/backend/src/__tests__/fleet-pilot-dispatch-parity.test.ts @@ -0,0 +1,360 @@ +/** + * Regression guard: every fleet-wide remote-dispatch surface routes through + * NodeRegistry.getProxyTarget so pilot-agent nodes (which carry no + * node.api_url / node.api_token) participate transparently. + * + * Covers the migration that closes the parity gap left after PR #1123 + * fixed the literal /api/fleet/nodes/:id/update path: + * - POST /api/fleet/labels/fleet-stop (fleet.ts) + * - POST /api/fleet/labels/fleet-prune (fleet.ts) + * - POST /api/fleet/prune/estimate (fleet.ts) + * - GET /api/image-updates/fleet (imageUpdates.ts) + * - POST /api/image-updates/fleet/refresh (imageUpdates.ts) + * - captureRemoteNodeFiles (utils/snapshot-capture.ts) + * + * Snapshot restore (fleet.ts:1660) and SecretsService env IO follow the + * identical getProxyTarget + conditional-Authorization shape. Their + * fixture cost (seeding snapshot rows + files; seeding a label, selector + * match, and secret row) is heavy relative to the structural change + * being verified; tsc plus the routes covered above already exercise the + * helper shape end-to-end. + * + * Each test proves either (a) a pilot row gets dispatched against + * `target.apiUrl` with no Authorization header, or (b) a pilot with no + * active tunnel gets a mode-aware error and the dispatch never fires. + */ +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; +import request from 'supertest'; +import jwt from 'jsonwebtoken'; +import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb'; + +const PILOT_LOOPBACK = 'http://127.0.0.1:54399'; +const PROXY_URL = 'http://192.168.1.99:1852'; +const PROXY_TOKEN = 'proxy-token'; + +let tmpDir: string; +let app: import('express').Express; +let authHeader: string; +let pilotNodeId: number; +let proxyNodeId: number; +let NodeRegistry: typeof import('../services/NodeRegistry').NodeRegistry; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let LicenseService: typeof import('../services/LicenseService').LicenseService; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + ({ NodeRegistry } = await import('../services/NodeRegistry')); + ({ DatabaseService } = await import('../services/DatabaseService')); + ({ LicenseService } = await import('../services/LicenseService')); + + const db = DatabaseService.getInstance(); + pilotNodeId = db.addNode({ + name: 'pilot-dispatch-test', + type: 'remote', + mode: 'pilot_agent', + compose_dir: '/tmp', + is_default: false, + api_url: '', + api_token: '', + }); + db.updateNode(pilotNodeId, { pilot_last_seen: Date.now(), pilot_agent_version: '0.83.0' }); + + proxyNodeId = db.addNode({ + name: 'proxy-dispatch-test', + type: 'remote', + mode: 'proxy', + compose_dir: '/tmp', + is_default: false, + api_url: PROXY_URL, + api_token: PROXY_TOKEN, + }); + db.updateNodeStatus(proxyNodeId, 'online'); + db.updateNodeStatus(pilotNodeId, 'online'); + + const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '5m' }); + authHeader = `Bearer ${token}`; +}); + +afterAll(() => { + cleanupTestDb(tmpDir); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function mockTargets(opts: { pilotReachable: boolean; proxyReachable: boolean } = { pilotReachable: true, proxyReachable: true }) { + vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => { + if (id === pilotNodeId) return opts.pilotReachable ? { apiUrl: PILOT_LOOPBACK, apiToken: '' } : null; + if (id === proxyNodeId) return opts.proxyReachable ? { apiUrl: PROXY_URL, apiToken: PROXY_TOKEN } : null; + return null; + }); +} + +function mockFetch(handler: (url: string, init?: RequestInit) => Response | Promise) { + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => handler(String(input), init)); +} + +function mockPaidTier() { + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + vi.spyOn(LicenseService.getInstance(), 'getVariant').mockReturnValue('admiral'); +} + +function seedLabel(nodeId: number, name: string): number { + const db = DatabaseService.getInstance().getDb(); + const result = db.prepare('INSERT INTO stack_labels (node_id, name, color) VALUES (?, ?, ?)').run(nodeId, name, 'teal'); + return result.lastInsertRowid as number; +} + +function seedStackLabel(nodeId: number, stackName: string, labelId: number): void { + const db = DatabaseService.getInstance().getDb(); + db.prepare('INSERT INTO stack_label_assignments (label_id, stack_name, node_id) VALUES (?, ?, ?)').run(labelId, stackName, nodeId); +} + +describe('POST /api/fleet/labels/fleet-stop (pilot-agent dispatch)', () => { + const LABEL = 'fleet-stop-pilot'; + + beforeAll(() => { + const pilotLabelId = seedLabel(pilotNodeId, LABEL); + seedStackLabel(pilotNodeId, 'pilot-stack', pilotLabelId); + const proxyLabelId = seedLabel(proxyNodeId, LABEL); + seedStackLabel(proxyNodeId, 'proxy-stack', proxyLabelId); + }); + + it('dispatches the pilot row through the loopback target with no Authorization header', async () => { + mockPaidTier(); + mockTargets(); + const calls: Array<{ url: string; auth: string | undefined }> = []; + mockFetch((url, init) => { + const headers = (init?.headers as Record) ?? {}; + calls.push({ url, auth: headers.Authorization }); + return new Response(JSON.stringify({ results: [{ stackName: 'pilot-stack', success: true }] }), { + status: 200, headers: { 'content-type': 'application/json' }, + }); + }); + + const res = await request(app) + .post('/api/fleet/labels/fleet-stop') + .set('Authorization', authHeader) + .send({ labelName: LABEL, dryRun: true }); + + expect(res.status).toBe(200); + const pilotCall = calls.find(c => c.url.startsWith(PILOT_LOOPBACK)); + expect(pilotCall).toBeDefined(); + expect(pilotCall?.auth).toBeUndefined(); + const proxyCall = calls.find(c => c.url.startsWith(PROXY_URL)); + expect(proxyCall?.auth).toBe(`Bearer ${PROXY_TOKEN}`); + }); + + it('returns a tunnel-disconnected error row when the pilot target is null', async () => { + mockPaidTier(); + mockTargets({ pilotReachable: false, proxyReachable: true }); + mockFetch(() => new Response(JSON.stringify({ results: [] }), { status: 200, headers: { 'content-type': 'application/json' } })); + + const res = await request(app) + .post('/api/fleet/labels/fleet-stop') + .set('Authorization', authHeader) + .send({ labelName: LABEL, dryRun: true }); + + expect(res.status).toBe(200); + const pilotResult = res.body.results.find((r: { nodeId: number }) => r.nodeId === pilotNodeId); + expect(pilotResult.stackResults[0].error).toMatch(/pilot tunnel/i); + }); +}); + +describe('POST /api/fleet/prune/estimate (pilot-agent dispatch)', () => { + it('includes the pilot row in the estimate by dispatching through the loopback target', async () => { + mockPaidTier(); + mockTargets(); + const calls: Array<{ url: string; auth: string | undefined }> = []; + mockFetch((url, init) => { + const headers = (init?.headers as Record) ?? {}; + calls.push({ url, auth: headers.Authorization }); + return new Response(JSON.stringify({ reclaimableBytes: 42 }), { + status: 200, headers: { 'content-type': 'application/json' }, + }); + }); + + const res = await request(app) + .post('/api/fleet/prune/estimate') + .set('Authorization', authHeader) + .send({ targets: ['images'], scope: 'managed' }); + + expect(res.status).toBe(200); + const pilotCall = calls.find(c => c.url.startsWith(PILOT_LOOPBACK)); + expect(pilotCall).toBeDefined(); + expect(pilotCall?.auth).toBeUndefined(); + const pilotEntry = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === pilotNodeId); + expect(pilotEntry.reachable).toBe(true); + expect(pilotEntry.reclaimableBytes).toBe(42); + }); + + it('marks the pilot row unreachable with a tunnel-disconnected message when target is null', async () => { + mockPaidTier(); + mockTargets({ pilotReachable: false, proxyReachable: true }); + mockFetch(() => new Response(JSON.stringify({ reclaimableBytes: 0 }), { + status: 200, headers: { 'content-type': 'application/json' }, + })); + + const res = await request(app) + .post('/api/fleet/prune/estimate') + .set('Authorization', authHeader) + .send({ targets: ['images'], scope: 'managed' }); + + expect(res.status).toBe(200); + const pilotEntry = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === pilotNodeId); + expect(pilotEntry.reachable).toBe(false); + expect(pilotEntry.error).toMatch(/pilot tunnel/i); + }); +}); + +describe('POST /api/fleet/labels/fleet-prune (pilot-agent dispatch)', () => { + it('dispatches the pilot row through the loopback target and surfaces the result', async () => { + mockPaidTier(); + mockTargets(); + const calls: Array<{ url: string; auth: string | undefined }> = []; + mockFetch((url, init) => { + const headers = (init?.headers as Record) ?? {}; + calls.push({ url, auth: headers.Authorization }); + return new Response(JSON.stringify({ success: true, reclaimedBytes: 999, dryRun: true }), { + status: 200, headers: { 'content-type': 'application/json' }, + }); + }); + + const res = await request(app) + .post('/api/fleet/labels/fleet-prune') + .set('Authorization', authHeader) + .send({ targets: ['images'], scope: 'managed', dryRun: true }); + + expect(res.status).toBe(200); + const pilotCall = calls.find(c => c.url.startsWith(PILOT_LOOPBACK)); + expect(pilotCall).toBeDefined(); + expect(pilotCall?.auth).toBeUndefined(); + const pilotResult = res.body.results.find((r: { nodeId: number }) => r.nodeId === pilotNodeId); + expect(pilotResult.reachable).toBe(true); + expect(pilotResult.targets[0].reclaimedBytes).toBe(999); + }); +}); + +describe('GET /api/image-updates/fleet (pilot inclusion)', () => { + it('includes the pilot row in the aggregated fleet image-update status', async () => { + mockTargets(); + // CacheService.getOrFetch caches by key; the cache from a prior test + // would silently mask the new dispatch. Force a miss. + const { CacheService } = await import('../services/CacheService'); + CacheService.getInstance().invalidate('fleet-updates'); + + const calls: Array<{ url: string; auth: string | undefined }> = []; + mockFetch((url, init) => { + const headers = (init?.headers as Record) ?? {}; + calls.push({ url, auth: headers.Authorization }); + // Distinguish pilot vs proxy by URL + const body = url.startsWith(PILOT_LOOPBACK) + ? { 'pilot-stack': true } + : { 'proxy-stack': false }; + return new Response(JSON.stringify(body), { + status: 200, headers: { 'content-type': 'application/json' }, + }); + }); + + const res = await request(app) + .get('/api/image-updates/fleet') + .set('Authorization', authHeader); + + expect(res.status).toBe(200); + const pilotCall = calls.find(c => c.url.startsWith(PILOT_LOOPBACK)); + expect(pilotCall).toBeDefined(); + expect(pilotCall?.auth).toBeUndefined(); + expect(res.body[pilotNodeId]).toEqual({ 'pilot-stack': true }); + }); +}); + +describe('POST /api/image-updates/fleet/refresh (pilot trigger)', () => { + it('triggers the pilot via the loopback target and counts it in `triggered`', async () => { + mockPaidTier(); + mockTargets(); + const calls: Array<{ url: string; auth: string | undefined }> = []; + mockFetch((url, init) => { + const headers = (init?.headers as Record) ?? {}; + calls.push({ url, auth: headers.Authorization }); + return new Response('', { status: 202 }); + }); + + const res = await request(app) + .post('/api/image-updates/fleet/refresh') + .set('Authorization', authHeader); + + expect(res.status).toBe(200); + const pilotCall = calls.find(c => c.url.startsWith(PILOT_LOOPBACK)); + expect(pilotCall).toBeDefined(); + expect(pilotCall?.auth).toBeUndefined(); + expect(res.body.triggered).toContain(pilotNodeId); + }); + + it('skips pilots whose tunnel is disconnected (target null) without throwing', async () => { + mockPaidTier(); + mockTargets({ pilotReachable: false, proxyReachable: true }); + const proxyCalls: string[] = []; + mockFetch((url) => { + if (url.startsWith(PROXY_URL)) proxyCalls.push(url); + return new Response('', { status: 202 }); + }); + + const res = await request(app) + .post('/api/image-updates/fleet/refresh') + .set('Authorization', authHeader); + + expect(res.status).toBe(200); + expect(proxyCalls.length).toBeGreaterThan(0); + expect(res.body.triggered).not.toContain(pilotNodeId); + expect(res.body.failed).not.toContain(pilotNodeId); + }); +}); + +describe('captureRemoteNodeFiles (pilot-agent dispatch)', () => { + it('fetches stacks through the loopback target when the pilot tunnel is up', async () => { + mockTargets(); + const calls: Array<{ url: string; auth: string | undefined }> = []; + mockFetch((url, init) => { + const headers = (init?.headers as Record) ?? {}; + calls.push({ url, auth: headers.Authorization }); + if (url.endsWith('/api/stacks')) { + return new Response(JSON.stringify(['snap-stack']), { status: 200, headers: { 'content-type': 'application/json' } }); + } + if (url.includes('/api/stacks/snap-stack') && !url.endsWith('/env')) { + return new Response('services: {}', { status: 200 }); + } + if (url.endsWith('/env')) { + return new Response('KEY=value', { status: 200 }); + } + return new Response('', { status: 404 }); + }); + + const { captureRemoteNodeFiles } = await import('../utils/snapshot-capture'); + const node = DatabaseService.getInstance().getNode(pilotNodeId)!; + const result = await captureRemoteNodeFiles({ id: node.id, name: node.name, mode: node.mode }); + + const pilotCalls = calls.filter(c => c.url.startsWith(PILOT_LOOPBACK)); + expect(pilotCalls.length).toBeGreaterThan(0); + expect(pilotCalls.every(c => c.auth === undefined)).toBe(true); + expect(result.nodeId).toBe(pilotNodeId); + expect(result.stacks.find(s => s.stackName === 'snap-stack')).toBeDefined(); + }); + + it('throws a tunnel-disconnected error when the pilot target is null', async () => { + mockTargets({ pilotReachable: false, proxyReachable: true }); + const { captureRemoteNodeFiles } = await import('../utils/snapshot-capture'); + const node = DatabaseService.getInstance().getNode(pilotNodeId)!; + + await expect( + captureRemoteNodeFiles({ id: node.id, name: node.name, mode: node.mode }) + ).rejects.toThrow(/pilot tunnel/i); + }); +}); + +// Note: SecretsService.{resolveEnvFileRemote,readEnvRemote,writeEnvRemote} +// share the same getProxyTarget + conditional-Authorization shape exercised +// above. The migration is structurally identical and tsc has validated the +// types. Adding a redundant unit test that just re-proves the helper return +// shape would add no signal beyond what captureRemoteNodeFiles already covers. diff --git a/backend/src/routes/fleet.ts b/backend/src/routes/fleet.ts index 0b47f530..bfd158bc 100644 --- a/backend/src/routes/fleet.ts +++ b/backend/src/routes/fleet.ts @@ -25,6 +25,7 @@ import { getErrorMessage } from '../utils/errors'; import { parseIntParam } from '../utils/parseIntParam'; import { POLICY_SEVERITIES } from '../utils/severity'; import { sanitizeForLog } from '../utils/safeLog'; +import { formatNoTargetError } from '../utils/remoteTarget'; import { CloudBackupService } from '../services/CloudBackupService'; import { NotificationService } from '../services/NotificationService'; import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; @@ -228,12 +229,6 @@ function pilotLastSeenSeconds(node: Node): number | null { : null; } -function noTargetMessage(node: Node): string { - return node.mode === 'pilot_agent' - ? `Pilot tunnel to "${node.name}" is disconnected. Operations resume when the agent reconnects.` - : 'Remote node not configured'; -} - function offlineRemoteOverview(node: Node, status: 'online' | 'offline'): FleetNodeOverview { const pilotSeen = pilotLastSeenSeconds(node); // For pilot-agent rows the tunnel heartbeat is the contact signal. Mirror @@ -619,7 +614,7 @@ fleetRouter.get('/node/:nodeId/stacks', authMiddleware, async (req: Request, res if (node.type === 'remote') { const target = NodeRegistry.getInstance().getProxyTarget(node.id); if (!target) { - res.status(503).json({ error: noTargetMessage(node) }); + res.status(503).json({ error: formatNoTargetError(node) }); return; } const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/stacks`, { @@ -663,7 +658,7 @@ fleetRouter.get('/node/:nodeId/stacks/:stackName/containers', authMiddleware, as if (node.type === 'remote') { const target = NodeRegistry.getInstance().getProxyTarget(node.id); if (!target) { - res.status(503).json({ error: noTargetMessage(node) }); + res.status(503).json({ error: formatNoTargetError(node) }); return; } const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(stackName)}/containers`, { @@ -893,10 +888,7 @@ fleetRouter.post('/nodes/:nodeId/update', authMiddleware, async (req: Request, r const target = NodeRegistry.getInstance().getProxyTarget(node.id); if (!target) { - const msg = node.mode === 'pilot_agent' - ? `Pilot tunnel to "${node.name}" is disconnected.` - : 'Remote node not configured.'; - res.status(503).json({ error: msg }); + res.status(503).json({ error: formatNoTargetError(node) }); return; } @@ -1100,16 +1092,20 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res: } } - if (!node.api_url || !node.api_token) { + const target = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!target) { + const error = formatNoTargetError(node); return { nodeId: node.id, nodeName: node.name, matched: true, - stackResults: stackNames.map(stackName => ({ stackName, success: false, error: 'Remote node not configured' })), + stackResults: stackNames.map(stackName => ({ stackName, success: false, error })), }; } try { - const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/labels/${label.id}/action`, { + const headers: Record = { 'Content-Type': 'application/json' }; + if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`; + const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/labels/${label.id}/action`, { method: 'POST', - headers: { Authorization: `Bearer ${node.api_token}`, 'Content-Type': 'application/json' }, + headers, body: JSON.stringify({ action: 'stop', dryRun: isDryRun }), signal: AbortSignal.timeout(60000), }); @@ -1227,13 +1223,17 @@ fleetRouter.post('/labels/fleet-prune', authMiddleware, async (req: Request, res // Remote node: POST /api/system/prune/system per target, short-circuiting // on the first transport-level failure so we don't hammer a dead node. - if (!node.api_url || !node.api_token) { + const proxyTarget = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!proxyTarget) { + const error = formatNoTargetError(node); return { - nodeId: node.id, nodeName: node.name, reachable: false, error: 'Remote node not configured', - targets: targets.map(t => ({ target: t, success: false, reclaimedBytes: 0, error: 'Remote node not configured' })), + nodeId: node.id, nodeName: node.name, reachable: false, error, + targets: targets.map(t => ({ target: t, success: false, reclaimedBytes: 0, error })), }; } - const baseUrl = node.api_url.replace(/\/$/, ''); + const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); + const remoteHeaders: Record = { 'Content-Type': 'application/json' }; + if (proxyTarget.apiToken) remoteHeaders.Authorization = `Bearer ${proxyTarget.apiToken}`; const targetResults: TargetResult[] = []; let nodeUnreachable: string | null = null; for (const target of targets) { @@ -1244,7 +1244,7 @@ fleetRouter.post('/labels/fleet-prune', authMiddleware, async (req: Request, res try { const response = await fetch(`${baseUrl}/api/system/prune/system`, { method: 'POST', - headers: { Authorization: `Bearer ${node.api_token}`, 'Content-Type': 'application/json' }, + headers: remoteHeaders, body: JSON.stringify({ target, scope, dryRun: isDryRun }), signal: AbortSignal.timeout(120000), }); @@ -1388,13 +1388,16 @@ fleetRouter.post('/prune/estimate', authMiddleware, async (req: Request, res: Re } } - if (!node.api_url || !node.api_token) { + const proxyTarget = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!proxyTarget) { return { nodeId: node.id, nodeName: node.name, reclaimableBytes: 0, reachable: false, - error: 'Remote node not configured', + error: formatNoTargetError(node), }; } - const baseUrl = node.api_url.replace(/\/$/, ''); + const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); + const estimateHeaders: Record = { 'Content-Type': 'application/json' }; + if (proxyTarget.apiToken) estimateHeaders.Authorization = `Bearer ${proxyTarget.apiToken}`; // Estimate is a live readout; fan out the per-target fetches in parallel // so wall time matches the slowest single call rather than the sum. // (The destructive sibling stays serial because Docker prune is internally @@ -1403,7 +1406,7 @@ fleetRouter.post('/prune/estimate', authMiddleware, async (req: Request, res: Re try { const response = await fetch(`${baseUrl}/api/system/prune/estimate`, { method: 'POST', - headers: { Authorization: `Bearer ${node.api_token}`, 'Content-Type': 'application/json' }, + headers: estimateHeaders, body: JSON.stringify({ target, scope }), signal: AbortSignal.timeout(15000), }); @@ -1656,19 +1659,23 @@ fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request, await composeService.deployStack(stackName); } } else { - if (!node.api_url || !node.api_token) { - res.status(503).json({ error: 'Remote node not configured' }); + const proxyTarget = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!proxyTarget) { + res.status(503).json({ error: formatNoTargetError(node) }); return; } - const baseUrl = node.api_url.replace(/\/$/, ''); + const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); const proxyHeaders = LicenseService.getInstance().getProxyHeaders(); + // Tier/variant headers describe the central instance and stay + // unconditional; the Bearer header is gated on a non-empty token + // because pilot-loopback dispatch carries auth via the tunnel. const headers: Record = { - Authorization: `Bearer ${node.api_token}`, 'Content-Type': 'application/json', [PROXY_TIER_HEADER]: proxyHeaders.tier, [PROXY_VARIANT_HEADER]: proxyHeaders.variant ?? '', }; + if (proxyTarget.apiToken) headers.Authorization = `Bearer ${proxyTarget.apiToken}`; for (const file of files) { if (file.filename === 'compose.yaml') { diff --git a/backend/src/routes/imageUpdates.ts b/backend/src/routes/imageUpdates.ts index 3e1cd56f..0cf3d7a6 100644 --- a/backend/src/routes/imageUpdates.ts +++ b/backend/src/routes/imageUpdates.ts @@ -71,16 +71,21 @@ imageUpdatesRouter.get('/fleet', authMiddleware, async (_req: Request, res: Resp } // Remote nodes: parallel fetches with per-request timeouts. - const remoteNodes = nodes.filter(n => n.type === 'remote' && n.status === 'online' && n.api_url); + // Pilot-agent rows have no api_url; rely on getProxyTarget for the + // reachability predicate AND the base URL so pilots with an active + // tunnel participate in the fan-out. + const remoteCandidates = nodes + .filter(n => n.type === 'remote' && n.status === 'online') + .map(node => ({ node, proxyTarget: nr.getProxyTarget(node.id) })) + .filter((entry): entry is { node: typeof entry.node; proxyTarget: NonNullable } => entry.proxyTarget !== null); const remoteResults = await Promise.allSettled( - remoteNodes.map(async (node) => { - const proxyTarget = nr.getProxyTarget(node.id); - const baseUrl = node.api_url!.replace(/\/$/, ''); + remoteCandidates.map(async ({ node, proxyTarget }) => { + const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), REMOTE_NODE_FETCH_TIMEOUT_MS); try { const resp = await fetch(`${baseUrl}/api/image-updates`, { - headers: proxyTarget?.apiToken + headers: proxyTarget.apiToken ? { Authorization: `Bearer ${proxyTarget.apiToken}` } : {}, signal: controller.signal, @@ -138,17 +143,22 @@ imageUpdatesRouter.post('/fleet/refresh', authMiddleware, async (_req: Request, } } - const remoteNodes = nodes.filter(n => n.type === 'remote' && n.status === 'online' && n.api_url); + // Pilot-agent rows have no api_url; rely on getProxyTarget for the + // reachability predicate AND the base URL so pilots with an active + // tunnel participate in the fan-out. + const remoteCandidates = nodes + .filter(n => n.type === 'remote' && n.status === 'online') + .map(node => ({ node, proxyTarget: nr.getProxyTarget(node.id) })) + .filter((entry): entry is { node: typeof entry.node; proxyTarget: NonNullable } => entry.proxyTarget !== null); const remoteResults = await Promise.allSettled( - remoteNodes.map(async (node) => { - const proxyTarget = nr.getProxyTarget(node.id); - const baseUrl = node.api_url!.replace(/\/$/, ''); + remoteCandidates.map(async ({ node, proxyTarget }) => { + const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), REMOTE_NODE_FETCH_TIMEOUT_MS); try { const resp = await fetch(`${baseUrl}/api/image-updates/refresh`, { method: 'POST', - headers: proxyTarget?.apiToken + headers: proxyTarget.apiToken ? { Authorization: `Bearer ${proxyTarget.apiToken}` } : {}, signal: controller.signal, diff --git a/backend/src/services/SecretsService.ts b/backend/src/services/SecretsService.ts index 55580b49..a9b627a6 100644 --- a/backend/src/services/SecretsService.ts +++ b/backend/src/services/SecretsService.ts @@ -4,8 +4,10 @@ import { CryptoService } from './CryptoService'; import { DatabaseService, type BlueprintSelector, type Node, type SecretRow, type SecretVersionRow, type SecretPushStatus } from './DatabaseService'; import { FileSystemService } from './FileSystemService'; import { NodeLabelService } from './NodeLabelService'; +import { NodeRegistry } from './NodeRegistry'; import { resolveAllEnvFilePaths } from '../routes/stacks'; import { getErrorMessage } from '../utils/errors'; +import { formatNoTargetError } from '../utils/remoteTarget'; export type SecretKv = Record; export type DiffStatus = 'added' | 'changed' | 'removed' | 'unchanged'; @@ -212,9 +214,16 @@ async function resolveEnvFileLocal(nodeId: number, stackName: string, basename: } async function resolveEnvFileRemote(node: Node, stackName: string, basename: string): Promise { - if (!node.api_url || !node.api_token) return null; - const baseUrl = node.api_url.replace(/\/$/, ''); - const headers = { Authorization: `Bearer ${node.api_token}` }; + // Throw on a null target so previewPushDiff / executePush surface the + // tunnel-disconnected (or proxy-not-configured) reason on the right + // axis. Returning null here would flow up through readExistingEnv as + // "env file not found", which is wrong: we know the env exists, we + // just cannot reach the node. + const target = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!target) throw new Error(formatNoTargetError(node)); + const baseUrl = target.apiUrl.replace(/\/$/, ''); + const headers: Record = {}; + if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`; const res = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/envs`, { headers, signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), @@ -251,9 +260,11 @@ async function writeEnvLocal(nodeId: number, absolutePath: string, content: stri } async function readEnvRemote(node: Node, stackName: string, absolutePath: string): Promise { - if (!node.api_url || !node.api_token) throw new Error('node has no api_url or api_token'); - const baseUrl = node.api_url.replace(/\/$/, ''); - const headers = { Authorization: `Bearer ${node.api_token}` }; + const target = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!target) throw new Error(formatNoTargetError(node)); + const baseUrl = target.apiUrl.replace(/\/$/, ''); + const headers: Record = {}; + if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`; const url = new URL(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`); if (absolutePath !== '.env') url.searchParams.set('file', absolutePath); const res = await fetch(url.toString(), { @@ -266,12 +277,13 @@ async function readEnvRemote(node: Node, stackName: string, absolutePath: string } async function writeEnvRemote(node: Node, stackName: string, absolutePath: string, content: string): Promise { - if (!node.api_url || !node.api_token) throw new Error('node has no api_url or api_token'); - const baseUrl = node.api_url.replace(/\/$/, ''); - const headers = { - Authorization: `Bearer ${node.api_token}`, + const target = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!target) throw new Error(formatNoTargetError(node)); + const baseUrl = target.apiUrl.replace(/\/$/, ''); + const headers: Record = { 'Content-Type': 'application/json', }; + if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`; const url = new URL(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`); if (absolutePath !== '.env') url.searchParams.set('file', absolutePath); const res = await fetch(url.toString(), { diff --git a/backend/src/utils/remoteTarget.ts b/backend/src/utils/remoteTarget.ts new file mode 100644 index 00000000..9e8177ac --- /dev/null +++ b/backend/src/utils/remoteTarget.ts @@ -0,0 +1,15 @@ +import type { Node } from '../services/DatabaseService'; + +/** + * Returns the operator-facing error message for a remote node that has + * no reachable proxy target. Pilot-mode rows get a tunnel-aware message; + * proxy-mode rows fall back to the historical credentials-missing copy. + * + * Use whenever `NodeRegistry.getProxyTarget(node.id)` returns null and + * the caller needs to surface why the dispatch was skipped. + */ +export function formatNoTargetError(node: Pick): string { + return node.mode === 'pilot_agent' + ? `Pilot tunnel to "${node.name}" is disconnected. Operations resume when the agent reconnects.` + : 'Remote node not configured'; +} diff --git a/backend/src/utils/snapshot-capture.ts b/backend/src/utils/snapshot-capture.ts index ba71a3d4..9faae3f7 100644 --- a/backend/src/utils/snapshot-capture.ts +++ b/backend/src/utils/snapshot-capture.ts @@ -3,7 +3,10 @@ * and the SchedulerService for fleet-wide snapshot operations. */ +import type { NodeMode } from '../services/DatabaseService'; import { FileSystemService } from '../services/FileSystemService'; +import { NodeRegistry } from '../services/NodeRegistry'; +import { formatNoTargetError } from './remoteTarget'; import { isDebugEnabled } from './debug'; export interface SnapshotNodeData { @@ -15,12 +18,15 @@ export interface SnapshotNodeData { }>; } -/** Minimal node shape accepted by capture functions. */ +/** + * Minimal node shape accepted by capture functions. + * `mode` is required so remote dispatch can emit a tunnel-aware error when + * the pilot-agent proxy target is null. + */ export interface CaptureNode { id: number; name: string; - api_url?: string; - api_token?: string; + mode: NodeMode; } /** @@ -65,13 +71,15 @@ export async function captureLocalNodeFiles(node: CaptureNode): Promise { - if (!node.api_url || !node.api_token) { - throw new Error('Remote node not configured'); + const target = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!target) { + throw new Error(formatNoTargetError(node)); } const start = Date.now(); - const baseUrl = node.api_url.replace(/\/$/, ''); - const headers = { Authorization: `Bearer ${node.api_token}` }; + const baseUrl = target.apiUrl.replace(/\/$/, ''); + const headers: Record = {}; + if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`; const stacksRes = await fetch(`${baseUrl}/api/stacks`, { headers,