fix(fleet): route remaining fleet dispatches through getProxyTarget for pilot-agent nodes (#1152)

* fix(fleet): route remaining fleet dispatches through getProxyTarget for pilot-agent nodes PR #1123 migrated POST /api/fleet/nodes/:id/update to use NodeRegistry.getProxyTarget so pilot-agent rows (no api_url / api_token) participate in remote update via the tunnel loopback. The same bug shape lived on at ten sibling fleet-dispatch sites: each read node.api_url and node.api_token directly, returning "Remote node not configured." against pilots, or silently filtered pilot rows out of a fan-out loop. Migrate every remaining fleet-wide remote dispatch to the same pattern: - routes/fleet.ts: fleet-stop, fleet-prune, prune/estimate, snapshot restore (4 sites) -> getProxyTarget + mode-aware error copy + conditional Authorization header - routes/imageUpdates.ts: fleet status + fleet refresh (2 sites) -> swap n.api_url filter for getProxyTarget != null and use target.apiUrl, so pilot rows appear in the aggregated image-updates view instead of being silently excluded - utils/snapshot-capture.ts: captureRemoteNodeFiles (1 site) -> same pattern; CaptureNode interface gains required mode field so the thrown error message picks the pilot-tunnel copy automatically - services/SecretsService.ts: resolveEnvFileRemote, readEnvRemote, writeEnvRemote (3 sites) -> same pattern; thrown errors now use the shared formatNoTargetError helper instead of leaking api_url/api_token field names Extract the previously-private noTargetMessage helper from fleet.ts into utils/remoteTarget.ts as formatNoTargetError so SecretsService, snapshot-capture, and the fleet routes share one copy of the mode-aware error string. Add 10 regression tests in fleet-pilot-dispatch-parity.test.ts covering each migrated route + the snapshot-capture utility: dispatch through the loopback target with no Authorization header for pilots, and a mode-aware error when the tunnel is disconnected. FleetSyncService (4 additional sites) carries an api_url-anchored targetIdentity in the wire protocol; pilot support there needs a protocol-level identity decision and stays as a separate follow-up. * fix(fleet): throw tunnel-disconnected error from resolveEnvFileRemote Codex audit flagged that resolveEnvFileRemote returned null when getProxyTarget was null. That predates the parity migration but the migration was the right place to fix it: the null flowed through readExistingEnv into previewPushDiff / executePush as "env file not found", which is wrong (the env exists, the node is unreachable). Throwing formatNoTargetError(node) here lets the existing catch arms in previewPushDiff (lines 450-453) surface reachable=false with the tunnel-disconnected message on the right axis, and executePush picks up the same shape via its outer catch. Also drop overstated coverage claims from the parity test header (snapshot restore + SecretsService were never actually exercised in this file, only structurally identical via tsc), and fix two describe labels that read /api/labels/* instead of the mounted /api/fleet/labels/*.
2026-07-26 11:49:16 +00:00 · 2026-05-22 00:30:51 -04:00
parent 60f893a81f
commit 2f2401df68
6 changed files with 468 additions and 56 deletions
@@ -0,0 +1,360 @@
+/**
+ * Regression guard: every fleet-wide remote-dispatch surface routes through
+ * NodeRegistry.getProxyTarget so pilot-agent nodes (which carry no
+ * node.api_url / node.api_token) participate transparently.
+ *
+ * Covers the migration that closes the parity gap left after PR #1123
+ * fixed the literal /api/fleet/nodes/:id/update path:
+ *   - POST /api/fleet/labels/fleet-stop      (fleet.ts)
+ *   - POST /api/fleet/labels/fleet-prune     (fleet.ts)
+ *   - POST /api/fleet/prune/estimate         (fleet.ts)
+ *   - GET  /api/image-updates/fleet          (imageUpdates.ts)
+ *   - POST /api/image-updates/fleet/refresh  (imageUpdates.ts)
+ *   - captureRemoteNodeFiles                 (utils/snapshot-capture.ts)
+ *
+ * Snapshot restore (fleet.ts:1660) and SecretsService env IO follow the
+ * identical getProxyTarget + conditional-Authorization shape. Their
+ * fixture cost (seeding snapshot rows + files; seeding a label, selector
+ * match, and secret row) is heavy relative to the structural change
+ * being verified; tsc plus the routes covered above already exercise the
+ * helper shape end-to-end.
+ *
+ * Each test proves either (a) a pilot row gets dispatched against
+ * `target.apiUrl` with no Authorization header, or (b) a pilot with no
+ * active tunnel gets a mode-aware error and the dispatch never fires.
+ */
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
+import request from 'supertest';
+import jwt from 'jsonwebtoken';
+import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
+
+const PILOT_LOOPBACK = 'http://127.0.0.1:54399';
+const PROXY_URL = 'http://192.168.1.99:1852';
+const PROXY_TOKEN = 'proxy-token';
+
+let tmpDir: string;
+let app: import('express').Express;
+let authHeader: string;
+let pilotNodeId: number;
+let proxyNodeId: number;
+let NodeRegistry: typeof import('../services/NodeRegistry').NodeRegistry;
+let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
+let LicenseService: typeof import('../services/LicenseService').LicenseService;
+
+beforeAll(async () => {
+  tmpDir = await setupTestDb();
+  ({ app } = await import('../index'));
+  ({ NodeRegistry } = await import('../services/NodeRegistry'));
+  ({ DatabaseService } = await import('../services/DatabaseService'));
+  ({ LicenseService } = await import('../services/LicenseService'));
+
+  const db = DatabaseService.getInstance();
+  pilotNodeId = db.addNode({
+    name: 'pilot-dispatch-test',
+    type: 'remote',
+    mode: 'pilot_agent',
+    compose_dir: '/tmp',
+    is_default: false,
+    api_url: '',
+    api_token: '',
+  });
+  db.updateNode(pilotNodeId, { pilot_last_seen: Date.now(), pilot_agent_version: '0.83.0' });
+
+  proxyNodeId = db.addNode({
+    name: 'proxy-dispatch-test',
+    type: 'remote',
+    mode: 'proxy',
+    compose_dir: '/tmp',
+    is_default: false,
+    api_url: PROXY_URL,
+    api_token: PROXY_TOKEN,
+  });
+  db.updateNodeStatus(proxyNodeId, 'online');
+  db.updateNodeStatus(pilotNodeId, 'online');
+
+  const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '5m' });
+  authHeader = `Bearer ${token}`;
+});
+
+afterAll(() => {
+  cleanupTestDb(tmpDir);
+});
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+function mockTargets(opts: { pilotReachable: boolean; proxyReachable: boolean } = { pilotReachable: true, proxyReachable: true }) {
+  vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => {
+    if (id === pilotNodeId) return opts.pilotReachable ? { apiUrl: PILOT_LOOPBACK, apiToken: '' } : null;
+    if (id === proxyNodeId) return opts.proxyReachable ? { apiUrl: PROXY_URL, apiToken: PROXY_TOKEN } : null;
+    return null;
+  });
+}
+
+function mockFetch(handler: (url: string, init?: RequestInit) => Response | Promise<Response>) {
+  vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => handler(String(input), init));
+}
+
+function mockPaidTier() {
+  vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
+  vi.spyOn(LicenseService.getInstance(), 'getVariant').mockReturnValue('admiral');
+}
+
+function seedLabel(nodeId: number, name: string): number {
+  const db = DatabaseService.getInstance().getDb();
+  const result = db.prepare('INSERT INTO stack_labels (node_id, name, color) VALUES (?, ?, ?)').run(nodeId, name, 'teal');
+  return result.lastInsertRowid as number;
+}
+
+function seedStackLabel(nodeId: number, stackName: string, labelId: number): void {
+  const db = DatabaseService.getInstance().getDb();
+  db.prepare('INSERT INTO stack_label_assignments (label_id, stack_name, node_id) VALUES (?, ?, ?)').run(labelId, stackName, nodeId);
+}
+
+describe('POST /api/fleet/labels/fleet-stop (pilot-agent dispatch)', () => {
+  const LABEL = 'fleet-stop-pilot';
+
+  beforeAll(() => {
+    const pilotLabelId = seedLabel(pilotNodeId, LABEL);
+    seedStackLabel(pilotNodeId, 'pilot-stack', pilotLabelId);
+    const proxyLabelId = seedLabel(proxyNodeId, LABEL);
+    seedStackLabel(proxyNodeId, 'proxy-stack', proxyLabelId);
+  });
+
+  it('dispatches the pilot row through the loopback target with no Authorization header', async () => {
+    mockPaidTier();
+    mockTargets();
+    const calls: Array<{ url: string; auth: string | undefined }> = [];
+    mockFetch((url, init) => {
+      const headers = (init?.headers as Record<string, string>) ?? {};
+      calls.push({ url, auth: headers.Authorization });
+      return new Response(JSON.stringify({ results: [{ stackName: 'pilot-stack', success: true }] }), {
+        status: 200, headers: { 'content-type': 'application/json' },
+      });
+    });
+
+    const res = await request(app)
+      .post('/api/fleet/labels/fleet-stop')
+      .set('Authorization', authHeader)
+      .send({ labelName: LABEL, dryRun: true });
+
+    expect(res.status).toBe(200);
+    const pilotCall = calls.find(c => c.url.startsWith(PILOT_LOOPBACK));
+    expect(pilotCall).toBeDefined();
+    expect(pilotCall?.auth).toBeUndefined();
+    const proxyCall = calls.find(c => c.url.startsWith(PROXY_URL));
+    expect(proxyCall?.auth).toBe(`Bearer ${PROXY_TOKEN}`);
+  });
+
+  it('returns a tunnel-disconnected error row when the pilot target is null', async () => {
+    mockPaidTier();
+    mockTargets({ pilotReachable: false, proxyReachable: true });
+    mockFetch(() => new Response(JSON.stringify({ results: [] }), { status: 200, headers: { 'content-type': 'application/json' } }));
+
+    const res = await request(app)
+      .post('/api/fleet/labels/fleet-stop')
+      .set('Authorization', authHeader)
+      .send({ labelName: LABEL, dryRun: true });
+
+    expect(res.status).toBe(200);
+    const pilotResult = res.body.results.find((r: { nodeId: number }) => r.nodeId === pilotNodeId);
+    expect(pilotResult.stackResults[0].error).toMatch(/pilot tunnel/i);
+  });
+});
+
+describe('POST /api/fleet/prune/estimate (pilot-agent dispatch)', () => {
+  it('includes the pilot row in the estimate by dispatching through the loopback target', async () => {
+    mockPaidTier();
+    mockTargets();
+    const calls: Array<{ url: string; auth: string | undefined }> = [];
+    mockFetch((url, init) => {
+      const headers = (init?.headers as Record<string, string>) ?? {};
+      calls.push({ url, auth: headers.Authorization });
+      return new Response(JSON.stringify({ reclaimableBytes: 42 }), {
+        status: 200, headers: { 'content-type': 'application/json' },
+      });
+    });
+
+    const res = await request(app)
+      .post('/api/fleet/prune/estimate')
+      .set('Authorization', authHeader)
+      .send({ targets: ['images'], scope: 'managed' });
+
+    expect(res.status).toBe(200);
+    const pilotCall = calls.find(c => c.url.startsWith(PILOT_LOOPBACK));
+    expect(pilotCall).toBeDefined();
+    expect(pilotCall?.auth).toBeUndefined();
+    const pilotEntry = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === pilotNodeId);
+    expect(pilotEntry.reachable).toBe(true);
+    expect(pilotEntry.reclaimableBytes).toBe(42);
+  });
+
+  it('marks the pilot row unreachable with a tunnel-disconnected message when target is null', async () => {
+    mockPaidTier();
+    mockTargets({ pilotReachable: false, proxyReachable: true });
+    mockFetch(() => new Response(JSON.stringify({ reclaimableBytes: 0 }), {
+      status: 200, headers: { 'content-type': 'application/json' },
+    }));
+
+    const res = await request(app)
+      .post('/api/fleet/prune/estimate')
+      .set('Authorization', authHeader)
+      .send({ targets: ['images'], scope: 'managed' });
+
+    expect(res.status).toBe(200);
+    const pilotEntry = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === pilotNodeId);
+    expect(pilotEntry.reachable).toBe(false);
+    expect(pilotEntry.error).toMatch(/pilot tunnel/i);
+  });
+});
+
+describe('POST /api/fleet/labels/fleet-prune (pilot-agent dispatch)', () => {
+  it('dispatches the pilot row through the loopback target and surfaces the result', async () => {
+    mockPaidTier();
+    mockTargets();
+    const calls: Array<{ url: string; auth: string | undefined }> = [];
+    mockFetch((url, init) => {
+      const headers = (init?.headers as Record<string, string>) ?? {};
+      calls.push({ url, auth: headers.Authorization });
+      return new Response(JSON.stringify({ success: true, reclaimedBytes: 999, dryRun: true }), {
+        status: 200, headers: { 'content-type': 'application/json' },
+      });
+    });
+
+    const res = await request(app)
+      .post('/api/fleet/labels/fleet-prune')
+      .set('Authorization', authHeader)
+      .send({ targets: ['images'], scope: 'managed', dryRun: true });
+
+    expect(res.status).toBe(200);
+    const pilotCall = calls.find(c => c.url.startsWith(PILOT_LOOPBACK));
+    expect(pilotCall).toBeDefined();
+    expect(pilotCall?.auth).toBeUndefined();
+    const pilotResult = res.body.results.find((r: { nodeId: number }) => r.nodeId === pilotNodeId);
+    expect(pilotResult.reachable).toBe(true);
+    expect(pilotResult.targets[0].reclaimedBytes).toBe(999);
+  });
+});
+
+describe('GET /api/image-updates/fleet (pilot inclusion)', () => {
+  it('includes the pilot row in the aggregated fleet image-update status', async () => {
+    mockTargets();
+    // CacheService.getOrFetch caches by key; the cache from a prior test
+    // would silently mask the new dispatch. Force a miss.
+    const { CacheService } = await import('../services/CacheService');
+    CacheService.getInstance().invalidate('fleet-updates');
+
+    const calls: Array<{ url: string; auth: string | undefined }> = [];
+    mockFetch((url, init) => {
+      const headers = (init?.headers as Record<string, string>) ?? {};
+      calls.push({ url, auth: headers.Authorization });
+      // Distinguish pilot vs proxy by URL
+      const body = url.startsWith(PILOT_LOOPBACK)
+        ? { 'pilot-stack': true }
+        : { 'proxy-stack': false };
+      return new Response(JSON.stringify(body), {
+        status: 200, headers: { 'content-type': 'application/json' },
+      });
+    });
+
+    const res = await request(app)
+      .get('/api/image-updates/fleet')
+      .set('Authorization', authHeader);
+
+    expect(res.status).toBe(200);
+    const pilotCall = calls.find(c => c.url.startsWith(PILOT_LOOPBACK));
+    expect(pilotCall).toBeDefined();
+    expect(pilotCall?.auth).toBeUndefined();
+    expect(res.body[pilotNodeId]).toEqual({ 'pilot-stack': true });
+  });
+});
+
+describe('POST /api/image-updates/fleet/refresh (pilot trigger)', () => {
+  it('triggers the pilot via the loopback target and counts it in `triggered`', async () => {
+    mockPaidTier();
+    mockTargets();
+    const calls: Array<{ url: string; auth: string | undefined }> = [];
+    mockFetch((url, init) => {
+      const headers = (init?.headers as Record<string, string>) ?? {};
+      calls.push({ url, auth: headers.Authorization });
+      return new Response('', { status: 202 });
+    });
+
+    const res = await request(app)
+      .post('/api/image-updates/fleet/refresh')
+      .set('Authorization', authHeader);
+
+    expect(res.status).toBe(200);
+    const pilotCall = calls.find(c => c.url.startsWith(PILOT_LOOPBACK));
+    expect(pilotCall).toBeDefined();
+    expect(pilotCall?.auth).toBeUndefined();
+    expect(res.body.triggered).toContain(pilotNodeId);
+  });
+
+  it('skips pilots whose tunnel is disconnected (target null) without throwing', async () => {
+    mockPaidTier();
+    mockTargets({ pilotReachable: false, proxyReachable: true });
+    const proxyCalls: string[] = [];
+    mockFetch((url) => {
+      if (url.startsWith(PROXY_URL)) proxyCalls.push(url);
+      return new Response('', { status: 202 });
+    });
+
+    const res = await request(app)
+      .post('/api/image-updates/fleet/refresh')
+      .set('Authorization', authHeader);
+
+    expect(res.status).toBe(200);
+    expect(proxyCalls.length).toBeGreaterThan(0);
+    expect(res.body.triggered).not.toContain(pilotNodeId);
+    expect(res.body.failed).not.toContain(pilotNodeId);
+  });
+});
+
+describe('captureRemoteNodeFiles (pilot-agent dispatch)', () => {
+  it('fetches stacks through the loopback target when the pilot tunnel is up', async () => {
+    mockTargets();
+    const calls: Array<{ url: string; auth: string | undefined }> = [];
+    mockFetch((url, init) => {
+      const headers = (init?.headers as Record<string, string>) ?? {};
+      calls.push({ url, auth: headers.Authorization });
+      if (url.endsWith('/api/stacks')) {
+        return new Response(JSON.stringify(['snap-stack']), { status: 200, headers: { 'content-type': 'application/json' } });
+      }
+      if (url.includes('/api/stacks/snap-stack') && !url.endsWith('/env')) {
+        return new Response('services: {}', { status: 200 });
+      }
+      if (url.endsWith('/env')) {
+        return new Response('KEY=value', { status: 200 });
+      }
+      return new Response('', { status: 404 });
+    });
+
+    const { captureRemoteNodeFiles } = await import('../utils/snapshot-capture');
+    const node = DatabaseService.getInstance().getNode(pilotNodeId)!;
+    const result = await captureRemoteNodeFiles({ id: node.id, name: node.name, mode: node.mode });
+
+    const pilotCalls = calls.filter(c => c.url.startsWith(PILOT_LOOPBACK));
+    expect(pilotCalls.length).toBeGreaterThan(0);
+    expect(pilotCalls.every(c => c.auth === undefined)).toBe(true);
+    expect(result.nodeId).toBe(pilotNodeId);
+    expect(result.stacks.find(s => s.stackName === 'snap-stack')).toBeDefined();
+  });
+
+  it('throws a tunnel-disconnected error when the pilot target is null', async () => {
+    mockTargets({ pilotReachable: false, proxyReachable: true });
+    const { captureRemoteNodeFiles } = await import('../utils/snapshot-capture');
+    const node = DatabaseService.getInstance().getNode(pilotNodeId)!;
+
+    await expect(
+      captureRemoteNodeFiles({ id: node.id, name: node.name, mode: node.mode })
+    ).rejects.toThrow(/pilot tunnel/i);
+  });
+});
+
+// Note: SecretsService.{resolveEnvFileRemote,readEnvRemote,writeEnvRemote}
+// share the same getProxyTarget + conditional-Authorization shape exercised
+// above. The migration is structurally identical and tsc has validated the
+// types. Adding a redundant unit test that just re-proves the helper return
+// shape would add no signal beyond what captureRemoteNodeFiles already covers.
@@ -25,6 +25,7 @@ import { getErrorMessage } from '../utils/errors';
 import { parseIntParam } from '../utils/parseIntParam';
 import { POLICY_SEVERITIES } from '../utils/severity';
 import { sanitizeForLog } from '../utils/safeLog';
+import { formatNoTargetError } from '../utils/remoteTarget';
 import { CloudBackupService } from '../services/CloudBackupService';
 import { NotificationService } from '../services/NotificationService';
 import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
@@ -228,12 +229,6 @@ function pilotLastSeenSeconds(node: Node): number | null {
    : null;
 }

-function noTargetMessage(node: Node): string {
-  return node.mode === 'pilot_agent'
-    ? `Pilot tunnel to "${node.name}" is disconnected. Operations resume when the agent reconnects.`
-    : 'Remote node not configured';
-}
-
 function offlineRemoteOverview(node: Node, status: 'online' | 'offline'): FleetNodeOverview {
  const pilotSeen = pilotLastSeenSeconds(node);
  // For pilot-agent rows the tunnel heartbeat is the contact signal. Mirror
@@ -619,7 +614,7 @@ fleetRouter.get('/node/:nodeId/stacks', authMiddleware, async (req: Request, res
    if (node.type === 'remote') {
      const target = NodeRegistry.getInstance().getProxyTarget(node.id);
      if (!target) {
-        res.status(503).json({ error: noTargetMessage(node) });
+        res.status(503).json({ error: formatNoTargetError(node) });
        return;
      }
      const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/stacks`, {
@@ -663,7 +658,7 @@ fleetRouter.get('/node/:nodeId/stacks/:stackName/containers', authMiddleware, as
    if (node.type === 'remote') {
      const target = NodeRegistry.getInstance().getProxyTarget(node.id);
      if (!target) {
-        res.status(503).json({ error: noTargetMessage(node) });
+        res.status(503).json({ error: formatNoTargetError(node) });
        return;
      }
      const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(stackName)}/containers`, {
@@ -893,10 +888,7 @@ fleetRouter.post('/nodes/:nodeId/update', authMiddleware, async (req: Request, r

    const target = NodeRegistry.getInstance().getProxyTarget(node.id);
    if (!target) {
-      const msg = node.mode === 'pilot_agent'
-        ? `Pilot tunnel to "${node.name}" is disconnected.`
-        : 'Remote node not configured.';
-      res.status(503).json({ error: msg });
+      res.status(503).json({ error: formatNoTargetError(node) });
      return;
    }

@@ -1100,16 +1092,20 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res:
        }
      }

-      if (!node.api_url || !node.api_token) {
+      const target = NodeRegistry.getInstance().getProxyTarget(node.id);
+      if (!target) {
+        const error = formatNoTargetError(node);
        return {
          nodeId: node.id, nodeName: node.name, matched: true,
-          stackResults: stackNames.map(stackName => ({ stackName, success: false, error: 'Remote node not configured' })),
+          stackResults: stackNames.map(stackName => ({ stackName, success: false, error })),
        };
      }
      try {
-        const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/labels/${label.id}/action`, {
+        const headers: Record<string, string> = { 'Content-Type': 'application/json' };
+        if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
+        const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/labels/${label.id}/action`, {
          method: 'POST',
-          headers: { Authorization: `Bearer ${node.api_token}`, 'Content-Type': 'application/json' },
+          headers,
          body: JSON.stringify({ action: 'stop', dryRun: isDryRun }),
          signal: AbortSignal.timeout(60000),
        });
@@ -1227,13 +1223,17 @@ fleetRouter.post('/labels/fleet-prune', authMiddleware, async (req: Request, res

      // Remote node: POST /api/system/prune/system per target, short-circuiting
      // on the first transport-level failure so we don't hammer a dead node.
-      if (!node.api_url || !node.api_token) {
+      const proxyTarget = NodeRegistry.getInstance().getProxyTarget(node.id);
+      if (!proxyTarget) {
+        const error = formatNoTargetError(node);
        return {
-          nodeId: node.id, nodeName: node.name, reachable: false, error: 'Remote node not configured',
-          targets: targets.map(t => ({ target: t, success: false, reclaimedBytes: 0, error: 'Remote node not configured' })),
+          nodeId: node.id, nodeName: node.name, reachable: false, error,
+          targets: targets.map(t => ({ target: t, success: false, reclaimedBytes: 0, error })),
        };
      }
-      const baseUrl = node.api_url.replace(/\/$/, '');
+      const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
+      const remoteHeaders: Record<string, string> = { 'Content-Type': 'application/json' };
+      if (proxyTarget.apiToken) remoteHeaders.Authorization = `Bearer ${proxyTarget.apiToken}`;
      const targetResults: TargetResult[] = [];
      let nodeUnreachable: string | null = null;
      for (const target of targets) {
@@ -1244,7 +1244,7 @@ fleetRouter.post('/labels/fleet-prune', authMiddleware, async (req: Request, res
        try {
          const response = await fetch(`${baseUrl}/api/system/prune/system`, {
            method: 'POST',
-            headers: { Authorization: `Bearer ${node.api_token}`, 'Content-Type': 'application/json' },
+            headers: remoteHeaders,
            body: JSON.stringify({ target, scope, dryRun: isDryRun }),
            signal: AbortSignal.timeout(120000),
          });
@@ -1388,13 +1388,16 @@ fleetRouter.post('/prune/estimate', authMiddleware, async (req: Request, res: Re
        }
      }

-      if (!node.api_url || !node.api_token) {
+      const proxyTarget = NodeRegistry.getInstance().getProxyTarget(node.id);
+      if (!proxyTarget) {
        return {
          nodeId: node.id, nodeName: node.name, reclaimableBytes: 0, reachable: false,
-          error: 'Remote node not configured',
+          error: formatNoTargetError(node),
        };
      }
-      const baseUrl = node.api_url.replace(/\/$/, '');
+      const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
+      const estimateHeaders: Record<string, string> = { 'Content-Type': 'application/json' };
+      if (proxyTarget.apiToken) estimateHeaders.Authorization = `Bearer ${proxyTarget.apiToken}`;
      // Estimate is a live readout; fan out the per-target fetches in parallel
      // so wall time matches the slowest single call rather than the sum.
      // (The destructive sibling stays serial because Docker prune is internally
@@ -1403,7 +1406,7 @@ fleetRouter.post('/prune/estimate', authMiddleware, async (req: Request, res: Re
        try {
          const response = await fetch(`${baseUrl}/api/system/prune/estimate`, {
            method: 'POST',
-            headers: { Authorization: `Bearer ${node.api_token}`, 'Content-Type': 'application/json' },
+            headers: estimateHeaders,
            body: JSON.stringify({ target, scope }),
            signal: AbortSignal.timeout(15000),
          });
@@ -1656,19 +1659,23 @@ fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request,
        await composeService.deployStack(stackName);
      }
    } else {
-      if (!node.api_url || !node.api_token) {
-        res.status(503).json({ error: 'Remote node not configured' });
+      const proxyTarget = NodeRegistry.getInstance().getProxyTarget(node.id);
+      if (!proxyTarget) {
+        res.status(503).json({ error: formatNoTargetError(node) });
        return;
      }

-      const baseUrl = node.api_url.replace(/\/$/, '');
+      const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
      const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
+      // Tier/variant headers describe the central instance and stay
+      // unconditional; the Bearer header is gated on a non-empty token
+      // because pilot-loopback dispatch carries auth via the tunnel.
      const headers: Record<string, string> = {
-        Authorization: `Bearer ${node.api_token}`,
        'Content-Type': 'application/json',
        [PROXY_TIER_HEADER]: proxyHeaders.tier,
        [PROXY_VARIANT_HEADER]: proxyHeaders.variant ?? '',
      };
+      if (proxyTarget.apiToken) headers.Authorization = `Bearer ${proxyTarget.apiToken}`;

      for (const file of files) {
        if (file.filename === 'compose.yaml') {
@@ -71,16 +71,21 @@ imageUpdatesRouter.get('/fleet', authMiddleware, async (_req: Request, res: Resp
        }

        // Remote nodes: parallel fetches with per-request timeouts.
-        const remoteNodes = nodes.filter(n => n.type === 'remote' && n.status === 'online' && n.api_url);
+        // Pilot-agent rows have no api_url; rely on getProxyTarget for the
+        // reachability predicate AND the base URL so pilots with an active
+        // tunnel participate in the fan-out.
+        const remoteCandidates = nodes
+          .filter(n => n.type === 'remote' && n.status === 'online')
+          .map(node => ({ node, proxyTarget: nr.getProxyTarget(node.id) }))
+          .filter((entry): entry is { node: typeof entry.node; proxyTarget: NonNullable<typeof entry.proxyTarget> } => entry.proxyTarget !== null);
        const remoteResults = await Promise.allSettled(
-          remoteNodes.map(async (node) => {
-            const proxyTarget = nr.getProxyTarget(node.id);
-            const baseUrl = node.api_url!.replace(/\/$/, '');
+          remoteCandidates.map(async ({ node, proxyTarget }) => {
+            const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
            const controller = new AbortController();
            const timeout = setTimeout(() => controller.abort(), REMOTE_NODE_FETCH_TIMEOUT_MS);
            try {
              const resp = await fetch(`${baseUrl}/api/image-updates`, {
-                headers: proxyTarget?.apiToken
+                headers: proxyTarget.apiToken
                  ? { Authorization: `Bearer ${proxyTarget.apiToken}` }
                  : {},
                signal: controller.signal,
@@ -138,17 +143,22 @@ imageUpdatesRouter.post('/fleet/refresh', authMiddleware, async (_req: Request,
    }
  }

-  const remoteNodes = nodes.filter(n => n.type === 'remote' && n.status === 'online' && n.api_url);
+  // Pilot-agent rows have no api_url; rely on getProxyTarget for the
+  // reachability predicate AND the base URL so pilots with an active
+  // tunnel participate in the fan-out.
+  const remoteCandidates = nodes
+    .filter(n => n.type === 'remote' && n.status === 'online')
+    .map(node => ({ node, proxyTarget: nr.getProxyTarget(node.id) }))
+    .filter((entry): entry is { node: typeof entry.node; proxyTarget: NonNullable<typeof entry.proxyTarget> } => entry.proxyTarget !== null);
  const remoteResults = await Promise.allSettled(
-    remoteNodes.map(async (node) => {
-      const proxyTarget = nr.getProxyTarget(node.id);
-      const baseUrl = node.api_url!.replace(/\/$/, '');
+    remoteCandidates.map(async ({ node, proxyTarget }) => {
+      const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
      const controller = new AbortController();
      const timeout = setTimeout(() => controller.abort(), REMOTE_NODE_FETCH_TIMEOUT_MS);
      try {
        const resp = await fetch(`${baseUrl}/api/image-updates/refresh`, {
          method: 'POST',
-          headers: proxyTarget?.apiToken
+          headers: proxyTarget.apiToken
            ? { Authorization: `Bearer ${proxyTarget.apiToken}` }
            : {},
          signal: controller.signal,
@@ -4,8 +4,10 @@ import { CryptoService } from './CryptoService';
 import { DatabaseService, type BlueprintSelector, type Node, type SecretRow, type SecretVersionRow, type SecretPushStatus } from './DatabaseService';
 import { FileSystemService } from './FileSystemService';
 import { NodeLabelService } from './NodeLabelService';
+import { NodeRegistry } from './NodeRegistry';
 import { resolveAllEnvFilePaths } from '../routes/stacks';
 import { getErrorMessage } from '../utils/errors';
+import { formatNoTargetError } from '../utils/remoteTarget';

 export type SecretKv = Record<string, string>;
 export type DiffStatus = 'added' | 'changed' | 'removed' | 'unchanged';
@@ -212,9 +214,16 @@ async function resolveEnvFileLocal(nodeId: number, stackName: string, basename:
 }

 async function resolveEnvFileRemote(node: Node, stackName: string, basename: string): Promise<ResolvedEnvFile | null> {
-    if (!node.api_url || !node.api_token) return null;
-    const baseUrl = node.api_url.replace(/\/$/, '');
-    const headers = { Authorization: `Bearer ${node.api_token}` };
+    // Throw on a null target so previewPushDiff / executePush surface the
+    // tunnel-disconnected (or proxy-not-configured) reason on the right
+    // axis. Returning null here would flow up through readExistingEnv as
+    // "env file not found", which is wrong: we know the env exists, we
+    // just cannot reach the node.
+    const target = NodeRegistry.getInstance().getProxyTarget(node.id);
+    if (!target) throw new Error(formatNoTargetError(node));
+    const baseUrl = target.apiUrl.replace(/\/$/, '');
+    const headers: Record<string, string> = {};
+    if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
    const res = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/envs`, {
        headers,
        signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
@@ -251,9 +260,11 @@ async function writeEnvLocal(nodeId: number, absolutePath: string, content: stri
 }

 async function readEnvRemote(node: Node, stackName: string, absolutePath: string): Promise<string> {
-    if (!node.api_url || !node.api_token) throw new Error('node has no api_url or api_token');
-    const baseUrl = node.api_url.replace(/\/$/, '');
-    const headers = { Authorization: `Bearer ${node.api_token}` };
+    const target = NodeRegistry.getInstance().getProxyTarget(node.id);
+    if (!target) throw new Error(formatNoTargetError(node));
+    const baseUrl = target.apiUrl.replace(/\/$/, '');
+    const headers: Record<string, string> = {};
+    if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
    const url = new URL(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`);
    if (absolutePath !== '.env') url.searchParams.set('file', absolutePath);
    const res = await fetch(url.toString(), {
@@ -266,12 +277,13 @@ async function readEnvRemote(node: Node, stackName: string, absolutePath: string
 }

 async function writeEnvRemote(node: Node, stackName: string, absolutePath: string, content: string): Promise<void> {
-    if (!node.api_url || !node.api_token) throw new Error('node has no api_url or api_token');
-    const baseUrl = node.api_url.replace(/\/$/, '');
-    const headers = {
-        Authorization: `Bearer ${node.api_token}`,
+    const target = NodeRegistry.getInstance().getProxyTarget(node.id);
+    if (!target) throw new Error(formatNoTargetError(node));
+    const baseUrl = target.apiUrl.replace(/\/$/, '');
+    const headers: Record<string, string> = {
        'Content-Type': 'application/json',
    };
+    if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
    const url = new URL(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`);
    if (absolutePath !== '.env') url.searchParams.set('file', absolutePath);
    const res = await fetch(url.toString(), {
@@ -0,0 +1,15 @@
+import type { Node } from '../services/DatabaseService';
+
+/**
+ * Returns the operator-facing error message for a remote node that has
+ * no reachable proxy target. Pilot-mode rows get a tunnel-aware message;
+ * proxy-mode rows fall back to the historical credentials-missing copy.
+ *
+ * Use whenever `NodeRegistry.getProxyTarget(node.id)` returns null and
+ * the caller needs to surface why the dispatch was skipped.
+ */
+export function formatNoTargetError(node: Pick<Node, 'mode' | 'name'>): string {
+  return node.mode === 'pilot_agent'
+    ? `Pilot tunnel to "${node.name}" is disconnected. Operations resume when the agent reconnects.`
+    : 'Remote node not configured';
+}
@@ -3,7 +3,10 @@
 * and the SchedulerService for fleet-wide snapshot operations.
 */

+import type { NodeMode } from '../services/DatabaseService';
 import { FileSystemService } from '../services/FileSystemService';
+import { NodeRegistry } from '../services/NodeRegistry';
+import { formatNoTargetError } from './remoteTarget';
 import { isDebugEnabled } from './debug';

 export interface SnapshotNodeData {
@@ -15,12 +18,15 @@ export interface SnapshotNodeData {
  }>;
 }

-/** Minimal node shape accepted by capture functions. */
+/**
+ * Minimal node shape accepted by capture functions.
+ * `mode` is required so remote dispatch can emit a tunnel-aware error when
+ * the pilot-agent proxy target is null.
+ */
 export interface CaptureNode {
  id: number;
  name: string;
-  api_url?: string;
-  api_token?: string;
+  mode: NodeMode;
 }

 /**
@@ -65,13 +71,15 @@ export async function captureLocalNodeFiles(node: CaptureNode): Promise<Snapshot
 * fetched are silently skipped.
 */
 export async function captureRemoteNodeFiles(node: CaptureNode): Promise<SnapshotNodeData> {
-  if (!node.api_url || !node.api_token) {
-    throw new Error('Remote node not configured');
+  const target = NodeRegistry.getInstance().getProxyTarget(node.id);
+  if (!target) {
+    throw new Error(formatNoTargetError(node));
  }

  const start = Date.now();
-  const baseUrl = node.api_url.replace(/\/$/, '');
-  const headers = { Authorization: `Bearer ${node.api_token}` };
+  const baseUrl = target.apiUrl.replace(/\/$/, '');
+  const headers: Record<string, string> = {};
+  if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;

  const stacksRes = await fetch(`${baseUrl}/api/stacks`, {
    headers,