fix(mesh): inspect remote stacks via HTTP proxy not Dockerode (#992)

MeshService.inspectStackServices used DockerController.getInstance(nodeId)
to enumerate compose-labeled containers, but NodeRegistry.getDocker
explicitly throws for any node with type='remote' by design. The throw
was silently caught and an empty array returned, so opt-in for any
remote node failed with a misleading "no running services" error and
refreshAliasCache only ever populated LOCAL aliases. Cross-node mesh
routing was therefore impossible end-to-end regardless of pilot-tunnel
state.

Split the inspector. inspectLocalStackServices keeps the Dockerode
listContainers path and always queries the local Docker daemon; it is
public so the new route can call it. inspectStackServices is now a
dispatcher: local nodes fall through to the Dockerode path, remote
nodes (proxy mode and pilot-agent) HTTP-fetch
/api/mesh/local-services/:stackName against the URL resolved by
NodeRegistry.getProxyTarget, with the persisted node_proxy Bearer
token and license tier headers attached. The remote's MeshService
enumerates its own LOCAL Docker daemon and returns the
{service, ports[]} envelope.

refreshAliasCache now inspects every opted-in stack in parallel via
Promise.allSettled so a slow or unreachable remote does not stall the
60-second loop. The new /api/mesh/local-services/:stackName route is
gated by requireAdmiral plus isValidStackName and always queries the
caller's own Sencho instance.

Together with PR #989 (proxy-side bridge dispatch) and PR #990
(agent-side loopback auth), this makes the central, bridge, agent,
local-Sencho mesh control plane functional end-to-end for both
proxy-mode and pilot-agent remotes.

backend/src/__tests__/mesh-inspect-remote.test.ts

@@ -0,0 +1,151 @@
+/**
+ * Regression guard for the C-3 fix: MeshService inspects remote nodes via the
+ * existing HTTP proxy chain rather than calling Dockerode directly (which
+ * NodeRegistry.getDocker explicitly throws for any remote node by design).
+ *
+ * Two behaviors covered:
+ *   1. For a local node, the dispatcher calls `inspectLocalStackServices`
+ *      (which uses the local Dockerode).
+ *   2. For a remote node, the dispatcher fetches `/api/mesh/local-services/:stackName`
+ *      against the resolved proxy target with the appropriate Authorization
+ *      and license tier headers, parses the JSON envelope, and returns the
+ *      decoded `services[]` array.
+ */
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
+import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
+
+let tmpDir: string;
+let MeshService: typeof import('../services/MeshService').MeshService;
+let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
+let NodeRegistry: typeof import('../services/NodeRegistry').NodeRegistry;
+
+beforeAll(async () => {
+    tmpDir = await setupTestDb();
+    ({ MeshService } = await import('../services/MeshService'));
+    ({ DatabaseService } = await import('../services/DatabaseService'));
+    ({ NodeRegistry } = await import('../services/NodeRegistry'));
+});
+
+afterAll(() => {
+    vi.restoreAllMocks();
+    cleanupTestDb(tmpDir);
+});
+
+afterEach(() => {
+    // Restore both spies and the global fetch patch so a sibling test file
+    // running in the same worker (e.g. fleet.test.ts) does not see a stale
+    // mocked fetch / getProxyTarget.
+    vi.restoreAllMocks();
+});
+
+describe('MeshService.inspectStackServices dispatch (C-3 fix)', () => {
+    it('uses the local Dockerode path for the local node', async () => {
+        const svc = MeshService.getInstance();
+        const db = DatabaseService.getInstance();
+        const localNodeId = db.getNodes()[0].id;
+
+        const localSpy = vi
+            .spyOn(svc, 'inspectLocalStackServices')
+            .mockResolvedValue([{ service: 'echo', ports: [9000] }]);
+        const fetchSpy = vi.spyOn(globalThis, 'fetch');
+
+        const out = await (svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> })
+            .inspectStackServices(localNodeId, 'audit-mesh-prod');
+
+        expect(localSpy).toHaveBeenCalledWith('audit-mesh-prod');
+        expect(fetchSpy).not.toHaveBeenCalled();
+        expect(out).toEqual([{ service: 'echo', ports: [9000] }]);
+    });
+
+    it('fetches /api/mesh/local-services for remote nodes and forwards the proxy target headers', async () => {
+        const svc = MeshService.getInstance();
+        const db = DatabaseService.getInstance();
+        const remoteNodeId = db.addNode({
+            name: 'inspect-remote-test',
+            type: 'remote',
+            mode: 'proxy',
+            compose_dir: '/tmp',
+            is_default: false,
+            api_url: 'https://remote.example.com:1852',
+            api_token: 'remote-tok',
+        });
+
+        // Force the registry to return a known target so we exercise the
+        // request shape rather than the registry's own resolution rules.
+        vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
+            apiUrl: 'https://remote.example.com:1852',
+            apiToken: 'remote-tok',
+        });
+
+        const fetchMock = vi
+            .spyOn(globalThis, 'fetch')
+            .mockResolvedValue(new Response(
+                JSON.stringify({ services: [{ service: 'echo', ports: [9001] }] }),
+                { status: 200, headers: { 'content-type': 'application/json' } },
+            ));
+
+        const out = await (svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> })
+            .inspectStackServices(remoteNodeId, 'audit-mesh-pilot');
+
+        expect(out).toEqual([{ service: 'echo', ports: [9001] }]);
+        expect(fetchMock).toHaveBeenCalledTimes(1);
+        const call = fetchMock.mock.calls[0];
+        expect(String(call[0])).toBe('https://remote.example.com:1852/api/mesh/local-services/audit-mesh-pilot');
+        const headers = (call[1] as { headers: Record<string, string> }).headers;
+        expect(headers['Authorization']).toBe('Bearer remote-tok');
+        expect(headers).toHaveProperty('x-sencho-tier');
+        expect(headers).toHaveProperty('x-sencho-variant');
+
+        db.deleteNode(remoteNodeId);
+    });
+
+    it('returns [] when the remote responds non-2xx', async () => {
+        const svc = MeshService.getInstance();
+        const db = DatabaseService.getInstance();
+        const remoteNodeId = db.addNode({
+            name: 'inspect-remote-fail',
+            type: 'remote',
+            mode: 'proxy',
+            compose_dir: '/tmp',
+            is_default: false,
+            api_url: 'https://remote.example.com:1852',
+            api_token: 'remote-tok',
+        });
+
+        vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
+            apiUrl: 'https://remote.example.com:1852',
+            apiToken: 'remote-tok',
+        });
+        vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('Internal Server Error', { status: 500 }));
+
+        const out = await (svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> })
+            .inspectStackServices(remoteNodeId, 'audit-mesh-pilot');
+
+        expect(out).toEqual([]);
+        db.deleteNode(remoteNodeId);
+    });
+
+    it('returns [] for a remote node with no active proxy target (e.g. pilot-agent tunnel down)', async () => {
+        const svc = MeshService.getInstance();
+        const db = DatabaseService.getInstance();
+        const remoteNodeId = db.addNode({
+            name: 'inspect-remote-down',
+            type: 'remote',
+            mode: 'pilot_agent',
+            compose_dir: '/tmp',
+            is_default: false,
+            api_url: '',
+            api_token: '',
+        });
+
+        vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue(null);
+        const fetchSpy = vi.spyOn(globalThis, 'fetch');
+
+        const out = await (svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> })
+            .inspectStackServices(remoteNodeId, 'audit-mesh-pilot');
+
+        expect(out).toEqual([]);
+        expect(fetchSpy).not.toHaveBeenCalled();
+        db.deleteNode(remoteNodeId);
+    });
+});

backend/src/routes/mesh.ts

@@ -4,6 +4,7 @@ import { NodeRegistry } from '../services/NodeRegistry';
 import { MeshError, MeshService } from '../services/MeshService';
 import { requireAdmin, requireAdmiral } from '../middleware/tierGates';
 import { sanitizeForLog } from '../utils/safeLog';
+import { isValidStackName } from '../utils/validation';
 
 export const meshRouter = Router();
 
@@ -49,6 +50,26 @@ meshRouter.post('/nodes/:nodeId/disable', async (req: Request, res: Response): P
     }
 });
 
+/**
+ * Returns the LOCAL Docker daemon's services for a stack with their listening
+ * ports. Always queries this Sencho instance's own Dockerode regardless of
+ * `x-node-id`. Central calls this endpoint against each remote node via the
+ * existing proxy chain (`NodeRegistry.getProxyTarget`) so it can build the
+ * cross-fleet alias cache without violating the local-only Dockerode rule.
+ */
+meshRouter.get('/local-services/:stackName', async (req: Request, res: Response): Promise<void> => {
+    if (!requireAdmiral(req, res)) return;
+    const stackName = req.params.stackName as string;
+    if (!isValidStackName(stackName)) { res.status(400).json({ error: 'Invalid stack name' }); return; }
+    try {
+        const services = await MeshService.getInstance().inspectLocalStackServices(stackName);
+        res.json({ services });
+    } catch (err) {
+        console.warn('[mesh] /local-services failed:', sanitizeForLog((err as Error).message));
+        res.status(500).json({ error: 'Failed to list local services' });
+    }
+});
+
 meshRouter.get('/nodes/:nodeId/stacks', async (req: Request, res: Response): Promise<void> => {
     if (!requireAdmiral(req, res)) return;
     const nodeId = Number.parseInt(req.params.nodeId as string, 10);

backend/src/services/MeshService.ts

@@ -5,6 +5,9 @@ import { EventEmitter } from 'events';
 import jwt from 'jsonwebtoken';
 import { DatabaseService } from './DatabaseService';
 import DockerController from './DockerController';
+import { LicenseService } from './LicenseService';
+import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from './license-headers';
+import { NodeRegistry } from './NodeRegistry';
 import { PilotTunnelManager } from './PilotTunnelManager';
 import { generateOverrideYaml, MeshAlias } from './MeshComposeOverride';
 import { sanitizeForLog } from '../utils/safeLog';
@@ -359,10 +362,20 @@ export class MeshService extends EventEmitter {
         const portMap = new Map<number, MeshGlobalAlias>();
         const stacks = db.listMeshStacks();
 
-        for (const row of stacks) {
-            const node = db.getNode(row.node_id);
-            if (!node) continue;
-            const services = await this.inspectStackServices(row.node_id, row.stack_name).catch(() => []);
+        // Inspect all stacks in parallel; each remote-node lookup involves an
+        // HTTP fetch with its own 5 s AbortSignal. Sequential awaiting would
+        // let one slow node stall the whole refresh, which is on a 60 s loop.
+        const inspections = await Promise.allSettled(
+            stacks.map(async (row) => {
+                const node = db.getNode(row.node_id);
+                if (!node) return null;
+                const services = await this.inspectStackServices(row.node_id, row.stack_name);
+                return { row, node, services };
+            }),
+        );
+        for (const result of inspections) {
+            if (result.status !== 'fulfilled' || !result.value) continue;
+            const { row, node, services } = result.value;
             for (const svc of services) {
                 const host = `${svc.service}.${row.stack_name}.${node.name}.sencho`;
                 for (const port of svc.ports) {
@@ -388,12 +401,14 @@ export class MeshService extends EventEmitter {
     }
 
     /**
-     * Inspect a stack on a node and return its running services with the ports
-     * they listen on. Uses Compose container labels.
+     * Inspect a stack and return its running services with the ports they
+     * listen on. For the LOCAL Docker daemon only — callers targeting a
+     * remote node must use {@link inspectStackServices}, which dispatches
+     * via the HTTP proxy to the remote's `/api/mesh/local-services/:stack`.
      */
-    private async inspectStackServices(nodeId: number, stackName: string): Promise<Array<{ service: string; ports: number[] }>> {
+    public async inspectLocalStackServices(stackName: string): Promise<Array<{ service: string; ports: number[] }>> {
         try {
-            const docker = DockerController.getInstance(nodeId).getDocker();
+            const docker = DockerController.getInstance().getDocker();
             const containers = await docker.listContainers({
                 all: true,
                 filters: { label: [`com.docker.compose.project=${stackName}`] },
@@ -410,7 +425,45 @@ export class MeshService extends EventEmitter {
             }
             return Array.from(byService.entries()).map(([service, ports]) => ({ service, ports: Array.from(ports) }));
         } catch (err) {
-            console.warn('[MeshService] inspectStackServices failed:', sanitizeForLog((err as Error).message));
+            console.warn('[MeshService] inspectLocalStackServices failed:', sanitizeForLog((err as Error).message));
+            return [];
+        }
+    }
+
+    /**
+     * Inspect a stack on a (possibly remote) node and return its running
+     * services with the ports they listen on. Local nodes hit Dockerode
+     * directly; remote nodes (proxy mode and pilot-agent) reach their own
+     * Sencho's `/api/mesh/local-services/:stackName` via the existing
+     * `NodeRegistry.getProxyTarget` resolution chain because Dockerode is not
+     * directly reachable for remote nodes by design.
+     */
+    private async inspectStackServices(nodeId: number, stackName: string): Promise<Array<{ service: string; ports: number[] }>> {
+        const node = DatabaseService.getInstance().getNode(nodeId);
+        if (!node) return [];
+        if (node.type !== 'remote') return this.inspectLocalStackServices(stackName);
+
+        const target = NodeRegistry.getInstance().getProxyTarget(nodeId);
+        if (!target) {
+            console.warn(`[MeshService] inspectStackServices: no proxy target for node ${nodeId} (${sanitizeForLog(node.name)})`);
+            return [];
+        }
+        try {
+            const url = `${target.apiUrl.replace(/\/$/, '')}/api/mesh/local-services/${encodeURIComponent(stackName)}`;
+            const headers: Record<string, string> = {};
+            if (target.apiToken) headers['Authorization'] = `Bearer ${target.apiToken}`;
+            const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
+            headers[PROXY_TIER_HEADER] = proxyHeaders.tier;
+            headers[PROXY_VARIANT_HEADER] = proxyHeaders.variant || '';
+            const res = await fetch(url, { headers, signal: AbortSignal.timeout(5_000) });
+            if (!res.ok) {
+                console.error(`[MeshService] inspectStackServices: HTTP ${res.status} from node ${nodeId} (${sanitizeForLog(node.name)})`);
+                return [];
+            }
+            const body = await res.json() as { services?: Array<{ service: string; ports: number[] }> };
+            return body.services ?? [];
+        } catch (err) {
+            console.error('[MeshService] inspectStackServices remote unreachable:', sanitizeForLog((err as Error).message));
             return [];
         }
     }