From 23bbee4f45ea08d133b5249a537e978fb94ad1b4 Mon Sep 17 00:00:00 2001 From: Anso Date: Sat, 9 May 2026 00:11:09 -0400 Subject: [PATCH] feat(mesh): replace host-mode with shared sencho_mesh Docker network (#1009) * feat(mesh): replace host-mode with shared sencho_mesh Docker network Phase D of the mesh redesign: drop the operator's `network_mode: host` requirement and the `host-gateway` extra_hosts pattern that did not work on cloud iptables-restrictive distros (OCI, etc.) or Docker Desktop. Each Sencho creates a shared `sencho_mesh` Docker bridge network on boot (default subnet 172.30.0.0/24, override via SENCHO_MESH_SUBNET), pins itself at `+2`, and attaches every meshed user service to the same bridge. Compose overrides now emit IP-based `extra_hosts` plus a top-level `networks` block declaring `sencho_mesh` external. Override delivery: central renders for local stacks; for remote stacks it sends the fleet alias list to the remote's new `PUT /api/mesh/local- override/:stackName` endpoint, which renders against the remote's OWN local senchoIp and writes under its OWN DATA_DIR. Each node may use a different subnet without coordination beyond the env var. Opt-in / opt-out now trigger an automatic redeploy of the affected stack via the existing deploy code path (local: ComposeService; remote: HTTP POST through proxyFetch). The frontend opt-in sheet shows a confirmation modal (ConfirmModal) before the mutation. Failed redeploys emit both a mesh activity event and a durable audit-log row. Hardening: - Reserve port 1852 at opt-in (prevents user containers from racing the Sencho API listener). - ensureMeshNetwork refuses to continue if `sencho_mesh` exists with a mismatched subnet rather than silently routing to the wrong IP. - Idempotent network connect/disconnect helpers in DockerController. - optInStack rolls back the DB row if the just-inserted stack's override push fails (no half-states surviving across calls). - regenerateOverridesForNode runs in parallel and skips the just- pushed stack on opt-in. Operator template: drop `network_mode: host`, restore `ports: ["1852:1852"]`. Mesh now works identically on Linux LAN, OCI, and Docker Desktop without firewall changes. Docs: rewrite docs/features/sencho-mesh.mdx around the shared bridge network, document SENCHO_MESH_SUBNET, surface the host-network-service opt-in restriction, and cross-link with the Pilot Agent docs. BREAKING CHANGE: the operator's `docker-compose.yml` no longer uses `network_mode: host`. After upgrading, redeploy any meshed stacks once so they pick up the new IP-based override and join `sencho_mesh`. * fix(mesh): wrap stackName with path.basename in local-override fs ops CodeQL flagged js/path-injection on the new applyLocalOverride and removeLocalOverride methods because they are publicly reachable and its data-flow model does not recognize isValidStackName / isPathWithinBase as sanitizers. The validation IS sufficient (the allowlist regex blocks path separators, the path-prefix check blocks escape), but path.basename is a model CodeQL recognizes and is purely defensive: for any input that already passes isValidStackName, basename is the identity. --- .../src/__tests__/docker-controller.test.ts | 87 ++++ .../__tests__/mesh-compose-override.test.ts | 56 ++- backend/src/__tests__/mesh-service.test.ts | 118 +++++ backend/src/routes/mesh.ts | 67 +++ backend/src/services/DockerController.ts | 62 +++ backend/src/services/MeshComposeOverride.ts | 41 +- backend/src/services/MeshService.ts | 473 +++++++++++++++++- docker-compose.yml | 13 +- docs/features/pilot-agent.mdx | 6 +- docs/features/sencho-mesh.mdx | 84 +++- .../src/components/fleet/MeshOptInSheet.tsx | 138 +++-- 11 files changed, 1023 insertions(+), 122 deletions(-) diff --git a/backend/src/__tests__/docker-controller.test.ts b/backend/src/__tests__/docker-controller.test.ts index 76bfa91d..c13774dc 100644 --- a/backend/src/__tests__/docker-controller.test.ts +++ b/backend/src/__tests__/docker-controller.test.ts @@ -663,3 +663,90 @@ describe('removeContainers', () => { expect(results).toEqual([]); }); }); + +// ── Network connect / disconnect helpers ─────────────────────────────── + +describe('DockerController - connectContainerToNetwork', () => { + it('attaches a container to a network with no static IP', async () => { + const connect = vi.fn().mockResolvedValue(undefined); + mockDocker.getNetwork.mockReturnValue({ connect }); + + const dc = DockerController.getInstance(1); + await dc.connectContainerToNetwork('sencho_mesh', 'sencho-host-1234'); + + expect(mockDocker.getNetwork).toHaveBeenCalledWith('sencho_mesh'); + expect(connect).toHaveBeenCalledWith({ Container: 'sencho-host-1234' }); + }); + + it('attaches with a static IPv4 address when provided', async () => { + const connect = vi.fn().mockResolvedValue(undefined); + mockDocker.getNetwork.mockReturnValue({ connect }); + + const dc = DockerController.getInstance(1); + await dc.connectContainerToNetwork('sencho_mesh', 'sencho-host-1234', { ipv4Address: '172.30.0.2' }); + + expect(connect).toHaveBeenCalledWith({ + Container: 'sencho-host-1234', + EndpointConfig: { IPAMConfig: { IPv4Address: '172.30.0.2' } }, + }); + }); + + it('treats 403 already-connected as success (idempotent)', async () => { + const connect = vi.fn().mockRejectedValue({ statusCode: 403, message: 'endpoint already exists' }); + mockDocker.getNetwork.mockReturnValue({ connect }); + + const dc = DockerController.getInstance(1); + await expect(dc.connectContainerToNetwork('sencho_mesh', 'sencho-host-1234')).resolves.toBeUndefined(); + }); + + it('rethrows non-idempotent errors', async () => { + const connect = vi.fn().mockRejectedValue({ statusCode: 500, message: 'server error' }); + mockDocker.getNetwork.mockReturnValue({ connect }); + + const dc = DockerController.getInstance(1); + await expect(dc.connectContainerToNetwork('sencho_mesh', 'sencho-host-1234')).rejects.toMatchObject({ + statusCode: 500, + }); + }); + + it('rethrows a 403 whose message is unrelated to already-attached', async () => { + const connect = vi.fn().mockRejectedValue({ statusCode: 403, message: 'host-mode container cannot join network' }); + mockDocker.getNetwork.mockReturnValue({ connect }); + + const dc = DockerController.getInstance(1); + await expect(dc.connectContainerToNetwork('sencho_mesh', 'sencho-host-1234')).rejects.toMatchObject({ + statusCode: 403, + }); + }); +}); + +describe('DockerController - disconnectContainerFromNetwork', () => { + it('detaches a container from a network with force=true', async () => { + const disconnect = vi.fn().mockResolvedValue(undefined); + mockDocker.getNetwork.mockReturnValue({ disconnect }); + + const dc = DockerController.getInstance(1); + await dc.disconnectContainerFromNetwork('sencho_mesh', 'sencho-host-1234'); + + expect(mockDocker.getNetwork).toHaveBeenCalledWith('sencho_mesh'); + expect(disconnect).toHaveBeenCalledWith({ Container: 'sencho-host-1234', Force: true }); + }); + + it('treats 404 not-connected as success (idempotent)', async () => { + const disconnect = vi.fn().mockRejectedValue({ statusCode: 404, message: 'no such network endpoint' }); + mockDocker.getNetwork.mockReturnValue({ disconnect }); + + const dc = DockerController.getInstance(1); + await expect(dc.disconnectContainerFromNetwork('sencho_mesh', 'sencho-host-1234')).resolves.toBeUndefined(); + }); + + it('rethrows non-idempotent errors', async () => { + const disconnect = vi.fn().mockRejectedValue({ statusCode: 500, message: 'server error' }); + mockDocker.getNetwork.mockReturnValue({ disconnect }); + + const dc = DockerController.getInstance(1); + await expect(dc.disconnectContainerFromNetwork('sencho_mesh', 'sencho-host-1234')).rejects.toMatchObject({ + statusCode: 500, + }); + }); +}); diff --git a/backend/src/__tests__/mesh-compose-override.test.ts b/backend/src/__tests__/mesh-compose-override.test.ts index c4715ca5..8bac650b 100644 --- a/backend/src/__tests__/mesh-compose-override.test.ts +++ b/backend/src/__tests__/mesh-compose-override.test.ts @@ -1,36 +1,64 @@ import { describe, expect, it } from 'vitest'; import * as YAML from 'yaml'; -import { buildAliasHosts, generateOverrideYaml } from '../services/MeshComposeOverride'; +import { buildAliasHosts, generateOverrideYaml, SENCHO_MESH_NETWORK } from '../services/MeshComposeOverride'; + +const SENCHO_IP = '172.30.0.2'; describe('generateOverrideYaml', () => { - it('emits services with extra_hosts pointing to host-gateway', () => { + it('emits services with extra_hosts pointing to the Sencho mesh IP', () => { const yaml = generateOverrideYaml({ services: ['web', 'cache'], aliases: [ { host: 'db.api.opsix.sencho' }, { host: 'etl.worker.opsix.sencho' }, ], + senchoIp: SENCHO_IP, }); const parsed = YAML.parse(yaml) as Record; - expect(parsed.networks).toBeUndefined(); - const services = parsed.services as Record; + const services = parsed.services as Record; expect(Object.keys(services).sort()).toEqual(['cache', 'web']); for (const svc of ['web', 'cache']) { expect(services[svc].extra_hosts).toEqual([ - 'db.api.opsix.sencho:host-gateway', - 'etl.worker.opsix.sencho:host-gateway', + `db.api.opsix.sencho:${SENCHO_IP}`, + `etl.worker.opsix.sencho:${SENCHO_IP}`, ]); } }); - it('emits empty service stubs when no aliases exist yet', () => { + it('attaches every service to the sencho_mesh network', () => { + const yaml = generateOverrideYaml({ + services: ['web', 'cache', 'worker'], + aliases: [{ host: 'db.api.opsix.sencho' }], + senchoIp: SENCHO_IP, + }); + const parsed = YAML.parse(yaml) as Record; + const services = parsed.services as Record; + for (const svc of ['web', 'cache', 'worker']) { + expect(services[svc].networks).toEqual([SENCHO_MESH_NETWORK]); + } + }); + + it('declares sencho_mesh as an external network at the top level', () => { const yaml = generateOverrideYaml({ services: ['web'], aliases: [], + senchoIp: SENCHO_IP, }); const parsed = YAML.parse(yaml) as Record; - const services = parsed.services as Record; + const networks = parsed.networks as Record; + expect(networks[SENCHO_MESH_NETWORK]).toEqual({ external: true }); + }); + + it('still attaches services to the network when no aliases exist yet', () => { + const yaml = generateOverrideYaml({ + services: ['web'], + aliases: [], + senchoIp: SENCHO_IP, + }); + const parsed = YAML.parse(yaml) as Record; + const services = parsed.services as Record; expect(services.web.extra_hosts).toBeUndefined(); + expect(services.web.networks).toEqual([SENCHO_MESH_NETWORK]); }); it('produces stable output regardless of input ordering', () => { @@ -40,6 +68,7 @@ describe('generateOverrideYaml', () => { { host: 'b.x.y.sencho' }, { host: 'a.x.y.sencho' }, ], + senchoIp: SENCHO_IP, }); const b = generateOverrideYaml({ services: ['cache', 'web'], @@ -47,9 +76,20 @@ describe('generateOverrideYaml', () => { { host: 'a.x.y.sencho' }, { host: 'b.x.y.sencho' }, ], + senchoIp: SENCHO_IP, }); expect(a).toBe(b); }); + + it('uses the senchoIp argument verbatim in extra_hosts entries', () => { + const customIp = '10.42.7.99'; + const yaml = generateOverrideYaml({ + services: ['web'], + aliases: [{ host: 'svc.stack.node.sencho' }], + senchoIp: customIp, + }); + expect(yaml).toContain(`svc.stack.node.sencho:${customIp}`); + }); }); describe('buildAliasHosts', () => { diff --git a/backend/src/__tests__/mesh-service.test.ts b/backend/src/__tests__/mesh-service.test.ts index 2bf0c9b5..d565b4b2 100644 --- a/backend/src/__tests__/mesh-service.test.ts +++ b/backend/src/__tests__/mesh-service.test.ts @@ -1,5 +1,6 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { getSenchoIpFromSubnet } from '../services/MeshService'; let tmpDir: string; let MeshService: typeof import('../services/MeshService').MeshService; @@ -26,6 +27,9 @@ beforeEach(() => { activeStreams: Map; routeErrorMap: Map; routeLatencyMap: Map; + senchoIp: string | null; + meshSubnet: string; + networkSetupError: string | null; }; svc.aliasCache = new Map(); svc.aliasByPort = new Map(); @@ -33,6 +37,9 @@ beforeEach(() => { svc.activeStreams = new Map(); svc.routeErrorMap = new Map(); svc.routeLatencyMap = new Map(); + svc.senchoIp = '172.30.0.2'; + svc.meshSubnet = '172.30.0.0/24'; + svc.networkSetupError = null; vi.restoreAllMocks(); }); @@ -227,3 +234,114 @@ describe('MeshService.testUpstream tunnel-down path', () => { expect(result.code).toBe('no_route'); }); }); + +describe('getSenchoIpFromSubnet', () => { + it('returns network+2 for the default /24', () => { + expect(getSenchoIpFromSubnet('172.30.0.0/24')).toBe('172.30.0.2'); + }); + + it('handles a custom /24 in a different range', () => { + expect(getSenchoIpFromSubnet('10.42.7.0/24')).toBe('10.42.7.2'); + }); + + it('handles a /16', () => { + expect(getSenchoIpFromSubnet('172.30.0.0/16')).toBe('172.30.0.2'); + }); + + it('masks the input IP to the network address before adding 2', () => { + // 172.30.0.50/24 → network 172.30.0.0 → +2 = 172.30.0.2 + expect(getSenchoIpFromSubnet('172.30.0.50/24')).toBe('172.30.0.2'); + }); + + it('rejects a malformed CIDR', () => { + expect(() => getSenchoIpFromSubnet('not-a-cidr')).toThrow(/Invalid mesh subnet/); + expect(() => getSenchoIpFromSubnet('172.30.0.0')).toThrow(/Invalid mesh subnet/); + }); + + it('rejects prefixes too narrow to host two addresses', () => { + expect(() => getSenchoIpFromSubnet('172.30.0.0/31')).toThrow(/Invalid mesh subnet/); + }); + + it('rejects out-of-range octets', () => { + expect(() => getSenchoIpFromSubnet('172.30.0.999/24')).toThrow(/Invalid mesh subnet/); + }); +}); + +describe('MeshService.ensureMeshNetwork', () => { + it('refuses to continue when sencho_mesh exists with a different subnet', async () => { + const svc = MeshService.getInstance(); + const dcModule = await import('../services/DockerController'); + const fakeController = { + createNetwork: vi.fn().mockRejectedValue({ statusCode: 409, message: 'network already exists' }), + inspectNetwork: vi.fn().mockResolvedValue({ IPAM: { Config: [{ Subnet: '10.99.0.0/24' }] } }), + }; + vi.spyOn(dcModule.default, 'getInstance').mockReturnValue(fakeController as unknown as ReturnType); + + await expect( + (svc as unknown as { ensureMeshNetwork: (s: string) => Promise }).ensureMeshNetwork('172.30.0.0/24'), + ).rejects.toThrow(/exists with subnet 10\.99\.0\.0\/24/); + }); + + it('treats 409 with matching subnet as idempotent success', async () => { + const svc = MeshService.getInstance(); + const dcModule = await import('../services/DockerController'); + const fakeController = { + createNetwork: vi.fn().mockRejectedValue({ statusCode: 409, message: 'network already exists' }), + inspectNetwork: vi.fn().mockResolvedValue({ IPAM: { Config: [{ Subnet: '172.30.0.0/24' }] } }), + }; + vi.spyOn(dcModule.default, 'getInstance').mockReturnValue(fakeController as unknown as ReturnType); + + await expect( + (svc as unknown as { ensureMeshNetwork: (s: string) => Promise }).ensureMeshNetwork('172.30.0.0/24'), + ).resolves.toBeUndefined(); + }); +}); + +describe('MeshService.optInStack rollback', () => { + it('rolls back the DB row when the just-inserted stack fails to push its override', async () => { + const svc = MeshService.getInstance(); + vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise }, 'inspectStackServices') + .mockResolvedValue([{ service: 'db', ports: [5432] }]); + vi.spyOn(svc, 'pushOverrideToNode') + .mockRejectedValue(new Error('simulated remote pilot offline')); + vi.spyOn(svc as unknown as { triggerRedeploy: (n: number, s: string, a: string) => void }, 'triggerRedeploy') + .mockImplementation(() => { /* noop */ }); + + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + + await expect(svc.optInStack(localNodeId, 'api', 'tester')) + .rejects.toThrow(/simulated remote pilot offline/); + expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(false); + }); +}); + +describe('MeshService.optInStack guard rails (network setup)', () => { + it('rejects opt-in when senchoIp is null (mesh data plane unavailable)', async () => { + const svc = MeshService.getInstance(); + (svc as unknown as { senchoIp: string | null }).senchoIp = null; + (svc as unknown as { networkSetupError: string | null }).networkSetupError = 'sencho_mesh subnet mismatch'; + + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + + await expect(svc.optInStack(localNodeId, 'api', 'tester')) + .rejects.toThrow(/sencho_mesh subnet mismatch/); + expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(false); + }); + + it('rejects opt-in when a service exposes the reserved Sencho API port', async () => { + const svc = MeshService.getInstance(); + vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise }, 'inspectStackServices') + .mockResolvedValue([{ service: 'web', ports: [1852] }]); + vi.spyOn(svc as unknown as { regenerateOverridesForNode: (n: number) => Promise }, 'regenerateOverridesForNode') + .mockResolvedValue(undefined); + + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + + await expect(svc.optInStack(localNodeId, 'api', 'tester')) + .rejects.toThrow(/port 1852 is reserved/); + expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(false); + }); +}); diff --git a/backend/src/routes/mesh.ts b/backend/src/routes/mesh.ts index 510cadca..d987c8ad 100644 --- a/backend/src/routes/mesh.ts +++ b/backend/src/routes/mesh.ts @@ -70,6 +70,69 @@ meshRouter.get('/local-services/:stackName', async (req: Request, res: Response) } }); +const MAX_ALIASES_PER_PUSH = 1024; + +/** + * Accepts a fleet-wide alias list from central and writes a mesh override + * for the named stack onto THIS Sencho's local DATA_DIR. The pilot looks + * up its own service names and uses its own static IP on `sencho_mesh`, + * so alias hostnames in user containers always resolve to the LOCAL + * Sencho IP on the deploying node. Always writes against the LOCAL + * Sencho's default node id. + */ +meshRouter.put('/local-override/:stackName', async (req: Request, res: Response): Promise => { + if (!requireAdmiral(req, res)) return; + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { res.status(400).json({ error: 'Invalid stack name' }); return; } + const body = req.body as { aliases?: unknown }; + if (!Array.isArray(body?.aliases)) { res.status(400).json({ error: 'Missing aliases array in body' }); return; } + if (body.aliases.length > MAX_ALIASES_PER_PUSH) { + res.status(413).json({ error: `Alias list exceeds ${MAX_ALIASES_PER_PUSH} entries` }); + return; + } + const aliases: { host: string }[] = []; + for (const entry of body.aliases) { + const host = (entry as { host?: unknown } | null | undefined)?.host; + if (typeof host !== 'string' || host.length === 0 || host.length > 253) { + // 253 octets is the DNS hostname ceiling. Defensive against a + // malicious or buggy central sending a multi-KB host string. + res.status(400).json({ error: 'Invalid alias entry' }); + return; + } + aliases.push({ host }); + } + try { + const written = await MeshService.getInstance().applyLocalOverride(stackName, aliases); + if (!written) { res.status(400).json({ error: 'Refused to write override (path validation failed)' }); return; } + res.json({ ok: true, path: written }); + } catch (err) { + if (err instanceof MeshError && err.code === 'push_failed') { + res.status(503).json({ error: err.message, code: err.code }); + return; + } + console.warn('[mesh] /local-override failed:', sanitizeForLog((err as Error).message)); + res.status(500).json({ error: 'Failed to write local override' }); + } +}); + +/** + * Delete a previously written local override. Mirror of the PUT endpoint; + * called by central when a stack is opted out so stale overrides do not + * linger on the deploying node. + */ +meshRouter.delete('/local-override/:stackName', async (req: Request, res: Response): Promise => { + if (!requireAdmiral(req, res)) return; + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { res.status(400).json({ error: 'Invalid stack name' }); return; } + try { + await MeshService.getInstance().removeLocalOverride(stackName); + res.json({ ok: true }); + } catch (err) { + console.warn('[mesh] DELETE /local-override failed:', sanitizeForLog((err as Error).message)); + res.status(500).json({ error: 'Failed to remove local override' }); + } +}); + meshRouter.get('/nodes/:nodeId/stacks', async (req: Request, res: Response): Promise => { if (!requireAdmiral(req, res)) return; const nodeId = Number.parseInt(req.params.nodeId as string, 10); @@ -105,6 +168,10 @@ meshRouter.post('/nodes/:nodeId/stacks/:stackName/opt-in', async (req: Request, res.status(409).json({ error: err.message, code: err.code }); return; } + if (err instanceof MeshError && err.code === 'push_failed') { + res.status(503).json({ error: err.message, code: err.code }); + return; + } if (err instanceof MeshError) { res.status(400).json({ error: err.message, code: err.code }); return; diff --git a/backend/src/services/DockerController.ts b/backend/src/services/DockerController.ts index 861eee94..ed0facdb 100644 --- a/backend/src/services/DockerController.ts +++ b/backend/src/services/DockerController.ts @@ -463,6 +463,68 @@ class DockerController { return await this.docker.createNetwork(options); } + /** + * Attach a container to a Docker network. Idempotent: if the container is + * already attached the call resolves silently. Optionally pins the + * container's IPv4 address inside the network so other services can use + * static `extra_hosts` entries against it. + */ + public async connectContainerToNetwork( + networkName: string, + containerId: string, + opts: { ipv4Address?: string } = {}, + ): Promise { + const network = this.docker.getNetwork(networkName); + const payload: { Container: string; EndpointConfig?: { IPAMConfig?: { IPv4Address: string } } } = { + Container: containerId, + }; + if (opts.ipv4Address) { + payload.EndpointConfig = { IPAMConfig: { IPv4Address: opts.ipv4Address } }; + } + try { + await network.connect(payload); + } catch (err) { + if (DockerController.isAlreadyConnectedError(err)) return; + throw err; + } + } + + /** + * Detach a container from a Docker network. Idempotent: if the container + * is not attached the call resolves silently. + */ + public async disconnectContainerFromNetwork( + networkName: string, + containerId: string, + ): Promise { + const network = this.docker.getNetwork(networkName); + try { + await network.disconnect({ Container: containerId, Force: true }); + } catch (err) { + if (DockerController.isNotConnectedError(err)) return; + throw err; + } + } + + private static isAlreadyConnectedError(err: unknown): boolean { + const e = err as { statusCode?: number; message?: string }; + const msg = (e?.message || '').toLowerCase(); + // Docker daemon returns 403 for several distinct cases (already + // attached, host-network containers, permission denied), so match the + // message body too rather than treating any 403 as idempotent success. + if (e?.statusCode === 403 && (msg.includes('already exists') || msg.includes('already attached'))) { + return true; + } + return msg.includes('already exists') || msg.includes('already attached'); + } + + private static isNotConnectedError(err: unknown): boolean { + const e = err as { statusCode?: number; message?: string }; + if (e?.statusCode === 404) return true; + const msg = (e?.message || '').toLowerCase(); + return msg.includes('is not connected') || msg.includes('no such container'); + } + public async getRunningContainers() { const containers = await this.docker.listContainers({ all: false }); return this.validateApiData(containers); diff --git a/backend/src/services/MeshComposeOverride.ts b/backend/src/services/MeshComposeOverride.ts index a8b0d8d4..f73041ee 100644 --- a/backend/src/services/MeshComposeOverride.ts +++ b/backend/src/services/MeshComposeOverride.ts @@ -4,13 +4,19 @@ import * as YAML from 'yaml'; * Sencho Mesh Compose override generator. * * Produces a YAML override applied with `docker compose -f compose.yml -f - * mesh.override.yml up` that injects cross-node alias entries into each - * opted-in service's /etc/hosts. The aliases resolve to the host gateway - * (`host-gateway`), which is where the local Sencho Mesh sidecar listens - * (host network mode). The user's source compose file is never mutated; the - * override lives in Sencho's data dir. + * mesh.override.yml up` that: + * - injects cross-node alias entries into each opted-in service's + * `/etc/hosts`, resolving every alias to the Sencho container's + * static IP on the internal `sencho_mesh` Docker network; + * - attaches each service to `sencho_mesh` so that IP is reachable + * from inside the user's container. + * + * The user's source compose file is never mutated; the override lives in + * Sencho's data dir and is regenerated whenever the alias set changes. */ +export const SENCHO_MESH_NETWORK = 'sencho_mesh'; + export interface MeshAlias { /** `...sencho` */ host: string; @@ -21,6 +27,8 @@ export interface MeshOverrideInput { services: string[]; /** Aliases this stack should be able to resolve. Order is normalized in output. */ aliases: MeshAlias[]; + /** Sencho's static IP on the `sencho_mesh` Docker network. */ + senchoIp: string; } /** @@ -31,20 +39,23 @@ export function generateOverrideYaml(input: MeshOverrideInput): string { const sortedServices = [...input.services].sort(); const sortedAliases = [...input.aliases].sort((a, b) => a.host.localeCompare(b.host)); - if (sortedAliases.length === 0) { - const services: Record = {}; - for (const svc of sortedServices) services[svc] = {}; - return YAML.stringify({ services }, { lineWidth: 0 }); - } - - const extraHostsList = sortedAliases.map((a) => `${a.host}:host-gateway`); - const services: Record = {}; for (const svc of sortedServices) { - services[svc] = { extra_hosts: extraHostsList }; + const entry: Record = { + networks: [SENCHO_MESH_NETWORK], + }; + if (sortedAliases.length > 0) { + entry.extra_hosts = sortedAliases.map((a) => `${a.host}:${input.senchoIp}`); + } + services[svc] = entry; } - return YAML.stringify({ services }, { lineWidth: 0 }); + const doc: Record = { + services, + networks: { [SENCHO_MESH_NETWORK]: { external: true } }, + }; + + return YAML.stringify(doc, { lineWidth: 0 }); } /** diff --git a/backend/src/services/MeshService.ts b/backend/src/services/MeshService.ts index 410448ac..7c2e7423 100644 --- a/backend/src/services/MeshService.ts +++ b/backend/src/services/MeshService.ts @@ -2,6 +2,7 @@ import net from 'net'; import path from 'path'; import fs from 'fs/promises'; import { EventEmitter } from 'events'; +import { ComposeService } from './ComposeService'; import { DatabaseService } from './DatabaseService'; import DockerController from './DockerController'; import { LicenseService } from './LicenseService'; @@ -9,14 +10,43 @@ import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from './license-headers'; import { MeshForwarder, type MeshForwarderHost } from './MeshForwarder'; import { NodeRegistry } from './NodeRegistry'; import { PilotTunnelManager } from './PilotTunnelManager'; -import { generateOverrideYaml, MeshAlias } from './MeshComposeOverride'; +import { generateOverrideYaml, MeshAlias, SENCHO_MESH_NETWORK } from './MeshComposeOverride'; import { sanitizeForLog } from '../utils/safeLog'; import { isPathWithinBase, isValidStackName } from '../utils/validation'; +import { PORT as SENCHO_LISTEN_PORT } from '../helpers/constants'; const ACTIVITY_BUFFER_SIZE = 1000; const ALIAS_REFRESH_INTERVAL_MS = 60_000; const PROBE_TIMEOUT_MS = 5_000; const SLOW_PROBE_THRESHOLD_MS = 500; +const DEFAULT_MESH_SUBNET = '172.30.0.0/24'; + +/** + * Returns the static IPv4 address Sencho will pin itself to on the mesh + * Docker network: ` + 2`. The Docker daemon assigns + * ` + 1` to the bridge gateway, so `+2` is the first usable host + * address. For the default `172.30.0.0/24` this is `172.30.0.2`. Throws + * on invalid CIDR or a prefix too narrow to host two addresses. + */ +export function getSenchoIpFromSubnet(subnet: string): string { + const cidr = subnet.trim().match(/^(\d+)\.(\d+)\.(\d+)\.(\d+)\/(\d+)$/); + if (!cidr) throw new Error(`Invalid mesh subnet CIDR: ${subnet}`); + const octets = [Number(cidr[1]), Number(cidr[2]), Number(cidr[3]), Number(cidr[4])]; + const prefix = Number(cidr[5]); + if (octets.some((o) => o < 0 || o > 255) || prefix < 8 || prefix > 30) { + throw new Error(`Invalid mesh subnet CIDR: ${subnet}`); + } + const ipInt = (octets[0] << 24) | (octets[1] << 16) | (octets[2] << 8) | octets[3]; + const mask = prefix === 0 ? 0 : (0xffffffff << (32 - prefix)) >>> 0; + const network = (ipInt & mask) >>> 0; + const sencho = (network + 2) >>> 0; + return [ + (sencho >>> 24) & 0xff, + (sencho >>> 16) & 0xff, + (sencho >>> 8) & 0xff, + sencho & 0xff, + ].join('.'); +} export type MeshActivitySource = 'pilot' | 'mesh'; export type MeshActivityLevel = 'info' | 'warn' | 'error'; @@ -109,17 +139,17 @@ interface ActiveStreamRecord { * container; one container per node now. * - opt-in / opt-out persistence and cascading override regeneration * - global alias aggregation (across the fleet via the existing HTTP - * proxy chain — see `inspectStackServices`) + * proxy chain, see `inspectStackServices`) * - cross-node TCP forwarding via `PilotTunnelManager` (central-side) * - probe + diagnostics + activity ring buffer * * V1 limitations: * - one cross-node alias per TCP port across the fleet (port-collision * check at opt-in) - * - aliases resolve via `host-gateway` extra_hosts; Sencho's container - * must run with `network_mode: host` for the forwarder's listeners to - * bind on the host's network where meshed containers' `host-gateway` - * entries point + * - aliases resolve to Sencho's static IP on the internal `sencho_mesh` + * Docker bridge network. Meshed user services join `sencho_mesh` so + * that IP is reachable from inside their containers without any + * host-firewall coordination. * - cross-node mesh routing is central → pilot in this phase. Pilot → * central and pilot ↔ pilot via central relay land in Phase B. */ @@ -135,6 +165,9 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { private routeLatencyMap = new Map(); private activityListeners = new Set<(e: MeshActivityEvent) => void>(); private readonly forwarder: MeshForwarder; + private senchoIp: string | null = null; + private meshSubnet: string = DEFAULT_MESH_SUBNET; + private networkSetupError: string | null = null; private constructor() { super(); @@ -158,6 +191,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { nodeId, message: `pilot tunnel up for node ${nodeId}`, })); + await this.setupMeshNetwork(); await this.refreshAliasCache(); await this.syncForwarderListeners(); this.aliasRefreshTimer = setInterval(() => { @@ -187,6 +221,128 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { await this.forwarder.shutdown(); } + public getSenchoIp(): string | null { + return this.senchoIp; + } + + public getMeshSubnet(): string { + return this.meshSubnet; + } + + public getNetworkSetupError(): string | null { + return this.networkSetupError; + } + + /** + * Idempotent setup of the shared `sencho_mesh` Docker bridge network and + * Sencho's static attachment to it. Called once at boot before alias + * cache refresh. Failures here disable mesh routing for the lifetime of + * the process (forwarder still binds, but `ensureStackOverride` short- + * circuits because there is no IP to point user containers at). + * + * Skipped entirely when Sencho is not running inside Docker (dev mode, + * detected by an unset HOSTNAME env var or by the inspect lookup + * failing). The forwarder still runs locally for unit-test coverage. + */ + private async setupMeshNetwork(): Promise { + const subnet = (process.env.SENCHO_MESH_SUBNET || DEFAULT_MESH_SUBNET).trim(); + try { + this.senchoIp = getSenchoIpFromSubnet(subnet); + this.meshSubnet = subnet; + } catch (err) { + this.networkSetupError = (err as Error).message; + console.warn('[Mesh]', this.networkSetupError); + this.senchoIp = null; + return; + } + + try { + await this.ensureMeshNetwork(subnet); + } catch (err) { + this.networkSetupError = (err as Error).message; + console.warn('[Mesh] mesh network setup failed:', sanitizeForLog(this.networkSetupError)); + this.senchoIp = null; + return; + } + + try { + await this.ensureSelfAttached(); + } catch (err) { + this.networkSetupError = (err as Error).message; + console.warn('[Mesh] self-attach failed:', sanitizeForLog(this.networkSetupError)); + this.senchoIp = null; + return; + } + + this.networkSetupError = null; + } + + /** + * Create `sencho_mesh` if it does not exist. If it does, validate the + * subnet matches `expectedSubnet`; on mismatch, refuse to continue. + * Silently using the wrong subnet would route traffic to the wrong IP. + */ + private async ensureMeshNetwork(expectedSubnet: string): Promise { + const dc = DockerController.getInstance(NodeRegistry.getInstance().getDefaultNodeId()); + try { + await dc.createNetwork({ + Name: SENCHO_MESH_NETWORK, + Driver: 'bridge', + Attachable: true, + IPAM: { Config: [{ Subnet: expectedSubnet }] }, + Labels: { 'io.sencho.mesh': 'true' }, + }); + return; + } catch (err) { + const e = err as { statusCode?: number; message?: string }; + if (e?.statusCode !== 409) throw err; + } + + const info = await dc.inspectNetwork(SENCHO_MESH_NETWORK) as { + IPAM?: { Config?: Array<{ Subnet?: string }> }; + }; + const existingSubnet = info?.IPAM?.Config?.[0]?.Subnet; + if (existingSubnet && existingSubnet !== expectedSubnet) { + throw new Error( + `${SENCHO_MESH_NETWORK} exists with subnet ${existingSubnet}, ` + + `expected ${expectedSubnet}. Remove the network or set SENCHO_MESH_SUBNET to match.`, + ); + } + } + + /** + * Connect Sencho's own container to `sencho_mesh` at the static IP. Uses + * the `HOSTNAME` env var (which Docker sets to the container's short ID + * by default) to identify the container, mirroring the + * SelfUpdateService pattern. Skipped in dev mode where HOSTNAME is + * the laptop hostname and the inspect lookup would fail. + */ + private async ensureSelfAttached(): Promise { + if (!this.senchoIp) return; + const hostname = process.env.HOSTNAME; + if (!hostname) { + console.log('[Mesh] HOSTNAME not set, mesh routing disabled (not running in Docker?)'); + this.senchoIp = null; + return; + } + const dc = DockerController.getInstance(NodeRegistry.getInstance().getDefaultNodeId()); + try { + await dc.connectContainerToNetwork(SENCHO_MESH_NETWORK, hostname, { ipv4Address: this.senchoIp }); + } catch (err) { + const e = err as { statusCode?: number; message?: string }; + if (e?.statusCode === 404) { + this.logActivity({ + source: 'mesh', level: 'warn', type: 'mesh.disable', + message: 'self-container lookup failed; mesh routing disabled (not running in Docker?)', + }); + console.warn('[Mesh] self-container lookup failed; mesh routing disabled (not running in Docker?)'); + this.senchoIp = null; + return; + } + throw err; + } + } + /** * Bind the forwarder's listeners to every alias port across the fleet * and release any listeners no longer in the alias set. Called from @@ -194,11 +350,11 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { * opt-in / opt-out / disable so the bound port set follows the DB * state. * - * Every meshed node binds every alias port — not just ports it owns — - * because meshed containers' `extra_hosts: :host-gateway` - * entries resolve to the SOURCE node's gateway, so the source node is - * where the inbound TCP connection lands. `handleAccept` then - * dispatches to the same-node fast path or the cross-node bridge based + * Every meshed node binds every alias port (not just ports it owns) + * because alias DNS entries resolve to the SOURCE node's Sencho IP, so + * the source node is where the inbound TCP connection lands. + * `handleAccept` then dispatches to the same-node fast path or the + * cross-node bridge based * on the resolved alias's owner. Fleet-wide port collisions are * blocked at opt-in time (`optInStack` checks `aliasByPort`), so * binding every alias port is unambiguous. @@ -269,6 +425,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { if (!isValidStackName(stackName)) { throw new MeshError('denied', `invalid stack name: ${stackName}`); } + if (!this.senchoIp) { + throw new MeshError( + 'denied', + this.networkSetupError || 'mesh data plane unavailable (mesh network setup did not complete)', + ); + } const db = DatabaseService.getInstance(); if (db.isMeshStackEnabled(nodeId, stackName)) return; @@ -279,6 +441,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { const newPorts = new Set(); for (const svc of services) for (const p of svc.ports) newPorts.add(p); + if (newPorts.has(SENCHO_LISTEN_PORT)) { + throw new MeshError( + 'port_collision', + `port ${SENCHO_LISTEN_PORT} is reserved for the Sencho API and cannot be used by a meshed service`, + ); + } for (const port of newPorts) { const existing = this.aliasByPort.get(port); if (existing) { @@ -292,7 +460,25 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { db.insertMeshStack(nodeId, stackName, actor); await this.refreshAliasCache(); await this.syncForwarderListeners(); - await this.regenerateOverridesForNode(nodeId); + + // Push the just-opted-in stack's override loudly. If this fails the + // DB state is invalid (alias claimed but remote pilot has no + // override file) so roll back rather than leave a half-state that + // future opt-in calls would short-circuit on `isMeshStackEnabled`. + try { + await this.pushOverrideToNode(nodeId, stackName); + } catch (err) { + db.deleteMeshStack(nodeId, stackName); + await this.refreshAliasCache(); + await this.syncForwarderListeners(); + throw err; + } + // Regenerate OTHER meshed stacks' overrides on the same node so + // they pick up the new alias entry. The just-opted-in stack was + // already pushed above; skip it to avoid a duplicate round-trip. + // Best-effort; per-stack failures are logged inside the helper. + await this.regenerateOverridesForNode(nodeId, stackName); + this.triggerRedeploy(nodeId, stackName, actor); this.logActivity({ source: 'mesh', level: 'info', type: 'opt_in', @@ -313,10 +499,11 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { const db = DatabaseService.getInstance(); if (!db.isMeshStackEnabled(nodeId, stackName)) return; db.deleteMeshStack(nodeId, stackName); - await this.removeStackOverride(nodeId, stackName); + await this.removeOverrideFromNode(nodeId, stackName); await this.refreshAliasCache(); await this.syncForwarderListeners(); await this.regenerateOverridesForNode(nodeId); + this.triggerRedeploy(nodeId, stackName, actor); this.logActivity({ source: 'mesh', level: 'info', type: 'opt_out', @@ -359,12 +546,14 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { if (!isValidStackName(stackName)) return null; const db = DatabaseService.getInstance(); if (!db.isMeshStackEnabled(nodeId, stackName)) return null; + if (!this.senchoIp) return null; const aliases: MeshAlias[] = Array.from(this.aliasCache.values()).map((a) => ({ host: a.host })); const services = await this.inspectStackServices(nodeId, stackName); const yaml = generateOverrideYaml({ services: services.map((s) => s.service), aliases, + senchoIp: this.senchoIp, }); const dir = this.overrideDirFor(nodeId); @@ -375,6 +564,55 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { return file; } + /** + * Render and write a mesh override on the LOCAL node's filesystem from + * a fleet-wide alias list supplied by central. The pilot looks up its + * own service names and uses its own static IP, so each node's + * override resolves alias hostnames to that node's local Sencho. + * This is critical because each node has its own `sencho_mesh` + * network with its own subnet. Returns the absolute path on success + * or null if path validation rejects the input. + */ + public async applyLocalOverride(stackName: string, aliases: MeshAlias[]): Promise { + if (!isValidStackName(stackName)) return null; + if (!this.senchoIp) { + throw new MeshError( + 'push_failed', + this.networkSetupError || 'mesh data plane unavailable on this node', + ); + } + const services = await this.inspectLocalStackServices(stackName); + const yaml = generateOverrideYaml({ + services: services.map((s) => s.service), + aliases, + senchoIp: this.senchoIp, + }); + + const localNodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const dir = this.overrideDirFor(localNodeId); + await fs.mkdir(dir, { recursive: true }); + // path.basename strips any directory component as defense-in-depth + // on top of isValidStackName + isPathWithinBase. Recognized by + // CodeQL's path-injection model. + const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`); + if (!isPathWithinBase(file, dir)) return null; + await fs.writeFile(file, yaml, 'utf8'); + return file; + } + + /** + * Delete a previously applied local override (mirror of + * `applyLocalOverride`). Used by central when a stack is opted out. + */ + public async removeLocalOverride(stackName: string): Promise { + if (!isValidStackName(stackName)) return; + const localNodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const dir = this.overrideDirFor(localNodeId); + const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`); + if (!isPathWithinBase(file, dir)) return; + try { await fs.unlink(file); } catch { /* ignore not-exist */ } + } + private async removeStackOverride(nodeId: number, stackName: string): Promise { if (!isValidStackName(stackName)) return; const dir = this.overrideDirFor(nodeId); @@ -388,16 +626,24 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { return path.join(dataDir, 'mesh', 'overrides', String(nodeId)); } - private async regenerateOverridesForNode(nodeId: number): Promise { + private async regenerateOverridesForNode(nodeId: number, skipStack?: string): Promise { const db = DatabaseService.getInstance(); const stacks = db.listMeshStacks(nodeId); - for (const s of stacks) { - try { - await this.ensureStackOverride(nodeId, s.stack_name); - } catch (err) { - console.warn('[MeshService] override regen failed:', sanitizeForLog((err as Error).message)); - } - } + // Push all overrides in parallel: each remote-node call is its own + // HTTP round-trip, so awaiting sequentially turns N stacks into N + // serialised PUTs. `allSettled` so a single failure does not abort + // the others. + await Promise.allSettled( + stacks + .filter((s) => s.stack_name !== skipStack) + .map(async (s) => { + try { + await this.pushOverrideToNode(nodeId, s.stack_name); + } catch (err) { + console.warn('[MeshService] override push failed:', sanitizeForLog((err as Error).message)); + } + }), + ); } // --- Alias aggregation --- @@ -448,7 +694,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { /** * Inspect a stack and return its running services with the ports they - * listen on. For the LOCAL Docker daemon only — callers targeting a + * listen on. For the LOCAL Docker daemon only; callers targeting a * remote node must use {@link inspectStackServices}, which dispatches * via the HTTP proxy to the remote's `/api/mesh/local-services/:stack`. */ @@ -514,6 +760,178 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { } } + /** + * Build a `fetch` against a remote Sencho's API with the bearer token + * and the proxy tier/variant headers in place. Centralizes the header + * shape so a future addition (license header, audit context) only + * needs to land in one place. + * + * `x-node-id` is deliberately NOT set: callers target the remote + * Sencho's own routes, which operate against the remote's local node + * id. The bearer token alone authenticates. + */ + private async proxyFetch( + nodeId: number, + method: 'GET' | 'PUT' | 'POST' | 'DELETE', + apiPath: string, + body: unknown, + timeoutMs: number, + ): Promise { + const target = NodeRegistry.getInstance().getProxyTarget(nodeId); + if (!target) throw new MeshError('push_failed', `no proxy target for node ${nodeId}`); + const url = `${target.apiUrl.replace(/\/$/, '')}${apiPath}`; + const headers: Record = {}; + if (body !== undefined) headers['Content-Type'] = 'application/json'; + if (target.apiToken) headers['Authorization'] = `Bearer ${target.apiToken}`; + const proxyHeaders = LicenseService.getInstance().getProxyHeaders(); + headers[PROXY_TIER_HEADER] = proxyHeaders.tier; + headers[PROXY_VARIANT_HEADER] = proxyHeaders.variant || ''; + return await fetch(url, { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + signal: AbortSignal.timeout(timeoutMs), + }); + } + + /** + * Place a mesh override for a stack on whichever node owns it: + * - local node: regenerates via `ensureStackOverride`, which uses + * central's own senchoIp (correct because central is the node + * deploying that stack). + * - remote node: sends the fleet-wide alias list to the remote's + * `PUT /api/mesh/local-override/:stackName`; the remote renders + * the YAML using its OWN local senchoIp and writes it under its + * own DATA_DIR. This is essential because each node has its own + * `sencho_mesh` network and may be configured with a different + * SENCHO_MESH_SUBNET, so alias hostnames must always resolve to + * the local Sencho IP on the deploying node. + * + * Throws on remote push failure so callers (opt-in / opt-out) can abort + * cleanly rather than silently leaving stale overrides. + */ + public async pushOverrideToNode(nodeId: number, stackName: string): Promise { + const node = DatabaseService.getInstance().getNode(nodeId); + if (!node) throw new MeshError('denied', `unknown node ${nodeId}`); + + if (node.type !== 'remote') { + await this.ensureStackOverride(nodeId, stackName); + return; + } + + const aliases: MeshAlias[] = Array.from(this.aliasCache.values()).map((a) => ({ host: a.host })); + const res = await this.proxyFetch( + nodeId, + 'PUT', + `/api/mesh/local-override/${encodeURIComponent(stackName)}`, + { aliases }, + 5_000, + ); + if (res.status === 404) { + throw new MeshError( + 'push_failed', + `node ${node.name} does not support mesh override push (upgrade required)`, + ); + } + if (!res.ok) { + throw new MeshError('push_failed', `HTTP ${res.status} from node ${node.name}`); + } + } + + /** + * Fire-and-forget redeploy of a stack on whichever node owns it. Used + * by opt-in and opt-out so the new alias entries reach the user + * containers' /etc/hosts without an operator manually clicking deploy. + * + * For local stacks: invokes `ComposeService.deployStack` directly. + * For remote stacks: HTTP POSTs to `/api/stacks/:name/deploy` + * via the same bearer-token pattern the rest of the proxy chain uses. + * + * Errors are logged to the mesh activity buffer rather than thrown so + * the opt-in or opt-out call site can return success quickly. The + * operator sees the redeploy progress through the existing deploy + * stream surfaces; if it fails, the activity log records why. + */ + public triggerRedeploy(nodeId: number, stackName: string, actor: string): void { + void this.runRedeploy(nodeId, stackName, actor).catch((err) => { + const reason = sanitizeForLog((err as Error).message); + this.logActivity({ + source: 'mesh', level: 'error', type: 'forwarder.error', + nodeId, + message: `mesh redeploy failed for ${stackName}: ${reason}`, + details: { actor, stackName }, + }); + // Also drop a durable audit row so an operator who walks away + // from the toast still has a trail. The activity ring buffer + // alone gets pruned at 1000 events. + DatabaseService.getInstance().insertAuditLog({ + timestamp: Date.now(), username: actor, method: 'POST', + path: `/api/mesh/nodes/${nodeId}/stacks/${stackName}/redeploy`, + status_code: 500, node_id: nodeId, ip_address: '127.0.0.1', + summary: `Sencho Mesh: redeploy failed for ${stackName}: ${reason}`, + }); + }); + } + + private async runRedeploy(nodeId: number, stackName: string, actor: string): Promise { + const node = DatabaseService.getInstance().getNode(nodeId); + if (!node) throw new Error(`unknown node ${nodeId}`); + + if (node.type !== 'remote') { + await ComposeService.getInstance(nodeId).deployStack(stackName); + this.logActivity({ + source: 'mesh', level: 'info', type: 'mesh.enable', + nodeId, + message: `mesh redeploy ok for ${stackName}`, + details: { actor, stackName }, + }); + return; + } + + // Mesh redeploys are bounded by docker compose's own runtime; pick a + // generous ceiling rather than the 5 s default used for control-plane + // calls so a slow image pull does not abort the redeploy. + const res = await this.proxyFetch( + nodeId, + 'POST', + `/api/stacks/${encodeURIComponent(stackName)}/deploy`, + {}, + 10 * 60 * 1000, + ); + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new Error(`HTTP ${res.status} from node ${node.name}: ${body.slice(0, 256)}`); + } + this.logActivity({ + source: 'mesh', level: 'info', type: 'mesh.enable', + nodeId, + message: `mesh redeploy ok for ${stackName}`, + details: { actor, stackName }, + }); + } + + public async removeOverrideFromNode(nodeId: number, stackName: string): Promise { + const node = DatabaseService.getInstance().getNode(nodeId); + if (!node) return; + + if (node.type !== 'remote') { + await this.removeStackOverride(nodeId, stackName); + return; + } + + try { + await this.proxyFetch( + nodeId, + 'DELETE', + `/api/mesh/local-override/${encodeURIComponent(stackName)}`, + undefined, + 5_000, + ); + } catch (err) { + console.warn('[MeshService] removeOverrideFromNode failed:', sanitizeForLog((err as Error).message)); + } + } + // --- Resolution + forwarding --- public resolveByLocalPort(port: number): MeshTarget | null { @@ -943,9 +1361,16 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { // WebSocket. See `docs/internal/architecture/mesh.md` for the new flow. } +export type MeshErrorCode = + | 'no_target' + | 'port_collision' + | 'denied' + | 'agent_error' + | 'push_failed'; + export class MeshError extends Error { - public readonly code: 'no_target' | 'port_collision' | 'denied' | 'agent_error'; - constructor(code: 'no_target' | 'port_collision' | 'denied' | 'agent_error', message: string) { + public readonly code: MeshErrorCode; + constructor(code: MeshErrorCode, message: string) { super(message); this.code = code; } diff --git a/docker-compose.yml b/docker-compose.yml index 57d67019..cc27edd7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,12 +4,13 @@ services: build: . container_name: sencho restart: unless-stopped - # Sencho Mesh listens on host ports for cross-stack alias traffic. Host - # network mode is required for mesh to function. If you do not use mesh, - # comment `network_mode: host` out and uncomment the `ports` block below. - network_mode: host - # ports: - # - "1852:1852" + # Publish only the Sencho UI / API port. Sencho Mesh runs on an internal + # `sencho_mesh` Docker network that Sencho creates and joins on boot, + # so cross-stack mesh routing does not require any extra host ports or + # firewall rules. Override the mesh subnet (default 172.30.0.0/24) with + # SENCHO_MESH_SUBNET if it conflicts with your network. + ports: + - "1852:1852" volumes: # Required: Docker Socket for container orchestration - /var/run/docker.sock:/var/run/docker.sock diff --git a/docs/features/pilot-agent.mdx b/docs/features/pilot-agent.mdx index 7964f7eb..82eff1f4 100644 --- a/docs/features/pilot-agent.mdx +++ b/docs/features/pilot-agent.mdx @@ -5,6 +5,10 @@ description: Add remote nodes behind NAT, residential networks, or corporate fir Pilot Agent mode connects a remote server to your primary Sencho instance through a single outbound WebSocket tunnel. Every request the primary sends to that node, HTTP or WebSocket, rides through this tunnel. The remote host never opens an inbound port and never needs a TLS certificate. + + Once a node is enrolled, you can wire stacks across nodes by hostname using Sencho Mesh, which rides on this same tunnel. + + ## When to use Pilot Agent Pick Pilot Agent when the remote host: @@ -95,7 +99,7 @@ The agent uses the bundle as the only trust anchor for the tunnel WebSocket. TLS Each tunnel has fixed protocol-level ceilings to keep one misbehaving agent from impacting the primary: - **Frame size:** individual WebSocket frames are capped at 8 MB. Single requests and responses larger than that are rejected and the tunnel reconnects. -- **Concurrent streams:** at most 1024 multiplexed HTTP, WebSocket, or Mesh-TCP streams per tunnel. Above the cap the bridge returns 503 and the agent rejects new incoming streams with a typed error frame. +- **Concurrent streams:** at most 1024 multiplexed HTTP and WebSocket streams per tunnel (mesh TCP streams share the same pool). Above the cap the bridge returns 503 and the agent rejects new incoming streams with a typed error frame. - **Stream idle:** any stream with no activity for 10 minutes is closed and removed. - **System-wide tunnels:** a single primary accepts up to 256 concurrent pilot tunnels. Past the soft warning at 128 the primary logs a WARN; past the hard cap of 256 new tunnels are refused with WebSocket close code 1013 (Try Again Later) so the agent backs off rather than tight-looping. diff --git a/docs/features/sencho-mesh.mdx b/docs/features/sencho-mesh.mdx index 9c4e581b..42d8895a 100644 --- a/docs/features/sencho-mesh.mdx +++ b/docs/features/sencho-mesh.mdx @@ -1,6 +1,6 @@ --- title: Sencho Mesh -description: Connect containers across nodes by hostname over the Pilot tunnel — no VPN, no firewall changes, no extra ports. +description: Connect containers across nodes by hostname over the Pilot tunnel, no VPN, no firewall changes, no extra ports. --- @@ -9,13 +9,21 @@ description: Connect containers across nodes by hostname over the Pilot tunnel Sencho Mesh makes a multi-node fleet feel like one machine. Opt a stack into the mesh and its services become reachable from any other meshed stack on the fleet by a stable hostname. Traffic rides the existing Pilot tunnel, so there are no new ports to open and no separate VPN to manage. + + Mesh rides on the Pilot tunnel. See Pilot Agent for how to enroll a node into your fleet. + + + + Mesh today requires nodes that connect to your primary in **Pilot Agent** mode. Support for Distributed API (proxy-mode) remotes is on the roadmap. + + ## How it works -Each node runs a small `sencho-mesh` sidecar container that listens on the host's network. When you opt a stack into the mesh, Sencho: +Each Sencho instance creates an internal Docker bridge network called `sencho_mesh` (default subnet `172.30.0.0/24`) on first boot and pins itself to a static IP on that network. When you opt a stack into the mesh, Sencho: -1. Generates a Compose override file that injects `extra_hosts` for every cross-node alias the fleet currently exposes. -2. Redeploys the stack with the override applied so its containers pick up the new entries. -3. Routes incoming traffic on the alias's port through the local sidecar, then over the Pilot tunnel to the destination node, where the sidecar dials the target container. +1. Generates a Compose override file that injects `extra_hosts` for every cross-node alias the fleet currently exposes, pointing each one at the local Sencho's static IP. +2. Attaches every service in the stack to the `sencho_mesh` network so the alias IP is reachable from inside the user's containers. +3. Redeploys the stack with the override applied so its containers pick up the new entries and the network attachment. Aliases follow a predictable scheme: @@ -23,22 +31,22 @@ Aliases follow a predictable scheme: ...sencho ``` -For example, a Postgres `db` service in a stack named `api` on a node named `opsix` is reachable as `db.api.opsix.sencho` from any other meshed stack. +A Postgres `db` service in a stack named `api` on a node named `opsix` is reachable as `db.api.opsix.sencho` from any other meshed stack. ## Enable the mesh -1. Open **Fleet → Traffic**. +1. Open **Fleet → Routing**. 2. Toggle **mesh** on for each node you want to participate. 3. Click **Add stack to mesh** on any node and tick the stacks whose services should be reachable cross-node. -Each opt-in triggers an automatic redeploy of every other meshed stack on that node so the new alias is added to their hosts files. The opt-in sheet warns you before it does. +Each opt-in or opt-out triggers an automatic redeploy of the affected stack so its `/etc/hosts` and network attachments refresh. The opt-in sheet shows a confirmation prompt before it starts the redeploy. ## What's exposed and what isn't Four guarantees: 1. **Only opted-in services are reachable.** A stack reaches another stack's services through the mesh only if both stacks have explicitly opted in. The Pilot agent on the target node refuses any request for a non-opted service. -2. **Aliases are not internet-reachable.** Sidecars listen only on the host network of each node. Nothing about the mesh exposes new ports beyond the host's existing firewall posture. +2. **Aliases are not internet-reachable.** Sencho listens on an internal Docker network; nothing about the mesh exposes new ports beyond the host's existing firewall posture. 3. **Traffic is encrypted in transit.** Cross-node bytes ride the existing Pilot WSS tunnel. 4. **Tier-gated and audit-logged.** Only Admiral users can configure the mesh. Enable, disable, opt-in, and opt-out events write durable rows to the audit log with the actor's identity. @@ -49,12 +57,25 @@ What the mesh does **not** do: - No rate limiting or quotas. - No persistent traffic metrics. Live diagnostics only. +## Customizing the mesh subnet + +Each Sencho creates `sencho_mesh` as a `/24` bridge network at `172.30.0.0/24` by default. If that range collides with an existing network on a host, override it with the `SENCHO_MESH_SUBNET` environment variable on that node: + +```yaml +services: + sencho: + environment: + - SENCHO_MESH_SUBNET=10.42.0.0/24 +``` + +Sencho's static IP on the network is ` + 2` (so `10.42.0.2` for the example above). Operators can configure each node independently; the override generator pushes alias hostnames that resolve to whichever IP the deploying node uses locally. + ## Test upstream Every alias row has a one-click **Test** button that runs a real probe across the same code path traffic uses. Result is shown inline: - **Green tick** with round-trip time when the path is healthy. -- **Red badge** with the failing stage (`sidecar`, `pilot_tunnel`, `agent_resolve`, `agent_dial`, or `target_port`) when something is wrong. +- **Red badge** with the failing stage (`pilot_tunnel`, `agent_resolve`, `agent_dial`, or `target_port`) when something is wrong. Use the Test button before assuming an issue is your application's fault. It tells you whether the mesh path itself is the problem. @@ -62,10 +83,9 @@ Use the Test button before assuming an issue is your application's fault. It tel Every node card has a **Diagnostics** button that opens a live view of: -- Sidecar liveness and pilot tunnel state on this node. +- Forwarder liveness and pilot tunnel state on this node. - Active TCP streams with byte counters and open age. - The resolver cache showing which aliases are registered. -- A **Restart sidecar** action. This is the first place to look when a connection isn't behaving as expected. @@ -77,18 +97,44 @@ The masthead has a **Mesh activity** button that opens the fleet-wide event log. A few things are deliberately out of scope for the first release: -- **One alias per TCP port across the fleet.** If two stacks expose the same port (e.g. two Postgres instances on 5432), only the first can be added to the mesh. The opt-in sheet shows a clear inline error if the second tries. -- **Pilot-to-pilot routing is not supported.** Mesh works for traffic between the central node and its pilots in either direction. Two pilot nodes cannot reach each other through the mesh. +- **One alias per TCP port across the fleet.** If two stacks expose the same port (for example two Postgres instances on 5432), only the first can be added to the mesh. The opt-in sheet shows a clear inline error if the second tries. +- **Sencho's API port is reserved.** A meshed service exposing port 1852 is rejected at opt-in to prevent collision with the Sencho UI / API listener. +- **Pilot-to-pilot routing rides through central.** Mesh works for traffic between the central node and its pilots in either direction, and pilot-to-pilot via central relay. Direct peer-to-peer pilot tunnels are not supported. +- **Stream pool shared with the Pilot tunnel.** Each Pilot tunnel multiplexes up to 1024 concurrent streams covering HTTP, WebSocket, and mesh TCP traffic. A heavy mesh workload counts against the same ceiling as ordinary fleet API traffic. See the [Pilot Agent](/features/pilot-agent) resource-limit notes for the full picture. - **No TLS termination, no blue/green cutover.** Layer 7 features land in a follow-up. ## Troubleshooting -**A route shows `tunnel down`.** The Pilot tunnel to the target node is gone. Check **Fleet → Overview** for the node's status. Mesh recovers automatically when the tunnel reconnects. + + + The Sencho instance failed to set up its `sencho_mesh` Docker network at boot. Check the Sencho container's logs for `[Mesh]` messages. Common causes: Sencho is running outside Docker (dev mode, mesh routing skipped); the Docker socket is not mounted; `sencho_mesh` already exists with a different subnet from a previous install. The mesh status surface on the Routing tab shows the specific reason. + -**A route shows `unreachable`.** The tunnel is up but the destination port did not answer. Check that the target stack is running and that its service is listening on the declared port. Click **Test** to see the exact failing stage. + + Sencho refuses to start the mesh data plane if a network named `sencho_mesh` exists with a subnet different from the one this Sencho is configured for. Either remove the existing network (`docker network rm sencho_mesh` on the affected host, after detaching any containers) or set `SENCHO_MESH_SUBNET` on this node to match the existing subnet. Restart the Sencho container so the setup runs again. + -**A route shows `not authorized`.** The destination stack is not opted into the mesh on its home node. Open Routing on that node and add the stack. + + Stacks whose services declare `network_mode: host` cannot join `sencho_mesh` and therefore cannot participate in the mesh. Switch the affected service to bridge networking and redeploy, or accept that the stack stays out of the mesh. + -**Adding a stack hangs.** Mesh redeploys peers on opt-in to refresh hostnames. A stuck redeploy usually means the stack itself failed to come back up. Check the stack's deploy logs. + + The Pilot tunnel to the target node is gone. Check **Fleet → Overview** for the node's status. Mesh recovers automatically when the tunnel reconnects. + -**The sidecar shows `off` for a node.** Click **Diagnostics → Restart sidecar** on the node card. If the sidecar still does not come up, check the node's `docker ps -a` output for `sencho-mesh-` and inspect its logs. + + The tunnel is up but the destination port did not answer. Check that the target stack is running and that its service is listening on the declared port. Click **Test** to see the exact failing stage. + + + + The destination stack is not opted into the mesh on its home node. Open Routing on that node and add the stack. + + + + Mesh redeploys the stack on opt-in to refresh hostnames and network attachments. A stuck redeploy usually means the stack itself failed to come back up. Check the stack's deploy logs. + + + + `172.30.0.0/24` is the default range for the `sencho_mesh` network and is rare in most setups. If it collides with an existing VPN, VLAN, or Docker network, set `SENCHO_MESH_SUBNET` on the Sencho service to a free `/24` and restart the container. Each node can be configured independently. + + diff --git a/frontend/src/components/fleet/MeshOptInSheet.tsx b/frontend/src/components/fleet/MeshOptInSheet.tsx index cc709429..ab54bc90 100644 --- a/frontend/src/components/fleet/MeshOptInSheet.tsx +++ b/frontend/src/components/fleet/MeshOptInSheet.tsx @@ -2,7 +2,8 @@ import { useEffect, useState } from 'react'; import { apiFetch } from '@/lib/api'; import { toast } from '@/components/ui/toast-store'; import { SystemSheet } from '@/components/ui/system-sheet'; -import { Checkbox } from '@/components/ui/checkbox'; +import { Button } from '@/components/ui/button'; +import { ConfirmModal } from '@/components/ui/modal'; import type { MeshStackEntry } from '@/types/mesh'; import { Loader2 } from 'lucide-react'; @@ -19,6 +20,7 @@ export function MeshOptInSheet({ open, onOpenChange, nodeId, nodeName, onChanged const [loading, setLoading] = useState(false); const [error, setError] = useState(null); const [pendingStack, setPendingStack] = useState(null); + const [confirmStack, setConfirmStack] = useState(null); useEffect(() => { if (!open) return; @@ -40,7 +42,7 @@ export function MeshOptInSheet({ open, onOpenChange, nodeId, nodeName, onChanged return () => { cancelled = true; }; }, [open, nodeId]); - const toggle = async (stack: MeshStackEntry) => { + const performToggle = async (stack: MeshStackEntry) => { setPendingStack(stack.name); setError(null); try { @@ -54,10 +56,17 @@ export function MeshOptInSheet({ open, onOpenChange, nodeId, nodeName, onChanged setError(body.error || 'Port already claimed by another mesh stack'); return; } + if (res.status === 503) { + const body = await res.json().catch(() => ({})) as { error?: string }; + setError(body.error || 'Mesh data plane unavailable on this node'); + return; + } if (!res.ok) throw new Error(`status ${res.status}`); setStacks((prev) => prev.map((s) => s.name === stack.name ? { ...s, optedIn: !stack.optedIn } : s)); onChanged(); - toast.success(stack.optedIn ? 'Stack removed from mesh' : 'Stack added to mesh'); + toast.success(stack.optedIn + ? `${stack.name} removed from mesh, redeploying` + : `${stack.name} added to mesh, redeploying`); } catch (err) { setError((err as Error).message); toast.error('Mesh update failed'); @@ -70,54 +79,85 @@ export function MeshOptInSheet({ open, onOpenChange, nodeId, nodeName, onChanged const meta = `${inMeshCount} of ${stacks.length} in mesh`; return ( - -
-

- Adding a stack lets its services be reached from other meshed stacks by hostname. - Toggling a stack redeploys it to refresh hostnames. -

+ <> + +
+

+ Adding a stack lets its services be reached from other meshed stacks by hostname. + Toggling a stack triggers a redeploy on its node so the routing override applies. +

- {loading && ( -
- Loading stacks… -
- )} - {error && ( -
- {error} -
- )} - {!loading && stacks.length === 0 && ( -
No stacks deployed on this node yet.
- )} - -
- {stacks.map((stack) => ( -
-
- { void toggle(stack); }} - /> - -
- {pendingStack === stack.name && } - {stack.optedIn && pendingStack !== stack.name && ( - in mesh - )} + {loading && ( +
+ Loading stacks…
- ))} + )} + {error && ( +
+ {error} +
+ )} + {!loading && stacks.length === 0 && ( +
No stacks deployed on this node yet.
+ )} + +
+ {stacks.map((stack) => ( +
+
+ {stack.name} + {stack.optedIn && pendingStack !== stack.name && ( + in mesh + )} +
+ {pendingStack === stack.name ? ( + + ) : ( + + )} +
+ ))} +
-
- + + + { if (!o) setConfirmStack(null); }} + variant={confirmStack?.optedIn ? 'destructive' : 'default'} + kicker={`Mesh / ${nodeName}`} + title={ + confirmStack?.optedIn + ? `Remove ${confirmStack.name} from mesh?` + : `Add ${confirmStack?.name ?? ''} to mesh?` + } + description={ + confirmStack?.optedIn + ? `${confirmStack.name} will be redeployed on ${nodeName} so its containers drop the mesh routing entries from /etc/hosts.` + : confirmStack + ? `${confirmStack.name} will be redeployed on ${nodeName} so its containers pick up the mesh routing entries.` + : undefined + } + confirmLabel={confirmStack?.optedIn ? 'Remove and redeploy' : 'Add and redeploy'} + onConfirm={() => { + if (confirmStack) void performToggle(confirmStack); + setConfirmStack(null); + }} + onCancel={() => setConfirmStack(null)} + /> + ); }