mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-11 03:06:54 +00:00
feat(mesh): replace host-mode with shared sencho_mesh Docker network (#1009)
* feat(mesh): replace host-mode with shared sencho_mesh Docker network Phase D of the mesh redesign: drop the operator's `network_mode: host` requirement and the `host-gateway` extra_hosts pattern that did not work on cloud iptables-restrictive distros (OCI, etc.) or Docker Desktop. Each Sencho creates a shared `sencho_mesh` Docker bridge network on boot (default subnet 172.30.0.0/24, override via SENCHO_MESH_SUBNET), pins itself at `<network>+2`, and attaches every meshed user service to the same bridge. Compose overrides now emit IP-based `extra_hosts` plus a top-level `networks` block declaring `sencho_mesh` external. Override delivery: central renders for local stacks; for remote stacks it sends the fleet alias list to the remote's new `PUT /api/mesh/local- override/:stackName` endpoint, which renders against the remote's OWN local senchoIp and writes under its OWN DATA_DIR. Each node may use a different subnet without coordination beyond the env var. Opt-in / opt-out now trigger an automatic redeploy of the affected stack via the existing deploy code path (local: ComposeService; remote: HTTP POST through proxyFetch). The frontend opt-in sheet shows a confirmation modal (ConfirmModal) before the mutation. Failed redeploys emit both a mesh activity event and a durable audit-log row. Hardening: - Reserve port 1852 at opt-in (prevents user containers from racing the Sencho API listener). - ensureMeshNetwork refuses to continue if `sencho_mesh` exists with a mismatched subnet rather than silently routing to the wrong IP. - Idempotent network connect/disconnect helpers in DockerController. - optInStack rolls back the DB row if the just-inserted stack's override push fails (no half-states surviving across calls). - regenerateOverridesForNode runs in parallel and skips the just- pushed stack on opt-in. Operator template: drop `network_mode: host`, restore `ports: ["1852:1852"]`. Mesh now works identically on Linux LAN, OCI, and Docker Desktop without firewall changes. Docs: rewrite docs/features/sencho-mesh.mdx around the shared bridge network, document SENCHO_MESH_SUBNET, surface the host-network-service opt-in restriction, and cross-link with the Pilot Agent docs. BREAKING CHANGE: the operator's `docker-compose.yml` no longer uses `network_mode: host`. After upgrading, redeploy any meshed stacks once so they pick up the new IP-based override and join `sencho_mesh`. * fix(mesh): wrap stackName with path.basename in local-override fs ops CodeQL flagged js/path-injection on the new applyLocalOverride and removeLocalOverride methods because they are publicly reachable and its data-flow model does not recognize isValidStackName / isPathWithinBase as sanitizers. The validation IS sufficient (the allowlist regex blocks path separators, the path-prefix check blocks escape), but path.basename is a model CodeQL recognizes and is purely defensive: for any input that already passes isValidStackName, basename is the identity.
This commit is contained in:
@@ -663,3 +663,90 @@ describe('removeContainers', () => {
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Network connect / disconnect helpers ───────────────────────────────
|
||||
|
||||
describe('DockerController - connectContainerToNetwork', () => {
|
||||
it('attaches a container to a network with no static IP', async () => {
|
||||
const connect = vi.fn().mockResolvedValue(undefined);
|
||||
mockDocker.getNetwork.mockReturnValue({ connect });
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
await dc.connectContainerToNetwork('sencho_mesh', 'sencho-host-1234');
|
||||
|
||||
expect(mockDocker.getNetwork).toHaveBeenCalledWith('sencho_mesh');
|
||||
expect(connect).toHaveBeenCalledWith({ Container: 'sencho-host-1234' });
|
||||
});
|
||||
|
||||
it('attaches with a static IPv4 address when provided', async () => {
|
||||
const connect = vi.fn().mockResolvedValue(undefined);
|
||||
mockDocker.getNetwork.mockReturnValue({ connect });
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
await dc.connectContainerToNetwork('sencho_mesh', 'sencho-host-1234', { ipv4Address: '172.30.0.2' });
|
||||
|
||||
expect(connect).toHaveBeenCalledWith({
|
||||
Container: 'sencho-host-1234',
|
||||
EndpointConfig: { IPAMConfig: { IPv4Address: '172.30.0.2' } },
|
||||
});
|
||||
});
|
||||
|
||||
it('treats 403 already-connected as success (idempotent)', async () => {
|
||||
const connect = vi.fn().mockRejectedValue({ statusCode: 403, message: 'endpoint already exists' });
|
||||
mockDocker.getNetwork.mockReturnValue({ connect });
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
await expect(dc.connectContainerToNetwork('sencho_mesh', 'sencho-host-1234')).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('rethrows non-idempotent errors', async () => {
|
||||
const connect = vi.fn().mockRejectedValue({ statusCode: 500, message: 'server error' });
|
||||
mockDocker.getNetwork.mockReturnValue({ connect });
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
await expect(dc.connectContainerToNetwork('sencho_mesh', 'sencho-host-1234')).rejects.toMatchObject({
|
||||
statusCode: 500,
|
||||
});
|
||||
});
|
||||
|
||||
it('rethrows a 403 whose message is unrelated to already-attached', async () => {
|
||||
const connect = vi.fn().mockRejectedValue({ statusCode: 403, message: 'host-mode container cannot join network' });
|
||||
mockDocker.getNetwork.mockReturnValue({ connect });
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
await expect(dc.connectContainerToNetwork('sencho_mesh', 'sencho-host-1234')).rejects.toMatchObject({
|
||||
statusCode: 403,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('DockerController - disconnectContainerFromNetwork', () => {
|
||||
it('detaches a container from a network with force=true', async () => {
|
||||
const disconnect = vi.fn().mockResolvedValue(undefined);
|
||||
mockDocker.getNetwork.mockReturnValue({ disconnect });
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
await dc.disconnectContainerFromNetwork('sencho_mesh', 'sencho-host-1234');
|
||||
|
||||
expect(mockDocker.getNetwork).toHaveBeenCalledWith('sencho_mesh');
|
||||
expect(disconnect).toHaveBeenCalledWith({ Container: 'sencho-host-1234', Force: true });
|
||||
});
|
||||
|
||||
it('treats 404 not-connected as success (idempotent)', async () => {
|
||||
const disconnect = vi.fn().mockRejectedValue({ statusCode: 404, message: 'no such network endpoint' });
|
||||
mockDocker.getNetwork.mockReturnValue({ disconnect });
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
await expect(dc.disconnectContainerFromNetwork('sencho_mesh', 'sencho-host-1234')).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('rethrows non-idempotent errors', async () => {
|
||||
const disconnect = vi.fn().mockRejectedValue({ statusCode: 500, message: 'server error' });
|
||||
mockDocker.getNetwork.mockReturnValue({ disconnect });
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
await expect(dc.disconnectContainerFromNetwork('sencho_mesh', 'sencho-host-1234')).rejects.toMatchObject({
|
||||
statusCode: 500,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,36 +1,64 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import * as YAML from 'yaml';
|
||||
import { buildAliasHosts, generateOverrideYaml } from '../services/MeshComposeOverride';
|
||||
import { buildAliasHosts, generateOverrideYaml, SENCHO_MESH_NETWORK } from '../services/MeshComposeOverride';
|
||||
|
||||
const SENCHO_IP = '172.30.0.2';
|
||||
|
||||
describe('generateOverrideYaml', () => {
|
||||
it('emits services with extra_hosts pointing to host-gateway', () => {
|
||||
it('emits services with extra_hosts pointing to the Sencho mesh IP', () => {
|
||||
const yaml = generateOverrideYaml({
|
||||
services: ['web', 'cache'],
|
||||
aliases: [
|
||||
{ host: 'db.api.opsix.sencho' },
|
||||
{ host: 'etl.worker.opsix.sencho' },
|
||||
],
|
||||
senchoIp: SENCHO_IP,
|
||||
});
|
||||
const parsed = YAML.parse(yaml) as Record<string, unknown>;
|
||||
expect(parsed.networks).toBeUndefined();
|
||||
const services = parsed.services as Record<string, { extra_hosts: string[] }>;
|
||||
const services = parsed.services as Record<string, { extra_hosts: string[]; networks: string[] }>;
|
||||
expect(Object.keys(services).sort()).toEqual(['cache', 'web']);
|
||||
for (const svc of ['web', 'cache']) {
|
||||
expect(services[svc].extra_hosts).toEqual([
|
||||
'db.api.opsix.sencho:host-gateway',
|
||||
'etl.worker.opsix.sencho:host-gateway',
|
||||
`db.api.opsix.sencho:${SENCHO_IP}`,
|
||||
`etl.worker.opsix.sencho:${SENCHO_IP}`,
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
it('emits empty service stubs when no aliases exist yet', () => {
|
||||
it('attaches every service to the sencho_mesh network', () => {
|
||||
const yaml = generateOverrideYaml({
|
||||
services: ['web', 'cache', 'worker'],
|
||||
aliases: [{ host: 'db.api.opsix.sencho' }],
|
||||
senchoIp: SENCHO_IP,
|
||||
});
|
||||
const parsed = YAML.parse(yaml) as Record<string, unknown>;
|
||||
const services = parsed.services as Record<string, { networks: string[] }>;
|
||||
for (const svc of ['web', 'cache', 'worker']) {
|
||||
expect(services[svc].networks).toEqual([SENCHO_MESH_NETWORK]);
|
||||
}
|
||||
});
|
||||
|
||||
it('declares sencho_mesh as an external network at the top level', () => {
|
||||
const yaml = generateOverrideYaml({
|
||||
services: ['web'],
|
||||
aliases: [],
|
||||
senchoIp: SENCHO_IP,
|
||||
});
|
||||
const parsed = YAML.parse(yaml) as Record<string, unknown>;
|
||||
const services = parsed.services as Record<string, { extra_hosts?: string[] }>;
|
||||
const networks = parsed.networks as Record<string, { external: boolean }>;
|
||||
expect(networks[SENCHO_MESH_NETWORK]).toEqual({ external: true });
|
||||
});
|
||||
|
||||
it('still attaches services to the network when no aliases exist yet', () => {
|
||||
const yaml = generateOverrideYaml({
|
||||
services: ['web'],
|
||||
aliases: [],
|
||||
senchoIp: SENCHO_IP,
|
||||
});
|
||||
const parsed = YAML.parse(yaml) as Record<string, unknown>;
|
||||
const services = parsed.services as Record<string, { extra_hosts?: string[]; networks: string[] }>;
|
||||
expect(services.web.extra_hosts).toBeUndefined();
|
||||
expect(services.web.networks).toEqual([SENCHO_MESH_NETWORK]);
|
||||
});
|
||||
|
||||
it('produces stable output regardless of input ordering', () => {
|
||||
@@ -40,6 +68,7 @@ describe('generateOverrideYaml', () => {
|
||||
{ host: 'b.x.y.sencho' },
|
||||
{ host: 'a.x.y.sencho' },
|
||||
],
|
||||
senchoIp: SENCHO_IP,
|
||||
});
|
||||
const b = generateOverrideYaml({
|
||||
services: ['cache', 'web'],
|
||||
@@ -47,9 +76,20 @@ describe('generateOverrideYaml', () => {
|
||||
{ host: 'a.x.y.sencho' },
|
||||
{ host: 'b.x.y.sencho' },
|
||||
],
|
||||
senchoIp: SENCHO_IP,
|
||||
});
|
||||
expect(a).toBe(b);
|
||||
});
|
||||
|
||||
it('uses the senchoIp argument verbatim in extra_hosts entries', () => {
|
||||
const customIp = '10.42.7.99';
|
||||
const yaml = generateOverrideYaml({
|
||||
services: ['web'],
|
||||
aliases: [{ host: 'svc.stack.node.sencho' }],
|
||||
senchoIp: customIp,
|
||||
});
|
||||
expect(yaml).toContain(`svc.stack.node.sencho:${customIp}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildAliasHosts', () => {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
import { getSenchoIpFromSubnet } from '../services/MeshService';
|
||||
|
||||
let tmpDir: string;
|
||||
let MeshService: typeof import('../services/MeshService').MeshService;
|
||||
@@ -26,6 +27,9 @@ beforeEach(() => {
|
||||
activeStreams: Map<number, unknown>;
|
||||
routeErrorMap: Map<string, unknown>;
|
||||
routeLatencyMap: Map<string, unknown>;
|
||||
senchoIp: string | null;
|
||||
meshSubnet: string;
|
||||
networkSetupError: string | null;
|
||||
};
|
||||
svc.aliasCache = new Map();
|
||||
svc.aliasByPort = new Map();
|
||||
@@ -33,6 +37,9 @@ beforeEach(() => {
|
||||
svc.activeStreams = new Map();
|
||||
svc.routeErrorMap = new Map();
|
||||
svc.routeLatencyMap = new Map();
|
||||
svc.senchoIp = '172.30.0.2';
|
||||
svc.meshSubnet = '172.30.0.0/24';
|
||||
svc.networkSetupError = null;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
@@ -227,3 +234,114 @@ describe('MeshService.testUpstream tunnel-down path', () => {
|
||||
expect(result.code).toBe('no_route');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getSenchoIpFromSubnet', () => {
|
||||
it('returns network+2 for the default /24', () => {
|
||||
expect(getSenchoIpFromSubnet('172.30.0.0/24')).toBe('172.30.0.2');
|
||||
});
|
||||
|
||||
it('handles a custom /24 in a different range', () => {
|
||||
expect(getSenchoIpFromSubnet('10.42.7.0/24')).toBe('10.42.7.2');
|
||||
});
|
||||
|
||||
it('handles a /16', () => {
|
||||
expect(getSenchoIpFromSubnet('172.30.0.0/16')).toBe('172.30.0.2');
|
||||
});
|
||||
|
||||
it('masks the input IP to the network address before adding 2', () => {
|
||||
// 172.30.0.50/24 → network 172.30.0.0 → +2 = 172.30.0.2
|
||||
expect(getSenchoIpFromSubnet('172.30.0.50/24')).toBe('172.30.0.2');
|
||||
});
|
||||
|
||||
it('rejects a malformed CIDR', () => {
|
||||
expect(() => getSenchoIpFromSubnet('not-a-cidr')).toThrow(/Invalid mesh subnet/);
|
||||
expect(() => getSenchoIpFromSubnet('172.30.0.0')).toThrow(/Invalid mesh subnet/);
|
||||
});
|
||||
|
||||
it('rejects prefixes too narrow to host two addresses', () => {
|
||||
expect(() => getSenchoIpFromSubnet('172.30.0.0/31')).toThrow(/Invalid mesh subnet/);
|
||||
});
|
||||
|
||||
it('rejects out-of-range octets', () => {
|
||||
expect(() => getSenchoIpFromSubnet('172.30.0.999/24')).toThrow(/Invalid mesh subnet/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MeshService.ensureMeshNetwork', () => {
|
||||
it('refuses to continue when sencho_mesh exists with a different subnet', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const dcModule = await import('../services/DockerController');
|
||||
const fakeController = {
|
||||
createNetwork: vi.fn().mockRejectedValue({ statusCode: 409, message: 'network already exists' }),
|
||||
inspectNetwork: vi.fn().mockResolvedValue({ IPAM: { Config: [{ Subnet: '10.99.0.0/24' }] } }),
|
||||
};
|
||||
vi.spyOn(dcModule.default, 'getInstance').mockReturnValue(fakeController as unknown as ReturnType<typeof dcModule.default.getInstance>);
|
||||
|
||||
await expect(
|
||||
(svc as unknown as { ensureMeshNetwork: (s: string) => Promise<void> }).ensureMeshNetwork('172.30.0.0/24'),
|
||||
).rejects.toThrow(/exists with subnet 10\.99\.0\.0\/24/);
|
||||
});
|
||||
|
||||
it('treats 409 with matching subnet as idempotent success', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const dcModule = await import('../services/DockerController');
|
||||
const fakeController = {
|
||||
createNetwork: vi.fn().mockRejectedValue({ statusCode: 409, message: 'network already exists' }),
|
||||
inspectNetwork: vi.fn().mockResolvedValue({ IPAM: { Config: [{ Subnet: '172.30.0.0/24' }] } }),
|
||||
};
|
||||
vi.spyOn(dcModule.default, 'getInstance').mockReturnValue(fakeController as unknown as ReturnType<typeof dcModule.default.getInstance>);
|
||||
|
||||
await expect(
|
||||
(svc as unknown as { ensureMeshNetwork: (s: string) => Promise<void> }).ensureMeshNetwork('172.30.0.0/24'),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('MeshService.optInStack rollback', () => {
|
||||
it('rolls back the DB row when the just-inserted stack fails to push its override', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> }, 'inspectStackServices')
|
||||
.mockResolvedValue([{ service: 'db', ports: [5432] }]);
|
||||
vi.spyOn(svc, 'pushOverrideToNode')
|
||||
.mockRejectedValue(new Error('simulated remote pilot offline'));
|
||||
vi.spyOn(svc as unknown as { triggerRedeploy: (n: number, s: string, a: string) => void }, 'triggerRedeploy')
|
||||
.mockImplementation(() => { /* noop */ });
|
||||
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
|
||||
await expect(svc.optInStack(localNodeId, 'api', 'tester'))
|
||||
.rejects.toThrow(/simulated remote pilot offline/);
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MeshService.optInStack guard rails (network setup)', () => {
|
||||
it('rejects opt-in when senchoIp is null (mesh data plane unavailable)', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
(svc as unknown as { senchoIp: string | null }).senchoIp = null;
|
||||
(svc as unknown as { networkSetupError: string | null }).networkSetupError = 'sencho_mesh subnet mismatch';
|
||||
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
|
||||
await expect(svc.optInStack(localNodeId, 'api', 'tester'))
|
||||
.rejects.toThrow(/sencho_mesh subnet mismatch/);
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects opt-in when a service exposes the reserved Sencho API port', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> }, 'inspectStackServices')
|
||||
.mockResolvedValue([{ service: 'web', ports: [1852] }]);
|
||||
vi.spyOn(svc as unknown as { regenerateOverridesForNode: (n: number) => Promise<void> }, 'regenerateOverridesForNode')
|
||||
.mockResolvedValue(undefined);
|
||||
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
|
||||
await expect(svc.optInStack(localNodeId, 'api', 'tester'))
|
||||
.rejects.toThrow(/port 1852 is reserved/);
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -70,6 +70,69 @@ meshRouter.get('/local-services/:stackName', async (req: Request, res: Response)
|
||||
}
|
||||
});
|
||||
|
||||
const MAX_ALIASES_PER_PUSH = 1024;
|
||||
|
||||
/**
|
||||
* Accepts a fleet-wide alias list from central and writes a mesh override
|
||||
* for the named stack onto THIS Sencho's local DATA_DIR. The pilot looks
|
||||
* up its own service names and uses its own static IP on `sencho_mesh`,
|
||||
* so alias hostnames in user containers always resolve to the LOCAL
|
||||
* Sencho IP on the deploying node. Always writes against the LOCAL
|
||||
* Sencho's default node id.
|
||||
*/
|
||||
meshRouter.put('/local-override/:stackName', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
const stackName = req.params.stackName as string;
|
||||
if (!isValidStackName(stackName)) { res.status(400).json({ error: 'Invalid stack name' }); return; }
|
||||
const body = req.body as { aliases?: unknown };
|
||||
if (!Array.isArray(body?.aliases)) { res.status(400).json({ error: 'Missing aliases array in body' }); return; }
|
||||
if (body.aliases.length > MAX_ALIASES_PER_PUSH) {
|
||||
res.status(413).json({ error: `Alias list exceeds ${MAX_ALIASES_PER_PUSH} entries` });
|
||||
return;
|
||||
}
|
||||
const aliases: { host: string }[] = [];
|
||||
for (const entry of body.aliases) {
|
||||
const host = (entry as { host?: unknown } | null | undefined)?.host;
|
||||
if (typeof host !== 'string' || host.length === 0 || host.length > 253) {
|
||||
// 253 octets is the DNS hostname ceiling. Defensive against a
|
||||
// malicious or buggy central sending a multi-KB host string.
|
||||
res.status(400).json({ error: 'Invalid alias entry' });
|
||||
return;
|
||||
}
|
||||
aliases.push({ host });
|
||||
}
|
||||
try {
|
||||
const written = await MeshService.getInstance().applyLocalOverride(stackName, aliases);
|
||||
if (!written) { res.status(400).json({ error: 'Refused to write override (path validation failed)' }); return; }
|
||||
res.json({ ok: true, path: written });
|
||||
} catch (err) {
|
||||
if (err instanceof MeshError && err.code === 'push_failed') {
|
||||
res.status(503).json({ error: err.message, code: err.code });
|
||||
return;
|
||||
}
|
||||
console.warn('[mesh] /local-override failed:', sanitizeForLog((err as Error).message));
|
||||
res.status(500).json({ error: 'Failed to write local override' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Delete a previously written local override. Mirror of the PUT endpoint;
|
||||
* called by central when a stack is opted out so stale overrides do not
|
||||
* linger on the deploying node.
|
||||
*/
|
||||
meshRouter.delete('/local-override/:stackName', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
const stackName = req.params.stackName as string;
|
||||
if (!isValidStackName(stackName)) { res.status(400).json({ error: 'Invalid stack name' }); return; }
|
||||
try {
|
||||
await MeshService.getInstance().removeLocalOverride(stackName);
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
console.warn('[mesh] DELETE /local-override failed:', sanitizeForLog((err as Error).message));
|
||||
res.status(500).json({ error: 'Failed to remove local override' });
|
||||
}
|
||||
});
|
||||
|
||||
meshRouter.get('/nodes/:nodeId/stacks', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
const nodeId = Number.parseInt(req.params.nodeId as string, 10);
|
||||
@@ -105,6 +168,10 @@ meshRouter.post('/nodes/:nodeId/stacks/:stackName/opt-in', async (req: Request,
|
||||
res.status(409).json({ error: err.message, code: err.code });
|
||||
return;
|
||||
}
|
||||
if (err instanceof MeshError && err.code === 'push_failed') {
|
||||
res.status(503).json({ error: err.message, code: err.code });
|
||||
return;
|
||||
}
|
||||
if (err instanceof MeshError) {
|
||||
res.status(400).json({ error: err.message, code: err.code });
|
||||
return;
|
||||
|
||||
@@ -463,6 +463,68 @@ class DockerController {
|
||||
return await this.docker.createNetwork(options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach a container to a Docker network. Idempotent: if the container is
|
||||
* already attached the call resolves silently. Optionally pins the
|
||||
* container's IPv4 address inside the network so other services can use
|
||||
* static `extra_hosts` entries against it.
|
||||
*/
|
||||
public async connectContainerToNetwork(
|
||||
networkName: string,
|
||||
containerId: string,
|
||||
opts: { ipv4Address?: string } = {},
|
||||
): Promise<void> {
|
||||
const network = this.docker.getNetwork(networkName);
|
||||
const payload: { Container: string; EndpointConfig?: { IPAMConfig?: { IPv4Address: string } } } = {
|
||||
Container: containerId,
|
||||
};
|
||||
if (opts.ipv4Address) {
|
||||
payload.EndpointConfig = { IPAMConfig: { IPv4Address: opts.ipv4Address } };
|
||||
}
|
||||
try {
|
||||
await network.connect(payload);
|
||||
} catch (err) {
|
||||
if (DockerController.isAlreadyConnectedError(err)) return;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detach a container from a Docker network. Idempotent: if the container
|
||||
* is not attached the call resolves silently.
|
||||
*/
|
||||
public async disconnectContainerFromNetwork(
|
||||
networkName: string,
|
||||
containerId: string,
|
||||
): Promise<void> {
|
||||
const network = this.docker.getNetwork(networkName);
|
||||
try {
|
||||
await network.disconnect({ Container: containerId, Force: true });
|
||||
} catch (err) {
|
||||
if (DockerController.isNotConnectedError(err)) return;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
private static isAlreadyConnectedError(err: unknown): boolean {
|
||||
const e = err as { statusCode?: number; message?: string };
|
||||
const msg = (e?.message || '').toLowerCase();
|
||||
// Docker daemon returns 403 for several distinct cases (already
|
||||
// attached, host-network containers, permission denied), so match the
|
||||
// message body too rather than treating any 403 as idempotent success.
|
||||
if (e?.statusCode === 403 && (msg.includes('already exists') || msg.includes('already attached'))) {
|
||||
return true;
|
||||
}
|
||||
return msg.includes('already exists') || msg.includes('already attached');
|
||||
}
|
||||
|
||||
private static isNotConnectedError(err: unknown): boolean {
|
||||
const e = err as { statusCode?: number; message?: string };
|
||||
if (e?.statusCode === 404) return true;
|
||||
const msg = (e?.message || '').toLowerCase();
|
||||
return msg.includes('is not connected') || msg.includes('no such container');
|
||||
}
|
||||
|
||||
public async getRunningContainers() {
|
||||
const containers = await this.docker.listContainers({ all: false });
|
||||
return this.validateApiData<any[]>(containers);
|
||||
|
||||
@@ -4,13 +4,19 @@ import * as YAML from 'yaml';
|
||||
* Sencho Mesh Compose override generator.
|
||||
*
|
||||
* Produces a YAML override applied with `docker compose -f compose.yml -f
|
||||
* mesh.override.yml up` that injects cross-node alias entries into each
|
||||
* opted-in service's /etc/hosts. The aliases resolve to the host gateway
|
||||
* (`host-gateway`), which is where the local Sencho Mesh sidecar listens
|
||||
* (host network mode). The user's source compose file is never mutated; the
|
||||
* override lives in Sencho's data dir.
|
||||
* mesh.override.yml up` that:
|
||||
* - injects cross-node alias entries into each opted-in service's
|
||||
* `/etc/hosts`, resolving every alias to the Sencho container's
|
||||
* static IP on the internal `sencho_mesh` Docker network;
|
||||
* - attaches each service to `sencho_mesh` so that IP is reachable
|
||||
* from inside the user's container.
|
||||
*
|
||||
* The user's source compose file is never mutated; the override lives in
|
||||
* Sencho's data dir and is regenerated whenever the alias set changes.
|
||||
*/
|
||||
|
||||
export const SENCHO_MESH_NETWORK = 'sencho_mesh';
|
||||
|
||||
export interface MeshAlias {
|
||||
/** `<service>.<stack>.<nodeName>.sencho` */
|
||||
host: string;
|
||||
@@ -21,6 +27,8 @@ export interface MeshOverrideInput {
|
||||
services: string[];
|
||||
/** Aliases this stack should be able to resolve. Order is normalized in output. */
|
||||
aliases: MeshAlias[];
|
||||
/** Sencho's static IP on the `sencho_mesh` Docker network. */
|
||||
senchoIp: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -31,20 +39,23 @@ export function generateOverrideYaml(input: MeshOverrideInput): string {
|
||||
const sortedServices = [...input.services].sort();
|
||||
const sortedAliases = [...input.aliases].sort((a, b) => a.host.localeCompare(b.host));
|
||||
|
||||
if (sortedAliases.length === 0) {
|
||||
const services: Record<string, unknown> = {};
|
||||
for (const svc of sortedServices) services[svc] = {};
|
||||
return YAML.stringify({ services }, { lineWidth: 0 });
|
||||
}
|
||||
|
||||
const extraHostsList = sortedAliases.map((a) => `${a.host}:host-gateway`);
|
||||
|
||||
const services: Record<string, unknown> = {};
|
||||
for (const svc of sortedServices) {
|
||||
services[svc] = { extra_hosts: extraHostsList };
|
||||
const entry: Record<string, unknown> = {
|
||||
networks: [SENCHO_MESH_NETWORK],
|
||||
};
|
||||
if (sortedAliases.length > 0) {
|
||||
entry.extra_hosts = sortedAliases.map((a) => `${a.host}:${input.senchoIp}`);
|
||||
}
|
||||
services[svc] = entry;
|
||||
}
|
||||
|
||||
return YAML.stringify({ services }, { lineWidth: 0 });
|
||||
const doc: Record<string, unknown> = {
|
||||
services,
|
||||
networks: { [SENCHO_MESH_NETWORK]: { external: true } },
|
||||
};
|
||||
|
||||
return YAML.stringify(doc, { lineWidth: 0 });
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -2,6 +2,7 @@ import net from 'net';
|
||||
import path from 'path';
|
||||
import fs from 'fs/promises';
|
||||
import { EventEmitter } from 'events';
|
||||
import { ComposeService } from './ComposeService';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import DockerController from './DockerController';
|
||||
import { LicenseService } from './LicenseService';
|
||||
@@ -9,14 +10,43 @@ import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from './license-headers';
|
||||
import { MeshForwarder, type MeshForwarderHost } from './MeshForwarder';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { PilotTunnelManager } from './PilotTunnelManager';
|
||||
import { generateOverrideYaml, MeshAlias } from './MeshComposeOverride';
|
||||
import { generateOverrideYaml, MeshAlias, SENCHO_MESH_NETWORK } from './MeshComposeOverride';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { isPathWithinBase, isValidStackName } from '../utils/validation';
|
||||
import { PORT as SENCHO_LISTEN_PORT } from '../helpers/constants';
|
||||
|
||||
const ACTIVITY_BUFFER_SIZE = 1000;
|
||||
const ALIAS_REFRESH_INTERVAL_MS = 60_000;
|
||||
const PROBE_TIMEOUT_MS = 5_000;
|
||||
const SLOW_PROBE_THRESHOLD_MS = 500;
|
||||
const DEFAULT_MESH_SUBNET = '172.30.0.0/24';
|
||||
|
||||
/**
|
||||
* Returns the static IPv4 address Sencho will pin itself to on the mesh
|
||||
* Docker network: `<network address> + 2`. The Docker daemon assigns
|
||||
* `<network> + 1` to the bridge gateway, so `+2` is the first usable host
|
||||
* address. For the default `172.30.0.0/24` this is `172.30.0.2`. Throws
|
||||
* on invalid CIDR or a prefix too narrow to host two addresses.
|
||||
*/
|
||||
export function getSenchoIpFromSubnet(subnet: string): string {
|
||||
const cidr = subnet.trim().match(/^(\d+)\.(\d+)\.(\d+)\.(\d+)\/(\d+)$/);
|
||||
if (!cidr) throw new Error(`Invalid mesh subnet CIDR: ${subnet}`);
|
||||
const octets = [Number(cidr[1]), Number(cidr[2]), Number(cidr[3]), Number(cidr[4])];
|
||||
const prefix = Number(cidr[5]);
|
||||
if (octets.some((o) => o < 0 || o > 255) || prefix < 8 || prefix > 30) {
|
||||
throw new Error(`Invalid mesh subnet CIDR: ${subnet}`);
|
||||
}
|
||||
const ipInt = (octets[0] << 24) | (octets[1] << 16) | (octets[2] << 8) | octets[3];
|
||||
const mask = prefix === 0 ? 0 : (0xffffffff << (32 - prefix)) >>> 0;
|
||||
const network = (ipInt & mask) >>> 0;
|
||||
const sencho = (network + 2) >>> 0;
|
||||
return [
|
||||
(sencho >>> 24) & 0xff,
|
||||
(sencho >>> 16) & 0xff,
|
||||
(sencho >>> 8) & 0xff,
|
||||
sencho & 0xff,
|
||||
].join('.');
|
||||
}
|
||||
|
||||
export type MeshActivitySource = 'pilot' | 'mesh';
|
||||
export type MeshActivityLevel = 'info' | 'warn' | 'error';
|
||||
@@ -109,17 +139,17 @@ interface ActiveStreamRecord {
|
||||
* container; one container per node now.
|
||||
* - opt-in / opt-out persistence and cascading override regeneration
|
||||
* - global alias aggregation (across the fleet via the existing HTTP
|
||||
* proxy chain — see `inspectStackServices`)
|
||||
* proxy chain, see `inspectStackServices`)
|
||||
* - cross-node TCP forwarding via `PilotTunnelManager` (central-side)
|
||||
* - probe + diagnostics + activity ring buffer
|
||||
*
|
||||
* V1 limitations:
|
||||
* - one cross-node alias per TCP port across the fleet (port-collision
|
||||
* check at opt-in)
|
||||
* - aliases resolve via `host-gateway` extra_hosts; Sencho's container
|
||||
* must run with `network_mode: host` for the forwarder's listeners to
|
||||
* bind on the host's network where meshed containers' `host-gateway`
|
||||
* entries point
|
||||
* - aliases resolve to Sencho's static IP on the internal `sencho_mesh`
|
||||
* Docker bridge network. Meshed user services join `sencho_mesh` so
|
||||
* that IP is reachable from inside their containers without any
|
||||
* host-firewall coordination.
|
||||
* - cross-node mesh routing is central → pilot in this phase. Pilot →
|
||||
* central and pilot ↔ pilot via central relay land in Phase B.
|
||||
*/
|
||||
@@ -135,6 +165,9 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
private routeLatencyMap = new Map<string, number>();
|
||||
private activityListeners = new Set<(e: MeshActivityEvent) => void>();
|
||||
private readonly forwarder: MeshForwarder;
|
||||
private senchoIp: string | null = null;
|
||||
private meshSubnet: string = DEFAULT_MESH_SUBNET;
|
||||
private networkSetupError: string | null = null;
|
||||
|
||||
private constructor() {
|
||||
super();
|
||||
@@ -158,6 +191,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
nodeId, message: `pilot tunnel up for node ${nodeId}`,
|
||||
}));
|
||||
|
||||
await this.setupMeshNetwork();
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
this.aliasRefreshTimer = setInterval(() => {
|
||||
@@ -187,6 +221,128 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
await this.forwarder.shutdown();
|
||||
}
|
||||
|
||||
public getSenchoIp(): string | null {
|
||||
return this.senchoIp;
|
||||
}
|
||||
|
||||
public getMeshSubnet(): string {
|
||||
return this.meshSubnet;
|
||||
}
|
||||
|
||||
public getNetworkSetupError(): string | null {
|
||||
return this.networkSetupError;
|
||||
}
|
||||
|
||||
/**
|
||||
* Idempotent setup of the shared `sencho_mesh` Docker bridge network and
|
||||
* Sencho's static attachment to it. Called once at boot before alias
|
||||
* cache refresh. Failures here disable mesh routing for the lifetime of
|
||||
* the process (forwarder still binds, but `ensureStackOverride` short-
|
||||
* circuits because there is no IP to point user containers at).
|
||||
*
|
||||
* Skipped entirely when Sencho is not running inside Docker (dev mode,
|
||||
* detected by an unset HOSTNAME env var or by the inspect lookup
|
||||
* failing). The forwarder still runs locally for unit-test coverage.
|
||||
*/
|
||||
private async setupMeshNetwork(): Promise<void> {
|
||||
const subnet = (process.env.SENCHO_MESH_SUBNET || DEFAULT_MESH_SUBNET).trim();
|
||||
try {
|
||||
this.senchoIp = getSenchoIpFromSubnet(subnet);
|
||||
this.meshSubnet = subnet;
|
||||
} catch (err) {
|
||||
this.networkSetupError = (err as Error).message;
|
||||
console.warn('[Mesh]', this.networkSetupError);
|
||||
this.senchoIp = null;
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.ensureMeshNetwork(subnet);
|
||||
} catch (err) {
|
||||
this.networkSetupError = (err as Error).message;
|
||||
console.warn('[Mesh] mesh network setup failed:', sanitizeForLog(this.networkSetupError));
|
||||
this.senchoIp = null;
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.ensureSelfAttached();
|
||||
} catch (err) {
|
||||
this.networkSetupError = (err as Error).message;
|
||||
console.warn('[Mesh] self-attach failed:', sanitizeForLog(this.networkSetupError));
|
||||
this.senchoIp = null;
|
||||
return;
|
||||
}
|
||||
|
||||
this.networkSetupError = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create `sencho_mesh` if it does not exist. If it does, validate the
|
||||
* subnet matches `expectedSubnet`; on mismatch, refuse to continue.
|
||||
* Silently using the wrong subnet would route traffic to the wrong IP.
|
||||
*/
|
||||
private async ensureMeshNetwork(expectedSubnet: string): Promise<void> {
|
||||
const dc = DockerController.getInstance(NodeRegistry.getInstance().getDefaultNodeId());
|
||||
try {
|
||||
await dc.createNetwork({
|
||||
Name: SENCHO_MESH_NETWORK,
|
||||
Driver: 'bridge',
|
||||
Attachable: true,
|
||||
IPAM: { Config: [{ Subnet: expectedSubnet }] },
|
||||
Labels: { 'io.sencho.mesh': 'true' },
|
||||
});
|
||||
return;
|
||||
} catch (err) {
|
||||
const e = err as { statusCode?: number; message?: string };
|
||||
if (e?.statusCode !== 409) throw err;
|
||||
}
|
||||
|
||||
const info = await dc.inspectNetwork(SENCHO_MESH_NETWORK) as {
|
||||
IPAM?: { Config?: Array<{ Subnet?: string }> };
|
||||
};
|
||||
const existingSubnet = info?.IPAM?.Config?.[0]?.Subnet;
|
||||
if (existingSubnet && existingSubnet !== expectedSubnet) {
|
||||
throw new Error(
|
||||
`${SENCHO_MESH_NETWORK} exists with subnet ${existingSubnet}, ` +
|
||||
`expected ${expectedSubnet}. Remove the network or set SENCHO_MESH_SUBNET to match.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect Sencho's own container to `sencho_mesh` at the static IP. Uses
|
||||
* the `HOSTNAME` env var (which Docker sets to the container's short ID
|
||||
* by default) to identify the container, mirroring the
|
||||
* SelfUpdateService pattern. Skipped in dev mode where HOSTNAME is
|
||||
* the laptop hostname and the inspect lookup would fail.
|
||||
*/
|
||||
private async ensureSelfAttached(): Promise<void> {
|
||||
if (!this.senchoIp) return;
|
||||
const hostname = process.env.HOSTNAME;
|
||||
if (!hostname) {
|
||||
console.log('[Mesh] HOSTNAME not set, mesh routing disabled (not running in Docker?)');
|
||||
this.senchoIp = null;
|
||||
return;
|
||||
}
|
||||
const dc = DockerController.getInstance(NodeRegistry.getInstance().getDefaultNodeId());
|
||||
try {
|
||||
await dc.connectContainerToNetwork(SENCHO_MESH_NETWORK, hostname, { ipv4Address: this.senchoIp });
|
||||
} catch (err) {
|
||||
const e = err as { statusCode?: number; message?: string };
|
||||
if (e?.statusCode === 404) {
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'warn', type: 'mesh.disable',
|
||||
message: 'self-container lookup failed; mesh routing disabled (not running in Docker?)',
|
||||
});
|
||||
console.warn('[Mesh] self-container lookup failed; mesh routing disabled (not running in Docker?)');
|
||||
this.senchoIp = null;
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Bind the forwarder's listeners to every alias port across the fleet
|
||||
* and release any listeners no longer in the alias set. Called from
|
||||
@@ -194,11 +350,11 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
* opt-in / opt-out / disable so the bound port set follows the DB
|
||||
* state.
|
||||
*
|
||||
* Every meshed node binds every alias port — not just ports it owns —
|
||||
* because meshed containers' `extra_hosts: <alias>:host-gateway`
|
||||
* entries resolve to the SOURCE node's gateway, so the source node is
|
||||
* where the inbound TCP connection lands. `handleAccept` then
|
||||
* dispatches to the same-node fast path or the cross-node bridge based
|
||||
* Every meshed node binds every alias port (not just ports it owns)
|
||||
* because alias DNS entries resolve to the SOURCE node's Sencho IP, so
|
||||
* the source node is where the inbound TCP connection lands.
|
||||
* `handleAccept` then dispatches to the same-node fast path or the
|
||||
* cross-node bridge based
|
||||
* on the resolved alias's owner. Fleet-wide port collisions are
|
||||
* blocked at opt-in time (`optInStack` checks `aliasByPort`), so
|
||||
* binding every alias port is unambiguous.
|
||||
@@ -269,6 +425,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
if (!isValidStackName(stackName)) {
|
||||
throw new MeshError('denied', `invalid stack name: ${stackName}`);
|
||||
}
|
||||
if (!this.senchoIp) {
|
||||
throw new MeshError(
|
||||
'denied',
|
||||
this.networkSetupError || 'mesh data plane unavailable (mesh network setup did not complete)',
|
||||
);
|
||||
}
|
||||
const db = DatabaseService.getInstance();
|
||||
if (db.isMeshStackEnabled(nodeId, stackName)) return;
|
||||
|
||||
@@ -279,6 +441,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
|
||||
const newPorts = new Set<number>();
|
||||
for (const svc of services) for (const p of svc.ports) newPorts.add(p);
|
||||
if (newPorts.has(SENCHO_LISTEN_PORT)) {
|
||||
throw new MeshError(
|
||||
'port_collision',
|
||||
`port ${SENCHO_LISTEN_PORT} is reserved for the Sencho API and cannot be used by a meshed service`,
|
||||
);
|
||||
}
|
||||
for (const port of newPorts) {
|
||||
const existing = this.aliasByPort.get(port);
|
||||
if (existing) {
|
||||
@@ -292,7 +460,25 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
db.insertMeshStack(nodeId, stackName, actor);
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
await this.regenerateOverridesForNode(nodeId);
|
||||
|
||||
// Push the just-opted-in stack's override loudly. If this fails the
|
||||
// DB state is invalid (alias claimed but remote pilot has no
|
||||
// override file) so roll back rather than leave a half-state that
|
||||
// future opt-in calls would short-circuit on `isMeshStackEnabled`.
|
||||
try {
|
||||
await this.pushOverrideToNode(nodeId, stackName);
|
||||
} catch (err) {
|
||||
db.deleteMeshStack(nodeId, stackName);
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
throw err;
|
||||
}
|
||||
// Regenerate OTHER meshed stacks' overrides on the same node so
|
||||
// they pick up the new alias entry. The just-opted-in stack was
|
||||
// already pushed above; skip it to avoid a duplicate round-trip.
|
||||
// Best-effort; per-stack failures are logged inside the helper.
|
||||
await this.regenerateOverridesForNode(nodeId, stackName);
|
||||
this.triggerRedeploy(nodeId, stackName, actor);
|
||||
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'opt_in',
|
||||
@@ -313,10 +499,11 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
const db = DatabaseService.getInstance();
|
||||
if (!db.isMeshStackEnabled(nodeId, stackName)) return;
|
||||
db.deleteMeshStack(nodeId, stackName);
|
||||
await this.removeStackOverride(nodeId, stackName);
|
||||
await this.removeOverrideFromNode(nodeId, stackName);
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
await this.regenerateOverridesForNode(nodeId);
|
||||
this.triggerRedeploy(nodeId, stackName, actor);
|
||||
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'opt_out',
|
||||
@@ -359,12 +546,14 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
if (!isValidStackName(stackName)) return null;
|
||||
const db = DatabaseService.getInstance();
|
||||
if (!db.isMeshStackEnabled(nodeId, stackName)) return null;
|
||||
if (!this.senchoIp) return null;
|
||||
|
||||
const aliases: MeshAlias[] = Array.from(this.aliasCache.values()).map((a) => ({ host: a.host }));
|
||||
const services = await this.inspectStackServices(nodeId, stackName);
|
||||
const yaml = generateOverrideYaml({
|
||||
services: services.map((s) => s.service),
|
||||
aliases,
|
||||
senchoIp: this.senchoIp,
|
||||
});
|
||||
|
||||
const dir = this.overrideDirFor(nodeId);
|
||||
@@ -375,6 +564,55 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
return file;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render and write a mesh override on the LOCAL node's filesystem from
|
||||
* a fleet-wide alias list supplied by central. The pilot looks up its
|
||||
* own service names and uses its own static IP, so each node's
|
||||
* override resolves alias hostnames to that node's local Sencho.
|
||||
* This is critical because each node has its own `sencho_mesh`
|
||||
* network with its own subnet. Returns the absolute path on success
|
||||
* or null if path validation rejects the input.
|
||||
*/
|
||||
public async applyLocalOverride(stackName: string, aliases: MeshAlias[]): Promise<string | null> {
|
||||
if (!isValidStackName(stackName)) return null;
|
||||
if (!this.senchoIp) {
|
||||
throw new MeshError(
|
||||
'push_failed',
|
||||
this.networkSetupError || 'mesh data plane unavailable on this node',
|
||||
);
|
||||
}
|
||||
const services = await this.inspectLocalStackServices(stackName);
|
||||
const yaml = generateOverrideYaml({
|
||||
services: services.map((s) => s.service),
|
||||
aliases,
|
||||
senchoIp: this.senchoIp,
|
||||
});
|
||||
|
||||
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
const dir = this.overrideDirFor(localNodeId);
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
// path.basename strips any directory component as defense-in-depth
|
||||
// on top of isValidStackName + isPathWithinBase. Recognized by
|
||||
// CodeQL's path-injection model.
|
||||
const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`);
|
||||
if (!isPathWithinBase(file, dir)) return null;
|
||||
await fs.writeFile(file, yaml, 'utf8');
|
||||
return file;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a previously applied local override (mirror of
|
||||
* `applyLocalOverride`). Used by central when a stack is opted out.
|
||||
*/
|
||||
public async removeLocalOverride(stackName: string): Promise<void> {
|
||||
if (!isValidStackName(stackName)) return;
|
||||
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
const dir = this.overrideDirFor(localNodeId);
|
||||
const file = path.resolve(dir, `${path.basename(stackName)}.override.yml`);
|
||||
if (!isPathWithinBase(file, dir)) return;
|
||||
try { await fs.unlink(file); } catch { /* ignore not-exist */ }
|
||||
}
|
||||
|
||||
private async removeStackOverride(nodeId: number, stackName: string): Promise<void> {
|
||||
if (!isValidStackName(stackName)) return;
|
||||
const dir = this.overrideDirFor(nodeId);
|
||||
@@ -388,16 +626,24 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
return path.join(dataDir, 'mesh', 'overrides', String(nodeId));
|
||||
}
|
||||
|
||||
private async regenerateOverridesForNode(nodeId: number): Promise<void> {
|
||||
private async regenerateOverridesForNode(nodeId: number, skipStack?: string): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const stacks = db.listMeshStacks(nodeId);
|
||||
for (const s of stacks) {
|
||||
try {
|
||||
await this.ensureStackOverride(nodeId, s.stack_name);
|
||||
} catch (err) {
|
||||
console.warn('[MeshService] override regen failed:', sanitizeForLog((err as Error).message));
|
||||
}
|
||||
}
|
||||
// Push all overrides in parallel: each remote-node call is its own
|
||||
// HTTP round-trip, so awaiting sequentially turns N stacks into N
|
||||
// serialised PUTs. `allSettled` so a single failure does not abort
|
||||
// the others.
|
||||
await Promise.allSettled(
|
||||
stacks
|
||||
.filter((s) => s.stack_name !== skipStack)
|
||||
.map(async (s) => {
|
||||
try {
|
||||
await this.pushOverrideToNode(nodeId, s.stack_name);
|
||||
} catch (err) {
|
||||
console.warn('[MeshService] override push failed:', sanitizeForLog((err as Error).message));
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// --- Alias aggregation ---
|
||||
@@ -448,7 +694,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
|
||||
/**
|
||||
* Inspect a stack and return its running services with the ports they
|
||||
* listen on. For the LOCAL Docker daemon only — callers targeting a
|
||||
* listen on. For the LOCAL Docker daemon only; callers targeting a
|
||||
* remote node must use {@link inspectStackServices}, which dispatches
|
||||
* via the HTTP proxy to the remote's `/api/mesh/local-services/:stack`.
|
||||
*/
|
||||
@@ -514,6 +760,178 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a `fetch` against a remote Sencho's API with the bearer token
|
||||
* and the proxy tier/variant headers in place. Centralizes the header
|
||||
* shape so a future addition (license header, audit context) only
|
||||
* needs to land in one place.
|
||||
*
|
||||
* `x-node-id` is deliberately NOT set: callers target the remote
|
||||
* Sencho's own routes, which operate against the remote's local node
|
||||
* id. The bearer token alone authenticates.
|
||||
*/
|
||||
private async proxyFetch(
|
||||
nodeId: number,
|
||||
method: 'GET' | 'PUT' | 'POST' | 'DELETE',
|
||||
apiPath: string,
|
||||
body: unknown,
|
||||
timeoutMs: number,
|
||||
): Promise<Response> {
|
||||
const target = NodeRegistry.getInstance().getProxyTarget(nodeId);
|
||||
if (!target) throw new MeshError('push_failed', `no proxy target for node ${nodeId}`);
|
||||
const url = `${target.apiUrl.replace(/\/$/, '')}${apiPath}`;
|
||||
const headers: Record<string, string> = {};
|
||||
if (body !== undefined) headers['Content-Type'] = 'application/json';
|
||||
if (target.apiToken) headers['Authorization'] = `Bearer ${target.apiToken}`;
|
||||
const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
|
||||
headers[PROXY_TIER_HEADER] = proxyHeaders.tier;
|
||||
headers[PROXY_VARIANT_HEADER] = proxyHeaders.variant || '';
|
||||
return await fetch(url, {
|
||||
method,
|
||||
headers,
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Place a mesh override for a stack on whichever node owns it:
|
||||
* - local node: regenerates via `ensureStackOverride`, which uses
|
||||
* central's own senchoIp (correct because central is the node
|
||||
* deploying that stack).
|
||||
* - remote node: sends the fleet-wide alias list to the remote's
|
||||
* `PUT /api/mesh/local-override/:stackName`; the remote renders
|
||||
* the YAML using its OWN local senchoIp and writes it under its
|
||||
* own DATA_DIR. This is essential because each node has its own
|
||||
* `sencho_mesh` network and may be configured with a different
|
||||
* SENCHO_MESH_SUBNET, so alias hostnames must always resolve to
|
||||
* the local Sencho IP on the deploying node.
|
||||
*
|
||||
* Throws on remote push failure so callers (opt-in / opt-out) can abort
|
||||
* cleanly rather than silently leaving stale overrides.
|
||||
*/
|
||||
public async pushOverrideToNode(nodeId: number, stackName: string): Promise<void> {
|
||||
const node = DatabaseService.getInstance().getNode(nodeId);
|
||||
if (!node) throw new MeshError('denied', `unknown node ${nodeId}`);
|
||||
|
||||
if (node.type !== 'remote') {
|
||||
await this.ensureStackOverride(nodeId, stackName);
|
||||
return;
|
||||
}
|
||||
|
||||
const aliases: MeshAlias[] = Array.from(this.aliasCache.values()).map((a) => ({ host: a.host }));
|
||||
const res = await this.proxyFetch(
|
||||
nodeId,
|
||||
'PUT',
|
||||
`/api/mesh/local-override/${encodeURIComponent(stackName)}`,
|
||||
{ aliases },
|
||||
5_000,
|
||||
);
|
||||
if (res.status === 404) {
|
||||
throw new MeshError(
|
||||
'push_failed',
|
||||
`node ${node.name} does not support mesh override push (upgrade required)`,
|
||||
);
|
||||
}
|
||||
if (!res.ok) {
|
||||
throw new MeshError('push_failed', `HTTP ${res.status} from node ${node.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fire-and-forget redeploy of a stack on whichever node owns it. Used
|
||||
* by opt-in and opt-out so the new alias entries reach the user
|
||||
* containers' /etc/hosts without an operator manually clicking deploy.
|
||||
*
|
||||
* For local stacks: invokes `ComposeService.deployStack` directly.
|
||||
* For remote stacks: HTTP POSTs to `<apiUrl>/api/stacks/:name/deploy`
|
||||
* via the same bearer-token pattern the rest of the proxy chain uses.
|
||||
*
|
||||
* Errors are logged to the mesh activity buffer rather than thrown so
|
||||
* the opt-in or opt-out call site can return success quickly. The
|
||||
* operator sees the redeploy progress through the existing deploy
|
||||
* stream surfaces; if it fails, the activity log records why.
|
||||
*/
|
||||
public triggerRedeploy(nodeId: number, stackName: string, actor: string): void {
|
||||
void this.runRedeploy(nodeId, stackName, actor).catch((err) => {
|
||||
const reason = sanitizeForLog((err as Error).message);
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'error', type: 'forwarder.error',
|
||||
nodeId,
|
||||
message: `mesh redeploy failed for ${stackName}: ${reason}`,
|
||||
details: { actor, stackName },
|
||||
});
|
||||
// Also drop a durable audit row so an operator who walks away
|
||||
// from the toast still has a trail. The activity ring buffer
|
||||
// alone gets pruned at 1000 events.
|
||||
DatabaseService.getInstance().insertAuditLog({
|
||||
timestamp: Date.now(), username: actor, method: 'POST',
|
||||
path: `/api/mesh/nodes/${nodeId}/stacks/${stackName}/redeploy`,
|
||||
status_code: 500, node_id: nodeId, ip_address: '127.0.0.1',
|
||||
summary: `Sencho Mesh: redeploy failed for ${stackName}: ${reason}`,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private async runRedeploy(nodeId: number, stackName: string, actor: string): Promise<void> {
|
||||
const node = DatabaseService.getInstance().getNode(nodeId);
|
||||
if (!node) throw new Error(`unknown node ${nodeId}`);
|
||||
|
||||
if (node.type !== 'remote') {
|
||||
await ComposeService.getInstance(nodeId).deployStack(stackName);
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'mesh.enable',
|
||||
nodeId,
|
||||
message: `mesh redeploy ok for ${stackName}`,
|
||||
details: { actor, stackName },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Mesh redeploys are bounded by docker compose's own runtime; pick a
|
||||
// generous ceiling rather than the 5 s default used for control-plane
|
||||
// calls so a slow image pull does not abort the redeploy.
|
||||
const res = await this.proxyFetch(
|
||||
nodeId,
|
||||
'POST',
|
||||
`/api/stacks/${encodeURIComponent(stackName)}/deploy`,
|
||||
{},
|
||||
10 * 60 * 1000,
|
||||
);
|
||||
if (!res.ok) {
|
||||
const body = await res.text().catch(() => '');
|
||||
throw new Error(`HTTP ${res.status} from node ${node.name}: ${body.slice(0, 256)}`);
|
||||
}
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'mesh.enable',
|
||||
nodeId,
|
||||
message: `mesh redeploy ok for ${stackName}`,
|
||||
details: { actor, stackName },
|
||||
});
|
||||
}
|
||||
|
||||
public async removeOverrideFromNode(nodeId: number, stackName: string): Promise<void> {
|
||||
const node = DatabaseService.getInstance().getNode(nodeId);
|
||||
if (!node) return;
|
||||
|
||||
if (node.type !== 'remote') {
|
||||
await this.removeStackOverride(nodeId, stackName);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.proxyFetch(
|
||||
nodeId,
|
||||
'DELETE',
|
||||
`/api/mesh/local-override/${encodeURIComponent(stackName)}`,
|
||||
undefined,
|
||||
5_000,
|
||||
);
|
||||
} catch (err) {
|
||||
console.warn('[MeshService] removeOverrideFromNode failed:', sanitizeForLog((err as Error).message));
|
||||
}
|
||||
}
|
||||
|
||||
// --- Resolution + forwarding ---
|
||||
|
||||
public resolveByLocalPort(port: number): MeshTarget | null {
|
||||
@@ -943,9 +1361,16 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
// WebSocket. See `docs/internal/architecture/mesh.md` for the new flow.
|
||||
}
|
||||
|
||||
export type MeshErrorCode =
|
||||
| 'no_target'
|
||||
| 'port_collision'
|
||||
| 'denied'
|
||||
| 'agent_error'
|
||||
| 'push_failed';
|
||||
|
||||
export class MeshError extends Error {
|
||||
public readonly code: 'no_target' | 'port_collision' | 'denied' | 'agent_error';
|
||||
constructor(code: 'no_target' | 'port_collision' | 'denied' | 'agent_error', message: string) {
|
||||
public readonly code: MeshErrorCode;
|
||||
constructor(code: MeshErrorCode, message: string) {
|
||||
super(message);
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user