From 1d7418a233796f61efc4739cd358298ab71f6438 Mon Sep 17 00:00:00 2001 From: Anso Date: Sun, 17 May 2026 19:02:02 -0400 Subject: [PATCH] fix(mesh): recompose previously-meshed containers when alias set changes (#1090) When a stack is opted into the mesh, every previously-meshed stack's override.yml on disk gains the new alias, but the running containers keep their stale /etc/hosts because extra_hosts is read at container creation time. Cross-stack DNS for the new alias silently fails until an operator manually redeploys each prior stack. Complete the cascade: after regenerateOverridesAcrossFleet finishes, fan out triggerRedeploy across every (node_id, stack_name) tuple in mesh_stacks, skipping the just-opted-in pair (the explicit triggerRedeploy at the end of optInStack covers it). Mirror the same cascade in optOutStack so the dropped alias exits every container's /etc/hosts. triggerRedeploy already dispatches local vs remote, already wraps runRedeploy in fire-and-forget error logging to the mesh activity ring and audit log, and already enforces compose policy gates. The cascade just reuses it. regenerateAllOverrides (boot and manual /regen-overrides) is intentionally NOT covered: a Sencho restart must not force a fleet-wide recompose; override files alone are sufficient there. Adds four unit tests in mesh-service.test.ts: cascade fires for every prior meshed stack on opt-in, no double-redeploy of the just-opted-in tuple, cascade is a no-op when no peers exist, cascade fires for every survivor on opt-out. --- backend/src/__tests__/mesh-service.test.ts | 115 +++++++++++++++++++++ backend/src/services/MeshService.ts | 71 +++++++++++++ 2 files changed, 186 insertions(+) diff --git a/backend/src/__tests__/mesh-service.test.ts b/backend/src/__tests__/mesh-service.test.ts index d97ab181..18fabc71 100644 --- a/backend/src/__tests__/mesh-service.test.ts +++ b/backend/src/__tests__/mesh-service.test.ts @@ -222,6 +222,121 @@ describe('MeshService.optInStack', () => { // was unlinked separately, so it must not appear in the cascade. expect(pushed.find((p) => p.stackName === 'to-remove')).toBeUndefined(); }); + + it('optInStack cascades recompose to every previously meshed stack across the fleet', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + const remoteNodeId = db.addNode({ + name: 'remote-pilot', type: 'remote', mode: 'pilot_agent', + compose_dir: '/tmp', is_default: false, api_url: '', api_token: '', + }); + db.insertMeshStack(localNodeId, 'local-existing', 'setup'); + db.insertMeshStack(remoteNodeId, 'remote-existing', 'setup'); + + vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise }, 'inspectStackServices') + .mockResolvedValue([{ service: 'web', ports: [8080] }]); + vi.spyOn(svc, 'pushOverrideToNode').mockResolvedValue(undefined); + const redeployed: Array<{ nodeId: number; stackName: string }> = []; + vi.spyOn(svc, 'triggerRedeploy').mockImplementation((nodeId, stackName) => { + redeployed.push({ nodeId, stackName }); + }); + + await svc.optInStack(localNodeId, 'new-stack', 'tester'); + + // The cascade must redeploy every previously meshed stack so its + // container's /etc/hosts picks up the newly-added alias. + expect(redeployed).toContainEqual({ nodeId: localNodeId, stackName: 'local-existing' }); + expect(redeployed).toContainEqual({ nodeId: remoteNodeId, stackName: 'remote-existing' }); + // The just-opted-in stack is still redeployed separately by the + // explicit triggerRedeploy at the end of optInStack. + expect(redeployed).toContainEqual({ nodeId: localNodeId, stackName: 'new-stack' }); + }); + + it('optInStack does not double-redeploy the just-opted-in tuple via the cascade path', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + const remoteNodeId = db.addNode({ + name: 'remote-pilot', type: 'remote', mode: 'pilot_agent', + compose_dir: '/tmp', is_default: false, api_url: '', api_token: '', + }); + db.insertMeshStack(remoteNodeId, 'remote-existing', 'setup'); + + vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise }, 'inspectStackServices') + .mockResolvedValue([{ service: 'web', ports: [8080] }]); + vi.spyOn(svc, 'pushOverrideToNode').mockResolvedValue(undefined); + const redeployed: Array<{ nodeId: number; stackName: string }> = []; + vi.spyOn(svc, 'triggerRedeploy').mockImplementation((nodeId, stackName) => { + redeployed.push({ nodeId, stackName }); + }); + + await svc.optInStack(localNodeId, 'new-stack', 'tester'); + + // new-stack appears exactly once: the explicit redeploy at the end + // of optInStack. The cascade walks db.listMeshStacks() (which now + // includes new-stack) but its skip tuple drops it. + const newStackHits = redeployed.filter( + (r) => r.nodeId === localNodeId && r.stackName === 'new-stack', + ); + expect(newStackHits).toHaveLength(1); + }); + + it('optInStack cascade is a no-op when no other meshed stacks exist', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + + vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise }, 'inspectStackServices') + .mockResolvedValue([{ service: 'web', ports: [8080] }]); + vi.spyOn(svc, 'pushOverrideToNode').mockResolvedValue(undefined); + const redeployed: Array<{ nodeId: number; stackName: string }> = []; + vi.spyOn(svc, 'triggerRedeploy').mockImplementation((nodeId, stackName) => { + redeployed.push({ nodeId, stackName }); + }); + + await svc.optInStack(localNodeId, 'first-stack', 'tester'); + + // Only the just-opted-in stack is redeployed; the cascade has no + // peers to recompose so the early return short-circuits before + // the summary activity entry fires. + expect(redeployed).toEqual([{ nodeId: localNodeId, stackName: 'first-stack' }]); + const cascadeEntries = svc.getActivity({ limit: 1000 }) + .filter((e) => e.message.startsWith('mesh cascade recompose')); + expect(cascadeEntries).toHaveLength(0); + }); + + it('optOutStack cascades recompose to every other meshed stack', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + const remoteNodeId = db.addNode({ + name: 'remote-pilot', type: 'remote', mode: 'pilot_agent', + compose_dir: '/tmp', is_default: false, api_url: '', api_token: '', + }); + db.insertMeshStack(localNodeId, 'to-remove', 'setup'); + db.insertMeshStack(localNodeId, 'local-survivor', 'setup'); + db.insertMeshStack(remoteNodeId, 'remote-survivor', 'setup'); + + vi.spyOn(svc, 'pushOverrideToNode').mockResolvedValue(undefined); + vi.spyOn(svc as unknown as { removeOverrideFromNode: (n: number, s: string) => Promise }, 'removeOverrideFromNode') + .mockResolvedValue(undefined); + const redeployed: Array<{ nodeId: number; stackName: string }> = []; + vi.spyOn(svc, 'triggerRedeploy').mockImplementation((nodeId, stackName) => { + redeployed.push({ nodeId, stackName }); + }); + + await svc.optOutStack(localNodeId, 'to-remove', 'tester'); + + // Surviving meshed stacks must recompose so their /etc/hosts drops + // the now-removed alias. + expect(redeployed).toContainEqual({ nodeId: localNodeId, stackName: 'local-survivor' }); + expect(redeployed).toContainEqual({ nodeId: remoteNodeId, stackName: 'remote-survivor' }); + // The opted-out stack is still redeployed separately by the + // explicit triggerRedeploy at the end of optOutStack so its own + // container's /etc/hosts is cleared. + expect(redeployed).toContainEqual({ nodeId: localNodeId, stackName: 'to-remove' }); + }); }); describe('MeshService activity log', () => { diff --git a/backend/src/services/MeshService.ts b/backend/src/services/MeshService.ts index c0a39780..63dfb2ec 100644 --- a/backend/src/services/MeshService.ts +++ b/backend/src/services/MeshService.ts @@ -795,6 +795,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { // avoid a duplicate round-trip. Best-effort; per-stack failures // surface as forwarder.error activity events. await this.regenerateOverridesAcrossFleet(nodeId, stackName); + // Recompose every previously-meshed container so the new alias + // actually lands in /etc/hosts. The override file alone is not + // enough: extra_hosts is read at container creation, so prior + // containers need to be recreated. Skip the just-opted-in tuple + // because the explicit triggerRedeploy below already covers it. + this.cascadeRecomposeAcrossFleet(nodeId, stackName, actor); this.triggerRedeploy(nodeId, stackName, actor); this.logActivity({ @@ -823,6 +829,13 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { // include it. Walk the remaining fleet-wide rows so every other // meshed stack regenerates its override without the dropped alias. await this.regenerateOverridesAcrossFleet(); + // Recompose every still-meshed container so the dropped alias + // exits /etc/hosts. Skip args are absent because the opted-out + // row is already gone from listMeshStacks; the explicit + // triggerRedeploy below recomposes the opted-out stack itself + // (with the override file removed) so its container drops the + // entries it owned. + this.cascadeRecomposeAcrossFleet(undefined, undefined, actor); this.triggerRedeploy(nodeId, stackName, actor); this.logActivity({ @@ -1080,6 +1093,64 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { ); } + /** + * Walk every `mesh_stacks` row across the fleet and fire a redeploy for + * each, skipping the (skipNodeId, skipStack) tuple. Called from + * optInStack / optOutStack after `regenerateOverridesAcrossFleet` so + * previously-meshed containers actually pick up the new or removed + * alias entries in `/etc/hosts`. Without this, override `.yml` files on + * disk reflect the new alias set but the running containers still hold + * the alias set they had at last compose, so cross-stack DNS silently + * fails until an operator redeploys every prior stack by hand. + * + * Each `triggerRedeploy` call is fire-and-forget. Local stacks route + * through `ComposeService.deployStack`, remote stacks through + * `POST /api/stacks/:name/deploy` via the proxy chain. Failures land in + * the mesh activity ring buffer and the audit log, so one slow or + * offline peer cannot block other targets. + * + * This is intentionally not invoked from `regenerateAllOverrides` + * (boot and manual `/regen-overrides`). A Sencho restart must not + * force a fleet-wide recompose of every meshed stack; the override + * files alone are sufficient there. + * + * Pacing: the cascade fans out in parallel. For the v1 mesh-stack + * counts (single-digit to low teens per host) this is fine; Docker's + * daemon serializes the contention that matters. If real-world fleets + * routinely exceed ~20 meshed stacks on one host, swap the loop for a + * `p-limit(4)` semaphore keyed on `node_id` (no test rewiring needed). + */ + private cascadeRecomposeAcrossFleet( + skipNodeId: number | undefined, + skipStack: string | undefined, + actor: string, + ): void { + const db = DatabaseService.getInstance(); + const targets = db.listMeshStacks().filter( + (s) => !(s.node_id === skipNodeId && s.stack_name === skipStack), + ); + if (targets.length === 0) return; + + for (const t of targets) { + this.triggerRedeploy(t.node_id, t.stack_name, actor); + } + + const nodeIds = new Set(targets.map((t) => t.node_id)); + const skippedNote = skipNodeId !== undefined && skipStack !== undefined + ? ` (skipped ${skipStack} on node ${skipNodeId})` + : ''; + this.logActivity({ + source: 'mesh', level: 'info', type: 'mesh.enable', + message: `mesh cascade recompose: ${targets.length} stack${targets.length === 1 ? '' : 's'} across ${nodeIds.size} node${nodeIds.size === 1 ? '' : 's'}${skippedNote}`, + details: { + cascadeRecomposes: targets.length, + nodeCount: nodeIds.size, + skipNodeId: skipNodeId ?? null, + skipStack: skipStack ?? null, + }, + }); + } + /** * Walk every `mesh_stacks` row across the fleet and re-push each override * to its owning node. Called once at boot so on-disk override files