From a32183d198e11ffa23bf26d3803129e2ee843962 Mon Sep 17 00:00:00 2001 From: Anso Date: Sun, 17 May 2026 03:48:58 -0400 Subject: [PATCH] fix(mesh): cascade override regen across all meshed nodes on opt-in/opt-out (#1079) Opting a stack into mesh on node A now propagates the new alias to every meshed node's override file, not just stacks on node A. Same on opt-out. Pre-fix, optInStack and optOutStack called regenerateOverridesForNode which only walks db.listMeshStacks(nodeId). Other meshed stacks on other nodes did not learn about the new alias until something else re-pushed their overrides; operators worked around it with a manual POST /api/mesh/regen-overrides plus a redeploy of each affected stack. New regenerateOverridesAcrossFleet helper iterates db.listMeshStacks() (no arg, fleet-wide) under Promise.allSettled, skipping the (nodeId, stackName) tuple opt-in just pushed loudly. Per-stack failures surface as forwarder.error activity events, matching the pattern used by regenerateAllOverrides. Offline remote nodes leave stale overrides until the next opt-in/opt-out, the next tunnel reconnect, or a manual rerun of the regen-overrides endpoint. regenerateOverridesForNode stays for the tunnel-up retry listener: a reconnecting pilot only needs its own stacks re-pushed, not the fleet. Tests: redirect five existing spies in mesh-service.test.ts to the new method; add two new cases asserting cross-node cascade for opt-in and opt-out. 52 tests in mesh-service.test.ts pass. --- backend/src/__tests__/mesh-service.test.ts | 67 ++++++++++++++++++++-- backend/src/services/MeshService.ts | 51 ++++++++++++++-- 2 files changed, 107 insertions(+), 11 deletions(-) diff --git a/backend/src/__tests__/mesh-service.test.ts b/backend/src/__tests__/mesh-service.test.ts index fb166ec5..d97ab181 100644 --- a/backend/src/__tests__/mesh-service.test.ts +++ b/backend/src/__tests__/mesh-service.test.ts @@ -57,7 +57,7 @@ describe('MeshService.optInStack', () => { vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise }, 'inspectStackServices') .mockResolvedValue([{ service: 'db', ports: [5432] }]); - vi.spyOn(svc as unknown as { regenerateOverridesForNode: (n: number) => Promise }, 'regenerateOverridesForNode') + vi.spyOn(svc as unknown as { regenerateOverridesAcrossFleet: (n?: number, s?: string) => Promise }, 'regenerateOverridesAcrossFleet') .mockResolvedValue(undefined); const db = DatabaseService.getInstance(); @@ -80,7 +80,7 @@ describe('MeshService.optInStack', () => { { service: 'web', ports: [] }, { service: 'db', ports: [] }, ]); - vi.spyOn(svc as unknown as { regenerateOverridesForNode: (n: number) => Promise }, 'regenerateOverridesForNode') + vi.spyOn(svc as unknown as { regenerateOverridesAcrossFleet: (n?: number, s?: string) => Promise }, 'regenerateOverridesAcrossFleet') .mockResolvedValue(undefined); await expect(svc.optInStack(localNodeId, 'silent', 'tester')) @@ -94,7 +94,7 @@ describe('MeshService.optInStack', () => { const svc = MeshService.getInstance(); vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise }, 'inspectStackServices') .mockResolvedValue([{ service: 'db', ports: [5432] }]); - vi.spyOn(svc as unknown as { regenerateOverridesForNode: (n: number) => Promise }, 'regenerateOverridesForNode') + vi.spyOn(svc as unknown as { regenerateOverridesAcrossFleet: (n?: number, s?: string) => Promise }, 'regenerateOverridesAcrossFleet') .mockResolvedValue(undefined); vi.spyOn(svc as unknown as { removeStackOverride: (n: number, s: string) => Promise }, 'removeStackOverride') .mockResolvedValue(undefined); @@ -165,6 +165,63 @@ describe('MeshService.optInStack', () => { db.deleteNode(remoteNodeId); } }); + + it('optInStack cascades override regen to every meshed node, not just the source node', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + const remoteNodeId = db.addNode({ + name: 'remote-pilot', type: 'remote', mode: 'pilot_agent', + compose_dir: '/tmp', is_default: false, api_url: '', api_token: '', + }); + db.insertMeshStack(remoteNodeId, 'remote-existing', 'setup'); + + vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise }, 'inspectStackServices') + .mockResolvedValue([{ service: 'web', ports: [8080] }]); + const pushed: Array<{ nodeId: number; stackName: string }> = []; + vi.spyOn(svc, 'pushOverrideToNode').mockImplementation(async (nodeId, stackName) => { + pushed.push({ nodeId, stackName }); + }); + vi.spyOn(svc as unknown as { triggerRedeploy: (n: number, s: string, a: string) => void }, 'triggerRedeploy') + .mockImplementation(() => { /* noop */ }); + + await svc.optInStack(localNodeId, 'new-stack', 'tester'); + + expect(pushed).toContainEqual({ nodeId: localNodeId, stackName: 'new-stack' }); + // Other meshed nodes' existing stacks must regenerate so they pick + // up the new alias entry. + expect(pushed).toContainEqual({ nodeId: remoteNodeId, stackName: 'remote-existing' }); + }); + + it('optOutStack cascades override regen to every other meshed node', async () => { + const svc = MeshService.getInstance(); + const db = DatabaseService.getInstance(); + const localNodeId = db.getNodes()[0].id; + const remoteNodeId = db.addNode({ + name: 'remote-pilot', type: 'remote', mode: 'pilot_agent', + compose_dir: '/tmp', is_default: false, api_url: '', api_token: '', + }); + db.insertMeshStack(localNodeId, 'to-remove', 'setup'); + db.insertMeshStack(remoteNodeId, 'remote-existing', 'setup'); + + const pushed: Array<{ nodeId: number; stackName: string }> = []; + vi.spyOn(svc, 'pushOverrideToNode').mockImplementation(async (nodeId, stackName) => { + pushed.push({ nodeId, stackName }); + }); + vi.spyOn(svc as unknown as { removeOverrideFromNode: (n: number, s: string) => Promise }, 'removeOverrideFromNode') + .mockResolvedValue(undefined); + vi.spyOn(svc as unknown as { triggerRedeploy: (n: number, s: string, a: string) => void }, 'triggerRedeploy') + .mockImplementation(() => { /* noop */ }); + + await svc.optOutStack(localNodeId, 'to-remove', 'tester'); + + // Other meshed nodes' existing stacks must regenerate so they drop + // the removed alias entry. + expect(pushed).toContainEqual({ nodeId: remoteNodeId, stackName: 'remote-existing' }); + // The removed row was deleted before the cascade ran, and its file + // was unlinked separately, so it must not appear in the cascade. + expect(pushed.find((p) => p.stackName === 'to-remove')).toBeUndefined(); + }); }); describe('MeshService activity log', () => { @@ -439,7 +496,7 @@ describe('MeshService.optInStack guard rails (network setup)', () => { const svc = MeshService.getInstance(); vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise }, 'inspectStackServices') .mockResolvedValue([{ service: 'web', ports: [1852] }]); - vi.spyOn(svc as unknown as { regenerateOverridesForNode: (n: number) => Promise }, 'regenerateOverridesForNode') + vi.spyOn(svc as unknown as { regenerateOverridesAcrossFleet: (n?: number, s?: string) => Promise }, 'regenerateOverridesAcrossFleet') .mockResolvedValue(undefined); const db = DatabaseService.getInstance(); @@ -589,7 +646,7 @@ describe('MeshService.regenerateAllOverrides (F6: boot-time regen)', () => { const pushSpy = vi.spyOn(svc, 'pushOverrideToNode').mockResolvedValue(undefined); vi.spyOn(svc as unknown as { triggerRedeploy: (n: number, s: string, a: string) => void }, 'triggerRedeploy') .mockImplementation(() => { /* noop */ }); - vi.spyOn(svc as unknown as { regenerateOverridesForNode: (n: number, skip?: string) => Promise }, 'regenerateOverridesForNode') + vi.spyOn(svc as unknown as { regenerateOverridesAcrossFleet: (n?: number, skip?: string) => Promise }, 'regenerateOverridesAcrossFleet') .mockResolvedValue(undefined); const [summary] = await Promise.all([ diff --git a/backend/src/services/MeshService.ts b/backend/src/services/MeshService.ts index 06c73a88..5ca6aab0 100644 --- a/backend/src/services/MeshService.ts +++ b/backend/src/services/MeshService.ts @@ -675,11 +675,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { await this.syncForwarderListeners(); throw err; } - // Regenerate OTHER meshed stacks' overrides on the same node so - // they pick up the new alias entry. The just-opted-in stack was - // already pushed above; skip it to avoid a duplicate round-trip. - // Best-effort; per-stack failures are logged inside the helper. - await this.regenerateOverridesForNode(nodeId, stackName); + // Regenerate every other meshed stack's override across the fleet + // so they pick up the new alias entry. The just-opted-in stack was + // already pushed above; skip the (nodeId, stackName) tuple to + // avoid a duplicate round-trip. Best-effort; per-stack failures + // surface as forwarder.error activity events. + await this.regenerateOverridesAcrossFleet(nodeId, stackName); this.triggerRedeploy(nodeId, stackName, actor); this.logActivity({ @@ -704,7 +705,10 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { await this.removeOverrideFromNode(nodeId, stackName); await this.refreshAliasCache(); await this.syncForwarderListeners(); - await this.regenerateOverridesForNode(nodeId); + // The opted-out row is already deleted, so listMeshStacks() will not + // include it. Walk the remaining fleet-wide rows so every other + // meshed stack regenerates its override without the dropped alias. + await this.regenerateOverridesAcrossFleet(); this.triggerRedeploy(nodeId, stackName, actor); this.logActivity({ @@ -927,6 +931,41 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { ); } + /** + * Walk every `mesh_stacks` row across the fleet and re-push each override. + * Called from optInStack / optOutStack so a new or removed alias + * propagates to every meshed node's override file in one pass, not just + * the node whose row changed. Best-effort: per-stack failures emit a + * forwarder.error activity event and the other nodes still get + * regenerated. An offline remote node leaves stale overrides until the + * next opt-in / opt-out, the next tunnel reconnect, or a manual + * `POST /api/mesh/regen-overrides`. + */ + private async regenerateOverridesAcrossFleet( + skipNodeId?: number, + skipStack?: string, + ): Promise { + const db = DatabaseService.getInstance(); + const stacks = db.listMeshStacks(); + await Promise.allSettled( + stacks + .filter((s) => !(s.node_id === skipNodeId && s.stack_name === skipStack)) + .map(async (s) => { + try { + await this.pushOverrideToNode(s.node_id, s.stack_name); + } catch (err) { + const message = sanitizeForLog((err as Error).message); + this.logActivity({ + source: 'mesh', level: 'warn', type: 'forwarder.error', + nodeId: s.node_id, + message: `cascade override push failed for ${s.stack_name}: ${message}`, + details: { stackName: s.stack_name }, + }); + } + }), + ); + } + /** * Walk every `mesh_stacks` row across the fleet and re-push each override * to its owning node. Called once at boot so on-disk override files