mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-07-26 11:49:16 +00:00
fix(mesh): recompose previously-meshed containers when alias set changes (#1090)
When a stack is opted into the mesh, every previously-meshed stack's override.yml on disk gains the new alias, but the running containers keep their stale /etc/hosts because extra_hosts is read at container creation time. Cross-stack DNS for the new alias silently fails until an operator manually redeploys each prior stack. Complete the cascade: after regenerateOverridesAcrossFleet finishes, fan out triggerRedeploy across every (node_id, stack_name) tuple in mesh_stacks, skipping the just-opted-in pair (the explicit triggerRedeploy at the end of optInStack covers it). Mirror the same cascade in optOutStack so the dropped alias exits every container's /etc/hosts. triggerRedeploy already dispatches local vs remote, already wraps runRedeploy in fire-and-forget error logging to the mesh activity ring and audit log, and already enforces compose policy gates. The cascade just reuses it. regenerateAllOverrides (boot and manual /regen-overrides) is intentionally NOT covered: a Sencho restart must not force a fleet-wide recompose; override files alone are sufficient there. Adds four unit tests in mesh-service.test.ts: cascade fires for every prior meshed stack on opt-in, no double-redeploy of the just-opted-in tuple, cascade is a no-op when no peers exist, cascade fires for every survivor on opt-out.
This commit is contained in:
Anso
@@ -222,6 +222,121 @@ describe('MeshService.optInStack', () => {
|
||||
// was unlinked separately, so it must not appear in the cascade.
|
||||
expect(pushed.find((p) => p.stackName === 'to-remove')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('optInStack cascades recompose to every previously meshed stack across the fleet', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
const remoteNodeId = db.addNode({
|
||||
name: 'remote-pilot', type: 'remote', mode: 'pilot_agent',
|
||||
compose_dir: '/tmp', is_default: false, api_url: '', api_token: '',
|
||||
});
|
||||
db.insertMeshStack(localNodeId, 'local-existing', 'setup');
|
||||
db.insertMeshStack(remoteNodeId, 'remote-existing', 'setup');
|
||||
|
||||
vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> }, 'inspectStackServices')
|
||||
.mockResolvedValue([{ service: 'web', ports: [8080] }]);
|
||||
vi.spyOn(svc, 'pushOverrideToNode').mockResolvedValue(undefined);
|
||||
const redeployed: Array<{ nodeId: number; stackName: string }> = [];
|
||||
vi.spyOn(svc, 'triggerRedeploy').mockImplementation((nodeId, stackName) => {
|
||||
redeployed.push({ nodeId, stackName });
|
||||
});
|
||||
|
||||
await svc.optInStack(localNodeId, 'new-stack', 'tester');
|
||||
|
||||
// The cascade must redeploy every previously meshed stack so its
|
||||
// container's /etc/hosts picks up the newly-added alias.
|
||||
expect(redeployed).toContainEqual({ nodeId: localNodeId, stackName: 'local-existing' });
|
||||
expect(redeployed).toContainEqual({ nodeId: remoteNodeId, stackName: 'remote-existing' });
|
||||
// The just-opted-in stack is still redeployed separately by the
|
||||
// explicit triggerRedeploy at the end of optInStack.
|
||||
expect(redeployed).toContainEqual({ nodeId: localNodeId, stackName: 'new-stack' });
|
||||
});
|
||||
|
||||
it('optInStack does not double-redeploy the just-opted-in tuple via the cascade path', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
const remoteNodeId = db.addNode({
|
||||
name: 'remote-pilot', type: 'remote', mode: 'pilot_agent',
|
||||
compose_dir: '/tmp', is_default: false, api_url: '', api_token: '',
|
||||
});
|
||||
db.insertMeshStack(remoteNodeId, 'remote-existing', 'setup');
|
||||
|
||||
vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> }, 'inspectStackServices')
|
||||
.mockResolvedValue([{ service: 'web', ports: [8080] }]);
|
||||
vi.spyOn(svc, 'pushOverrideToNode').mockResolvedValue(undefined);
|
||||
const redeployed: Array<{ nodeId: number; stackName: string }> = [];
|
||||
vi.spyOn(svc, 'triggerRedeploy').mockImplementation((nodeId, stackName) => {
|
||||
redeployed.push({ nodeId, stackName });
|
||||
});
|
||||
|
||||
await svc.optInStack(localNodeId, 'new-stack', 'tester');
|
||||
|
||||
// new-stack appears exactly once: the explicit redeploy at the end
|
||||
// of optInStack. The cascade walks db.listMeshStacks() (which now
|
||||
// includes new-stack) but its skip tuple drops it.
|
||||
const newStackHits = redeployed.filter(
|
||||
(r) => r.nodeId === localNodeId && r.stackName === 'new-stack',
|
||||
);
|
||||
expect(newStackHits).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('optInStack cascade is a no-op when no other meshed stacks exist', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
|
||||
vi.spyOn(svc as unknown as { inspectStackServices: (n: number, s: string) => Promise<unknown> }, 'inspectStackServices')
|
||||
.mockResolvedValue([{ service: 'web', ports: [8080] }]);
|
||||
vi.spyOn(svc, 'pushOverrideToNode').mockResolvedValue(undefined);
|
||||
const redeployed: Array<{ nodeId: number; stackName: string }> = [];
|
||||
vi.spyOn(svc, 'triggerRedeploy').mockImplementation((nodeId, stackName) => {
|
||||
redeployed.push({ nodeId, stackName });
|
||||
});
|
||||
|
||||
await svc.optInStack(localNodeId, 'first-stack', 'tester');
|
||||
|
||||
// Only the just-opted-in stack is redeployed; the cascade has no
|
||||
// peers to recompose so the early return short-circuits before
|
||||
// the summary activity entry fires.
|
||||
expect(redeployed).toEqual([{ nodeId: localNodeId, stackName: 'first-stack' }]);
|
||||
const cascadeEntries = svc.getActivity({ limit: 1000 })
|
||||
.filter((e) => e.message.startsWith('mesh cascade recompose'));
|
||||
expect(cascadeEntries).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('optOutStack cascades recompose to every other meshed stack', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
const remoteNodeId = db.addNode({
|
||||
name: 'remote-pilot', type: 'remote', mode: 'pilot_agent',
|
||||
compose_dir: '/tmp', is_default: false, api_url: '', api_token: '',
|
||||
});
|
||||
db.insertMeshStack(localNodeId, 'to-remove', 'setup');
|
||||
db.insertMeshStack(localNodeId, 'local-survivor', 'setup');
|
||||
db.insertMeshStack(remoteNodeId, 'remote-survivor', 'setup');
|
||||
|
||||
vi.spyOn(svc, 'pushOverrideToNode').mockResolvedValue(undefined);
|
||||
vi.spyOn(svc as unknown as { removeOverrideFromNode: (n: number, s: string) => Promise<void> }, 'removeOverrideFromNode')
|
||||
.mockResolvedValue(undefined);
|
||||
const redeployed: Array<{ nodeId: number; stackName: string }> = [];
|
||||
vi.spyOn(svc, 'triggerRedeploy').mockImplementation((nodeId, stackName) => {
|
||||
redeployed.push({ nodeId, stackName });
|
||||
});
|
||||
|
||||
await svc.optOutStack(localNodeId, 'to-remove', 'tester');
|
||||
|
||||
// Surviving meshed stacks must recompose so their /etc/hosts drops
|
||||
// the now-removed alias.
|
||||
expect(redeployed).toContainEqual({ nodeId: localNodeId, stackName: 'local-survivor' });
|
||||
expect(redeployed).toContainEqual({ nodeId: remoteNodeId, stackName: 'remote-survivor' });
|
||||
// The opted-out stack is still redeployed separately by the
|
||||
// explicit triggerRedeploy at the end of optOutStack so its own
|
||||
// container's /etc/hosts is cleared.
|
||||
expect(redeployed).toContainEqual({ nodeId: localNodeId, stackName: 'to-remove' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('MeshService activity log', () => {
|
||||
|
||||
@@ -795,6 +795,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
// avoid a duplicate round-trip. Best-effort; per-stack failures
|
||||
// surface as forwarder.error activity events.
|
||||
await this.regenerateOverridesAcrossFleet(nodeId, stackName);
|
||||
// Recompose every previously-meshed container so the new alias
|
||||
// actually lands in /etc/hosts. The override file alone is not
|
||||
// enough: extra_hosts is read at container creation, so prior
|
||||
// containers need to be recreated. Skip the just-opted-in tuple
|
||||
// because the explicit triggerRedeploy below already covers it.
|
||||
this.cascadeRecomposeAcrossFleet(nodeId, stackName, actor);
|
||||
this.triggerRedeploy(nodeId, stackName, actor);
|
||||
|
||||
this.logActivity({
|
||||
@@ -823,6 +829,13 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
// include it. Walk the remaining fleet-wide rows so every other
|
||||
// meshed stack regenerates its override without the dropped alias.
|
||||
await this.regenerateOverridesAcrossFleet();
|
||||
// Recompose every still-meshed container so the dropped alias
|
||||
// exits /etc/hosts. Skip args are absent because the opted-out
|
||||
// row is already gone from listMeshStacks; the explicit
|
||||
// triggerRedeploy below recomposes the opted-out stack itself
|
||||
// (with the override file removed) so its container drops the
|
||||
// entries it owned.
|
||||
this.cascadeRecomposeAcrossFleet(undefined, undefined, actor);
|
||||
this.triggerRedeploy(nodeId, stackName, actor);
|
||||
|
||||
this.logActivity({
|
||||
@@ -1080,6 +1093,64 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk every `mesh_stacks` row across the fleet and fire a redeploy for
|
||||
* each, skipping the (skipNodeId, skipStack) tuple. Called from
|
||||
* optInStack / optOutStack after `regenerateOverridesAcrossFleet` so
|
||||
* previously-meshed containers actually pick up the new or removed
|
||||
* alias entries in `/etc/hosts`. Without this, override `.yml` files on
|
||||
* disk reflect the new alias set but the running containers still hold
|
||||
* the alias set they had at last compose, so cross-stack DNS silently
|
||||
* fails until an operator redeploys every prior stack by hand.
|
||||
*
|
||||
* Each `triggerRedeploy` call is fire-and-forget. Local stacks route
|
||||
* through `ComposeService.deployStack`, remote stacks through
|
||||
* `POST /api/stacks/:name/deploy` via the proxy chain. Failures land in
|
||||
* the mesh activity ring buffer and the audit log, so one slow or
|
||||
* offline peer cannot block other targets.
|
||||
*
|
||||
* This is intentionally not invoked from `regenerateAllOverrides`
|
||||
* (boot and manual `/regen-overrides`). A Sencho restart must not
|
||||
* force a fleet-wide recompose of every meshed stack; the override
|
||||
* files alone are sufficient there.
|
||||
*
|
||||
* Pacing: the cascade fans out in parallel. For the v1 mesh-stack
|
||||
* counts (single-digit to low teens per host) this is fine; Docker's
|
||||
* daemon serializes the contention that matters. If real-world fleets
|
||||
* routinely exceed ~20 meshed stacks on one host, swap the loop for a
|
||||
* `p-limit(4)` semaphore keyed on `node_id` (no test rewiring needed).
|
||||
*/
|
||||
private cascadeRecomposeAcrossFleet(
|
||||
skipNodeId: number | undefined,
|
||||
skipStack: string | undefined,
|
||||
actor: string,
|
||||
): void {
|
||||
const db = DatabaseService.getInstance();
|
||||
const targets = db.listMeshStacks().filter(
|
||||
(s) => !(s.node_id === skipNodeId && s.stack_name === skipStack),
|
||||
);
|
||||
if (targets.length === 0) return;
|
||||
|
||||
for (const t of targets) {
|
||||
this.triggerRedeploy(t.node_id, t.stack_name, actor);
|
||||
}
|
||||
|
||||
const nodeIds = new Set(targets.map((t) => t.node_id));
|
||||
const skippedNote = skipNodeId !== undefined && skipStack !== undefined
|
||||
? ` (skipped ${skipStack} on node ${skipNodeId})`
|
||||
: '';
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'mesh.enable',
|
||||
message: `mesh cascade recompose: ${targets.length} stack${targets.length === 1 ? '' : 's'} across ${nodeIds.size} node${nodeIds.size === 1 ? '' : 's'}${skippedNote}`,
|
||||
details: {
|
||||
cascadeRecomposes: targets.length,
|
||||
nodeCount: nodeIds.size,
|
||||
skipNodeId: skipNodeId ?? null,
|
||||
skipStack: skipStack ?? null,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk every `mesh_stacks` row across the fleet and re-push each override
|
||||
* to its owning node. Called once at boot so on-disk override files
|
||||
|
||||
Reference in New Issue
Block a user