mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-07 09:24:09 +00:00
fix(mesh): trust central for cross-node dial auth and regenerate overrides at boot (#1014)
Two bugs in the same Phase D follow-up surface, fixed together because they both block declaring B-verify complete on the production fleet. Cross-node mesh dials returned `denied` at the agent. The pilot's `tcp_open` handler in `agent.ts::resolveMeshTarget` consulted the local SQLite `mesh_stacks` table, which is no longer written to under the post-Phase D control plane (state lives only on central). Drop the check. The pilot tunnel JWT (scope `pilot_tunnel`, signed with central's `auth_jwt_secret`) authenticates the caller; the same trust model already applies to filesystem ops, exec, and container control over the same tunnel. Threat-model trade-off: a leaked `pilot_tunnel` JWT or compromised central can now dial any compose-managed service on the pilot. Containers without `com.docker.compose.project` + `com.docker.compose.service` labels remain unreachable via this path. `MeshService.start()` did not regenerate compose override files at boot. After a Sencho restart with missing overrides on disk, meshed user containers had no `extra_hosts` / `networks: [sencho_mesh]` injection until each stack was opted out and back in. Add `regenerateAllOverrides()` that walks every `mesh_stacks` row and re-pushes via `pushOverrideToNode`. Best-effort: per-stack failures log to the mesh activity buffer; `MeshService.start()` is fire-and-forget at startup so a slow remote node does not delay boot. Tests: - `pilot-agent-mesh-resolve.test.ts` (new): mocks dockerode and proves `resolveMeshTarget` no longer returns `denied` with an empty `mesh_stacks` table. - `mesh-service.test.ts`: three new cases for `regenerateAllOverrides` - fan-out across the fleet, skip when `senchoIp` is null, log per-stack warning on push failure without throwing.
This commit is contained in:
@@ -345,3 +345,67 @@ describe('MeshService.optInStack guard rails (network setup)', () => {
|
||||
expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MeshService.regenerateAllOverrides (F6: boot-time regen)', () => {
|
||||
it('pushes every mesh_stacks row across the fleet', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
const remoteNodeId = db.addNode({
|
||||
name: 'remote-pilot', type: 'remote', mode: 'pilot_agent',
|
||||
compose_dir: '/tmp', is_default: false, api_url: '', api_token: '',
|
||||
});
|
||||
|
||||
db.insertMeshStack(localNodeId, 'audit-mesh-prod', 'tester');
|
||||
db.insertMeshStack(remoteNodeId, 'audit-mesh-pilot', 'tester');
|
||||
|
||||
const pushSpy = vi.spyOn(svc, 'pushOverrideToNode').mockResolvedValue(undefined);
|
||||
|
||||
try {
|
||||
await (svc as unknown as { regenerateAllOverrides: () => Promise<void> }).regenerateAllOverrides();
|
||||
|
||||
expect(pushSpy).toHaveBeenCalledTimes(2);
|
||||
expect(pushSpy).toHaveBeenCalledWith(localNodeId, 'audit-mesh-prod');
|
||||
expect(pushSpy).toHaveBeenCalledWith(remoteNodeId, 'audit-mesh-pilot');
|
||||
|
||||
const activity = svc.getActivity({ limit: 100 });
|
||||
expect(activity.some((e) => e.message === 'boot regenerated 2 override(s)')).toBe(true);
|
||||
} finally {
|
||||
db.deleteNode(remoteNodeId);
|
||||
}
|
||||
});
|
||||
|
||||
it('skips entirely when senchoIp is null (network setup failed)', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
(svc as unknown as { senchoIp: string | null }).senchoIp = null;
|
||||
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
db.insertMeshStack(localNodeId, 'audit-mesh-prod', 'tester');
|
||||
|
||||
const pushSpy = vi.spyOn(svc, 'pushOverrideToNode').mockResolvedValue(undefined);
|
||||
|
||||
await (svc as unknown as { regenerateAllOverrides: () => Promise<void> }).regenerateAllOverrides();
|
||||
|
||||
expect(pushSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('logs a warning per stack when push fails but does not throw', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = db.getNodes()[0].id;
|
||||
|
||||
db.insertMeshStack(localNodeId, 'audit-mesh-prod', 'tester');
|
||||
|
||||
vi.spyOn(svc, 'pushOverrideToNode').mockRejectedValue(new Error('remote node offline'));
|
||||
|
||||
await expect(
|
||||
(svc as unknown as { regenerateAllOverrides: () => Promise<void> }).regenerateAllOverrides(),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
const activity = svc.getActivity({ limit: 100 });
|
||||
expect(activity.some((e) =>
|
||||
e.level === 'warn' && /boot override regen failed for audit-mesh-prod/.test(e.message),
|
||||
)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* F7 regression: pilot agent's mesh dial path no longer denies based on the
|
||||
* pilot's local `mesh_stacks` table. Central is the sole authority for
|
||||
* mesh opt-in (state lives in central's SQLite); the pilot resolves the
|
||||
* target container by Compose labels and dials it directly. The tunnel JWT
|
||||
* authenticates the caller, so per-stack gating on the pilot would only
|
||||
* deny legitimate central-issued dials whenever Phase D's central-only
|
||||
* state model is in effect.
|
||||
*/
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
|
||||
const listContainersMock = vi.fn();
|
||||
|
||||
vi.mock('dockerode', () => {
|
||||
function Docker(this: unknown) {
|
||||
(this as { listContainers: typeof listContainersMock }).listContainers = listContainersMock;
|
||||
}
|
||||
return { default: Docker };
|
||||
});
|
||||
|
||||
let tmpDir: string;
|
||||
let PilotAgent: typeof import('../pilot/agent').PilotAgent;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
|
||||
interface ResolveResult {
|
||||
ok: boolean;
|
||||
host?: string;
|
||||
port?: number;
|
||||
err?: string;
|
||||
}
|
||||
|
||||
function makeAgent(): import('../pilot/agent').PilotAgent {
|
||||
return new PilotAgent({
|
||||
primaryUrl: 'http://primary.invalid',
|
||||
loopbackPort: 1,
|
||||
initialToken: 'irrelevant',
|
||||
enrolling: false,
|
||||
});
|
||||
}
|
||||
|
||||
function callResolve(agent: import('../pilot/agent').PilotAgent, stack: string, service: string, port: number): Promise<ResolveResult> {
|
||||
const fn = (agent as unknown as { resolveMeshTarget: (s: string, sv: string, p: number) => Promise<ResolveResult> }).resolveMeshTarget.bind(agent);
|
||||
return fn(stack, service, port);
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ PilotAgent } = await import('../pilot/agent'));
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
listContainersMock.mockReset();
|
||||
DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_stacks').run();
|
||||
});
|
||||
|
||||
describe('PilotAgent.resolveMeshTarget (F7: trust central)', () => {
|
||||
it('returns the container IP when mesh_stacks is empty (central is the gate)', async () => {
|
||||
listContainersMock.mockResolvedValue([
|
||||
{ NetworkSettings: { Networks: { sencho_mesh: { IPAddress: '172.30.0.5' } } } },
|
||||
]);
|
||||
|
||||
const agent = makeAgent();
|
||||
const result = await callResolve(agent, 'audit-mesh-pilot', 'echo', 9001);
|
||||
|
||||
expect(result.err).not.toBe('denied');
|
||||
expect(result.ok).toBe(true);
|
||||
expect(result.host).toBe('172.30.0.5');
|
||||
expect(result.port).toBe(9001);
|
||||
});
|
||||
|
||||
it('queries dockerode with the Compose project + service label filter', async () => {
|
||||
listContainersMock.mockResolvedValue([
|
||||
{ NetworkSettings: { Networks: { bridge: { IPAddress: '10.0.0.7' } } } },
|
||||
]);
|
||||
|
||||
const agent = makeAgent();
|
||||
await callResolve(agent, 'api', 'db', 5432);
|
||||
|
||||
expect(listContainersMock).toHaveBeenCalledTimes(1);
|
||||
const args = listContainersMock.mock.calls[0][0] as { filters: { label: string[] } };
|
||||
expect(args.filters.label).toEqual([
|
||||
'com.docker.compose.project=api',
|
||||
'com.docker.compose.service=db',
|
||||
]);
|
||||
});
|
||||
|
||||
it('returns no_target (not denied) when dockerode finds no matching container', async () => {
|
||||
listContainersMock.mockResolvedValue([]);
|
||||
|
||||
const agent = makeAgent();
|
||||
const result = await callResolve(agent, 'missing', 'svc', 8080);
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.err).toBe('no_target');
|
||||
});
|
||||
|
||||
it('returns agent_error (not denied) when dockerode throws', async () => {
|
||||
listContainersMock.mockRejectedValue(new Error('docker daemon unreachable'));
|
||||
|
||||
const agent = makeAgent();
|
||||
const result = await callResolve(agent, 'api', 'db', 5432);
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.err).toBe('agent_error');
|
||||
});
|
||||
});
|
||||
+10
-15
@@ -562,9 +562,11 @@ export class PilotAgent {
|
||||
|
||||
// --- Sencho Mesh TCP dispatch (tunnel -> Compose service container) ---
|
||||
//
|
||||
// PR 1 rejects every tcp_open with mesh_not_enabled; the dial path is
|
||||
// exercised by tests via setMeshResolver but never lit in production until
|
||||
// PR 2 wires Dockerode resolution gated by the local mesh_stacks table.
|
||||
// Central is the sole authority for mesh opt-in (state lives in central's
|
||||
// SQLite mesh_stacks table). The pilot resolves a target by Compose
|
||||
// container labels and dials directly. The tunnel JWT (scope
|
||||
// 'pilot_tunnel') gates the WS upgrade itself, so any tcp_open frame on
|
||||
// an open tunnel is trusted to originate from central.
|
||||
|
||||
private async onTcpOpen(frame: Extract<ReturnType<typeof decodeJsonFrame>, { t: 'tcp_open' }>): Promise<void> {
|
||||
const ws = this.ws;
|
||||
@@ -724,11 +726,11 @@ export class PilotAgent {
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a mesh target by consulting the local mesh_stacks opt-in table
|
||||
* and Compose container labels. Refuses if the target stack is not opted
|
||||
* in on this node (defense-in-depth: the primary is trusted, but we also
|
||||
* gate at the agent so a leaked tunnel token cannot reach unauthorized
|
||||
* services).
|
||||
* Resolves a mesh target by Compose container labels and returns the
|
||||
* container's first usable IP. Central has already validated that the
|
||||
* target stack is opted in before issuing the dial; the tunnel JWT
|
||||
* (scope 'pilot_tunnel') authenticates the caller, so this handler
|
||||
* does no per-stack gating of its own.
|
||||
*/
|
||||
private async resolveMeshTarget(
|
||||
stack: string,
|
||||
@@ -736,15 +738,8 @@ export class PilotAgent {
|
||||
port: number,
|
||||
): Promise<MeshResolveResult> {
|
||||
try {
|
||||
const { DatabaseService } = await import('../services/DatabaseService');
|
||||
const { NodeRegistry } = await import('../services/NodeRegistry');
|
||||
const dockerodeMod = await import('dockerode');
|
||||
const Docker = (dockerodeMod as { default: new (opts?: unknown) => { listContainers: (opts?: unknown) => Promise<unknown[]> } }).default;
|
||||
const db = DatabaseService.getInstance();
|
||||
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
if (!db.isMeshStackEnabled(localNodeId, stack)) {
|
||||
return { ok: false, err: 'denied' };
|
||||
}
|
||||
const docker = new Docker();
|
||||
const containers = (await docker.listContainers({
|
||||
filters: { label: [`com.docker.compose.project=${stack}`, `com.docker.compose.service=${service}`] },
|
||||
|
||||
@@ -194,6 +194,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
await this.setupMeshNetwork();
|
||||
await this.refreshAliasCache();
|
||||
await this.syncForwarderListeners();
|
||||
await this.regenerateAllOverrides();
|
||||
this.aliasRefreshTimer = setInterval(() => {
|
||||
void (async () => {
|
||||
try {
|
||||
@@ -646,6 +647,38 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk every `mesh_stacks` row across the fleet and re-push each override
|
||||
* to its owning node. Called once at boot so on-disk override files
|
||||
* survive a Sencho restart even if they were lost (image rebuild, volume
|
||||
* reset, manual cleanup). Best-effort: failures are logged per-stack and
|
||||
* other nodes still get regenerated. An offline remote node leaves stale
|
||||
* overrides until the next opt-in / opt-out on that node.
|
||||
*/
|
||||
private async regenerateAllOverrides(): Promise<void> {
|
||||
if (!this.senchoIp) return;
|
||||
const db = DatabaseService.getInstance();
|
||||
const stacks = db.listMeshStacks();
|
||||
await Promise.allSettled(
|
||||
stacks.map(async (s) => {
|
||||
try {
|
||||
await this.pushOverrideToNode(s.node_id, s.stack_name);
|
||||
} catch (err) {
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'warn', type: 'forwarder.error',
|
||||
nodeId: s.node_id,
|
||||
message: `boot override regen failed for ${s.stack_name}: ${sanitizeForLog((err as Error).message)}`,
|
||||
details: { stackName: s.stack_name },
|
||||
});
|
||||
}
|
||||
}),
|
||||
);
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'info', type: 'mesh.enable',
|
||||
message: `boot regenerated ${stacks.length} override(s)`,
|
||||
});
|
||||
}
|
||||
|
||||
// --- Alias aggregation ---
|
||||
|
||||
public async refreshAliasCache(): Promise<void> {
|
||||
|
||||
Reference in New Issue
Block a user