fix(mesh): trust central for cross-node dial auth and regenerate overrides at boot (#1014)

Two bugs in the same Phase D follow-up surface, fixed together because they
both block declaring B-verify complete on the production fleet.

Cross-node mesh dials returned `denied` at the agent. The pilot's
`tcp_open` handler in `agent.ts::resolveMeshTarget` consulted the local
SQLite `mesh_stacks` table, which is no longer written to under the
post-Phase D control plane (state lives only on central). Drop the check.
The pilot tunnel JWT (scope `pilot_tunnel`, signed with central's
`auth_jwt_secret`) authenticates the caller; the same trust model already
applies to filesystem ops, exec, and container control over the same
tunnel.

Threat-model trade-off: a leaked `pilot_tunnel` JWT or compromised
central can now dial any compose-managed service on the pilot. Containers
without `com.docker.compose.project` + `com.docker.compose.service`
labels remain unreachable via this path.

`MeshService.start()` did not regenerate compose override files at boot.
After a Sencho restart with missing overrides on disk, meshed user
containers had no `extra_hosts` / `networks: [sencho_mesh]` injection
until each stack was opted out and back in. Add `regenerateAllOverrides()`
that walks every `mesh_stacks` row and re-pushes via `pushOverrideToNode`.
Best-effort: per-stack failures log to the mesh activity buffer;
`MeshService.start()` is fire-and-forget at startup so a slow remote
node does not delay boot.

Tests:
- `pilot-agent-mesh-resolve.test.ts` (new): mocks dockerode and proves
  `resolveMeshTarget` no longer returns `denied` with an empty
  `mesh_stacks` table.
- `mesh-service.test.ts`: three new cases for `regenerateAllOverrides` -
  fan-out across the fleet, skip when `senchoIp` is null, log per-stack
  warning on push failure without throwing.
This commit is contained in:
Anso
2026-05-09 03:27:39 -04:00
committed by GitHub
parent fc05a818d0
commit 0947a80cda
4 changed files with 219 additions and 15 deletions
@@ -345,3 +345,67 @@ describe('MeshService.optInStack guard rails (network setup)', () => {
expect(db.isMeshStackEnabled(localNodeId, 'api')).toBe(false);
});
});
describe('MeshService.regenerateAllOverrides (F6: boot-time regen)', () => {
it('pushes every mesh_stacks row across the fleet', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
const remoteNodeId = db.addNode({
name: 'remote-pilot', type: 'remote', mode: 'pilot_agent',
compose_dir: '/tmp', is_default: false, api_url: '', api_token: '',
});
db.insertMeshStack(localNodeId, 'audit-mesh-prod', 'tester');
db.insertMeshStack(remoteNodeId, 'audit-mesh-pilot', 'tester');
const pushSpy = vi.spyOn(svc, 'pushOverrideToNode').mockResolvedValue(undefined);
try {
await (svc as unknown as { regenerateAllOverrides: () => Promise<void> }).regenerateAllOverrides();
expect(pushSpy).toHaveBeenCalledTimes(2);
expect(pushSpy).toHaveBeenCalledWith(localNodeId, 'audit-mesh-prod');
expect(pushSpy).toHaveBeenCalledWith(remoteNodeId, 'audit-mesh-pilot');
const activity = svc.getActivity({ limit: 100 });
expect(activity.some((e) => e.message === 'boot regenerated 2 override(s)')).toBe(true);
} finally {
db.deleteNode(remoteNodeId);
}
});
it('skips entirely when senchoIp is null (network setup failed)', async () => {
const svc = MeshService.getInstance();
(svc as unknown as { senchoIp: string | null }).senchoIp = null;
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
db.insertMeshStack(localNodeId, 'audit-mesh-prod', 'tester');
const pushSpy = vi.spyOn(svc, 'pushOverrideToNode').mockResolvedValue(undefined);
await (svc as unknown as { regenerateAllOverrides: () => Promise<void> }).regenerateAllOverrides();
expect(pushSpy).not.toHaveBeenCalled();
});
it('logs a warning per stack when push fails but does not throw', async () => {
const svc = MeshService.getInstance();
const db = DatabaseService.getInstance();
const localNodeId = db.getNodes()[0].id;
db.insertMeshStack(localNodeId, 'audit-mesh-prod', 'tester');
vi.spyOn(svc, 'pushOverrideToNode').mockRejectedValue(new Error('remote node offline'));
await expect(
(svc as unknown as { regenerateAllOverrides: () => Promise<void> }).regenerateAllOverrides(),
).resolves.toBeUndefined();
const activity = svc.getActivity({ limit: 100 });
expect(activity.some((e) =>
e.level === 'warn' && /boot override regen failed for audit-mesh-prod/.test(e.message),
)).toBe(true);
});
});
@@ -0,0 +1,112 @@
/**
* F7 regression: pilot agent's mesh dial path no longer denies based on the
* pilot's local `mesh_stacks` table. Central is the sole authority for
* mesh opt-in (state lives in central's SQLite); the pilot resolves the
* target container by Compose labels and dials it directly. The tunnel JWT
* authenticates the caller, so per-stack gating on the pilot would only
* deny legitimate central-issued dials whenever Phase D's central-only
* state model is in effect.
*/
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
const listContainersMock = vi.fn();
vi.mock('dockerode', () => {
function Docker(this: unknown) {
(this as { listContainers: typeof listContainersMock }).listContainers = listContainersMock;
}
return { default: Docker };
});
let tmpDir: string;
let PilotAgent: typeof import('../pilot/agent').PilotAgent;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
interface ResolveResult {
ok: boolean;
host?: string;
port?: number;
err?: string;
}
function makeAgent(): import('../pilot/agent').PilotAgent {
return new PilotAgent({
primaryUrl: 'http://primary.invalid',
loopbackPort: 1,
initialToken: 'irrelevant',
enrolling: false,
});
}
function callResolve(agent: import('../pilot/agent').PilotAgent, stack: string, service: string, port: number): Promise<ResolveResult> {
const fn = (agent as unknown as { resolveMeshTarget: (s: string, sv: string, p: number) => Promise<ResolveResult> }).resolveMeshTarget.bind(agent);
return fn(stack, service, port);
}
beforeAll(async () => {
tmpDir = await setupTestDb();
({ PilotAgent } = await import('../pilot/agent'));
({ DatabaseService } = await import('../services/DatabaseService'));
});
afterAll(() => {
cleanupTestDb(tmpDir);
});
afterEach(() => {
listContainersMock.mockReset();
DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_stacks').run();
});
describe('PilotAgent.resolveMeshTarget (F7: trust central)', () => {
it('returns the container IP when mesh_stacks is empty (central is the gate)', async () => {
listContainersMock.mockResolvedValue([
{ NetworkSettings: { Networks: { sencho_mesh: { IPAddress: '172.30.0.5' } } } },
]);
const agent = makeAgent();
const result = await callResolve(agent, 'audit-mesh-pilot', 'echo', 9001);
expect(result.err).not.toBe('denied');
expect(result.ok).toBe(true);
expect(result.host).toBe('172.30.0.5');
expect(result.port).toBe(9001);
});
it('queries dockerode with the Compose project + service label filter', async () => {
listContainersMock.mockResolvedValue([
{ NetworkSettings: { Networks: { bridge: { IPAddress: '10.0.0.7' } } } },
]);
const agent = makeAgent();
await callResolve(agent, 'api', 'db', 5432);
expect(listContainersMock).toHaveBeenCalledTimes(1);
const args = listContainersMock.mock.calls[0][0] as { filters: { label: string[] } };
expect(args.filters.label).toEqual([
'com.docker.compose.project=api',
'com.docker.compose.service=db',
]);
});
it('returns no_target (not denied) when dockerode finds no matching container', async () => {
listContainersMock.mockResolvedValue([]);
const agent = makeAgent();
const result = await callResolve(agent, 'missing', 'svc', 8080);
expect(result.ok).toBe(false);
expect(result.err).toBe('no_target');
});
it('returns agent_error (not denied) when dockerode throws', async () => {
listContainersMock.mockRejectedValue(new Error('docker daemon unreachable'));
const agent = makeAgent();
const result = await callResolve(agent, 'api', 'db', 5432);
expect(result.ok).toBe(false);
expect(result.err).toBe('agent_error');
});
});
+10 -15
View File
@@ -562,9 +562,11 @@ export class PilotAgent {
// --- Sencho Mesh TCP dispatch (tunnel -> Compose service container) ---
//
// PR 1 rejects every tcp_open with mesh_not_enabled; the dial path is
// exercised by tests via setMeshResolver but never lit in production until
// PR 2 wires Dockerode resolution gated by the local mesh_stacks table.
// Central is the sole authority for mesh opt-in (state lives in central's
// SQLite mesh_stacks table). The pilot resolves a target by Compose
// container labels and dials directly. The tunnel JWT (scope
// 'pilot_tunnel') gates the WS upgrade itself, so any tcp_open frame on
// an open tunnel is trusted to originate from central.
private async onTcpOpen(frame: Extract<ReturnType<typeof decodeJsonFrame>, { t: 'tcp_open' }>): Promise<void> {
const ws = this.ws;
@@ -724,11 +726,11 @@ export class PilotAgent {
}
/**
* Resolves a mesh target by consulting the local mesh_stacks opt-in table
* and Compose container labels. Refuses if the target stack is not opted
* in on this node (defense-in-depth: the primary is trusted, but we also
* gate at the agent so a leaked tunnel token cannot reach unauthorized
* services).
* Resolves a mesh target by Compose container labels and returns the
* container's first usable IP. Central has already validated that the
* target stack is opted in before issuing the dial; the tunnel JWT
* (scope 'pilot_tunnel') authenticates the caller, so this handler
* does no per-stack gating of its own.
*/
private async resolveMeshTarget(
stack: string,
@@ -736,15 +738,8 @@ export class PilotAgent {
port: number,
): Promise<MeshResolveResult> {
try {
const { DatabaseService } = await import('../services/DatabaseService');
const { NodeRegistry } = await import('../services/NodeRegistry');
const dockerodeMod = await import('dockerode');
const Docker = (dockerodeMod as { default: new (opts?: unknown) => { listContainers: (opts?: unknown) => Promise<unknown[]> } }).default;
const db = DatabaseService.getInstance();
const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
if (!db.isMeshStackEnabled(localNodeId, stack)) {
return { ok: false, err: 'denied' };
}
const docker = new Docker();
const containers = (await docker.listContainers({
filters: { label: [`com.docker.compose.project=${stack}`, `com.docker.compose.service=${service}`] },
+33
View File
@@ -194,6 +194,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
await this.setupMeshNetwork();
await this.refreshAliasCache();
await this.syncForwarderListeners();
await this.regenerateAllOverrides();
this.aliasRefreshTimer = setInterval(() => {
void (async () => {
try {
@@ -646,6 +647,38 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
);
}
/**
* Walk every `mesh_stacks` row across the fleet and re-push each override
* to its owning node. Called once at boot so on-disk override files
* survive a Sencho restart even if they were lost (image rebuild, volume
* reset, manual cleanup). Best-effort: failures are logged per-stack and
* other nodes still get regenerated. An offline remote node leaves stale
* overrides until the next opt-in / opt-out on that node.
*/
private async regenerateAllOverrides(): Promise<void> {
if (!this.senchoIp) return;
const db = DatabaseService.getInstance();
const stacks = db.listMeshStacks();
await Promise.allSettled(
stacks.map(async (s) => {
try {
await this.pushOverrideToNode(s.node_id, s.stack_name);
} catch (err) {
this.logActivity({
source: 'mesh', level: 'warn', type: 'forwarder.error',
nodeId: s.node_id,
message: `boot override regen failed for ${s.stack_name}: ${sanitizeForLog((err as Error).message)}`,
details: { stackName: s.stack_name },
});
}
}),
);
this.logActivity({
source: 'mesh', level: 'info', type: 'mesh.enable',
message: `boot regenerated ${stacks.length} override(s)`,
});
}
// --- Alias aggregation ---
public async refreshAliasCache(): Promise<void> {