fix(mesh): auto-fallback through candidate subnets when default overlaps (#1156)

The default mesh subnet 172.30.0.0/24 is fully contained in linuxserver/*
default networks (sonarr_default 172.30.0.0/16, etc.), so libnetwork
rejects the IPAM allocation with "Pool overlaps with other one on this
address space" on a typical homelab Docker host. The single hard-coded
default left first-run operators with a silently broken mesh.

MeshService.setupMeshNetwork now resolves the subnet via three paths:

1. Operator-explicit (SENCHO_MESH_SUBNET set): use that subnet, strict.
   Pre-existing sencho_mesh with a different subnet still raises
   subnet_mismatch.
2. Adopt-existing (env unset, sencho_mesh already on the daemon): adopt
   the existing subnet. Docker is the source of truth across restarts.
3. Candidate iteration (env unset, no existing network): walk
   172.30.0.0/24, 172.31.0.0/24, 10.42.0.0/24, 10.43.0.0/24 in order.
   First subnet Docker accepts wins. If every candidate overlaps,
   record subnet_overlap with a message naming every attempt.

The dashboard's Fleet Heartbeat card now surfaces the down state via a
compact banner above the per-node rows, plus a "mesh down" counter
suffix on the right of the title. The existing Routing-tab banner is
extracted into a shared MeshDataPlaneBanner component with tab and
card variants. Dashboard polling is gated on Admiral tier so non-paid
users do not fire the Admiral-only /mesh/status endpoint.

Six new tests in mesh-setup-error-classification cover: iterates past
first overlap, all candidates overlap, adopts existing network,
inspectNetwork non-404 failure classified as attach_failed, env-matches-
existing skip-create, and operator-explicit strict (no fallback).

Fixes F-1 in the pre-1.0 audit. Closes the silent-failure mode that
left the mesh down on the most common homelab Docker layout.
This commit is contained in:
Anso
2026-05-22 13:20:27 -04:00
committed by GitHub
parent 606bdb6b67
commit 1a03cf82af
9 changed files with 454 additions and 110 deletions
@@ -656,36 +656,6 @@ describe('getSenchoIpFromSubnet', () => {
});
});
describe('MeshService.ensureMeshNetwork', () => {
it('refuses to continue when sencho_mesh exists with a different subnet', async () => {
const svc = MeshService.getInstance();
const dcModule = await import('../services/DockerController');
const fakeController = {
createNetwork: vi.fn().mockRejectedValue({ statusCode: 409, message: 'network already exists' }),
inspectNetwork: vi.fn().mockResolvedValue({ IPAM: { Config: [{ Subnet: '10.99.0.0/24' }] } }),
};
vi.spyOn(dcModule.default, 'getInstance').mockReturnValue(fakeController as unknown as ReturnType<typeof dcModule.default.getInstance>);
await expect(
(svc as unknown as { ensureMeshNetwork: (s: string) => Promise<void> }).ensureMeshNetwork('172.30.0.0/24'),
).rejects.toThrow(/exists with subnet 10\.99\.0\.0\/24/);
});
it('treats 409 with matching subnet as idempotent success', async () => {
const svc = MeshService.getInstance();
const dcModule = await import('../services/DockerController');
const fakeController = {
createNetwork: vi.fn().mockRejectedValue({ statusCode: 409, message: 'network already exists' }),
inspectNetwork: vi.fn().mockResolvedValue({ IPAM: { Config: [{ Subnet: '172.30.0.0/24' }] } }),
};
vi.spyOn(dcModule.default, 'getInstance').mockReturnValue(fakeController as unknown as ReturnType<typeof dcModule.default.getInstance>);
await expect(
(svc as unknown as { ensureMeshNetwork: (s: string) => Promise<void> }).ensureMeshNetwork('172.30.0.0/24'),
).resolves.toBeUndefined();
});
});
describe('MeshService.optInStack rollback', () => {
it('rolls back the DB row when the just-inserted stack fails to push its override', async () => {
const svc = MeshService.getInstance();
@@ -204,3 +204,127 @@ describe('MeshService.setupMeshNetwork failure classification', () => {
expect(svc.getNetworkSetupError()).toMatch(/overlap/i);
});
});
describe('MeshService.setupMeshNetwork subnet auto-fallback', () => {
it('iterates past the first overlapping candidate when SENCHO_MESH_SUBNET is unset', async () => {
delete process.env.SENCHO_MESH_SUBNET;
process.env.HOSTNAME = 'sencho';
const overlap = Object.assign(
new Error('Pool overlaps with other one on this address space'),
{ statusCode: 500 },
);
const createNetwork = vi.fn()
.mockRejectedValueOnce(overlap)
.mockResolvedValueOnce(undefined);
const inspectNetwork = vi.fn().mockRejectedValue({ statusCode: 404, message: 'no such network' });
mockDocker({ createNetwork, inspectNetwork });
const svc = MeshService.getInstance();
await callSetup(svc);
const status = svc.getDataPlaneStatus();
expect(status.ok).toBe(true);
expect(status.subnet).toBe('172.31.0.0/24');
expect(createNetwork).toHaveBeenCalledTimes(2);
});
it('records subnet_overlap with every tried candidate when all candidates overlap', async () => {
delete process.env.SENCHO_MESH_SUBNET;
process.env.HOSTNAME = 'sencho';
const overlap = Object.assign(
new Error('Pool overlaps with other one on this address space'),
{ statusCode: 500 },
);
const createNetwork = vi.fn().mockRejectedValue(overlap);
const inspectNetwork = vi.fn().mockRejectedValue({ statusCode: 404, message: 'no such network' });
mockDocker({ createNetwork, inspectNetwork });
const svc = MeshService.getInstance();
await callSetup(svc);
const status = svc.getDataPlaneStatus();
expect(status.ok).toBe(false);
expect(status.reason).toBe('subnet_overlap');
expect(createNetwork).toHaveBeenCalledTimes(4);
expect(status.message).toContain('172.30.0.0/24');
expect(status.message).toContain('172.31.0.0/24');
expect(status.message).toContain('10.42.0.0/24');
expect(status.message).toContain('10.43.0.0/24');
expect(status.message).toMatch(/SENCHO_MESH_SUBNET/);
});
it('adopts an existing sencho_mesh subnet when SENCHO_MESH_SUBNET is unset', async () => {
delete process.env.SENCHO_MESH_SUBNET;
process.env.HOSTNAME = 'sencho';
const createNetwork = vi.fn();
const inspectNetwork = vi.fn().mockResolvedValue({
IPAM: { Config: [{ Subnet: '192.168.42.0/24' }] },
});
mockDocker({ createNetwork, inspectNetwork });
const svc = MeshService.getInstance();
await callSetup(svc);
const status = svc.getDataPlaneStatus();
expect(status.ok).toBe(true);
expect(status.subnet).toBe('192.168.42.0/24');
expect(createNetwork).not.toHaveBeenCalled();
});
it('classifies a non-404 inspectNetwork failure as attach_failed without trying to create', async () => {
delete process.env.SENCHO_MESH_SUBNET;
process.env.HOSTNAME = 'sencho';
const createNetwork = vi.fn();
const inspectNetwork = vi.fn().mockRejectedValue(
Object.assign(new Error('daemon unresponsive'), { statusCode: 500 }),
);
mockDocker({ createNetwork, inspectNetwork });
const svc = MeshService.getInstance();
await callSetup(svc);
const status = svc.getDataPlaneStatus();
expect(status.ok).toBe(false);
expect(status.reason).toBe('attach_failed');
expect(createNetwork).not.toHaveBeenCalled();
});
it('skips create when SENCHO_MESH_SUBNET matches the existing sencho_mesh subnet', async () => {
process.env.SENCHO_MESH_SUBNET = '172.30.0.0/24';
process.env.HOSTNAME = 'sencho';
const createNetwork = vi.fn();
const inspectNetwork = vi.fn().mockResolvedValue({
IPAM: { Config: [{ Subnet: '172.30.0.0/24' }] },
});
mockDocker({ createNetwork, inspectNetwork });
const svc = MeshService.getInstance();
await callSetup(svc);
const status = svc.getDataPlaneStatus();
expect(status.ok).toBe(true);
expect(status.subnet).toBe('172.30.0.0/24');
expect(createNetwork).not.toHaveBeenCalled();
});
it('keeps the operator-explicit path strict (no candidate fallback)', async () => {
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
process.env.HOSTNAME = 'sencho';
const overlap = Object.assign(
new Error('Pool overlaps with other one on this address space'),
{ statusCode: 500 },
);
const createNetwork = vi.fn().mockRejectedValue(overlap);
const inspectNetwork = vi.fn().mockRejectedValue({ statusCode: 404, message: 'no such network' });
mockDocker({ createNetwork, inspectNetwork });
const svc = MeshService.getInstance();
await callSetup(svc);
const status = svc.getDataPlaneStatus();
expect(status.ok).toBe(false);
expect(status.reason).toBe('subnet_overlap');
expect(status.subnet).toBe('10.42.0.0/24');
expect(createNetwork).toHaveBeenCalledTimes(1);
});
});
+175 -36
View File
@@ -27,6 +27,23 @@ const PROBE_TIMEOUT_MS = 5_000;
const SLOW_PROBE_THRESHOLD_MS = 500;
const DEFAULT_MESH_SUBNET = '172.30.0.0/24';
/**
* Subnets attempted in order when SENCHO_MESH_SUBNET is unset and no
* `sencho_mesh` network already exists. Each is a `/24` chosen to dodge the
* usual homelab Docker patterns: `172.30.0.0/24` matches the prior default,
* `172.31.0.0/24` sits one above it, and the `10.42`/`10.43` pair lands well
* outside both the linuxserver/* `172.30.0.0/16` family and the typical
* `192.168.x` LAN range. The first candidate that Docker accepts wins; the
* chosen subnet persists implicitly through the `sencho_mesh` network on the
* Docker daemon (next boot adopts it via the inspect path).
*/
export const MESH_SUBNET_CANDIDATES = [
'172.30.0.0/24',
'172.31.0.0/24',
'10.42.0.0/24',
'10.43.0.0/24',
];
const REACHABLE_REASON: Record<DialFailureCode, string> = {
auth_failed: 'api token rejected by remote',
endpoint_not_found: 'remote does not support proxy mesh',
@@ -404,12 +421,13 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
const dpReason = this.dataPlaneStatus.reason;
const dataPlane = this.senchoIp ? 'ok' : `unavailable (${dpReason}: ${this.networkSetupError ?? 'unknown'})`;
const subnetSuffix = this.senchoIp ? `, subnet ${this.meshSubnet}` : '';
const summaryLevel: MeshActivityLevel = dpReason === 'ok'
? 'info'
: dpReason === 'not_in_docker' ? 'warn' : 'error';
this.logActivity({
source: 'mesh', level: summaryLevel, type: 'mesh.enable',
message: `MeshService started (data plane ${dataPlane}, self nodeId ${this.selfCentralNodeId})`,
message: `MeshService started (data plane ${dataPlane}${subnetSuffix}, self nodeId ${this.selfCentralNodeId})`,
});
}
@@ -581,28 +599,145 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
* Skipped entirely when Sencho is not running inside Docker (dev mode,
* detected by an unset HOSTNAME env var or by the inspect lookup
* failing). The forwarder still runs locally for unit-test coverage.
*
* Three subnet-resolution paths:
* 1. **Operator-explicit.** `SENCHO_MESH_SUBNET` is set. Use exactly
* that subnet; a pre-existing `sencho_mesh` with a different subnet
* raises `subnet_mismatch`. Preserves the loud-config-error case.
* 2. **Adopt-existing.** `SENCHO_MESH_SUBNET` is unset and
* `sencho_mesh` already exists on the Docker daemon. Adopt its
* subnet (Docker is the source of truth across restarts).
* 3. **Candidate iteration.** Neither of the above. Walk
* `MESH_SUBNET_CANDIDATES` in order; first subnet Docker accepts
* wins. If every candidate overlaps an existing network, record
* `subnet_overlap` with a message naming every attempted subnet.
*/
private async setupMeshNetwork(): Promise<void> {
const subnet = (process.env.SENCHO_MESH_SUBNET || DEFAULT_MESH_SUBNET).trim();
const envSubnet = process.env.SENCHO_MESH_SUBNET?.trim() || null;
// Validate the operator-supplied CIDR before any Docker call. An
// invalid env var is the operator's problem, not the daemon's, and
// reporting `subnet_invalid` from here means the diagnostic stays
// accurate even if the daemon is also broken.
if (envSubnet) {
try {
this.senchoIp = getSenchoIpFromSubnet(envSubnet);
this.meshSubnet = envSubnet;
} catch (err) {
this.recordSetupFailure('subnet_invalid', err, 'error', envSubnet);
return;
}
}
let existingSubnet: string | null;
try {
this.senchoIp = getSenchoIpFromSubnet(subnet);
this.meshSubnet = subnet;
existingSubnet = await this.inspectExistingMeshSubnet();
} catch (err) {
this.recordSetupFailure('subnet_invalid', err, 'error', subnet);
// A genuinely broken Docker daemon (404s return null, see
// `inspectExistingMeshSubnet`). Classify as `attach_failed`;
// calling create would just hit the same error one layer down.
this.recordSetupFailure(
'attach_failed',
err,
'error',
envSubnet ?? DEFAULT_MESH_SUBNET,
);
return;
}
try {
await this.ensureMeshNetwork(subnet);
} catch (err) {
this.recordSetupFailure(this.classifyMeshNetworkError(err), err, 'error', subnet);
return;
if (envSubnet) {
if (existingSubnet && existingSubnet !== envSubnet) {
this.recordSetupFailure(
'subnet_mismatch',
new Error(
`${SENCHO_MESH_NETWORK} exists with subnet ${existingSubnet}, ` +
`expected ${envSubnet}. Remove the network or set SENCHO_MESH_SUBNET to match.`,
),
'error',
envSubnet,
);
return;
}
if (!existingSubnet) {
try {
await this.createMeshNetwork(envSubnet);
} catch (err) {
this.recordSetupFailure(
this.classifyMeshNetworkError(err),
err,
'error',
envSubnet,
);
return;
}
}
} else if (existingSubnet) {
try {
this.senchoIp = getSenchoIpFromSubnet(existingSubnet);
this.meshSubnet = existingSubnet;
} catch (err) {
this.recordSetupFailure('subnet_invalid', err, 'error', existingSubnet);
return;
}
} else {
const tried: string[] = [];
let chosen: string | null = null;
let lastErr: unknown = null;
for (const candidate of MESH_SUBNET_CANDIDATES) {
tried.push(candidate);
try {
await this.createMeshNetwork(candidate);
chosen = candidate;
break;
} catch (err) {
const cls = this.classifyMeshNetworkError(err);
if (cls === 'subnet_overlap') {
lastErr = err;
continue;
}
// Non-overlap failures (e.g. daemon attach error) are not
// helped by trying another candidate; bail with the typed
// reason for this candidate. A 409 from another process
// racing to create `sencho_mesh` between our inspect and
// our create will classify as `attach_failed` here; the
// race is rare enough that we accept the bail and let the
// next process start adopt the now-existing network.
this.recordSetupFailure(cls, err, 'error', candidate);
return;
}
}
if (!chosen) {
this.recordSetupFailure(
'subnet_overlap',
new Error(
`every candidate subnet overlaps an existing Docker network on this host ` +
`(tried ${tried.join(', ')}). Set SENCHO_MESH_SUBNET to a free /24 and restart.` +
(lastErr instanceof Error ? ` Last error: ${lastErr.message}` : ''),
),
'error',
tried[tried.length - 1],
);
return;
}
try {
this.senchoIp = getSenchoIpFromSubnet(chosen);
this.meshSubnet = chosen;
} catch (err) {
// Hard-coded candidates are well-formed; defensive only.
this.recordSetupFailure('subnet_invalid', err, 'error', chosen);
return;
}
}
try {
await this.ensureSelfAttached();
} catch (err) {
this.recordSetupFailure(this.classifySelfAttachError(err), err, 'error', subnet);
this.recordSetupFailure(
this.classifySelfAttachError(err),
err,
'error',
this.meshSubnet,
);
return;
}
@@ -614,40 +749,44 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
if (!this.senchoIp) return;
this.networkSetupError = null;
this.dataPlaneStatus = { ok: true, reason: 'ok', message: null, subnet };
this.dataPlaneStatus = { ok: true, reason: 'ok', message: null, subnet: this.meshSubnet };
}
/**
* Create `sencho_mesh` if it does not exist. If it does, validate the
* subnet matches `expectedSubnet`; on mismatch, refuse to continue.
* Silently using the wrong subnet would route traffic to the wrong IP.
* Return the subnet of an existing `sencho_mesh` network, or null if the
* network does not exist. Docker's inspect endpoint surfaces 404 for
* the missing case; any other error is re-raised so the caller can
* classify it as `attach_failed`.
*/
private async ensureMeshNetwork(expectedSubnet: string): Promise<void> {
private async inspectExistingMeshSubnet(): Promise<string | null> {
const dc = DockerController.getInstance(NodeRegistry.getInstance().getDefaultNodeId());
try {
await dc.createNetwork({
Name: SENCHO_MESH_NETWORK,
Driver: 'bridge',
Attachable: true,
IPAM: { Config: [{ Subnet: expectedSubnet }] },
Labels: { 'io.sencho.mesh': 'true' },
});
return;
const info = await dc.inspectNetwork(SENCHO_MESH_NETWORK) as {
IPAM?: { Config?: Array<{ Subnet?: string }> };
} | undefined;
return info?.IPAM?.Config?.[0]?.Subnet ?? null;
} catch (err) {
const e = err as { statusCode?: number; message?: string };
if (e?.statusCode !== 409) throw err;
const e = err as { statusCode?: number };
if (e?.statusCode === 404) return null;
throw err;
}
}
const info = await dc.inspectNetwork(SENCHO_MESH_NETWORK) as {
IPAM?: { Config?: Array<{ Subnet?: string }> };
};
const existingSubnet = info?.IPAM?.Config?.[0]?.Subnet;
if (existingSubnet && existingSubnet !== expectedSubnet) {
throw new Error(
`${SENCHO_MESH_NETWORK} exists with subnet ${existingSubnet}, ` +
`expected ${expectedSubnet}. Remove the network or set SENCHO_MESH_SUBNET to match.`,
);
}
/**
* Create the `sencho_mesh` bridge network with the given subnet. Throws
* the raw Dockerode error (including the 500 pool-overlap that
* `classifyMeshNetworkError` recognises) so callers can decide whether
* to retry on another candidate or bail.
*/
private async createMeshNetwork(subnet: string): Promise<void> {
const dc = DockerController.getInstance(NodeRegistry.getInstance().getDefaultNodeId());
await dc.createNetwork({
Name: SENCHO_MESH_NETWORK,
Driver: 'bridge',
Attachable: true,
IPAM: { Config: [{ Subnet: subnet }] },
Labels: { 'io.sencho.mesh': 'true' },
});
}
/**