mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-09 10:21:03 +00:00
fix(mesh): auto-fallback through candidate subnets when default overlaps (#1156)
The default mesh subnet 172.30.0.0/24 is fully contained in linuxserver/* default networks (sonarr_default 172.30.0.0/16, etc.), so libnetwork rejects the IPAM allocation with "Pool overlaps with other one on this address space" on a typical homelab Docker host. The single hard-coded default left first-run operators with a silently broken mesh. MeshService.setupMeshNetwork now resolves the subnet via three paths: 1. Operator-explicit (SENCHO_MESH_SUBNET set): use that subnet, strict. Pre-existing sencho_mesh with a different subnet still raises subnet_mismatch. 2. Adopt-existing (env unset, sencho_mesh already on the daemon): adopt the existing subnet. Docker is the source of truth across restarts. 3. Candidate iteration (env unset, no existing network): walk 172.30.0.0/24, 172.31.0.0/24, 10.42.0.0/24, 10.43.0.0/24 in order. First subnet Docker accepts wins. If every candidate overlaps, record subnet_overlap with a message naming every attempt. The dashboard's Fleet Heartbeat card now surfaces the down state via a compact banner above the per-node rows, plus a "mesh down" counter suffix on the right of the title. The existing Routing-tab banner is extracted into a shared MeshDataPlaneBanner component with tab and card variants. Dashboard polling is gated on Admiral tier so non-paid users do not fire the Admiral-only /mesh/status endpoint. Six new tests in mesh-setup-error-classification cover: iterates past first overlap, all candidates overlap, adopts existing network, inspectNetwork non-404 failure classified as attach_failed, env-matches- existing skip-create, and operator-explicit strict (no fallback). Fixes F-1 in the pre-1.0 audit. Closes the silent-failure mode that left the mesh down on the most common homelab Docker layout.
This commit is contained in:
@@ -656,36 +656,6 @@ describe('getSenchoIpFromSubnet', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('MeshService.ensureMeshNetwork', () => {
|
||||
it('refuses to continue when sencho_mesh exists with a different subnet', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const dcModule = await import('../services/DockerController');
|
||||
const fakeController = {
|
||||
createNetwork: vi.fn().mockRejectedValue({ statusCode: 409, message: 'network already exists' }),
|
||||
inspectNetwork: vi.fn().mockResolvedValue({ IPAM: { Config: [{ Subnet: '10.99.0.0/24' }] } }),
|
||||
};
|
||||
vi.spyOn(dcModule.default, 'getInstance').mockReturnValue(fakeController as unknown as ReturnType<typeof dcModule.default.getInstance>);
|
||||
|
||||
await expect(
|
||||
(svc as unknown as { ensureMeshNetwork: (s: string) => Promise<void> }).ensureMeshNetwork('172.30.0.0/24'),
|
||||
).rejects.toThrow(/exists with subnet 10\.99\.0\.0\/24/);
|
||||
});
|
||||
|
||||
it('treats 409 with matching subnet as idempotent success', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
const dcModule = await import('../services/DockerController');
|
||||
const fakeController = {
|
||||
createNetwork: vi.fn().mockRejectedValue({ statusCode: 409, message: 'network already exists' }),
|
||||
inspectNetwork: vi.fn().mockResolvedValue({ IPAM: { Config: [{ Subnet: '172.30.0.0/24' }] } }),
|
||||
};
|
||||
vi.spyOn(dcModule.default, 'getInstance').mockReturnValue(fakeController as unknown as ReturnType<typeof dcModule.default.getInstance>);
|
||||
|
||||
await expect(
|
||||
(svc as unknown as { ensureMeshNetwork: (s: string) => Promise<void> }).ensureMeshNetwork('172.30.0.0/24'),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('MeshService.optInStack rollback', () => {
|
||||
it('rolls back the DB row when the just-inserted stack fails to push its override', async () => {
|
||||
const svc = MeshService.getInstance();
|
||||
|
||||
@@ -204,3 +204,127 @@ describe('MeshService.setupMeshNetwork failure classification', () => {
|
||||
expect(svc.getNetworkSetupError()).toMatch(/overlap/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MeshService.setupMeshNetwork subnet auto-fallback', () => {
|
||||
it('iterates past the first overlapping candidate when SENCHO_MESH_SUBNET is unset', async () => {
|
||||
delete process.env.SENCHO_MESH_SUBNET;
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
const overlap = Object.assign(
|
||||
new Error('Pool overlaps with other one on this address space'),
|
||||
{ statusCode: 500 },
|
||||
);
|
||||
const createNetwork = vi.fn()
|
||||
.mockRejectedValueOnce(overlap)
|
||||
.mockResolvedValueOnce(undefined);
|
||||
const inspectNetwork = vi.fn().mockRejectedValue({ statusCode: 404, message: 'no such network' });
|
||||
mockDocker({ createNetwork, inspectNetwork });
|
||||
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.ok).toBe(true);
|
||||
expect(status.subnet).toBe('172.31.0.0/24');
|
||||
expect(createNetwork).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('records subnet_overlap with every tried candidate when all candidates overlap', async () => {
|
||||
delete process.env.SENCHO_MESH_SUBNET;
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
const overlap = Object.assign(
|
||||
new Error('Pool overlaps with other one on this address space'),
|
||||
{ statusCode: 500 },
|
||||
);
|
||||
const createNetwork = vi.fn().mockRejectedValue(overlap);
|
||||
const inspectNetwork = vi.fn().mockRejectedValue({ statusCode: 404, message: 'no such network' });
|
||||
mockDocker({ createNetwork, inspectNetwork });
|
||||
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.ok).toBe(false);
|
||||
expect(status.reason).toBe('subnet_overlap');
|
||||
expect(createNetwork).toHaveBeenCalledTimes(4);
|
||||
expect(status.message).toContain('172.30.0.0/24');
|
||||
expect(status.message).toContain('172.31.0.0/24');
|
||||
expect(status.message).toContain('10.42.0.0/24');
|
||||
expect(status.message).toContain('10.43.0.0/24');
|
||||
expect(status.message).toMatch(/SENCHO_MESH_SUBNET/);
|
||||
});
|
||||
|
||||
it('adopts an existing sencho_mesh subnet when SENCHO_MESH_SUBNET is unset', async () => {
|
||||
delete process.env.SENCHO_MESH_SUBNET;
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
const createNetwork = vi.fn();
|
||||
const inspectNetwork = vi.fn().mockResolvedValue({
|
||||
IPAM: { Config: [{ Subnet: '192.168.42.0/24' }] },
|
||||
});
|
||||
mockDocker({ createNetwork, inspectNetwork });
|
||||
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.ok).toBe(true);
|
||||
expect(status.subnet).toBe('192.168.42.0/24');
|
||||
expect(createNetwork).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('classifies a non-404 inspectNetwork failure as attach_failed without trying to create', async () => {
|
||||
delete process.env.SENCHO_MESH_SUBNET;
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
const createNetwork = vi.fn();
|
||||
const inspectNetwork = vi.fn().mockRejectedValue(
|
||||
Object.assign(new Error('daemon unresponsive'), { statusCode: 500 }),
|
||||
);
|
||||
mockDocker({ createNetwork, inspectNetwork });
|
||||
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.ok).toBe(false);
|
||||
expect(status.reason).toBe('attach_failed');
|
||||
expect(createNetwork).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skips create when SENCHO_MESH_SUBNET matches the existing sencho_mesh subnet', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '172.30.0.0/24';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
const createNetwork = vi.fn();
|
||||
const inspectNetwork = vi.fn().mockResolvedValue({
|
||||
IPAM: { Config: [{ Subnet: '172.30.0.0/24' }] },
|
||||
});
|
||||
mockDocker({ createNetwork, inspectNetwork });
|
||||
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.ok).toBe(true);
|
||||
expect(status.subnet).toBe('172.30.0.0/24');
|
||||
expect(createNetwork).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps the operator-explicit path strict (no candidate fallback)', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
const overlap = Object.assign(
|
||||
new Error('Pool overlaps with other one on this address space'),
|
||||
{ statusCode: 500 },
|
||||
);
|
||||
const createNetwork = vi.fn().mockRejectedValue(overlap);
|
||||
const inspectNetwork = vi.fn().mockRejectedValue({ statusCode: 404, message: 'no such network' });
|
||||
mockDocker({ createNetwork, inspectNetwork });
|
||||
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.ok).toBe(false);
|
||||
expect(status.reason).toBe('subnet_overlap');
|
||||
expect(status.subnet).toBe('10.42.0.0/24');
|
||||
expect(createNetwork).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -27,6 +27,23 @@ const PROBE_TIMEOUT_MS = 5_000;
|
||||
const SLOW_PROBE_THRESHOLD_MS = 500;
|
||||
const DEFAULT_MESH_SUBNET = '172.30.0.0/24';
|
||||
|
||||
/**
|
||||
* Subnets attempted in order when SENCHO_MESH_SUBNET is unset and no
|
||||
* `sencho_mesh` network already exists. Each is a `/24` chosen to dodge the
|
||||
* usual homelab Docker patterns: `172.30.0.0/24` matches the prior default,
|
||||
* `172.31.0.0/24` sits one above it, and the `10.42`/`10.43` pair lands well
|
||||
* outside both the linuxserver/* `172.30.0.0/16` family and the typical
|
||||
* `192.168.x` LAN range. The first candidate that Docker accepts wins; the
|
||||
* chosen subnet persists implicitly through the `sencho_mesh` network on the
|
||||
* Docker daemon (next boot adopts it via the inspect path).
|
||||
*/
|
||||
export const MESH_SUBNET_CANDIDATES = [
|
||||
'172.30.0.0/24',
|
||||
'172.31.0.0/24',
|
||||
'10.42.0.0/24',
|
||||
'10.43.0.0/24',
|
||||
];
|
||||
|
||||
const REACHABLE_REASON: Record<DialFailureCode, string> = {
|
||||
auth_failed: 'api token rejected by remote',
|
||||
endpoint_not_found: 'remote does not support proxy mesh',
|
||||
@@ -404,12 +421,13 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
|
||||
const dpReason = this.dataPlaneStatus.reason;
|
||||
const dataPlane = this.senchoIp ? 'ok' : `unavailable (${dpReason}: ${this.networkSetupError ?? 'unknown'})`;
|
||||
const subnetSuffix = this.senchoIp ? `, subnet ${this.meshSubnet}` : '';
|
||||
const summaryLevel: MeshActivityLevel = dpReason === 'ok'
|
||||
? 'info'
|
||||
: dpReason === 'not_in_docker' ? 'warn' : 'error';
|
||||
this.logActivity({
|
||||
source: 'mesh', level: summaryLevel, type: 'mesh.enable',
|
||||
message: `MeshService started (data plane ${dataPlane}, self nodeId ${this.selfCentralNodeId})`,
|
||||
message: `MeshService started (data plane ${dataPlane}${subnetSuffix}, self nodeId ${this.selfCentralNodeId})`,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -581,28 +599,145 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
* Skipped entirely when Sencho is not running inside Docker (dev mode,
|
||||
* detected by an unset HOSTNAME env var or by the inspect lookup
|
||||
* failing). The forwarder still runs locally for unit-test coverage.
|
||||
*
|
||||
* Three subnet-resolution paths:
|
||||
* 1. **Operator-explicit.** `SENCHO_MESH_SUBNET` is set. Use exactly
|
||||
* that subnet; a pre-existing `sencho_mesh` with a different subnet
|
||||
* raises `subnet_mismatch`. Preserves the loud-config-error case.
|
||||
* 2. **Adopt-existing.** `SENCHO_MESH_SUBNET` is unset and
|
||||
* `sencho_mesh` already exists on the Docker daemon. Adopt its
|
||||
* subnet (Docker is the source of truth across restarts).
|
||||
* 3. **Candidate iteration.** Neither of the above. Walk
|
||||
* `MESH_SUBNET_CANDIDATES` in order; first subnet Docker accepts
|
||||
* wins. If every candidate overlaps an existing network, record
|
||||
* `subnet_overlap` with a message naming every attempted subnet.
|
||||
*/
|
||||
private async setupMeshNetwork(): Promise<void> {
|
||||
const subnet = (process.env.SENCHO_MESH_SUBNET || DEFAULT_MESH_SUBNET).trim();
|
||||
const envSubnet = process.env.SENCHO_MESH_SUBNET?.trim() || null;
|
||||
|
||||
// Validate the operator-supplied CIDR before any Docker call. An
|
||||
// invalid env var is the operator's problem, not the daemon's, and
|
||||
// reporting `subnet_invalid` from here means the diagnostic stays
|
||||
// accurate even if the daemon is also broken.
|
||||
if (envSubnet) {
|
||||
try {
|
||||
this.senchoIp = getSenchoIpFromSubnet(envSubnet);
|
||||
this.meshSubnet = envSubnet;
|
||||
} catch (err) {
|
||||
this.recordSetupFailure('subnet_invalid', err, 'error', envSubnet);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let existingSubnet: string | null;
|
||||
try {
|
||||
this.senchoIp = getSenchoIpFromSubnet(subnet);
|
||||
this.meshSubnet = subnet;
|
||||
existingSubnet = await this.inspectExistingMeshSubnet();
|
||||
} catch (err) {
|
||||
this.recordSetupFailure('subnet_invalid', err, 'error', subnet);
|
||||
// A genuinely broken Docker daemon (404s return null, see
|
||||
// `inspectExistingMeshSubnet`). Classify as `attach_failed`;
|
||||
// calling create would just hit the same error one layer down.
|
||||
this.recordSetupFailure(
|
||||
'attach_failed',
|
||||
err,
|
||||
'error',
|
||||
envSubnet ?? DEFAULT_MESH_SUBNET,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.ensureMeshNetwork(subnet);
|
||||
} catch (err) {
|
||||
this.recordSetupFailure(this.classifyMeshNetworkError(err), err, 'error', subnet);
|
||||
return;
|
||||
if (envSubnet) {
|
||||
if (existingSubnet && existingSubnet !== envSubnet) {
|
||||
this.recordSetupFailure(
|
||||
'subnet_mismatch',
|
||||
new Error(
|
||||
`${SENCHO_MESH_NETWORK} exists with subnet ${existingSubnet}, ` +
|
||||
`expected ${envSubnet}. Remove the network or set SENCHO_MESH_SUBNET to match.`,
|
||||
),
|
||||
'error',
|
||||
envSubnet,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!existingSubnet) {
|
||||
try {
|
||||
await this.createMeshNetwork(envSubnet);
|
||||
} catch (err) {
|
||||
this.recordSetupFailure(
|
||||
this.classifyMeshNetworkError(err),
|
||||
err,
|
||||
'error',
|
||||
envSubnet,
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} else if (existingSubnet) {
|
||||
try {
|
||||
this.senchoIp = getSenchoIpFromSubnet(existingSubnet);
|
||||
this.meshSubnet = existingSubnet;
|
||||
} catch (err) {
|
||||
this.recordSetupFailure('subnet_invalid', err, 'error', existingSubnet);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
const tried: string[] = [];
|
||||
let chosen: string | null = null;
|
||||
let lastErr: unknown = null;
|
||||
for (const candidate of MESH_SUBNET_CANDIDATES) {
|
||||
tried.push(candidate);
|
||||
try {
|
||||
await this.createMeshNetwork(candidate);
|
||||
chosen = candidate;
|
||||
break;
|
||||
} catch (err) {
|
||||
const cls = this.classifyMeshNetworkError(err);
|
||||
if (cls === 'subnet_overlap') {
|
||||
lastErr = err;
|
||||
continue;
|
||||
}
|
||||
// Non-overlap failures (e.g. daemon attach error) are not
|
||||
// helped by trying another candidate; bail with the typed
|
||||
// reason for this candidate. A 409 from another process
|
||||
// racing to create `sencho_mesh` between our inspect and
|
||||
// our create will classify as `attach_failed` here; the
|
||||
// race is rare enough that we accept the bail and let the
|
||||
// next process start adopt the now-existing network.
|
||||
this.recordSetupFailure(cls, err, 'error', candidate);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (!chosen) {
|
||||
this.recordSetupFailure(
|
||||
'subnet_overlap',
|
||||
new Error(
|
||||
`every candidate subnet overlaps an existing Docker network on this host ` +
|
||||
`(tried ${tried.join(', ')}). Set SENCHO_MESH_SUBNET to a free /24 and restart.` +
|
||||
(lastErr instanceof Error ? ` Last error: ${lastErr.message}` : ''),
|
||||
),
|
||||
'error',
|
||||
tried[tried.length - 1],
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.senchoIp = getSenchoIpFromSubnet(chosen);
|
||||
this.meshSubnet = chosen;
|
||||
} catch (err) {
|
||||
// Hard-coded candidates are well-formed; defensive only.
|
||||
this.recordSetupFailure('subnet_invalid', err, 'error', chosen);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await this.ensureSelfAttached();
|
||||
} catch (err) {
|
||||
this.recordSetupFailure(this.classifySelfAttachError(err), err, 'error', subnet);
|
||||
this.recordSetupFailure(
|
||||
this.classifySelfAttachError(err),
|
||||
err,
|
||||
'error',
|
||||
this.meshSubnet,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -614,40 +749,44 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
if (!this.senchoIp) return;
|
||||
|
||||
this.networkSetupError = null;
|
||||
this.dataPlaneStatus = { ok: true, reason: 'ok', message: null, subnet };
|
||||
this.dataPlaneStatus = { ok: true, reason: 'ok', message: null, subnet: this.meshSubnet };
|
||||
}
|
||||
|
||||
/**
|
||||
* Create `sencho_mesh` if it does not exist. If it does, validate the
|
||||
* subnet matches `expectedSubnet`; on mismatch, refuse to continue.
|
||||
* Silently using the wrong subnet would route traffic to the wrong IP.
|
||||
* Return the subnet of an existing `sencho_mesh` network, or null if the
|
||||
* network does not exist. Docker's inspect endpoint surfaces 404 for
|
||||
* the missing case; any other error is re-raised so the caller can
|
||||
* classify it as `attach_failed`.
|
||||
*/
|
||||
private async ensureMeshNetwork(expectedSubnet: string): Promise<void> {
|
||||
private async inspectExistingMeshSubnet(): Promise<string | null> {
|
||||
const dc = DockerController.getInstance(NodeRegistry.getInstance().getDefaultNodeId());
|
||||
try {
|
||||
await dc.createNetwork({
|
||||
Name: SENCHO_MESH_NETWORK,
|
||||
Driver: 'bridge',
|
||||
Attachable: true,
|
||||
IPAM: { Config: [{ Subnet: expectedSubnet }] },
|
||||
Labels: { 'io.sencho.mesh': 'true' },
|
||||
});
|
||||
return;
|
||||
const info = await dc.inspectNetwork(SENCHO_MESH_NETWORK) as {
|
||||
IPAM?: { Config?: Array<{ Subnet?: string }> };
|
||||
} | undefined;
|
||||
return info?.IPAM?.Config?.[0]?.Subnet ?? null;
|
||||
} catch (err) {
|
||||
const e = err as { statusCode?: number; message?: string };
|
||||
if (e?.statusCode !== 409) throw err;
|
||||
const e = err as { statusCode?: number };
|
||||
if (e?.statusCode === 404) return null;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const info = await dc.inspectNetwork(SENCHO_MESH_NETWORK) as {
|
||||
IPAM?: { Config?: Array<{ Subnet?: string }> };
|
||||
};
|
||||
const existingSubnet = info?.IPAM?.Config?.[0]?.Subnet;
|
||||
if (existingSubnet && existingSubnet !== expectedSubnet) {
|
||||
throw new Error(
|
||||
`${SENCHO_MESH_NETWORK} exists with subnet ${existingSubnet}, ` +
|
||||
`expected ${expectedSubnet}. Remove the network or set SENCHO_MESH_SUBNET to match.`,
|
||||
);
|
||||
}
|
||||
/**
|
||||
* Create the `sencho_mesh` bridge network with the given subnet. Throws
|
||||
* the raw Dockerode error (including the 500 pool-overlap that
|
||||
* `classifyMeshNetworkError` recognises) so callers can decide whether
|
||||
* to retry on another candidate or bail.
|
||||
*/
|
||||
private async createMeshNetwork(subnet: string): Promise<void> {
|
||||
const dc = DockerController.getInstance(NodeRegistry.getInstance().getDefaultNodeId());
|
||||
await dc.createNetwork({
|
||||
Name: SENCHO_MESH_NETWORK,
|
||||
Driver: 'bridge',
|
||||
Attachable: true,
|
||||
IPAM: { Config: [{ Subnet: subnet }] },
|
||||
Labels: { 'io.sencho.mesh': 'true' },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -202,15 +202,19 @@ These are the explicit boundaries of the v1 mesh.
|
||||
|
||||
<AccordionGroup>
|
||||
<Accordion title="Mesh data plane is down">
|
||||
The Routing tab shows a red banner when the local Sencho's `sencho_mesh` setup did not complete. The banner names the specific reason; the same reason appears in the mesh activity log and on `/api/health` as `mesh.dataPlane.reason`. The fix depends on which reason fired:
|
||||
The Routing tab shows a red banner when the local Sencho's `sencho_mesh` setup did not complete, and the dashboard's Fleet Heartbeat card surfaces the same failure as a compact status line. The reason also appears in the mesh activity log and on `/api/health` as `mesh.dataPlane.reason`.
|
||||
|
||||
- `subnet_overlap`: the requested CIDR overlaps another Docker bridge network on this host. Run `docker network ls -q | xargs -L1 docker network inspect --format '{{.Name}} {{range .IPAM.Config}}{{.Subnet}} {{end}}'` to list every existing subnet, then set `SENCHO_MESH_SUBNET` to a free `/24` (for example `10.42.0.0/24`) and recreate the Sencho container.
|
||||
- `subnet_mismatch`: `sencho_mesh` already exists with a different subnet. Either remove the network (`docker network rm sencho_mesh` after detaching any containers) or set `SENCHO_MESH_SUBNET` to match the existing subnet.
|
||||
**Subnet selection.** When `SENCHO_MESH_SUBNET` is unset, Sencho tries `172.30.0.0/24`, then `172.31.0.0/24`, then `10.42.0.0/24`, then `10.43.0.0/24` in order, and keeps the first subnet Docker accepts. If `sencho_mesh` already exists on the daemon, Sencho adopts that network's subnet without touching the candidate list. Set `SENCHO_MESH_SUBNET` only when you need to force a specific CIDR.
|
||||
|
||||
The fix depends on which reason fired:
|
||||
|
||||
- `subnet_overlap`: every candidate subnet overlaps an existing Docker bridge network on this host. Run `docker network ls -q | xargs -L1 docker network inspect --format '{{.Name}} {{range .IPAM.Config}}{{.Subnet}} {{end}}'` to list every existing subnet, then set `SENCHO_MESH_SUBNET` to a free `/24` outside the candidate list and recreate the Sencho container.
|
||||
- `subnet_mismatch`: `SENCHO_MESH_SUBNET` is set explicitly, but `sencho_mesh` already exists with a different subnet. Either remove the network (`docker network rm sencho_mesh` after detaching any containers) or change `SENCHO_MESH_SUBNET` to match the existing subnet. Unset the variable to let Sencho adopt whatever is on disk.
|
||||
- `subnet_invalid`: `SENCHO_MESH_SUBNET` is not a valid CIDR. Fix the value (it must look like `10.42.0.0/24`) and recreate the container.
|
||||
- `ip_in_use`: another container is squatting the IP Sencho wants on its mesh subnet. Find the squatting container with `docker network inspect sencho_mesh`, detach or remove it, then restart Sencho.
|
||||
- `attach_failed`: the Docker daemon refused the network attachment for a reason that does not match the patterns above. The full error appears in the activity log entry and on `/api/health`.
|
||||
|
||||
The `mesh.dataPlane.subnet` field shows the CIDR Sencho tried to use, so the operator can verify which subnet is configured before changing anything.
|
||||
The `mesh.dataPlane.subnet` field shows the CIDR Sencho settled on (or the last candidate it tried), so the operator can verify which subnet is configured before changing anything.
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Opt-in rejects with 'host-network service'">
|
||||
|
||||
@@ -2,7 +2,9 @@ import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
|
||||
import { Radio, CheckCircle2 } from 'lucide-react';
|
||||
import { formatRelativeTime } from '@/lib/utils';
|
||||
import { useFleetHeartbeat } from './useFleetHeartbeat';
|
||||
import { useMeshDataPlane } from './useMeshDataPlane';
|
||||
import { useNodes } from '@/context/NodeContext';
|
||||
import { MeshDataPlaneBanner } from '@/components/fleet/MeshDataPlaneBanner';
|
||||
import type { FleetNodeOverview } from './useFleetHeartbeat';
|
||||
import type { Node } from '@/context/NodeContext';
|
||||
|
||||
@@ -54,7 +56,11 @@ function SkeletonRow() {
|
||||
|
||||
export function FleetHeartbeat() {
|
||||
const { nodes: overviewNodes, loading, error } = useFleetHeartbeat();
|
||||
const { status: meshDataPlane } = useMeshDataPlane();
|
||||
const { nodes: contextNodes } = useNodes();
|
||||
const meshDown = meshDataPlane?.ok === false
|
||||
&& meshDataPlane.reason !== 'not_in_docker'
|
||||
&& meshDataPlane.reason !== 'not_started';
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
@@ -102,11 +108,15 @@ export function FleetHeartbeat() {
|
||||
{unreachableCount > 0 && (
|
||||
<span className="text-destructive"> · {unreachableCount} unreachable</span>
|
||||
)}
|
||||
{meshDown && (
|
||||
<span className="text-destructive"> · mesh down</span>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent className="pt-0">
|
||||
{meshDown && <MeshDataPlaneBanner status={meshDataPlane} variant="card" />}
|
||||
{sorted.length === 0 ? (
|
||||
<div className="flex items-center justify-center gap-2 py-6 text-stat-subtitle">
|
||||
<CheckCircle2 className="h-4 w-4 text-success" strokeWidth={1.5} />
|
||||
|
||||
@@ -9,6 +9,7 @@ export { StackRestartMap } from './StackRestartMap';
|
||||
export { useDashboardData } from './useDashboardData';
|
||||
export { useConfigurationStatus } from './useConfigurationStatus';
|
||||
export { useFleetHeartbeat } from './useFleetHeartbeat';
|
||||
export { useMeshDataPlane } from './useMeshDataPlane';
|
||||
export { useStackRestartMap } from './useStackRestartMap';
|
||||
export type * from './types';
|
||||
export type { ConfigurationStatus as ConfigurationStatusPayload } from './useConfigurationStatus';
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import { apiFetch } from '@/lib/api';
|
||||
import { useAuth } from '@/context/AuthContext';
|
||||
import { visibilityInterval } from '@/lib/utils';
|
||||
import type { MeshDataPlaneStatus } from '@/types/mesh';
|
||||
|
||||
export interface MeshDataPlaneResult {
|
||||
status: MeshDataPlaneStatus | null;
|
||||
loading: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll `/mesh/status` for the local data-plane health so dashboard surfaces
|
||||
* can flag a down mesh without opening the Routing tab. The endpoint is
|
||||
* Admiral-gated, so the hook short-circuits on non-Admiral tiers (no
|
||||
* request fired, no banner rendered). On the rare 403 from an Admiral
|
||||
* tier (token race during downgrade) we leave `status` at null. 30 s
|
||||
* cadence matches `useFleetHeartbeat` so the dashboard refresh feel is
|
||||
* consistent.
|
||||
*/
|
||||
export function useMeshDataPlane(): MeshDataPlaneResult {
|
||||
const { permissions } = useAuth();
|
||||
const isAdmiral = permissions?.isAdmiral ?? false;
|
||||
const [status, setStatus] = useState<MeshDataPlaneStatus | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
|
||||
const fetchStatus = useCallback(async () => {
|
||||
try {
|
||||
const res = await apiFetch('/mesh/status', { localOnly: true });
|
||||
if (res.status === 403) {
|
||||
setStatus(null);
|
||||
return;
|
||||
}
|
||||
if (!res.ok) return;
|
||||
const body = await res.json() as { localDataPlane?: MeshDataPlaneStatus };
|
||||
if (body.localDataPlane) setStatus(body.localDataPlane);
|
||||
} catch {
|
||||
// Background poll; transient network errors stay silent so the
|
||||
// card does not flicker on every refresh failure.
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isAdmiral) {
|
||||
setStatus(null);
|
||||
setLoading(false);
|
||||
return;
|
||||
}
|
||||
void fetchStatus();
|
||||
return visibilityInterval(() => { void fetchStatus(); }, 30_000);
|
||||
}, [isAdmiral, fetchStatus]);
|
||||
|
||||
return { status, loading };
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import { AlertTriangle } from 'lucide-react';
|
||||
import type { MeshDataPlaneStatus } from '@/types/mesh';
|
||||
|
||||
type Reason = MeshDataPlaneStatus['reason'];
|
||||
type ActionableReason = Exclude<Reason, 'ok' | 'not_started' | 'not_in_docker'>;
|
||||
|
||||
const HEADLINES: Record<ActionableReason, (status: MeshDataPlaneStatus) => string> = {
|
||||
subnet_invalid: () => 'SENCHO_MESH_SUBNET is not a valid CIDR.',
|
||||
subnet_overlap: (s) => `Mesh subnet ${s.subnet} overlaps an existing Docker network on this host.`,
|
||||
subnet_mismatch: (s) => `sencho_mesh already exists with a different subnet than ${s.subnet}.`,
|
||||
ip_in_use: (s) => `Another container is using Sencho's address on ${s.subnet}.`,
|
||||
attach_failed: () => 'Sencho could not attach to its own mesh network.',
|
||||
};
|
||||
|
||||
function isActionable(reason: Reason): reason is ActionableReason {
|
||||
return reason !== 'ok' && reason !== 'not_started' && reason !== 'not_in_docker';
|
||||
}
|
||||
|
||||
export type MeshDataPlaneBannerVariant = 'tab' | 'card';
|
||||
|
||||
/**
|
||||
* Surfaces the local mesh data-plane failure with the operator-actionable
|
||||
* recovery hint. Suppresses the dev-mode `not_in_docker` and the transient
|
||||
* `not_started` states so the operator only sees a banner when there is
|
||||
* something to fix.
|
||||
*
|
||||
* - `tab` variant: the full block shown at the top of the Routing tab.
|
||||
* - `card` variant: a single-line strip designed to sit inside a dashboard
|
||||
* card (e.g. Fleet Heartbeat).
|
||||
*/
|
||||
export function MeshDataPlaneBanner({
|
||||
status,
|
||||
variant = 'tab',
|
||||
}: {
|
||||
status: MeshDataPlaneStatus | null;
|
||||
variant?: MeshDataPlaneBannerVariant;
|
||||
}) {
|
||||
if (!status || status.ok || !isActionable(status.reason)) return null;
|
||||
const headline = HEADLINES[status.reason](status);
|
||||
|
||||
if (variant === 'card') {
|
||||
return (
|
||||
<div className="rounded-md border border-destructive/40 bg-destructive/10 px-2.5 py-1.5 mb-2 text-xs text-destructive flex items-start gap-2">
|
||||
<AlertTriangle className="h-3.5 w-3.5 mt-0.5 shrink-0" strokeWidth={1.75} />
|
||||
<div className="min-w-0 leading-relaxed">
|
||||
<span className="font-medium">Mesh data plane down · </span>
|
||||
<span className="font-mono text-[11px]">{status.reason}</span>
|
||||
<span> · </span>
|
||||
<span>{headline}</span>
|
||||
<span> Set </span>
|
||||
<code className="font-mono bg-destructive/15 px-1 rounded text-[10px]">SENCHO_MESH_SUBNET</code>
|
||||
<span> to a free </span>
|
||||
<code className="font-mono bg-destructive/15 px-1 rounded text-[10px]">/24</code>
|
||||
<span> and restart Sencho.</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="rounded-lg border border-destructive/40 bg-destructive/10 p-3 text-sm text-destructive flex items-start gap-3 shadow-card-bevel">
|
||||
<AlertTriangle className="h-4 w-4 mt-0.5 shrink-0" strokeWidth={1.75} />
|
||||
<div className="min-w-0 space-y-1">
|
||||
<div className="font-medium">Mesh data plane is down</div>
|
||||
<div className="text-xs leading-relaxed text-destructive/90">
|
||||
{headline}
|
||||
{' '}
|
||||
Set <code className="font-mono bg-destructive/15 px-1 py-0.5 rounded text-[11px]">SENCHO_MESH_SUBNET</code> to a free <code className="font-mono bg-destructive/15 px-1 py-0.5 rounded text-[11px]">/24</code> (for example <code className="font-mono bg-destructive/15 px-1 py-0.5 rounded text-[11px]">10.42.0.0/24</code>) and restart the Sencho container.
|
||||
</div>
|
||||
{status.message ? (
|
||||
<div className="text-[11px] font-mono text-destructive/80 truncate">{status.message}</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -3,7 +3,8 @@ import { apiFetch } from '@/lib/api';
|
||||
import { visibilityInterval } from '@/lib/utils';
|
||||
import { toast } from '@/components/ui/toast-store';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { AlertTriangle, ArrowLeftRight, Loader2, ScrollText, Table2, Network } from 'lucide-react';
|
||||
import { ArrowLeftRight, Loader2, ScrollText, Table2, Network } from 'lucide-react';
|
||||
import { MeshDataPlaneBanner } from './MeshDataPlaneBanner';
|
||||
import { RoutingNodeCard } from './RoutingNodeCard';
|
||||
import { MeshOptInSheet } from './MeshOptInSheet';
|
||||
import { MeshRouteDetailSheet } from './MeshRouteDetailSheet';
|
||||
@@ -158,7 +159,7 @@ export function RoutingTab() {
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<RoutingMasthead meshedNodes={meshedNodes} reachableNodes={reachableNodes} totalAliases={totalAliases} onShowActivity={() => setActivityOpen(true)} />
|
||||
<DataPlaneBanner status={localDataPlane} />
|
||||
<MeshDataPlaneBanner status={localDataPlane} variant="tab" />
|
||||
<div className="flex flex-col items-center justify-center py-12 rounded border border-dashed border-card-border bg-card/50">
|
||||
<ArrowLeftRight className="w-12 h-12 text-stat-subtitle mb-4" />
|
||||
<div className="text-lg font-display italic mb-2">Mesh containers across nodes</div>
|
||||
@@ -199,7 +200,7 @@ export function RoutingTab() {
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<RoutingMasthead meshedNodes={meshedNodes} reachableNodes={reachableNodes} totalAliases={totalAliases} onShowActivity={() => setActivityOpen(true)} />
|
||||
<DataPlaneBanner status={localDataPlane} />
|
||||
<MeshDataPlaneBanner status={localDataPlane} variant="tab" />
|
||||
<div className="flex flex-wrap items-center gap-3">
|
||||
<SegmentedControl<RoutingViewMode>
|
||||
value={viewMode}
|
||||
@@ -260,43 +261,6 @@ export function RoutingTab() {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Visible when the local Sencho's mesh data plane is down. Distinguishes the
|
||||
* common reasons (subnet conflict, IP-in-use, invalid CIDR) so the operator
|
||||
* does not have to dig through the activity log to know how to recover. The
|
||||
* `not_in_docker` warn-level case is intentionally suppressed: it is the
|
||||
* expected condition for dev-mode startup.
|
||||
*/
|
||||
function DataPlaneBanner({ status }: { status: MeshDataPlaneStatus | null }) {
|
||||
if (!status || status.ok || status.reason === 'not_in_docker' || status.reason === 'not_started') {
|
||||
return null;
|
||||
}
|
||||
const headlines: Record<Exclude<MeshDataPlaneStatus['reason'], 'ok' | 'not_started' | 'not_in_docker'>, string> = {
|
||||
subnet_invalid: 'SENCHO_MESH_SUBNET is not a valid CIDR.',
|
||||
subnet_overlap: `Mesh subnet ${status.subnet} overlaps another Docker network on this host.`,
|
||||
subnet_mismatch: `sencho_mesh already exists with a different subnet than ${status.subnet}.`,
|
||||
ip_in_use: `Another container is using Sencho's address on ${status.subnet}.`,
|
||||
attach_failed: 'Sencho could not attach to its own mesh network.',
|
||||
};
|
||||
const reason = status.reason as keyof typeof headlines;
|
||||
return (
|
||||
<div className="rounded-lg border border-destructive/40 bg-destructive/10 p-3 text-sm text-destructive flex items-start gap-3 shadow-card-bevel">
|
||||
<AlertTriangle className="h-4 w-4 mt-0.5 shrink-0" strokeWidth={1.75} />
|
||||
<div className="min-w-0 space-y-1">
|
||||
<div className="font-medium">Mesh data plane is down</div>
|
||||
<div className="text-xs leading-relaxed text-destructive/90">
|
||||
{headlines[reason] ?? 'Mesh setup did not complete.'}
|
||||
{' '}
|
||||
Set <code className="font-mono bg-destructive/15 px-1 py-0.5 rounded text-[11px]">SENCHO_MESH_SUBNET</code> to a free <code className="font-mono bg-destructive/15 px-1 py-0.5 rounded text-[11px]">/24</code> (for example <code className="font-mono bg-destructive/15 px-1 py-0.5 rounded text-[11px]">10.42.0.0/24</code>) and restart the Sencho container.
|
||||
</div>
|
||||
{status.message ? (
|
||||
<div className="text-[11px] font-mono text-destructive/80 truncate">{status.message}</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function RoutingMasthead({ meshedNodes, reachableNodes, totalAliases, onShowActivity }: {
|
||||
meshedNodes: number; reachableNodes: number; totalAliases: number; onShowActivity: () => void;
|
||||
}) {
|
||||
|
||||
Reference in New Issue
Block a user