fix(mesh): surface data-plane failures in health, meta, and Routing tab (#1088)

The three previously-silent console.warn paths in MeshService.setupMeshNetwork
now route through a typed recordSetupFailure helper that classifies the
failure (subnet_invalid, subnet_overlap, subnet_mismatch, ip_in_use,
attach_failed, not_in_docker), emits a mesh.disable activity entry at the
matching level (error for real failures, warn for the expected dev-mode
not_in_docker case), and strips mesh_proxy_callback_bootstrap from
advertised capabilities via CapabilityRegistry.

/api/health gains a mesh.dataPlane block carrying the typed status.
/api/mesh/status carries localDataPlane at the top level so the Routing tab
renders a red banner with an operator-actionable recovery hint (set
SENCHO_MESH_SUBNET to a free /24 and recreate the container) when the data
plane is down. The success path re-enables the capability and flips the
status to ok.

Generalizes the previously-documented F-0 failure mode (IP-in-subnet
collision) to also cover the subnet-pool-overlap case where another Docker
bridge on the host already owns the requested CIDR.
This commit is contained in:
Anso
2026-05-17 17:31:57 -04:00
committed by GitHub
parent f11e5ef58c
commit 578cac89da
9 changed files with 497 additions and 36 deletions
@@ -1,7 +1,17 @@
import { describe, it, expect } from 'vitest';
import { getActiveCapabilities, applyPilotModeCapabilityFilter, enableCapability } from '../services/CapabilityRegistry';
import { describe, it, expect, afterEach } from 'vitest';
import {
getActiveCapabilities,
applyPilotModeCapabilityFilter,
enableCapability,
disableCapability,
} from '../services/CapabilityRegistry';
describe('mesh_proxy_callback_bootstrap capability', () => {
afterEach(() => {
// Ensure the runtime override does not leak between tests.
enableCapability('mesh_proxy_callback_bootstrap');
});
it('is registered in the default CAPABILITIES list', () => {
expect(getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap');
});
@@ -12,4 +22,16 @@ describe('mesh_proxy_callback_bootstrap capability', () => {
enableCapability('host-console');
enableCapability('self-update');
});
it('is stripped from the active capabilities list when the data plane is disabled', () => {
disableCapability('mesh_proxy_callback_bootstrap');
expect(getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
});
it('returns to the advertised list once enableCapability is called again', () => {
disableCapability('mesh_proxy_callback_bootstrap');
expect(getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
enableCapability('mesh_proxy_callback_bootstrap');
expect(getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap');
});
});
+36
View File
@@ -38,6 +38,42 @@ describe('GET /api/health', () => {
expect(res.status).not.toBe(401);
expect(res.status).not.toBe(403);
});
it('reports mesh.dataPlane as a typed status object', async () => {
const res = await request(app).get('/api/health');
expect(res.body.mesh).toBeDefined();
expect(res.body.mesh.dataPlane).toBeDefined();
expect(typeof res.body.mesh.dataPlane.ok).toBe('boolean');
expect(typeof res.body.mesh.dataPlane.reason).toBe('string');
expect(typeof res.body.mesh.dataPlane.subnet).toBe('string');
// `message` is string or null
expect(['string', 'object']).toContain(typeof res.body.mesh.dataPlane.message);
});
it('reflects an injected data-plane failure', async () => {
const { MeshService } = await import('../services/MeshService');
const svc = MeshService.getInstance() as unknown as {
dataPlaneStatus: { ok: boolean; reason: string; message: string | null; subnet: string };
};
const prev = { ...svc.dataPlaneStatus };
svc.dataPlaneStatus = {
ok: false,
reason: 'subnet_overlap',
message: 'Pool overlaps with other one on this address space',
subnet: '172.30.0.0/24',
};
try {
const res = await request(app).get('/api/health');
expect(res.status).toBe(200);
expect(res.body.status).toBe('ok'); // process is up even when mesh is down
expect(res.body.mesh.dataPlane.ok).toBe(false);
expect(res.body.mesh.dataPlane.reason).toBe('subnet_overlap');
expect(res.body.mesh.dataPlane.subnet).toBe('172.30.0.0/24');
expect(res.body.mesh.dataPlane.message).toContain('overlap');
} finally {
svc.dataPlaneStatus = prev;
}
});
});
describe('GET /api/meta experimental flag', () => {
@@ -0,0 +1,218 @@
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import type { MeshActivityEvent, MeshDataPlaneStatus } from '../services/MeshService';
let tmpDir: string;
let MeshService: typeof import('../services/MeshService').MeshService;
let DockerController: typeof import('../services/DockerController').default;
let capability: typeof import('../services/CapabilityRegistry');
beforeAll(async () => {
tmpDir = await setupTestDb();
({ MeshService } = await import('../services/MeshService'));
({ default: DockerController } = await import('../services/DockerController'));
capability = await import('../services/CapabilityRegistry');
});
afterAll(() => {
cleanupTestDb(tmpDir);
});
type MutableSvc = {
activity: MeshActivityEvent[];
senchoIp: string | null;
meshSubnet: string;
networkSetupError: string | null;
dataPlaneStatus: MeshDataPlaneStatus;
};
let prevSubnetEnv: string | undefined;
let prevHostnameEnv: string | undefined;
beforeEach(() => {
prevSubnetEnv = process.env.SENCHO_MESH_SUBNET;
prevHostnameEnv = process.env.HOSTNAME;
const svc = MeshService.getInstance() as unknown as MutableSvc;
svc.activity = [];
svc.senchoIp = null;
svc.meshSubnet = '172.30.0.0/24';
svc.networkSetupError = null;
svc.dataPlaneStatus = {
ok: false,
reason: 'not_started',
message: 'mesh data plane has not initialized yet',
subnet: '',
};
capability.enableCapability('mesh_proxy_callback_bootstrap');
});
afterEach(() => {
if (prevSubnetEnv === undefined) delete process.env.SENCHO_MESH_SUBNET;
else process.env.SENCHO_MESH_SUBNET = prevSubnetEnv;
if (prevHostnameEnv === undefined) delete process.env.HOSTNAME;
else process.env.HOSTNAME = prevHostnameEnv;
capability.enableCapability('mesh_proxy_callback_bootstrap');
vi.restoreAllMocks();
});
function callSetup(svc: import('../services/MeshService').MeshService): Promise<void> {
return (svc as unknown as { setupMeshNetwork: () => Promise<void> }).setupMeshNetwork();
}
function lastDisable(svc: import('../services/MeshService').MeshService): MeshActivityEvent | undefined {
const all = (svc as unknown as MutableSvc).activity;
return all.filter((e) => e.type === 'mesh.disable').slice(-1)[0];
}
type FakeController = {
createNetwork: ReturnType<typeof vi.fn>;
inspectNetwork: ReturnType<typeof vi.fn>;
connectContainerToNetwork: ReturnType<typeof vi.fn>;
};
function mockDocker(overrides: Partial<FakeController> = {}): FakeController {
const fake: FakeController = {
createNetwork: vi.fn().mockResolvedValue(undefined),
inspectNetwork: vi.fn(),
connectContainerToNetwork: vi.fn().mockResolvedValue(undefined),
...overrides,
};
vi.spyOn(DockerController, 'getInstance').mockReturnValue(
fake as unknown as ReturnType<typeof DockerController.getInstance>,
);
return fake;
}
describe('MeshService.setupMeshNetwork failure classification', () => {
it('classifies an invalid CIDR as subnet_invalid', async () => {
process.env.SENCHO_MESH_SUBNET = 'not-a-cidr';
process.env.HOSTNAME = 'sencho';
mockDocker();
const svc = MeshService.getInstance();
await callSetup(svc);
const status = svc.getDataPlaneStatus();
expect(status.ok).toBe(false);
expect(status.reason).toBe('subnet_invalid');
expect(status.subnet).toBe('not-a-cidr');
expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
const entry = lastDisable(svc);
expect(entry?.level).toBe('error');
expect(entry?.details?.reason).toBe('subnet_invalid');
});
it('classifies a Docker pool-overlap error as subnet_overlap', async () => {
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
process.env.HOSTNAME = 'sencho';
mockDocker({
createNetwork: vi.fn().mockRejectedValue(
Object.assign(new Error('Pool overlaps with other one on this address space'), { statusCode: 500 }),
),
});
const svc = MeshService.getInstance();
await callSetup(svc);
const status = svc.getDataPlaneStatus();
expect(status.reason).toBe('subnet_overlap');
expect(status.subnet).toBe('10.42.0.0/24');
expect(status.message).toMatch(/overlap/i);
expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
const entry = lastDisable(svc);
expect(entry?.level).toBe('error');
expect(entry?.details?.reason).toBe('subnet_overlap');
});
it('classifies an existing-network mismatch as subnet_mismatch', async () => {
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
process.env.HOSTNAME = 'sencho';
mockDocker({
createNetwork: vi.fn().mockRejectedValue({ statusCode: 409, message: 'network already exists' }),
inspectNetwork: vi.fn().mockResolvedValue({ IPAM: { Config: [{ Subnet: '172.30.0.0/24' }] } }),
});
const svc = MeshService.getInstance();
await callSetup(svc);
const status = svc.getDataPlaneStatus();
expect(status.reason).toBe('subnet_mismatch');
expect(status.message).toMatch(/exists with subnet/i);
});
it('classifies an address-already-in-use attach error as ip_in_use', async () => {
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
process.env.HOSTNAME = 'sencho';
mockDocker({
connectContainerToNetwork: vi.fn().mockRejectedValue(new Error('Address already in use')),
});
const svc = MeshService.getInstance();
await callSetup(svc);
expect(svc.getDataPlaneStatus().reason).toBe('ip_in_use');
expect(lastDisable(svc)?.level).toBe('error');
});
it('classifies a generic attach failure as attach_failed', async () => {
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
process.env.HOSTNAME = 'sencho';
mockDocker({
connectContainerToNetwork: vi.fn().mockRejectedValue(new Error('Docker daemon explosion')),
});
const svc = MeshService.getInstance();
await callSetup(svc);
expect(svc.getDataPlaneStatus().reason).toBe('attach_failed');
});
it('records the HOSTNAME-unset path as not_in_docker at level warn', async () => {
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
delete process.env.HOSTNAME;
mockDocker();
const svc = MeshService.getInstance();
await callSetup(svc);
const status = svc.getDataPlaneStatus();
expect(status.ok).toBe(false);
expect(status.reason).toBe('not_in_docker');
const entry = lastDisable(svc);
expect(entry?.level).toBe('warn');
expect(entry?.details?.reason).toBe('not_in_docker');
// Capability is still stripped even in the warn-level not_in_docker
// case; pilot processes outside Docker must not advertise it.
expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
});
it('records the 404-on-inspect path as not_in_docker at level warn', async () => {
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
process.env.HOSTNAME = 'laptop-hostname';
mockDocker({
connectContainerToNetwork: vi.fn().mockRejectedValue({ statusCode: 404, message: 'no such container' }),
});
const svc = MeshService.getInstance();
await callSetup(svc);
const status = svc.getDataPlaneStatus();
expect(status.reason).toBe('not_in_docker');
expect(lastDisable(svc)?.level).toBe('warn');
});
it('records success as ok and re-enables the capability', async () => {
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
process.env.HOSTNAME = 'sencho';
// Force the capability off first to prove the success path flips it back on.
capability.disableCapability('mesh_proxy_callback_bootstrap');
mockDocker();
const svc = MeshService.getInstance();
await callSetup(svc);
const status = svc.getDataPlaneStatus();
expect(status.ok).toBe(true);
expect(status.reason).toBe('ok');
expect(status.subnet).toBe('10.42.0.0/24');
expect(status.message).toBeNull();
expect(capability.getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap');
});
it('preserves the legacy networkSetupError getter on failure', async () => {
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
process.env.HOSTNAME = 'sencho';
mockDocker({
createNetwork: vi.fn().mockRejectedValue(
Object.assign(new Error('Pool overlaps with other one on this address space'), { statusCode: 500 }),
),
});
const svc = MeshService.getInstance();
await callSetup(svc);
expect(svc.getNetworkSetupError()).toMatch(/overlap/i);
});
});
+3 -2
View File
@@ -16,8 +16,9 @@ function actorFor(req: Request): string {
meshRouter.get('/status', async (_req: Request, res: Response): Promise<void> => {
if (!requireAdmiral(_req, res)) return;
try {
const status = await MeshService.getInstance().getStatus();
res.json({ nodes: status });
const mesh = MeshService.getInstance();
const status = await mesh.getStatus();
res.json({ nodes: status, localDataPlane: mesh.getDataPlaneStatus() });
} catch (err) {
console.warn('[mesh] /status failed:', sanitizeForLog((err as Error).message));
res.status(500).json({ error: 'Failed to load mesh status' });
+13 -1
View File
@@ -1,5 +1,6 @@
import { Router, type Request, type Response } from 'express';
import { getActiveCapabilities, getSenchoVersion } from '../services/CapabilityRegistry';
import { MeshService } from '../services/MeshService';
import SelfUpdateService from '../services/SelfUpdateService';
// Captured at boot. Exposed via /api/health and /api/meta so the Fleet update
@@ -9,8 +10,19 @@ const processStartedAt = Date.now();
export const metaRouter = Router();
// Public health endpoint (no auth). Used by Docker HEALTHCHECK and uptime monitors.
// The `mesh.dataPlane` block reports whether `MeshService.setupMeshNetwork`
// completed successfully; an `ok: false` value means cross-node mesh routing
// is disabled on this node and the operator should consult the activity log
// or set `SENCHO_MESH_SUBNET` to a free /24 and restart the container.
metaRouter.get('/health', (_req: Request, res: Response): void => {
res.json({ status: 'ok', uptime: process.uptime(), startedAt: processStartedAt });
res.json({
status: 'ok',
uptime: process.uptime(),
startedAt: processStartedAt,
mesh: {
dataPlane: MeshService.getInstance().getDataPlaneStatus(),
},
});
});
// Public meta endpoint. Returns this instance's version and supported
+133 -19
View File
@@ -14,6 +14,7 @@ import { NodeRegistry } from './NodeRegistry';
import { PilotTunnelManager } from './PilotTunnelManager';
import { MeshProxyTunnelDialer, type DialFailureCode } from './MeshProxyTunnelDialer';
import { generateOverrideYaml, MeshAlias, SENCHO_MESH_NETWORK } from './MeshComposeOverride';
import { disableCapability, enableCapability } from './CapabilityRegistry';
import { lookupContainerIp } from '../mesh/containerLookup';
import { sanitizeForLog } from '../utils/safeLog';
import { isPathWithinBase, isValidStackName } from '../utils/validation';
@@ -61,6 +62,29 @@ export function getSenchoIpFromSubnet(subnet: string): string {
].join('.');
}
/**
* Discriminator for why the mesh data plane is or is not healthy. Set by
* `setupMeshNetwork` and exposed through `getDataPlaneStatus()` so
* `/api/health` and `/api/meta` can surface the state without parsing the
* raw error string.
*/
export type MeshDataPlaneReason =
| 'ok'
| 'not_started' // MeshService.start() has not finished setupMeshNetwork yet
| 'subnet_invalid' // SENCHO_MESH_SUBNET did not parse
| 'subnet_overlap' // Docker refused the IPAM pool, another network owns the CIDR
| 'subnet_mismatch' // sencho_mesh already exists with a different subnet
| 'ip_in_use' // another container squats <network>+2
| 'attach_failed' // self-attach failed for any other reason
| 'not_in_docker'; // HOSTNAME unset or self-container lookup returned 404
export interface MeshDataPlaneStatus {
ok: boolean;
reason: MeshDataPlaneReason;
message: string | null;
subnet: string;
}
export type MeshActivitySource = 'pilot' | 'mesh';
export type MeshActivityLevel = 'info' | 'warn' | 'error';
export type MeshActivityType =
@@ -228,6 +252,17 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
private senchoIp: string | null = null;
private meshSubnet: string = DEFAULT_MESH_SUBNET;
private networkSetupError: string | null = null;
// Discriminator-typed mirror of networkSetupError. Both stay in sync via
// `recordSetupFailure` / the setupMeshNetwork success path. The discriminator
// is consumed by /api/health and the Routing tab; networkSetupError is
// preserved for callers that already read the raw error string (optInStack,
// applyLocalOverride, regenerateAllOverrides).
private dataPlaneStatus: MeshDataPlaneStatus = {
ok: false,
reason: 'not_started',
message: 'mesh data plane has not initialized yet',
subnet: '',
};
// On a pilot node, central's DB id for this node (e.g. 14). Used by
// handleAccept to decide same-node vs cross-node; the pilotAliasOverlay
// carries nodeIds from central's perspective, so comparing against the
@@ -335,9 +370,13 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
})();
}, ALIAS_REFRESH_INTERVAL_MS);
const dataPlane = this.senchoIp ? 'ok' : `unavailable (${this.networkSetupError ?? 'unknown'})`;
const dpReason = this.dataPlaneStatus.reason;
const dataPlane = this.senchoIp ? 'ok' : `unavailable (${dpReason}: ${this.networkSetupError ?? 'unknown'})`;
const summaryLevel: MeshActivityLevel = dpReason === 'ok'
? 'info'
: dpReason === 'not_in_docker' ? 'warn' : 'error';
this.logActivity({
source: 'mesh', level: this.senchoIp ? 'info' : 'warn', type: 'mesh.enable',
source: 'mesh', level: summaryLevel, type: 'mesh.enable',
message: `MeshService started (data plane ${dataPlane}, self nodeId ${this.selfCentralNodeId})`,
});
}
@@ -429,6 +468,74 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
return this.networkSetupError;
}
/**
* Typed mirror of `networkSetupError` for consumers that need a
* discriminator (e.g. `/api/health` and the Routing tab). Always returns
* a value: `{ ok: true, reason: 'ok' }` once `setupMeshNetwork` completes
* successfully, otherwise a typed failure shape.
*/
public getDataPlaneStatus(): MeshDataPlaneStatus {
return this.dataPlaneStatus;
}
/**
* Single recording path for every mesh-setup failure. Keeps the legacy
* `networkSetupError` string in sync, sets the typed `dataPlaneStatus`,
* strips `mesh_proxy_callback_bootstrap` from advertised capabilities,
* and emits a `mesh.disable` activity entry. Callers pass `level: 'warn'`
* for expected conditions (`not_in_docker` in dev mode) and `'error'` for
* real failures.
*/
private recordSetupFailure(
reason: Exclude<MeshDataPlaneReason, 'ok' | 'not_started'>,
err: unknown,
level: MeshActivityLevel,
// The subnet_invalid path fires before `this.meshSubnet` is assigned,
// so callers must pass the value they tried explicitly; deriving from
// the field would silently report DEFAULT_MESH_SUBNET instead of the
// bad CIDR the operator actually configured.
subnet: string,
): void {
const message = err instanceof Error ? err.message : String(err);
this.networkSetupError = message;
this.dataPlaneStatus = { ok: false, reason, message, subnet };
this.senchoIp = null;
disableCapability('mesh_proxy_callback_bootstrap');
this.logActivity({
source: 'mesh',
level,
type: 'mesh.disable',
message: `mesh data plane unavailable (${reason}): ${sanitizeForLog(message)}`,
details: { reason, subnet },
});
}
/**
* Classify a throw from `ensureMeshNetwork` into a typed reason by matching
* on the error message. Docker's "pool overlaps with other one on this
* address space" surfaces as a 500 when another bridge owns the requested
* CIDR; the subnet-mismatch error is thrown synchronously from
* `ensureMeshNetwork` itself and contains the literal "exists with subnet".
*/
private classifyMeshNetworkError(err: unknown): 'subnet_overlap' | 'subnet_mismatch' | 'attach_failed' {
const m = err instanceof Error ? err.message : String(err);
if (/overlap/i.test(m)) return 'subnet_overlap';
if (/exists with subnet/i.test(m)) return 'subnet_mismatch';
return 'attach_failed';
}
/**
* Classify a throw from `ensureSelfAttached` (after its non-throwing
* not-in-Docker paths). Docker's "Address already in use" /
* "no available addresses" come back when another container squats
* `<network>+2`; anything else is a generic attach failure.
*/
private classifySelfAttachError(err: unknown): 'ip_in_use' | 'attach_failed' {
const m = err instanceof Error ? err.message : String(err);
if (/already in use|no available addresses|address already/i.test(m)) return 'ip_in_use';
return 'attach_failed';
}
/**
* Idempotent setup of the shared `sencho_mesh` Docker bridge network and
* Sencho's static attachment to it. Called once at boot before alias
@@ -446,31 +553,34 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
this.senchoIp = getSenchoIpFromSubnet(subnet);
this.meshSubnet = subnet;
} catch (err) {
this.networkSetupError = (err as Error).message;
console.warn('[Mesh]', this.networkSetupError);
this.senchoIp = null;
this.recordSetupFailure('subnet_invalid', err, 'error', subnet);
return;
}
try {
await this.ensureMeshNetwork(subnet);
} catch (err) {
this.networkSetupError = (err as Error).message;
console.warn('[Mesh] mesh network setup failed:', sanitizeForLog(this.networkSetupError));
this.senchoIp = null;
this.recordSetupFailure(this.classifyMeshNetworkError(err), err, 'error', subnet);
return;
}
try {
await this.ensureSelfAttached();
} catch (err) {
this.networkSetupError = (err as Error).message;
console.warn('[Mesh] self-attach failed:', sanitizeForLog(this.networkSetupError));
this.senchoIp = null;
this.recordSetupFailure(this.classifySelfAttachError(err), err, 'error', subnet);
return;
}
// `ensureSelfAttached` has non-throwing paths for the not-in-Docker
// case (HOSTNAME unset / inspect 404). Those paths call
// `recordSetupFailure` directly and leave `senchoIp` null, so a
// null here means the data plane is intentionally disabled (dev
// mode), not that the success path should run.
if (!this.senchoIp) return;
this.networkSetupError = null;
this.dataPlaneStatus = { ok: true, reason: 'ok', message: null, subnet };
enableCapability('mesh_proxy_callback_bootstrap');
}
/**
@@ -517,8 +627,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
if (!this.senchoIp) return;
const hostname = process.env.HOSTNAME;
if (!hostname) {
console.log('[Mesh] HOSTNAME not set, mesh routing disabled (not running in Docker?)');
this.senchoIp = null;
this.recordSetupFailure(
'not_in_docker',
new Error('HOSTNAME unset; mesh routing disabled (not running in Docker?)'),
'warn',
this.meshSubnet,
);
return;
}
const dc = DockerController.getInstance(NodeRegistry.getInstance().getDefaultNodeId());
@@ -527,12 +641,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
} catch (err) {
const e = err as { statusCode?: number; message?: string };
if (e?.statusCode === 404) {
this.logActivity({
source: 'mesh', level: 'warn', type: 'mesh.disable',
message: 'self-container lookup failed; mesh routing disabled (not running in Docker?)',
});
console.warn('[Mesh] self-container lookup failed; mesh routing disabled (not running in Docker?)');
this.senchoIp = null;
this.recordSetupFailure(
'not_in_docker',
new Error('self-container lookup failed (404); mesh routing disabled (not running in Docker?)'),
'warn',
this.meshSubnet,
);
return;
}
throw err;
+9 -9
View File
@@ -138,12 +138,16 @@ A few things are deliberately out of scope for the first release:
## Troubleshooting
<AccordionGroup>
<Accordion title="Adding a stack rejects with 'mesh data plane unavailable'">
The Sencho instance failed to set up its `sencho_mesh` Docker network at boot. Check the Sencho container's logs for `[Mesh]` messages. Common causes: Sencho is running outside Docker (dev mode, mesh routing skipped); the Docker socket is not mounted; `sencho_mesh` already exists with a different subnet from a previous install. The mesh status surface on the Routing tab shows the specific reason.
</Accordion>
<Accordion title="Mesh data plane is down">
The Routing tab shows a red banner when this Sencho's `sencho_mesh` setup did not complete. The banner names the specific reason; the same reason appears in the mesh activity log and on `/api/health` as `mesh.dataPlane.reason`. The fix depends on which reason fired:
<Accordion title="`sencho_mesh` exists with a different subnet">
Sencho refuses to start the mesh data plane if a network named `sencho_mesh` exists with a subnet different from the one this Sencho is configured for. Either remove the existing network (`docker network rm sencho_mesh` on the affected host, after detaching any containers) or set `SENCHO_MESH_SUBNET` on this node to match the existing subnet. Restart the Sencho container so the setup runs again.
- `subnet_overlap`: the requested CIDR overlaps another Docker bridge network on this host. Run `docker network ls -q | xargs -L1 docker network inspect --format '{{.Name}} {{range .IPAM.Config}}{{.Subnet}} {{end}}'` to list every existing subnet, then set `SENCHO_MESH_SUBNET` to a free `/24` (for example `10.42.0.0/24`) and recreate the Sencho container.
- `subnet_mismatch`: `sencho_mesh` already exists with a different subnet. Either remove the network (`docker network rm sencho_mesh` after detaching any containers) or set `SENCHO_MESH_SUBNET` to match the existing subnet.
- `subnet_invalid`: `SENCHO_MESH_SUBNET` is not a valid CIDR. Fix the value (it must look like `10.42.0.0/24`) and recreate the container.
- `ip_in_use`: another container is squatting the IP Sencho wants on its mesh subnet. Find the squatting container with `docker network inspect sencho_mesh`, detach or remove it, then restart Sencho.
- `attach_failed`: the Docker daemon refused the network attachment for a reason that does not match the patterns above. The full error appears in the activity log entry and on `/api/health`.
The `mesh.dataPlane.subnet` field shows the CIDR Sencho tried to use, so the operator can verify which subnet is configured before changing anything.
</Accordion>
<Accordion title="Opt-in rejects with 'host-network service'">
@@ -166,10 +170,6 @@ A few things are deliberately out of scope for the first release:
Mesh redeploys the stack on opt-in to refresh hostnames and network attachments. A stuck redeploy usually means the stack itself failed to come back up. Check the stack's deploy logs.
</Accordion>
<Accordion title="Subnet conflict with another network on the host">
`172.30.0.0/24` is the default range for the `sencho_mesh` network and is rare in most setups. If it collides with an existing VPN, VLAN, or Docker network, set `SENCHO_MESH_SUBNET` on the Sencho service to a free `/24` and restart the container. Each node can be configured independently.
</Accordion>
<Accordion title="A Distributed API node shows `unreachable` on the Routing tab">
Central could not open a mesh tunnel to this node. The badge tooltip shows the specific reason. While a node is in this state the **mesh toggle and Add stack to mesh action on the node card are disabled** so a redeploy is not triggered against an unreachable target. Common causes:
+44 -3
View File
@@ -3,7 +3,7 @@ import { apiFetch } from '@/lib/api';
import { visibilityInterval } from '@/lib/utils';
import { toast } from '@/components/ui/toast-store';
import { Button } from '@/components/ui/button';
import { ArrowLeftRight, Loader2, ScrollText, Table2, Network } from 'lucide-react';
import { AlertTriangle, ArrowLeftRight, Loader2, ScrollText, Table2, Network } from 'lucide-react';
import { RoutingNodeCard } from './RoutingNodeCard';
import { MeshOptInSheet } from './MeshOptInSheet';
import { MeshRouteDetailSheet } from './MeshRouteDetailSheet';
@@ -12,7 +12,7 @@ import { MeshActivitySheet } from './MeshActivitySheet';
import { MeshTopologyGraph, type MeshGraphEdgeMode } from './MeshTopologyGraph';
import { MeshStackTopologySheet } from './MeshStackTopologySheet';
import { SegmentedControl } from '@/components/ui/segmented-control';
import type { MeshAlias, MeshNodeStatus, MeshProbeResult } from '@/types/mesh';
import type { MeshAlias, MeshDataPlaneStatus, MeshNodeStatus, MeshProbeResult } from '@/types/mesh';
type RoutingViewMode = 'table' | 'graph';
@@ -46,6 +46,7 @@ function readStoredEdgeMode(): MeshGraphEdgeMode {
export function RoutingTab() {
const [status, setStatus] = useState<MeshNodeStatus[]>([]);
const [localDataPlane, setLocalDataPlane] = useState<MeshDataPlaneStatus | null>(null);
const [aliases, setAliases] = useState<MeshAlias[]>([]);
const [loading, setLoading] = useState(true);
const [optInNode, setOptInNode] = useState<{ id: number; name: string } | null>(null);
@@ -63,8 +64,9 @@ export function RoutingTab() {
apiFetch('/mesh/aliases', { localOnly: true }),
]);
if (statusRes.ok) {
const body = await statusRes.json() as { nodes: MeshNodeStatus[] };
const body = await statusRes.json() as { nodes: MeshNodeStatus[]; localDataPlane?: MeshDataPlaneStatus };
setStatus(body.nodes);
if (body.localDataPlane) setLocalDataPlane(body.localDataPlane);
}
if (aliasesRes.ok) {
const body = await aliasesRes.json() as { aliases: MeshAlias[] };
@@ -156,6 +158,7 @@ export function RoutingTab() {
return (
<div className="space-y-4">
<RoutingMasthead meshedNodes={meshedNodes} reachableNodes={reachableNodes} totalAliases={totalAliases} onShowActivity={() => setActivityOpen(true)} />
<DataPlaneBanner status={localDataPlane} />
<div className="flex flex-col items-center justify-center py-12 rounded border border-dashed border-card-border bg-card/50">
<ArrowLeftRight className="w-12 h-12 text-stat-subtitle mb-4" />
<div className="text-lg font-display italic mb-2">Mesh containers across nodes</div>
@@ -196,6 +199,7 @@ export function RoutingTab() {
return (
<div className="space-y-4">
<RoutingMasthead meshedNodes={meshedNodes} reachableNodes={reachableNodes} totalAliases={totalAliases} onShowActivity={() => setActivityOpen(true)} />
<DataPlaneBanner status={localDataPlane} />
<div className="flex flex-wrap items-center gap-3">
<SegmentedControl<RoutingViewMode>
value={viewMode}
@@ -256,6 +260,43 @@ export function RoutingTab() {
);
}
/**
* Visible when the local Sencho's mesh data plane is down. Distinguishes the
* common reasons (subnet conflict, IP-in-use, invalid CIDR) so the operator
* does not have to dig through the activity log to know how to recover. The
* `not_in_docker` warn-level case is intentionally suppressed: it is the
* expected condition for dev-mode startup.
*/
function DataPlaneBanner({ status }: { status: MeshDataPlaneStatus | null }) {
if (!status || status.ok || status.reason === 'not_in_docker' || status.reason === 'not_started') {
return null;
}
const headlines: Record<Exclude<MeshDataPlaneStatus['reason'], 'ok' | 'not_started' | 'not_in_docker'>, string> = {
subnet_invalid: 'SENCHO_MESH_SUBNET is not a valid CIDR.',
subnet_overlap: `Mesh subnet ${status.subnet} overlaps another Docker network on this host.`,
subnet_mismatch: `sencho_mesh already exists with a different subnet than ${status.subnet}.`,
ip_in_use: `Another container is using Sencho's address on ${status.subnet}.`,
attach_failed: 'Sencho could not attach to its own mesh network.',
};
const reason = status.reason as keyof typeof headlines;
return (
<div className="rounded-lg border border-destructive/40 bg-destructive/10 p-3 text-sm text-destructive flex items-start gap-3 shadow-card-bevel">
<AlertTriangle className="h-4 w-4 mt-0.5 shrink-0" strokeWidth={1.75} />
<div className="min-w-0 space-y-1">
<div className="font-medium">Mesh data plane is down</div>
<div className="text-xs leading-relaxed text-destructive/90">
{headlines[reason] ?? 'Mesh setup did not complete.'}
{' '}
Set <code className="font-mono bg-destructive/15 px-1 py-0.5 rounded text-[11px]">SENCHO_MESH_SUBNET</code> to a free <code className="font-mono bg-destructive/15 px-1 py-0.5 rounded text-[11px]">/24</code> (for example <code className="font-mono bg-destructive/15 px-1 py-0.5 rounded text-[11px]">10.42.0.0/24</code>) and restart the Sencho container.
</div>
{status.message ? (
<div className="text-[11px] font-mono text-destructive/80 truncate">{status.message}</div>
) : null}
</div>
</div>
);
}
function RoutingMasthead({ meshedNodes, reachableNodes, totalAliases, onShowActivity }: {
meshedNodes: number; reachableNodes: number; totalAliases: number; onShowActivity: () => void;
}) {
+17
View File
@@ -1,5 +1,22 @@
export type MeshRoutePillState = 'healthy' | 'degraded' | 'unreachable' | 'tunnel-down' | 'not-authorized';
export type MeshDataPlaneReason =
| 'ok'
| 'not_started'
| 'subnet_invalid'
| 'subnet_overlap'
| 'subnet_mismatch'
| 'ip_in_use'
| 'attach_failed'
| 'not_in_docker';
export interface MeshDataPlaneStatus {
ok: boolean;
reason: MeshDataPlaneReason;
message: string | null;
subnet: string;
}
export interface MeshAlias {
host: string;
nodeId: number;