mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-07-26 11:49:16 +00:00
fix(mesh): surface data-plane failures in health, meta, and Routing tab (#1088)
The three previously-silent console.warn paths in MeshService.setupMeshNetwork now route through a typed recordSetupFailure helper that classifies the failure (subnet_invalid, subnet_overlap, subnet_mismatch, ip_in_use, attach_failed, not_in_docker), emits a mesh.disable activity entry at the matching level (error for real failures, warn for the expected dev-mode not_in_docker case), and strips mesh_proxy_callback_bootstrap from advertised capabilities via CapabilityRegistry. /api/health gains a mesh.dataPlane block carrying the typed status. /api/mesh/status carries localDataPlane at the top level so the Routing tab renders a red banner with an operator-actionable recovery hint (set SENCHO_MESH_SUBNET to a free /24 and recreate the container) when the data plane is down. The success path re-enables the capability and flips the status to ok. Generalizes the previously-documented F-0 failure mode (IP-in-subnet collision) to also cover the subnet-pool-overlap case where another Docker bridge on the host already owns the requested CIDR.
This commit is contained in:
@@ -1,7 +1,17 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { getActiveCapabilities, applyPilotModeCapabilityFilter, enableCapability } from '../services/CapabilityRegistry';
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import {
|
||||
getActiveCapabilities,
|
||||
applyPilotModeCapabilityFilter,
|
||||
enableCapability,
|
||||
disableCapability,
|
||||
} from '../services/CapabilityRegistry';
|
||||
|
||||
describe('mesh_proxy_callback_bootstrap capability', () => {
|
||||
afterEach(() => {
|
||||
// Ensure the runtime override does not leak between tests.
|
||||
enableCapability('mesh_proxy_callback_bootstrap');
|
||||
});
|
||||
|
||||
it('is registered in the default CAPABILITIES list', () => {
|
||||
expect(getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap');
|
||||
});
|
||||
@@ -12,4 +22,16 @@ describe('mesh_proxy_callback_bootstrap capability', () => {
|
||||
enableCapability('host-console');
|
||||
enableCapability('self-update');
|
||||
});
|
||||
|
||||
it('is stripped from the active capabilities list when the data plane is disabled', () => {
|
||||
disableCapability('mesh_proxy_callback_bootstrap');
|
||||
expect(getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
|
||||
});
|
||||
|
||||
it('returns to the advertised list once enableCapability is called again', () => {
|
||||
disableCapability('mesh_proxy_callback_bootstrap');
|
||||
expect(getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
|
||||
enableCapability('mesh_proxy_callback_bootstrap');
|
||||
expect(getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -38,6 +38,42 @@ describe('GET /api/health', () => {
|
||||
expect(res.status).not.toBe(401);
|
||||
expect(res.status).not.toBe(403);
|
||||
});
|
||||
|
||||
it('reports mesh.dataPlane as a typed status object', async () => {
|
||||
const res = await request(app).get('/api/health');
|
||||
expect(res.body.mesh).toBeDefined();
|
||||
expect(res.body.mesh.dataPlane).toBeDefined();
|
||||
expect(typeof res.body.mesh.dataPlane.ok).toBe('boolean');
|
||||
expect(typeof res.body.mesh.dataPlane.reason).toBe('string');
|
||||
expect(typeof res.body.mesh.dataPlane.subnet).toBe('string');
|
||||
// `message` is string or null
|
||||
expect(['string', 'object']).toContain(typeof res.body.mesh.dataPlane.message);
|
||||
});
|
||||
|
||||
it('reflects an injected data-plane failure', async () => {
|
||||
const { MeshService } = await import('../services/MeshService');
|
||||
const svc = MeshService.getInstance() as unknown as {
|
||||
dataPlaneStatus: { ok: boolean; reason: string; message: string | null; subnet: string };
|
||||
};
|
||||
const prev = { ...svc.dataPlaneStatus };
|
||||
svc.dataPlaneStatus = {
|
||||
ok: false,
|
||||
reason: 'subnet_overlap',
|
||||
message: 'Pool overlaps with other one on this address space',
|
||||
subnet: '172.30.0.0/24',
|
||||
};
|
||||
try {
|
||||
const res = await request(app).get('/api/health');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.status).toBe('ok'); // process is up even when mesh is down
|
||||
expect(res.body.mesh.dataPlane.ok).toBe(false);
|
||||
expect(res.body.mesh.dataPlane.reason).toBe('subnet_overlap');
|
||||
expect(res.body.mesh.dataPlane.subnet).toBe('172.30.0.0/24');
|
||||
expect(res.body.mesh.dataPlane.message).toContain('overlap');
|
||||
} finally {
|
||||
svc.dataPlaneStatus = prev;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/meta experimental flag', () => {
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
import type { MeshActivityEvent, MeshDataPlaneStatus } from '../services/MeshService';
|
||||
|
||||
let tmpDir: string;
|
||||
let MeshService: typeof import('../services/MeshService').MeshService;
|
||||
let DockerController: typeof import('../services/DockerController').default;
|
||||
let capability: typeof import('../services/CapabilityRegistry');
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ MeshService } = await import('../services/MeshService'));
|
||||
({ default: DockerController } = await import('../services/DockerController'));
|
||||
capability = await import('../services/CapabilityRegistry');
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
type MutableSvc = {
|
||||
activity: MeshActivityEvent[];
|
||||
senchoIp: string | null;
|
||||
meshSubnet: string;
|
||||
networkSetupError: string | null;
|
||||
dataPlaneStatus: MeshDataPlaneStatus;
|
||||
};
|
||||
|
||||
let prevSubnetEnv: string | undefined;
|
||||
let prevHostnameEnv: string | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
prevSubnetEnv = process.env.SENCHO_MESH_SUBNET;
|
||||
prevHostnameEnv = process.env.HOSTNAME;
|
||||
const svc = MeshService.getInstance() as unknown as MutableSvc;
|
||||
svc.activity = [];
|
||||
svc.senchoIp = null;
|
||||
svc.meshSubnet = '172.30.0.0/24';
|
||||
svc.networkSetupError = null;
|
||||
svc.dataPlaneStatus = {
|
||||
ok: false,
|
||||
reason: 'not_started',
|
||||
message: 'mesh data plane has not initialized yet',
|
||||
subnet: '',
|
||||
};
|
||||
capability.enableCapability('mesh_proxy_callback_bootstrap');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (prevSubnetEnv === undefined) delete process.env.SENCHO_MESH_SUBNET;
|
||||
else process.env.SENCHO_MESH_SUBNET = prevSubnetEnv;
|
||||
if (prevHostnameEnv === undefined) delete process.env.HOSTNAME;
|
||||
else process.env.HOSTNAME = prevHostnameEnv;
|
||||
capability.enableCapability('mesh_proxy_callback_bootstrap');
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function callSetup(svc: import('../services/MeshService').MeshService): Promise<void> {
|
||||
return (svc as unknown as { setupMeshNetwork: () => Promise<void> }).setupMeshNetwork();
|
||||
}
|
||||
|
||||
function lastDisable(svc: import('../services/MeshService').MeshService): MeshActivityEvent | undefined {
|
||||
const all = (svc as unknown as MutableSvc).activity;
|
||||
return all.filter((e) => e.type === 'mesh.disable').slice(-1)[0];
|
||||
}
|
||||
|
||||
type FakeController = {
|
||||
createNetwork: ReturnType<typeof vi.fn>;
|
||||
inspectNetwork: ReturnType<typeof vi.fn>;
|
||||
connectContainerToNetwork: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
|
||||
function mockDocker(overrides: Partial<FakeController> = {}): FakeController {
|
||||
const fake: FakeController = {
|
||||
createNetwork: vi.fn().mockResolvedValue(undefined),
|
||||
inspectNetwork: vi.fn(),
|
||||
connectContainerToNetwork: vi.fn().mockResolvedValue(undefined),
|
||||
...overrides,
|
||||
};
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue(
|
||||
fake as unknown as ReturnType<typeof DockerController.getInstance>,
|
||||
);
|
||||
return fake;
|
||||
}
|
||||
|
||||
describe('MeshService.setupMeshNetwork failure classification', () => {
|
||||
it('classifies an invalid CIDR as subnet_invalid', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = 'not-a-cidr';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
mockDocker();
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.ok).toBe(false);
|
||||
expect(status.reason).toBe('subnet_invalid');
|
||||
expect(status.subnet).toBe('not-a-cidr');
|
||||
expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
|
||||
const entry = lastDisable(svc);
|
||||
expect(entry?.level).toBe('error');
|
||||
expect(entry?.details?.reason).toBe('subnet_invalid');
|
||||
});
|
||||
|
||||
it('classifies a Docker pool-overlap error as subnet_overlap', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
mockDocker({
|
||||
createNetwork: vi.fn().mockRejectedValue(
|
||||
Object.assign(new Error('Pool overlaps with other one on this address space'), { statusCode: 500 }),
|
||||
),
|
||||
});
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.reason).toBe('subnet_overlap');
|
||||
expect(status.subnet).toBe('10.42.0.0/24');
|
||||
expect(status.message).toMatch(/overlap/i);
|
||||
expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
|
||||
const entry = lastDisable(svc);
|
||||
expect(entry?.level).toBe('error');
|
||||
expect(entry?.details?.reason).toBe('subnet_overlap');
|
||||
});
|
||||
|
||||
it('classifies an existing-network mismatch as subnet_mismatch', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
mockDocker({
|
||||
createNetwork: vi.fn().mockRejectedValue({ statusCode: 409, message: 'network already exists' }),
|
||||
inspectNetwork: vi.fn().mockResolvedValue({ IPAM: { Config: [{ Subnet: '172.30.0.0/24' }] } }),
|
||||
});
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.reason).toBe('subnet_mismatch');
|
||||
expect(status.message).toMatch(/exists with subnet/i);
|
||||
});
|
||||
|
||||
it('classifies an address-already-in-use attach error as ip_in_use', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
mockDocker({
|
||||
connectContainerToNetwork: vi.fn().mockRejectedValue(new Error('Address already in use')),
|
||||
});
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
expect(svc.getDataPlaneStatus().reason).toBe('ip_in_use');
|
||||
expect(lastDisable(svc)?.level).toBe('error');
|
||||
});
|
||||
|
||||
it('classifies a generic attach failure as attach_failed', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
mockDocker({
|
||||
connectContainerToNetwork: vi.fn().mockRejectedValue(new Error('Docker daemon explosion')),
|
||||
});
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
expect(svc.getDataPlaneStatus().reason).toBe('attach_failed');
|
||||
});
|
||||
|
||||
it('records the HOSTNAME-unset path as not_in_docker at level warn', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
delete process.env.HOSTNAME;
|
||||
mockDocker();
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.ok).toBe(false);
|
||||
expect(status.reason).toBe('not_in_docker');
|
||||
const entry = lastDisable(svc);
|
||||
expect(entry?.level).toBe('warn');
|
||||
expect(entry?.details?.reason).toBe('not_in_docker');
|
||||
// Capability is still stripped even in the warn-level not_in_docker
|
||||
// case; pilot processes outside Docker must not advertise it.
|
||||
expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
|
||||
});
|
||||
|
||||
it('records the 404-on-inspect path as not_in_docker at level warn', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
process.env.HOSTNAME = 'laptop-hostname';
|
||||
mockDocker({
|
||||
connectContainerToNetwork: vi.fn().mockRejectedValue({ statusCode: 404, message: 'no such container' }),
|
||||
});
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.reason).toBe('not_in_docker');
|
||||
expect(lastDisable(svc)?.level).toBe('warn');
|
||||
});
|
||||
|
||||
it('records success as ok and re-enables the capability', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
// Force the capability off first to prove the success path flips it back on.
|
||||
capability.disableCapability('mesh_proxy_callback_bootstrap');
|
||||
mockDocker();
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.ok).toBe(true);
|
||||
expect(status.reason).toBe('ok');
|
||||
expect(status.subnet).toBe('10.42.0.0/24');
|
||||
expect(status.message).toBeNull();
|
||||
expect(capability.getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap');
|
||||
});
|
||||
|
||||
it('preserves the legacy networkSetupError getter on failure', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
mockDocker({
|
||||
createNetwork: vi.fn().mockRejectedValue(
|
||||
Object.assign(new Error('Pool overlaps with other one on this address space'), { statusCode: 500 }),
|
||||
),
|
||||
});
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
expect(svc.getNetworkSetupError()).toMatch(/overlap/i);
|
||||
});
|
||||
});
|
||||
@@ -16,8 +16,9 @@ function actorFor(req: Request): string {
|
||||
meshRouter.get('/status', async (_req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(_req, res)) return;
|
||||
try {
|
||||
const status = await MeshService.getInstance().getStatus();
|
||||
res.json({ nodes: status });
|
||||
const mesh = MeshService.getInstance();
|
||||
const status = await mesh.getStatus();
|
||||
res.json({ nodes: status, localDataPlane: mesh.getDataPlaneStatus() });
|
||||
} catch (err) {
|
||||
console.warn('[mesh] /status failed:', sanitizeForLog((err as Error).message));
|
||||
res.status(500).json({ error: 'Failed to load mesh status' });
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { getActiveCapabilities, getSenchoVersion } from '../services/CapabilityRegistry';
|
||||
import { MeshService } from '../services/MeshService';
|
||||
import SelfUpdateService from '../services/SelfUpdateService';
|
||||
|
||||
// Captured at boot. Exposed via /api/health and /api/meta so the Fleet update
|
||||
@@ -9,8 +10,19 @@ const processStartedAt = Date.now();
|
||||
export const metaRouter = Router();
|
||||
|
||||
// Public health endpoint (no auth). Used by Docker HEALTHCHECK and uptime monitors.
|
||||
// The `mesh.dataPlane` block reports whether `MeshService.setupMeshNetwork`
|
||||
// completed successfully; an `ok: false` value means cross-node mesh routing
|
||||
// is disabled on this node and the operator should consult the activity log
|
||||
// or set `SENCHO_MESH_SUBNET` to a free /24 and restart the container.
|
||||
metaRouter.get('/health', (_req: Request, res: Response): void => {
|
||||
res.json({ status: 'ok', uptime: process.uptime(), startedAt: processStartedAt });
|
||||
res.json({
|
||||
status: 'ok',
|
||||
uptime: process.uptime(),
|
||||
startedAt: processStartedAt,
|
||||
mesh: {
|
||||
dataPlane: MeshService.getInstance().getDataPlaneStatus(),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
// Public meta endpoint. Returns this instance's version and supported
|
||||
|
||||
@@ -14,6 +14,7 @@ import { NodeRegistry } from './NodeRegistry';
|
||||
import { PilotTunnelManager } from './PilotTunnelManager';
|
||||
import { MeshProxyTunnelDialer, type DialFailureCode } from './MeshProxyTunnelDialer';
|
||||
import { generateOverrideYaml, MeshAlias, SENCHO_MESH_NETWORK } from './MeshComposeOverride';
|
||||
import { disableCapability, enableCapability } from './CapabilityRegistry';
|
||||
import { lookupContainerIp } from '../mesh/containerLookup';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { isPathWithinBase, isValidStackName } from '../utils/validation';
|
||||
@@ -61,6 +62,29 @@ export function getSenchoIpFromSubnet(subnet: string): string {
|
||||
].join('.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Discriminator for why the mesh data plane is or is not healthy. Set by
|
||||
* `setupMeshNetwork` and exposed through `getDataPlaneStatus()` so
|
||||
* `/api/health` and `/api/meta` can surface the state without parsing the
|
||||
* raw error string.
|
||||
*/
|
||||
export type MeshDataPlaneReason =
|
||||
| 'ok'
|
||||
| 'not_started' // MeshService.start() has not finished setupMeshNetwork yet
|
||||
| 'subnet_invalid' // SENCHO_MESH_SUBNET did not parse
|
||||
| 'subnet_overlap' // Docker refused the IPAM pool, another network owns the CIDR
|
||||
| 'subnet_mismatch' // sencho_mesh already exists with a different subnet
|
||||
| 'ip_in_use' // another container squats <network>+2
|
||||
| 'attach_failed' // self-attach failed for any other reason
|
||||
| 'not_in_docker'; // HOSTNAME unset or self-container lookup returned 404
|
||||
|
||||
export interface MeshDataPlaneStatus {
|
||||
ok: boolean;
|
||||
reason: MeshDataPlaneReason;
|
||||
message: string | null;
|
||||
subnet: string;
|
||||
}
|
||||
|
||||
export type MeshActivitySource = 'pilot' | 'mesh';
|
||||
export type MeshActivityLevel = 'info' | 'warn' | 'error';
|
||||
export type MeshActivityType =
|
||||
@@ -228,6 +252,17 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
private senchoIp: string | null = null;
|
||||
private meshSubnet: string = DEFAULT_MESH_SUBNET;
|
||||
private networkSetupError: string | null = null;
|
||||
// Discriminator-typed mirror of networkSetupError. Both stay in sync via
|
||||
// `recordSetupFailure` / the setupMeshNetwork success path. The discriminator
|
||||
// is consumed by /api/health and the Routing tab; networkSetupError is
|
||||
// preserved for callers that already read the raw error string (optInStack,
|
||||
// applyLocalOverride, regenerateAllOverrides).
|
||||
private dataPlaneStatus: MeshDataPlaneStatus = {
|
||||
ok: false,
|
||||
reason: 'not_started',
|
||||
message: 'mesh data plane has not initialized yet',
|
||||
subnet: '',
|
||||
};
|
||||
// On a pilot node, central's DB id for this node (e.g. 14). Used by
|
||||
// handleAccept to decide same-node vs cross-node; the pilotAliasOverlay
|
||||
// carries nodeIds from central's perspective, so comparing against the
|
||||
@@ -335,9 +370,13 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
})();
|
||||
}, ALIAS_REFRESH_INTERVAL_MS);
|
||||
|
||||
const dataPlane = this.senchoIp ? 'ok' : `unavailable (${this.networkSetupError ?? 'unknown'})`;
|
||||
const dpReason = this.dataPlaneStatus.reason;
|
||||
const dataPlane = this.senchoIp ? 'ok' : `unavailable (${dpReason}: ${this.networkSetupError ?? 'unknown'})`;
|
||||
const summaryLevel: MeshActivityLevel = dpReason === 'ok'
|
||||
? 'info'
|
||||
: dpReason === 'not_in_docker' ? 'warn' : 'error';
|
||||
this.logActivity({
|
||||
source: 'mesh', level: this.senchoIp ? 'info' : 'warn', type: 'mesh.enable',
|
||||
source: 'mesh', level: summaryLevel, type: 'mesh.enable',
|
||||
message: `MeshService started (data plane ${dataPlane}, self nodeId ${this.selfCentralNodeId})`,
|
||||
});
|
||||
}
|
||||
@@ -429,6 +468,74 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
return this.networkSetupError;
|
||||
}
|
||||
|
||||
/**
|
||||
* Typed mirror of `networkSetupError` for consumers that need a
|
||||
* discriminator (e.g. `/api/health` and the Routing tab). Always returns
|
||||
* a value: `{ ok: true, reason: 'ok' }` once `setupMeshNetwork` completes
|
||||
* successfully, otherwise a typed failure shape.
|
||||
*/
|
||||
public getDataPlaneStatus(): MeshDataPlaneStatus {
|
||||
return this.dataPlaneStatus;
|
||||
}
|
||||
|
||||
/**
|
||||
* Single recording path for every mesh-setup failure. Keeps the legacy
|
||||
* `networkSetupError` string in sync, sets the typed `dataPlaneStatus`,
|
||||
* strips `mesh_proxy_callback_bootstrap` from advertised capabilities,
|
||||
* and emits a `mesh.disable` activity entry. Callers pass `level: 'warn'`
|
||||
* for expected conditions (`not_in_docker` in dev mode) and `'error'` for
|
||||
* real failures.
|
||||
*/
|
||||
private recordSetupFailure(
|
||||
reason: Exclude<MeshDataPlaneReason, 'ok' | 'not_started'>,
|
||||
err: unknown,
|
||||
level: MeshActivityLevel,
|
||||
// The subnet_invalid path fires before `this.meshSubnet` is assigned,
|
||||
// so callers must pass the value they tried explicitly; deriving from
|
||||
// the field would silently report DEFAULT_MESH_SUBNET instead of the
|
||||
// bad CIDR the operator actually configured.
|
||||
subnet: string,
|
||||
): void {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
this.networkSetupError = message;
|
||||
this.dataPlaneStatus = { ok: false, reason, message, subnet };
|
||||
this.senchoIp = null;
|
||||
disableCapability('mesh_proxy_callback_bootstrap');
|
||||
this.logActivity({
|
||||
source: 'mesh',
|
||||
level,
|
||||
type: 'mesh.disable',
|
||||
message: `mesh data plane unavailable (${reason}): ${sanitizeForLog(message)}`,
|
||||
details: { reason, subnet },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a throw from `ensureMeshNetwork` into a typed reason by matching
|
||||
* on the error message. Docker's "pool overlaps with other one on this
|
||||
* address space" surfaces as a 500 when another bridge owns the requested
|
||||
* CIDR; the subnet-mismatch error is thrown synchronously from
|
||||
* `ensureMeshNetwork` itself and contains the literal "exists with subnet".
|
||||
*/
|
||||
private classifyMeshNetworkError(err: unknown): 'subnet_overlap' | 'subnet_mismatch' | 'attach_failed' {
|
||||
const m = err instanceof Error ? err.message : String(err);
|
||||
if (/overlap/i.test(m)) return 'subnet_overlap';
|
||||
if (/exists with subnet/i.test(m)) return 'subnet_mismatch';
|
||||
return 'attach_failed';
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a throw from `ensureSelfAttached` (after its non-throwing
|
||||
* not-in-Docker paths). Docker's "Address already in use" /
|
||||
* "no available addresses" come back when another container squats
|
||||
* `<network>+2`; anything else is a generic attach failure.
|
||||
*/
|
||||
private classifySelfAttachError(err: unknown): 'ip_in_use' | 'attach_failed' {
|
||||
const m = err instanceof Error ? err.message : String(err);
|
||||
if (/already in use|no available addresses|address already/i.test(m)) return 'ip_in_use';
|
||||
return 'attach_failed';
|
||||
}
|
||||
|
||||
/**
|
||||
* Idempotent setup of the shared `sencho_mesh` Docker bridge network and
|
||||
* Sencho's static attachment to it. Called once at boot before alias
|
||||
@@ -446,31 +553,34 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
this.senchoIp = getSenchoIpFromSubnet(subnet);
|
||||
this.meshSubnet = subnet;
|
||||
} catch (err) {
|
||||
this.networkSetupError = (err as Error).message;
|
||||
console.warn('[Mesh]', this.networkSetupError);
|
||||
this.senchoIp = null;
|
||||
this.recordSetupFailure('subnet_invalid', err, 'error', subnet);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.ensureMeshNetwork(subnet);
|
||||
} catch (err) {
|
||||
this.networkSetupError = (err as Error).message;
|
||||
console.warn('[Mesh] mesh network setup failed:', sanitizeForLog(this.networkSetupError));
|
||||
this.senchoIp = null;
|
||||
this.recordSetupFailure(this.classifyMeshNetworkError(err), err, 'error', subnet);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.ensureSelfAttached();
|
||||
} catch (err) {
|
||||
this.networkSetupError = (err as Error).message;
|
||||
console.warn('[Mesh] self-attach failed:', sanitizeForLog(this.networkSetupError));
|
||||
this.senchoIp = null;
|
||||
this.recordSetupFailure(this.classifySelfAttachError(err), err, 'error', subnet);
|
||||
return;
|
||||
}
|
||||
|
||||
// `ensureSelfAttached` has non-throwing paths for the not-in-Docker
|
||||
// case (HOSTNAME unset / inspect 404). Those paths call
|
||||
// `recordSetupFailure` directly and leave `senchoIp` null, so a
|
||||
// null here means the data plane is intentionally disabled (dev
|
||||
// mode), not that the success path should run.
|
||||
if (!this.senchoIp) return;
|
||||
|
||||
this.networkSetupError = null;
|
||||
this.dataPlaneStatus = { ok: true, reason: 'ok', message: null, subnet };
|
||||
enableCapability('mesh_proxy_callback_bootstrap');
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -517,8 +627,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
if (!this.senchoIp) return;
|
||||
const hostname = process.env.HOSTNAME;
|
||||
if (!hostname) {
|
||||
console.log('[Mesh] HOSTNAME not set, mesh routing disabled (not running in Docker?)');
|
||||
this.senchoIp = null;
|
||||
this.recordSetupFailure(
|
||||
'not_in_docker',
|
||||
new Error('HOSTNAME unset; mesh routing disabled (not running in Docker?)'),
|
||||
'warn',
|
||||
this.meshSubnet,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const dc = DockerController.getInstance(NodeRegistry.getInstance().getDefaultNodeId());
|
||||
@@ -527,12 +641,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
} catch (err) {
|
||||
const e = err as { statusCode?: number; message?: string };
|
||||
if (e?.statusCode === 404) {
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'warn', type: 'mesh.disable',
|
||||
message: 'self-container lookup failed; mesh routing disabled (not running in Docker?)',
|
||||
});
|
||||
console.warn('[Mesh] self-container lookup failed; mesh routing disabled (not running in Docker?)');
|
||||
this.senchoIp = null;
|
||||
this.recordSetupFailure(
|
||||
'not_in_docker',
|
||||
new Error('self-container lookup failed (404); mesh routing disabled (not running in Docker?)'),
|
||||
'warn',
|
||||
this.meshSubnet,
|
||||
);
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
|
||||
@@ -138,12 +138,16 @@ A few things are deliberately out of scope for the first release:
|
||||
## Troubleshooting
|
||||
|
||||
<AccordionGroup>
|
||||
<Accordion title="Adding a stack rejects with 'mesh data plane unavailable'">
|
||||
The Sencho instance failed to set up its `sencho_mesh` Docker network at boot. Check the Sencho container's logs for `[Mesh]` messages. Common causes: Sencho is running outside Docker (dev mode, mesh routing skipped); the Docker socket is not mounted; `sencho_mesh` already exists with a different subnet from a previous install. The mesh status surface on the Routing tab shows the specific reason.
|
||||
</Accordion>
|
||||
<Accordion title="Mesh data plane is down">
|
||||
The Routing tab shows a red banner when this Sencho's `sencho_mesh` setup did not complete. The banner names the specific reason; the same reason appears in the mesh activity log and on `/api/health` as `mesh.dataPlane.reason`. The fix depends on which reason fired:
|
||||
|
||||
<Accordion title="`sencho_mesh` exists with a different subnet">
|
||||
Sencho refuses to start the mesh data plane if a network named `sencho_mesh` exists with a subnet different from the one this Sencho is configured for. Either remove the existing network (`docker network rm sencho_mesh` on the affected host, after detaching any containers) or set `SENCHO_MESH_SUBNET` on this node to match the existing subnet. Restart the Sencho container so the setup runs again.
|
||||
- `subnet_overlap`: the requested CIDR overlaps another Docker bridge network on this host. Run `docker network ls -q | xargs -L1 docker network inspect --format '{{.Name}} {{range .IPAM.Config}}{{.Subnet}} {{end}}'` to list every existing subnet, then set `SENCHO_MESH_SUBNET` to a free `/24` (for example `10.42.0.0/24`) and recreate the Sencho container.
|
||||
- `subnet_mismatch`: `sencho_mesh` already exists with a different subnet. Either remove the network (`docker network rm sencho_mesh` after detaching any containers) or set `SENCHO_MESH_SUBNET` to match the existing subnet.
|
||||
- `subnet_invalid`: `SENCHO_MESH_SUBNET` is not a valid CIDR. Fix the value (it must look like `10.42.0.0/24`) and recreate the container.
|
||||
- `ip_in_use`: another container is squatting the IP Sencho wants on its mesh subnet. Find the squatting container with `docker network inspect sencho_mesh`, detach or remove it, then restart Sencho.
|
||||
- `attach_failed`: the Docker daemon refused the network attachment for a reason that does not match the patterns above. The full error appears in the activity log entry and on `/api/health`.
|
||||
|
||||
The `mesh.dataPlane.subnet` field shows the CIDR Sencho tried to use, so the operator can verify which subnet is configured before changing anything.
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Opt-in rejects with 'host-network service'">
|
||||
@@ -166,10 +170,6 @@ A few things are deliberately out of scope for the first release:
|
||||
Mesh redeploys the stack on opt-in to refresh hostnames and network attachments. A stuck redeploy usually means the stack itself failed to come back up. Check the stack's deploy logs.
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Subnet conflict with another network on the host">
|
||||
`172.30.0.0/24` is the default range for the `sencho_mesh` network and is rare in most setups. If it collides with an existing VPN, VLAN, or Docker network, set `SENCHO_MESH_SUBNET` on the Sencho service to a free `/24` and restart the container. Each node can be configured independently.
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="A Distributed API node shows `unreachable` on the Routing tab">
|
||||
Central could not open a mesh tunnel to this node. The badge tooltip shows the specific reason. While a node is in this state the **mesh toggle and Add stack to mesh action on the node card are disabled** so a redeploy is not triggered against an unreachable target. Common causes:
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import { apiFetch } from '@/lib/api';
|
||||
import { visibilityInterval } from '@/lib/utils';
|
||||
import { toast } from '@/components/ui/toast-store';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { ArrowLeftRight, Loader2, ScrollText, Table2, Network } from 'lucide-react';
|
||||
import { AlertTriangle, ArrowLeftRight, Loader2, ScrollText, Table2, Network } from 'lucide-react';
|
||||
import { RoutingNodeCard } from './RoutingNodeCard';
|
||||
import { MeshOptInSheet } from './MeshOptInSheet';
|
||||
import { MeshRouteDetailSheet } from './MeshRouteDetailSheet';
|
||||
@@ -12,7 +12,7 @@ import { MeshActivitySheet } from './MeshActivitySheet';
|
||||
import { MeshTopologyGraph, type MeshGraphEdgeMode } from './MeshTopologyGraph';
|
||||
import { MeshStackTopologySheet } from './MeshStackTopologySheet';
|
||||
import { SegmentedControl } from '@/components/ui/segmented-control';
|
||||
import type { MeshAlias, MeshNodeStatus, MeshProbeResult } from '@/types/mesh';
|
||||
import type { MeshAlias, MeshDataPlaneStatus, MeshNodeStatus, MeshProbeResult } from '@/types/mesh';
|
||||
|
||||
type RoutingViewMode = 'table' | 'graph';
|
||||
|
||||
@@ -46,6 +46,7 @@ function readStoredEdgeMode(): MeshGraphEdgeMode {
|
||||
|
||||
export function RoutingTab() {
|
||||
const [status, setStatus] = useState<MeshNodeStatus[]>([]);
|
||||
const [localDataPlane, setLocalDataPlane] = useState<MeshDataPlaneStatus | null>(null);
|
||||
const [aliases, setAliases] = useState<MeshAlias[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [optInNode, setOptInNode] = useState<{ id: number; name: string } | null>(null);
|
||||
@@ -63,8 +64,9 @@ export function RoutingTab() {
|
||||
apiFetch('/mesh/aliases', { localOnly: true }),
|
||||
]);
|
||||
if (statusRes.ok) {
|
||||
const body = await statusRes.json() as { nodes: MeshNodeStatus[] };
|
||||
const body = await statusRes.json() as { nodes: MeshNodeStatus[]; localDataPlane?: MeshDataPlaneStatus };
|
||||
setStatus(body.nodes);
|
||||
if (body.localDataPlane) setLocalDataPlane(body.localDataPlane);
|
||||
}
|
||||
if (aliasesRes.ok) {
|
||||
const body = await aliasesRes.json() as { aliases: MeshAlias[] };
|
||||
@@ -156,6 +158,7 @@ export function RoutingTab() {
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<RoutingMasthead meshedNodes={meshedNodes} reachableNodes={reachableNodes} totalAliases={totalAliases} onShowActivity={() => setActivityOpen(true)} />
|
||||
<DataPlaneBanner status={localDataPlane} />
|
||||
<div className="flex flex-col items-center justify-center py-12 rounded border border-dashed border-card-border bg-card/50">
|
||||
<ArrowLeftRight className="w-12 h-12 text-stat-subtitle mb-4" />
|
||||
<div className="text-lg font-display italic mb-2">Mesh containers across nodes</div>
|
||||
@@ -196,6 +199,7 @@ export function RoutingTab() {
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<RoutingMasthead meshedNodes={meshedNodes} reachableNodes={reachableNodes} totalAliases={totalAliases} onShowActivity={() => setActivityOpen(true)} />
|
||||
<DataPlaneBanner status={localDataPlane} />
|
||||
<div className="flex flex-wrap items-center gap-3">
|
||||
<SegmentedControl<RoutingViewMode>
|
||||
value={viewMode}
|
||||
@@ -256,6 +260,43 @@ export function RoutingTab() {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Visible when the local Sencho's mesh data plane is down. Distinguishes the
|
||||
* common reasons (subnet conflict, IP-in-use, invalid CIDR) so the operator
|
||||
* does not have to dig through the activity log to know how to recover. The
|
||||
* `not_in_docker` warn-level case is intentionally suppressed: it is the
|
||||
* expected condition for dev-mode startup.
|
||||
*/
|
||||
function DataPlaneBanner({ status }: { status: MeshDataPlaneStatus | null }) {
|
||||
if (!status || status.ok || status.reason === 'not_in_docker' || status.reason === 'not_started') {
|
||||
return null;
|
||||
}
|
||||
const headlines: Record<Exclude<MeshDataPlaneStatus['reason'], 'ok' | 'not_started' | 'not_in_docker'>, string> = {
|
||||
subnet_invalid: 'SENCHO_MESH_SUBNET is not a valid CIDR.',
|
||||
subnet_overlap: `Mesh subnet ${status.subnet} overlaps another Docker network on this host.`,
|
||||
subnet_mismatch: `sencho_mesh already exists with a different subnet than ${status.subnet}.`,
|
||||
ip_in_use: `Another container is using Sencho's address on ${status.subnet}.`,
|
||||
attach_failed: 'Sencho could not attach to its own mesh network.',
|
||||
};
|
||||
const reason = status.reason as keyof typeof headlines;
|
||||
return (
|
||||
<div className="rounded-lg border border-destructive/40 bg-destructive/10 p-3 text-sm text-destructive flex items-start gap-3 shadow-card-bevel">
|
||||
<AlertTriangle className="h-4 w-4 mt-0.5 shrink-0" strokeWidth={1.75} />
|
||||
<div className="min-w-0 space-y-1">
|
||||
<div className="font-medium">Mesh data plane is down</div>
|
||||
<div className="text-xs leading-relaxed text-destructive/90">
|
||||
{headlines[reason] ?? 'Mesh setup did not complete.'}
|
||||
{' '}
|
||||
Set <code className="font-mono bg-destructive/15 px-1 py-0.5 rounded text-[11px]">SENCHO_MESH_SUBNET</code> to a free <code className="font-mono bg-destructive/15 px-1 py-0.5 rounded text-[11px]">/24</code> (for example <code className="font-mono bg-destructive/15 px-1 py-0.5 rounded text-[11px]">10.42.0.0/24</code>) and restart the Sencho container.
|
||||
</div>
|
||||
{status.message ? (
|
||||
<div className="text-[11px] font-mono text-destructive/80 truncate">{status.message}</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function RoutingMasthead({ meshedNodes, reachableNodes, totalAliases, onShowActivity }: {
|
||||
meshedNodes: number; reachableNodes: number; totalAliases: number; onShowActivity: () => void;
|
||||
}) {
|
||||
|
||||
@@ -1,5 +1,22 @@
|
||||
export type MeshRoutePillState = 'healthy' | 'degraded' | 'unreachable' | 'tunnel-down' | 'not-authorized';
|
||||
|
||||
export type MeshDataPlaneReason =
|
||||
| 'ok'
|
||||
| 'not_started'
|
||||
| 'subnet_invalid'
|
||||
| 'subnet_overlap'
|
||||
| 'subnet_mismatch'
|
||||
| 'ip_in_use'
|
||||
| 'attach_failed'
|
||||
| 'not_in_docker';
|
||||
|
||||
export interface MeshDataPlaneStatus {
|
||||
ok: boolean;
|
||||
reason: MeshDataPlaneReason;
|
||||
message: string | null;
|
||||
subnet: string;
|
||||
}
|
||||
|
||||
export interface MeshAlias {
|
||||
host: string;
|
||||
nodeId: number;
|
||||
|
||||
Reference in New Issue
Block a user