diff --git a/backend/src/__tests__/capability-mesh-callback-bootstrap.test.ts b/backend/src/__tests__/capability-mesh-callback-bootstrap.test.ts index c876103c..df5e4705 100644 --- a/backend/src/__tests__/capability-mesh-callback-bootstrap.test.ts +++ b/backend/src/__tests__/capability-mesh-callback-bootstrap.test.ts @@ -1,7 +1,17 @@ -import { describe, it, expect } from 'vitest'; -import { getActiveCapabilities, applyPilotModeCapabilityFilter, enableCapability } from '../services/CapabilityRegistry'; +import { describe, it, expect, afterEach } from 'vitest'; +import { + getActiveCapabilities, + applyPilotModeCapabilityFilter, + enableCapability, + disableCapability, +} from '../services/CapabilityRegistry'; describe('mesh_proxy_callback_bootstrap capability', () => { + afterEach(() => { + // Ensure the runtime override does not leak between tests. + enableCapability('mesh_proxy_callback_bootstrap'); + }); + it('is registered in the default CAPABILITIES list', () => { expect(getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap'); }); @@ -12,4 +22,16 @@ describe('mesh_proxy_callback_bootstrap capability', () => { enableCapability('host-console'); enableCapability('self-update'); }); + + it('is stripped from the active capabilities list when the data plane is disabled', () => { + disableCapability('mesh_proxy_callback_bootstrap'); + expect(getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap'); + }); + + it('returns to the advertised list once enableCapability is called again', () => { + disableCapability('mesh_proxy_callback_bootstrap'); + expect(getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap'); + enableCapability('mesh_proxy_callback_bootstrap'); + expect(getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap'); + }); }); diff --git a/backend/src/__tests__/health.test.ts b/backend/src/__tests__/health.test.ts index b383e6a6..7dd1012d 100644 --- a/backend/src/__tests__/health.test.ts +++ b/backend/src/__tests__/health.test.ts @@ -38,6 +38,42 @@ describe('GET /api/health', () => { expect(res.status).not.toBe(401); expect(res.status).not.toBe(403); }); + + it('reports mesh.dataPlane as a typed status object', async () => { + const res = await request(app).get('/api/health'); + expect(res.body.mesh).toBeDefined(); + expect(res.body.mesh.dataPlane).toBeDefined(); + expect(typeof res.body.mesh.dataPlane.ok).toBe('boolean'); + expect(typeof res.body.mesh.dataPlane.reason).toBe('string'); + expect(typeof res.body.mesh.dataPlane.subnet).toBe('string'); + // `message` is string or null + expect(['string', 'object']).toContain(typeof res.body.mesh.dataPlane.message); + }); + + it('reflects an injected data-plane failure', async () => { + const { MeshService } = await import('../services/MeshService'); + const svc = MeshService.getInstance() as unknown as { + dataPlaneStatus: { ok: boolean; reason: string; message: string | null; subnet: string }; + }; + const prev = { ...svc.dataPlaneStatus }; + svc.dataPlaneStatus = { + ok: false, + reason: 'subnet_overlap', + message: 'Pool overlaps with other one on this address space', + subnet: '172.30.0.0/24', + }; + try { + const res = await request(app).get('/api/health'); + expect(res.status).toBe(200); + expect(res.body.status).toBe('ok'); // process is up even when mesh is down + expect(res.body.mesh.dataPlane.ok).toBe(false); + expect(res.body.mesh.dataPlane.reason).toBe('subnet_overlap'); + expect(res.body.mesh.dataPlane.subnet).toBe('172.30.0.0/24'); + expect(res.body.mesh.dataPlane.message).toContain('overlap'); + } finally { + svc.dataPlaneStatus = prev; + } + }); }); describe('GET /api/meta experimental flag', () => { diff --git a/backend/src/__tests__/mesh-setup-error-classification.test.ts b/backend/src/__tests__/mesh-setup-error-classification.test.ts new file mode 100644 index 00000000..1dea6cbb --- /dev/null +++ b/backend/src/__tests__/mesh-setup-error-classification.test.ts @@ -0,0 +1,218 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import type { MeshActivityEvent, MeshDataPlaneStatus } from '../services/MeshService'; + +let tmpDir: string; +let MeshService: typeof import('../services/MeshService').MeshService; +let DockerController: typeof import('../services/DockerController').default; +let capability: typeof import('../services/CapabilityRegistry'); + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ MeshService } = await import('../services/MeshService')); + ({ default: DockerController } = await import('../services/DockerController')); + capability = await import('../services/CapabilityRegistry'); +}); + +afterAll(() => { + cleanupTestDb(tmpDir); +}); + +type MutableSvc = { + activity: MeshActivityEvent[]; + senchoIp: string | null; + meshSubnet: string; + networkSetupError: string | null; + dataPlaneStatus: MeshDataPlaneStatus; +}; + +let prevSubnetEnv: string | undefined; +let prevHostnameEnv: string | undefined; + +beforeEach(() => { + prevSubnetEnv = process.env.SENCHO_MESH_SUBNET; + prevHostnameEnv = process.env.HOSTNAME; + const svc = MeshService.getInstance() as unknown as MutableSvc; + svc.activity = []; + svc.senchoIp = null; + svc.meshSubnet = '172.30.0.0/24'; + svc.networkSetupError = null; + svc.dataPlaneStatus = { + ok: false, + reason: 'not_started', + message: 'mesh data plane has not initialized yet', + subnet: '', + }; + capability.enableCapability('mesh_proxy_callback_bootstrap'); +}); + +afterEach(() => { + if (prevSubnetEnv === undefined) delete process.env.SENCHO_MESH_SUBNET; + else process.env.SENCHO_MESH_SUBNET = prevSubnetEnv; + if (prevHostnameEnv === undefined) delete process.env.HOSTNAME; + else process.env.HOSTNAME = prevHostnameEnv; + capability.enableCapability('mesh_proxy_callback_bootstrap'); + vi.restoreAllMocks(); +}); + +function callSetup(svc: import('../services/MeshService').MeshService): Promise { + return (svc as unknown as { setupMeshNetwork: () => Promise }).setupMeshNetwork(); +} + +function lastDisable(svc: import('../services/MeshService').MeshService): MeshActivityEvent | undefined { + const all = (svc as unknown as MutableSvc).activity; + return all.filter((e) => e.type === 'mesh.disable').slice(-1)[0]; +} + +type FakeController = { + createNetwork: ReturnType; + inspectNetwork: ReturnType; + connectContainerToNetwork: ReturnType; +}; + +function mockDocker(overrides: Partial = {}): FakeController { + const fake: FakeController = { + createNetwork: vi.fn().mockResolvedValue(undefined), + inspectNetwork: vi.fn(), + connectContainerToNetwork: vi.fn().mockResolvedValue(undefined), + ...overrides, + }; + vi.spyOn(DockerController, 'getInstance').mockReturnValue( + fake as unknown as ReturnType, + ); + return fake; +} + +describe('MeshService.setupMeshNetwork failure classification', () => { + it('classifies an invalid CIDR as subnet_invalid', async () => { + process.env.SENCHO_MESH_SUBNET = 'not-a-cidr'; + process.env.HOSTNAME = 'sencho'; + mockDocker(); + const svc = MeshService.getInstance(); + await callSetup(svc); + const status = svc.getDataPlaneStatus(); + expect(status.ok).toBe(false); + expect(status.reason).toBe('subnet_invalid'); + expect(status.subnet).toBe('not-a-cidr'); + expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap'); + const entry = lastDisable(svc); + expect(entry?.level).toBe('error'); + expect(entry?.details?.reason).toBe('subnet_invalid'); + }); + + it('classifies a Docker pool-overlap error as subnet_overlap', async () => { + process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24'; + process.env.HOSTNAME = 'sencho'; + mockDocker({ + createNetwork: vi.fn().mockRejectedValue( + Object.assign(new Error('Pool overlaps with other one on this address space'), { statusCode: 500 }), + ), + }); + const svc = MeshService.getInstance(); + await callSetup(svc); + const status = svc.getDataPlaneStatus(); + expect(status.reason).toBe('subnet_overlap'); + expect(status.subnet).toBe('10.42.0.0/24'); + expect(status.message).toMatch(/overlap/i); + expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap'); + const entry = lastDisable(svc); + expect(entry?.level).toBe('error'); + expect(entry?.details?.reason).toBe('subnet_overlap'); + }); + + it('classifies an existing-network mismatch as subnet_mismatch', async () => { + process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24'; + process.env.HOSTNAME = 'sencho'; + mockDocker({ + createNetwork: vi.fn().mockRejectedValue({ statusCode: 409, message: 'network already exists' }), + inspectNetwork: vi.fn().mockResolvedValue({ IPAM: { Config: [{ Subnet: '172.30.0.0/24' }] } }), + }); + const svc = MeshService.getInstance(); + await callSetup(svc); + const status = svc.getDataPlaneStatus(); + expect(status.reason).toBe('subnet_mismatch'); + expect(status.message).toMatch(/exists with subnet/i); + }); + + it('classifies an address-already-in-use attach error as ip_in_use', async () => { + process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24'; + process.env.HOSTNAME = 'sencho'; + mockDocker({ + connectContainerToNetwork: vi.fn().mockRejectedValue(new Error('Address already in use')), + }); + const svc = MeshService.getInstance(); + await callSetup(svc); + expect(svc.getDataPlaneStatus().reason).toBe('ip_in_use'); + expect(lastDisable(svc)?.level).toBe('error'); + }); + + it('classifies a generic attach failure as attach_failed', async () => { + process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24'; + process.env.HOSTNAME = 'sencho'; + mockDocker({ + connectContainerToNetwork: vi.fn().mockRejectedValue(new Error('Docker daemon explosion')), + }); + const svc = MeshService.getInstance(); + await callSetup(svc); + expect(svc.getDataPlaneStatus().reason).toBe('attach_failed'); + }); + + it('records the HOSTNAME-unset path as not_in_docker at level warn', async () => { + process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24'; + delete process.env.HOSTNAME; + mockDocker(); + const svc = MeshService.getInstance(); + await callSetup(svc); + const status = svc.getDataPlaneStatus(); + expect(status.ok).toBe(false); + expect(status.reason).toBe('not_in_docker'); + const entry = lastDisable(svc); + expect(entry?.level).toBe('warn'); + expect(entry?.details?.reason).toBe('not_in_docker'); + // Capability is still stripped even in the warn-level not_in_docker + // case; pilot processes outside Docker must not advertise it. + expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap'); + }); + + it('records the 404-on-inspect path as not_in_docker at level warn', async () => { + process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24'; + process.env.HOSTNAME = 'laptop-hostname'; + mockDocker({ + connectContainerToNetwork: vi.fn().mockRejectedValue({ statusCode: 404, message: 'no such container' }), + }); + const svc = MeshService.getInstance(); + await callSetup(svc); + const status = svc.getDataPlaneStatus(); + expect(status.reason).toBe('not_in_docker'); + expect(lastDisable(svc)?.level).toBe('warn'); + }); + + it('records success as ok and re-enables the capability', async () => { + process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24'; + process.env.HOSTNAME = 'sencho'; + // Force the capability off first to prove the success path flips it back on. + capability.disableCapability('mesh_proxy_callback_bootstrap'); + mockDocker(); + const svc = MeshService.getInstance(); + await callSetup(svc); + const status = svc.getDataPlaneStatus(); + expect(status.ok).toBe(true); + expect(status.reason).toBe('ok'); + expect(status.subnet).toBe('10.42.0.0/24'); + expect(status.message).toBeNull(); + expect(capability.getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap'); + }); + + it('preserves the legacy networkSetupError getter on failure', async () => { + process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24'; + process.env.HOSTNAME = 'sencho'; + mockDocker({ + createNetwork: vi.fn().mockRejectedValue( + Object.assign(new Error('Pool overlaps with other one on this address space'), { statusCode: 500 }), + ), + }); + const svc = MeshService.getInstance(); + await callSetup(svc); + expect(svc.getNetworkSetupError()).toMatch(/overlap/i); + }); +}); diff --git a/backend/src/routes/mesh.ts b/backend/src/routes/mesh.ts index 896bb0cc..68c39c09 100644 --- a/backend/src/routes/mesh.ts +++ b/backend/src/routes/mesh.ts @@ -16,8 +16,9 @@ function actorFor(req: Request): string { meshRouter.get('/status', async (_req: Request, res: Response): Promise => { if (!requireAdmiral(_req, res)) return; try { - const status = await MeshService.getInstance().getStatus(); - res.json({ nodes: status }); + const mesh = MeshService.getInstance(); + const status = await mesh.getStatus(); + res.json({ nodes: status, localDataPlane: mesh.getDataPlaneStatus() }); } catch (err) { console.warn('[mesh] /status failed:', sanitizeForLog((err as Error).message)); res.status(500).json({ error: 'Failed to load mesh status' }); diff --git a/backend/src/routes/meta.ts b/backend/src/routes/meta.ts index 4f05ecfd..7a903101 100644 --- a/backend/src/routes/meta.ts +++ b/backend/src/routes/meta.ts @@ -1,5 +1,6 @@ import { Router, type Request, type Response } from 'express'; import { getActiveCapabilities, getSenchoVersion } from '../services/CapabilityRegistry'; +import { MeshService } from '../services/MeshService'; import SelfUpdateService from '../services/SelfUpdateService'; // Captured at boot. Exposed via /api/health and /api/meta so the Fleet update @@ -9,8 +10,19 @@ const processStartedAt = Date.now(); export const metaRouter = Router(); // Public health endpoint (no auth). Used by Docker HEALTHCHECK and uptime monitors. +// The `mesh.dataPlane` block reports whether `MeshService.setupMeshNetwork` +// completed successfully; an `ok: false` value means cross-node mesh routing +// is disabled on this node and the operator should consult the activity log +// or set `SENCHO_MESH_SUBNET` to a free /24 and restart the container. metaRouter.get('/health', (_req: Request, res: Response): void => { - res.json({ status: 'ok', uptime: process.uptime(), startedAt: processStartedAt }); + res.json({ + status: 'ok', + uptime: process.uptime(), + startedAt: processStartedAt, + mesh: { + dataPlane: MeshService.getInstance().getDataPlaneStatus(), + }, + }); }); // Public meta endpoint. Returns this instance's version and supported diff --git a/backend/src/services/MeshService.ts b/backend/src/services/MeshService.ts index d4bf0fdc..c0a39780 100644 --- a/backend/src/services/MeshService.ts +++ b/backend/src/services/MeshService.ts @@ -14,6 +14,7 @@ import { NodeRegistry } from './NodeRegistry'; import { PilotTunnelManager } from './PilotTunnelManager'; import { MeshProxyTunnelDialer, type DialFailureCode } from './MeshProxyTunnelDialer'; import { generateOverrideYaml, MeshAlias, SENCHO_MESH_NETWORK } from './MeshComposeOverride'; +import { disableCapability, enableCapability } from './CapabilityRegistry'; import { lookupContainerIp } from '../mesh/containerLookup'; import { sanitizeForLog } from '../utils/safeLog'; import { isPathWithinBase, isValidStackName } from '../utils/validation'; @@ -61,6 +62,29 @@ export function getSenchoIpFromSubnet(subnet: string): string { ].join('.'); } +/** + * Discriminator for why the mesh data plane is or is not healthy. Set by + * `setupMeshNetwork` and exposed through `getDataPlaneStatus()` so + * `/api/health` and `/api/meta` can surface the state without parsing the + * raw error string. + */ +export type MeshDataPlaneReason = + | 'ok' + | 'not_started' // MeshService.start() has not finished setupMeshNetwork yet + | 'subnet_invalid' // SENCHO_MESH_SUBNET did not parse + | 'subnet_overlap' // Docker refused the IPAM pool, another network owns the CIDR + | 'subnet_mismatch' // sencho_mesh already exists with a different subnet + | 'ip_in_use' // another container squats +2 + | 'attach_failed' // self-attach failed for any other reason + | 'not_in_docker'; // HOSTNAME unset or self-container lookup returned 404 + +export interface MeshDataPlaneStatus { + ok: boolean; + reason: MeshDataPlaneReason; + message: string | null; + subnet: string; +} + export type MeshActivitySource = 'pilot' | 'mesh'; export type MeshActivityLevel = 'info' | 'warn' | 'error'; export type MeshActivityType = @@ -228,6 +252,17 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { private senchoIp: string | null = null; private meshSubnet: string = DEFAULT_MESH_SUBNET; private networkSetupError: string | null = null; + // Discriminator-typed mirror of networkSetupError. Both stay in sync via + // `recordSetupFailure` / the setupMeshNetwork success path. The discriminator + // is consumed by /api/health and the Routing tab; networkSetupError is + // preserved for callers that already read the raw error string (optInStack, + // applyLocalOverride, regenerateAllOverrides). + private dataPlaneStatus: MeshDataPlaneStatus = { + ok: false, + reason: 'not_started', + message: 'mesh data plane has not initialized yet', + subnet: '', + }; // On a pilot node, central's DB id for this node (e.g. 14). Used by // handleAccept to decide same-node vs cross-node; the pilotAliasOverlay // carries nodeIds from central's perspective, so comparing against the @@ -335,9 +370,13 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { })(); }, ALIAS_REFRESH_INTERVAL_MS); - const dataPlane = this.senchoIp ? 'ok' : `unavailable (${this.networkSetupError ?? 'unknown'})`; + const dpReason = this.dataPlaneStatus.reason; + const dataPlane = this.senchoIp ? 'ok' : `unavailable (${dpReason}: ${this.networkSetupError ?? 'unknown'})`; + const summaryLevel: MeshActivityLevel = dpReason === 'ok' + ? 'info' + : dpReason === 'not_in_docker' ? 'warn' : 'error'; this.logActivity({ - source: 'mesh', level: this.senchoIp ? 'info' : 'warn', type: 'mesh.enable', + source: 'mesh', level: summaryLevel, type: 'mesh.enable', message: `MeshService started (data plane ${dataPlane}, self nodeId ${this.selfCentralNodeId})`, }); } @@ -429,6 +468,74 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { return this.networkSetupError; } + /** + * Typed mirror of `networkSetupError` for consumers that need a + * discriminator (e.g. `/api/health` and the Routing tab). Always returns + * a value: `{ ok: true, reason: 'ok' }` once `setupMeshNetwork` completes + * successfully, otherwise a typed failure shape. + */ + public getDataPlaneStatus(): MeshDataPlaneStatus { + return this.dataPlaneStatus; + } + + /** + * Single recording path for every mesh-setup failure. Keeps the legacy + * `networkSetupError` string in sync, sets the typed `dataPlaneStatus`, + * strips `mesh_proxy_callback_bootstrap` from advertised capabilities, + * and emits a `mesh.disable` activity entry. Callers pass `level: 'warn'` + * for expected conditions (`not_in_docker` in dev mode) and `'error'` for + * real failures. + */ + private recordSetupFailure( + reason: Exclude, + err: unknown, + level: MeshActivityLevel, + // The subnet_invalid path fires before `this.meshSubnet` is assigned, + // so callers must pass the value they tried explicitly; deriving from + // the field would silently report DEFAULT_MESH_SUBNET instead of the + // bad CIDR the operator actually configured. + subnet: string, + ): void { + const message = err instanceof Error ? err.message : String(err); + this.networkSetupError = message; + this.dataPlaneStatus = { ok: false, reason, message, subnet }; + this.senchoIp = null; + disableCapability('mesh_proxy_callback_bootstrap'); + this.logActivity({ + source: 'mesh', + level, + type: 'mesh.disable', + message: `mesh data plane unavailable (${reason}): ${sanitizeForLog(message)}`, + details: { reason, subnet }, + }); + } + + /** + * Classify a throw from `ensureMeshNetwork` into a typed reason by matching + * on the error message. Docker's "pool overlaps with other one on this + * address space" surfaces as a 500 when another bridge owns the requested + * CIDR; the subnet-mismatch error is thrown synchronously from + * `ensureMeshNetwork` itself and contains the literal "exists with subnet". + */ + private classifyMeshNetworkError(err: unknown): 'subnet_overlap' | 'subnet_mismatch' | 'attach_failed' { + const m = err instanceof Error ? err.message : String(err); + if (/overlap/i.test(m)) return 'subnet_overlap'; + if (/exists with subnet/i.test(m)) return 'subnet_mismatch'; + return 'attach_failed'; + } + + /** + * Classify a throw from `ensureSelfAttached` (after its non-throwing + * not-in-Docker paths). Docker's "Address already in use" / + * "no available addresses" come back when another container squats + * `+2`; anything else is a generic attach failure. + */ + private classifySelfAttachError(err: unknown): 'ip_in_use' | 'attach_failed' { + const m = err instanceof Error ? err.message : String(err); + if (/already in use|no available addresses|address already/i.test(m)) return 'ip_in_use'; + return 'attach_failed'; + } + /** * Idempotent setup of the shared `sencho_mesh` Docker bridge network and * Sencho's static attachment to it. Called once at boot before alias @@ -446,31 +553,34 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { this.senchoIp = getSenchoIpFromSubnet(subnet); this.meshSubnet = subnet; } catch (err) { - this.networkSetupError = (err as Error).message; - console.warn('[Mesh]', this.networkSetupError); - this.senchoIp = null; + this.recordSetupFailure('subnet_invalid', err, 'error', subnet); return; } try { await this.ensureMeshNetwork(subnet); } catch (err) { - this.networkSetupError = (err as Error).message; - console.warn('[Mesh] mesh network setup failed:', sanitizeForLog(this.networkSetupError)); - this.senchoIp = null; + this.recordSetupFailure(this.classifyMeshNetworkError(err), err, 'error', subnet); return; } try { await this.ensureSelfAttached(); } catch (err) { - this.networkSetupError = (err as Error).message; - console.warn('[Mesh] self-attach failed:', sanitizeForLog(this.networkSetupError)); - this.senchoIp = null; + this.recordSetupFailure(this.classifySelfAttachError(err), err, 'error', subnet); return; } + // `ensureSelfAttached` has non-throwing paths for the not-in-Docker + // case (HOSTNAME unset / inspect 404). Those paths call + // `recordSetupFailure` directly and leave `senchoIp` null, so a + // null here means the data plane is intentionally disabled (dev + // mode), not that the success path should run. + if (!this.senchoIp) return; + this.networkSetupError = null; + this.dataPlaneStatus = { ok: true, reason: 'ok', message: null, subnet }; + enableCapability('mesh_proxy_callback_bootstrap'); } /** @@ -517,8 +627,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { if (!this.senchoIp) return; const hostname = process.env.HOSTNAME; if (!hostname) { - console.log('[Mesh] HOSTNAME not set, mesh routing disabled (not running in Docker?)'); - this.senchoIp = null; + this.recordSetupFailure( + 'not_in_docker', + new Error('HOSTNAME unset; mesh routing disabled (not running in Docker?)'), + 'warn', + this.meshSubnet, + ); return; } const dc = DockerController.getInstance(NodeRegistry.getInstance().getDefaultNodeId()); @@ -527,12 +641,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { } catch (err) { const e = err as { statusCode?: number; message?: string }; if (e?.statusCode === 404) { - this.logActivity({ - source: 'mesh', level: 'warn', type: 'mesh.disable', - message: 'self-container lookup failed; mesh routing disabled (not running in Docker?)', - }); - console.warn('[Mesh] self-container lookup failed; mesh routing disabled (not running in Docker?)'); - this.senchoIp = null; + this.recordSetupFailure( + 'not_in_docker', + new Error('self-container lookup failed (404); mesh routing disabled (not running in Docker?)'), + 'warn', + this.meshSubnet, + ); return; } throw err; diff --git a/docs/features/sencho-mesh.mdx b/docs/features/sencho-mesh.mdx index b168850f..d4d3dec4 100644 --- a/docs/features/sencho-mesh.mdx +++ b/docs/features/sencho-mesh.mdx @@ -138,12 +138,16 @@ A few things are deliberately out of scope for the first release: ## Troubleshooting - - The Sencho instance failed to set up its `sencho_mesh` Docker network at boot. Check the Sencho container's logs for `[Mesh]` messages. Common causes: Sencho is running outside Docker (dev mode, mesh routing skipped); the Docker socket is not mounted; `sencho_mesh` already exists with a different subnet from a previous install. The mesh status surface on the Routing tab shows the specific reason. - + + The Routing tab shows a red banner when this Sencho's `sencho_mesh` setup did not complete. The banner names the specific reason; the same reason appears in the mesh activity log and on `/api/health` as `mesh.dataPlane.reason`. The fix depends on which reason fired: - - Sencho refuses to start the mesh data plane if a network named `sencho_mesh` exists with a subnet different from the one this Sencho is configured for. Either remove the existing network (`docker network rm sencho_mesh` on the affected host, after detaching any containers) or set `SENCHO_MESH_SUBNET` on this node to match the existing subnet. Restart the Sencho container so the setup runs again. + - `subnet_overlap`: the requested CIDR overlaps another Docker bridge network on this host. Run `docker network ls -q | xargs -L1 docker network inspect --format '{{.Name}} {{range .IPAM.Config}}{{.Subnet}} {{end}}'` to list every existing subnet, then set `SENCHO_MESH_SUBNET` to a free `/24` (for example `10.42.0.0/24`) and recreate the Sencho container. + - `subnet_mismatch`: `sencho_mesh` already exists with a different subnet. Either remove the network (`docker network rm sencho_mesh` after detaching any containers) or set `SENCHO_MESH_SUBNET` to match the existing subnet. + - `subnet_invalid`: `SENCHO_MESH_SUBNET` is not a valid CIDR. Fix the value (it must look like `10.42.0.0/24`) and recreate the container. + - `ip_in_use`: another container is squatting the IP Sencho wants on its mesh subnet. Find the squatting container with `docker network inspect sencho_mesh`, detach or remove it, then restart Sencho. + - `attach_failed`: the Docker daemon refused the network attachment for a reason that does not match the patterns above. The full error appears in the activity log entry and on `/api/health`. + + The `mesh.dataPlane.subnet` field shows the CIDR Sencho tried to use, so the operator can verify which subnet is configured before changing anything. @@ -166,10 +170,6 @@ A few things are deliberately out of scope for the first release: Mesh redeploys the stack on opt-in to refresh hostnames and network attachments. A stuck redeploy usually means the stack itself failed to come back up. Check the stack's deploy logs. - - `172.30.0.0/24` is the default range for the `sencho_mesh` network and is rare in most setups. If it collides with an existing VPN, VLAN, or Docker network, set `SENCHO_MESH_SUBNET` on the Sencho service to a free `/24` and restart the container. Each node can be configured independently. - - Central could not open a mesh tunnel to this node. The badge tooltip shows the specific reason. While a node is in this state the **mesh toggle and Add stack to mesh action on the node card are disabled** so a redeploy is not triggered against an unreachable target. Common causes: diff --git a/frontend/src/components/fleet/RoutingTab.tsx b/frontend/src/components/fleet/RoutingTab.tsx index 0fabe64b..1f904559 100644 --- a/frontend/src/components/fleet/RoutingTab.tsx +++ b/frontend/src/components/fleet/RoutingTab.tsx @@ -3,7 +3,7 @@ import { apiFetch } from '@/lib/api'; import { visibilityInterval } from '@/lib/utils'; import { toast } from '@/components/ui/toast-store'; import { Button } from '@/components/ui/button'; -import { ArrowLeftRight, Loader2, ScrollText, Table2, Network } from 'lucide-react'; +import { AlertTriangle, ArrowLeftRight, Loader2, ScrollText, Table2, Network } from 'lucide-react'; import { RoutingNodeCard } from './RoutingNodeCard'; import { MeshOptInSheet } from './MeshOptInSheet'; import { MeshRouteDetailSheet } from './MeshRouteDetailSheet'; @@ -12,7 +12,7 @@ import { MeshActivitySheet } from './MeshActivitySheet'; import { MeshTopologyGraph, type MeshGraphEdgeMode } from './MeshTopologyGraph'; import { MeshStackTopologySheet } from './MeshStackTopologySheet'; import { SegmentedControl } from '@/components/ui/segmented-control'; -import type { MeshAlias, MeshNodeStatus, MeshProbeResult } from '@/types/mesh'; +import type { MeshAlias, MeshDataPlaneStatus, MeshNodeStatus, MeshProbeResult } from '@/types/mesh'; type RoutingViewMode = 'table' | 'graph'; @@ -46,6 +46,7 @@ function readStoredEdgeMode(): MeshGraphEdgeMode { export function RoutingTab() { const [status, setStatus] = useState([]); + const [localDataPlane, setLocalDataPlane] = useState(null); const [aliases, setAliases] = useState([]); const [loading, setLoading] = useState(true); const [optInNode, setOptInNode] = useState<{ id: number; name: string } | null>(null); @@ -63,8 +64,9 @@ export function RoutingTab() { apiFetch('/mesh/aliases', { localOnly: true }), ]); if (statusRes.ok) { - const body = await statusRes.json() as { nodes: MeshNodeStatus[] }; + const body = await statusRes.json() as { nodes: MeshNodeStatus[]; localDataPlane?: MeshDataPlaneStatus }; setStatus(body.nodes); + if (body.localDataPlane) setLocalDataPlane(body.localDataPlane); } if (aliasesRes.ok) { const body = await aliasesRes.json() as { aliases: MeshAlias[] }; @@ -156,6 +158,7 @@ export function RoutingTab() { return (
setActivityOpen(true)} /> +
Mesh containers across nodes
@@ -196,6 +199,7 @@ export function RoutingTab() { return (
setActivityOpen(true)} /> +
value={viewMode} @@ -256,6 +260,43 @@ export function RoutingTab() { ); } +/** + * Visible when the local Sencho's mesh data plane is down. Distinguishes the + * common reasons (subnet conflict, IP-in-use, invalid CIDR) so the operator + * does not have to dig through the activity log to know how to recover. The + * `not_in_docker` warn-level case is intentionally suppressed: it is the + * expected condition for dev-mode startup. + */ +function DataPlaneBanner({ status }: { status: MeshDataPlaneStatus | null }) { + if (!status || status.ok || status.reason === 'not_in_docker' || status.reason === 'not_started') { + return null; + } + const headlines: Record, string> = { + subnet_invalid: 'SENCHO_MESH_SUBNET is not a valid CIDR.', + subnet_overlap: `Mesh subnet ${status.subnet} overlaps another Docker network on this host.`, + subnet_mismatch: `sencho_mesh already exists with a different subnet than ${status.subnet}.`, + ip_in_use: `Another container is using Sencho's address on ${status.subnet}.`, + attach_failed: 'Sencho could not attach to its own mesh network.', + }; + const reason = status.reason as keyof typeof headlines; + return ( +
+ +
+
Mesh data plane is down
+
+ {headlines[reason] ?? 'Mesh setup did not complete.'} + {' '} + Set SENCHO_MESH_SUBNET to a free /24 (for example 10.42.0.0/24) and restart the Sencho container. +
+ {status.message ? ( +
{status.message}
+ ) : null} +
+
+ ); +} + function RoutingMasthead({ meshedNodes, reachableNodes, totalAliases, onShowActivity }: { meshedNodes: number; reachableNodes: number; totalAliases: number; onShowActivity: () => void; }) { diff --git a/frontend/src/types/mesh.ts b/frontend/src/types/mesh.ts index 729ee0d1..cf91b853 100644 --- a/frontend/src/types/mesh.ts +++ b/frontend/src/types/mesh.ts @@ -1,5 +1,22 @@ export type MeshRoutePillState = 'healthy' | 'degraded' | 'unreachable' | 'tunnel-down' | 'not-authorized'; +export type MeshDataPlaneReason = + | 'ok' + | 'not_started' + | 'subnet_invalid' + | 'subnet_overlap' + | 'subnet_mismatch' + | 'ip_in_use' + | 'attach_failed' + | 'not_in_docker'; + +export interface MeshDataPlaneStatus { + ok: boolean; + reason: MeshDataPlaneReason; + message: string | null; + subnet: string; +} + export interface MeshAlias { host: string; nodeId: number;