mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-10 18:56:53 +00:00
fix(mesh): surface data-plane failures in health, meta, and Routing tab (#1088)
The three previously-silent console.warn paths in MeshService.setupMeshNetwork now route through a typed recordSetupFailure helper that classifies the failure (subnet_invalid, subnet_overlap, subnet_mismatch, ip_in_use, attach_failed, not_in_docker), emits a mesh.disable activity entry at the matching level (error for real failures, warn for the expected dev-mode not_in_docker case), and strips mesh_proxy_callback_bootstrap from advertised capabilities via CapabilityRegistry. /api/health gains a mesh.dataPlane block carrying the typed status. /api/mesh/status carries localDataPlane at the top level so the Routing tab renders a red banner with an operator-actionable recovery hint (set SENCHO_MESH_SUBNET to a free /24 and recreate the container) when the data plane is down. The success path re-enables the capability and flips the status to ok. Generalizes the previously-documented F-0 failure mode (IP-in-subnet collision) to also cover the subnet-pool-overlap case where another Docker bridge on the host already owns the requested CIDR.
This commit is contained in:
@@ -1,7 +1,17 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { getActiveCapabilities, applyPilotModeCapabilityFilter, enableCapability } from '../services/CapabilityRegistry';
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import {
|
||||
getActiveCapabilities,
|
||||
applyPilotModeCapabilityFilter,
|
||||
enableCapability,
|
||||
disableCapability,
|
||||
} from '../services/CapabilityRegistry';
|
||||
|
||||
describe('mesh_proxy_callback_bootstrap capability', () => {
|
||||
afterEach(() => {
|
||||
// Ensure the runtime override does not leak between tests.
|
||||
enableCapability('mesh_proxy_callback_bootstrap');
|
||||
});
|
||||
|
||||
it('is registered in the default CAPABILITIES list', () => {
|
||||
expect(getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap');
|
||||
});
|
||||
@@ -12,4 +22,16 @@ describe('mesh_proxy_callback_bootstrap capability', () => {
|
||||
enableCapability('host-console');
|
||||
enableCapability('self-update');
|
||||
});
|
||||
|
||||
it('is stripped from the active capabilities list when the data plane is disabled', () => {
|
||||
disableCapability('mesh_proxy_callback_bootstrap');
|
||||
expect(getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
|
||||
});
|
||||
|
||||
it('returns to the advertised list once enableCapability is called again', () => {
|
||||
disableCapability('mesh_proxy_callback_bootstrap');
|
||||
expect(getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
|
||||
enableCapability('mesh_proxy_callback_bootstrap');
|
||||
expect(getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -38,6 +38,42 @@ describe('GET /api/health', () => {
|
||||
expect(res.status).not.toBe(401);
|
||||
expect(res.status).not.toBe(403);
|
||||
});
|
||||
|
||||
it('reports mesh.dataPlane as a typed status object', async () => {
|
||||
const res = await request(app).get('/api/health');
|
||||
expect(res.body.mesh).toBeDefined();
|
||||
expect(res.body.mesh.dataPlane).toBeDefined();
|
||||
expect(typeof res.body.mesh.dataPlane.ok).toBe('boolean');
|
||||
expect(typeof res.body.mesh.dataPlane.reason).toBe('string');
|
||||
expect(typeof res.body.mesh.dataPlane.subnet).toBe('string');
|
||||
// `message` is string or null
|
||||
expect(['string', 'object']).toContain(typeof res.body.mesh.dataPlane.message);
|
||||
});
|
||||
|
||||
it('reflects an injected data-plane failure', async () => {
|
||||
const { MeshService } = await import('../services/MeshService');
|
||||
const svc = MeshService.getInstance() as unknown as {
|
||||
dataPlaneStatus: { ok: boolean; reason: string; message: string | null; subnet: string };
|
||||
};
|
||||
const prev = { ...svc.dataPlaneStatus };
|
||||
svc.dataPlaneStatus = {
|
||||
ok: false,
|
||||
reason: 'subnet_overlap',
|
||||
message: 'Pool overlaps with other one on this address space',
|
||||
subnet: '172.30.0.0/24',
|
||||
};
|
||||
try {
|
||||
const res = await request(app).get('/api/health');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.status).toBe('ok'); // process is up even when mesh is down
|
||||
expect(res.body.mesh.dataPlane.ok).toBe(false);
|
||||
expect(res.body.mesh.dataPlane.reason).toBe('subnet_overlap');
|
||||
expect(res.body.mesh.dataPlane.subnet).toBe('172.30.0.0/24');
|
||||
expect(res.body.mesh.dataPlane.message).toContain('overlap');
|
||||
} finally {
|
||||
svc.dataPlaneStatus = prev;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/meta experimental flag', () => {
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
import type { MeshActivityEvent, MeshDataPlaneStatus } from '../services/MeshService';
|
||||
|
||||
let tmpDir: string;
|
||||
let MeshService: typeof import('../services/MeshService').MeshService;
|
||||
let DockerController: typeof import('../services/DockerController').default;
|
||||
let capability: typeof import('../services/CapabilityRegistry');
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ MeshService } = await import('../services/MeshService'));
|
||||
({ default: DockerController } = await import('../services/DockerController'));
|
||||
capability = await import('../services/CapabilityRegistry');
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
type MutableSvc = {
|
||||
activity: MeshActivityEvent[];
|
||||
senchoIp: string | null;
|
||||
meshSubnet: string;
|
||||
networkSetupError: string | null;
|
||||
dataPlaneStatus: MeshDataPlaneStatus;
|
||||
};
|
||||
|
||||
let prevSubnetEnv: string | undefined;
|
||||
let prevHostnameEnv: string | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
prevSubnetEnv = process.env.SENCHO_MESH_SUBNET;
|
||||
prevHostnameEnv = process.env.HOSTNAME;
|
||||
const svc = MeshService.getInstance() as unknown as MutableSvc;
|
||||
svc.activity = [];
|
||||
svc.senchoIp = null;
|
||||
svc.meshSubnet = '172.30.0.0/24';
|
||||
svc.networkSetupError = null;
|
||||
svc.dataPlaneStatus = {
|
||||
ok: false,
|
||||
reason: 'not_started',
|
||||
message: 'mesh data plane has not initialized yet',
|
||||
subnet: '',
|
||||
};
|
||||
capability.enableCapability('mesh_proxy_callback_bootstrap');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (prevSubnetEnv === undefined) delete process.env.SENCHO_MESH_SUBNET;
|
||||
else process.env.SENCHO_MESH_SUBNET = prevSubnetEnv;
|
||||
if (prevHostnameEnv === undefined) delete process.env.HOSTNAME;
|
||||
else process.env.HOSTNAME = prevHostnameEnv;
|
||||
capability.enableCapability('mesh_proxy_callback_bootstrap');
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function callSetup(svc: import('../services/MeshService').MeshService): Promise<void> {
|
||||
return (svc as unknown as { setupMeshNetwork: () => Promise<void> }).setupMeshNetwork();
|
||||
}
|
||||
|
||||
function lastDisable(svc: import('../services/MeshService').MeshService): MeshActivityEvent | undefined {
|
||||
const all = (svc as unknown as MutableSvc).activity;
|
||||
return all.filter((e) => e.type === 'mesh.disable').slice(-1)[0];
|
||||
}
|
||||
|
||||
type FakeController = {
|
||||
createNetwork: ReturnType<typeof vi.fn>;
|
||||
inspectNetwork: ReturnType<typeof vi.fn>;
|
||||
connectContainerToNetwork: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
|
||||
function mockDocker(overrides: Partial<FakeController> = {}): FakeController {
|
||||
const fake: FakeController = {
|
||||
createNetwork: vi.fn().mockResolvedValue(undefined),
|
||||
inspectNetwork: vi.fn(),
|
||||
connectContainerToNetwork: vi.fn().mockResolvedValue(undefined),
|
||||
...overrides,
|
||||
};
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue(
|
||||
fake as unknown as ReturnType<typeof DockerController.getInstance>,
|
||||
);
|
||||
return fake;
|
||||
}
|
||||
|
||||
describe('MeshService.setupMeshNetwork failure classification', () => {
|
||||
it('classifies an invalid CIDR as subnet_invalid', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = 'not-a-cidr';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
mockDocker();
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.ok).toBe(false);
|
||||
expect(status.reason).toBe('subnet_invalid');
|
||||
expect(status.subnet).toBe('not-a-cidr');
|
||||
expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
|
||||
const entry = lastDisable(svc);
|
||||
expect(entry?.level).toBe('error');
|
||||
expect(entry?.details?.reason).toBe('subnet_invalid');
|
||||
});
|
||||
|
||||
it('classifies a Docker pool-overlap error as subnet_overlap', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
mockDocker({
|
||||
createNetwork: vi.fn().mockRejectedValue(
|
||||
Object.assign(new Error('Pool overlaps with other one on this address space'), { statusCode: 500 }),
|
||||
),
|
||||
});
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.reason).toBe('subnet_overlap');
|
||||
expect(status.subnet).toBe('10.42.0.0/24');
|
||||
expect(status.message).toMatch(/overlap/i);
|
||||
expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
|
||||
const entry = lastDisable(svc);
|
||||
expect(entry?.level).toBe('error');
|
||||
expect(entry?.details?.reason).toBe('subnet_overlap');
|
||||
});
|
||||
|
||||
it('classifies an existing-network mismatch as subnet_mismatch', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
mockDocker({
|
||||
createNetwork: vi.fn().mockRejectedValue({ statusCode: 409, message: 'network already exists' }),
|
||||
inspectNetwork: vi.fn().mockResolvedValue({ IPAM: { Config: [{ Subnet: '172.30.0.0/24' }] } }),
|
||||
});
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.reason).toBe('subnet_mismatch');
|
||||
expect(status.message).toMatch(/exists with subnet/i);
|
||||
});
|
||||
|
||||
it('classifies an address-already-in-use attach error as ip_in_use', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
mockDocker({
|
||||
connectContainerToNetwork: vi.fn().mockRejectedValue(new Error('Address already in use')),
|
||||
});
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
expect(svc.getDataPlaneStatus().reason).toBe('ip_in_use');
|
||||
expect(lastDisable(svc)?.level).toBe('error');
|
||||
});
|
||||
|
||||
it('classifies a generic attach failure as attach_failed', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
mockDocker({
|
||||
connectContainerToNetwork: vi.fn().mockRejectedValue(new Error('Docker daemon explosion')),
|
||||
});
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
expect(svc.getDataPlaneStatus().reason).toBe('attach_failed');
|
||||
});
|
||||
|
||||
it('records the HOSTNAME-unset path as not_in_docker at level warn', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
delete process.env.HOSTNAME;
|
||||
mockDocker();
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.ok).toBe(false);
|
||||
expect(status.reason).toBe('not_in_docker');
|
||||
const entry = lastDisable(svc);
|
||||
expect(entry?.level).toBe('warn');
|
||||
expect(entry?.details?.reason).toBe('not_in_docker');
|
||||
// Capability is still stripped even in the warn-level not_in_docker
|
||||
// case; pilot processes outside Docker must not advertise it.
|
||||
expect(capability.getActiveCapabilities()).not.toContain('mesh_proxy_callback_bootstrap');
|
||||
});
|
||||
|
||||
it('records the 404-on-inspect path as not_in_docker at level warn', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
process.env.HOSTNAME = 'laptop-hostname';
|
||||
mockDocker({
|
||||
connectContainerToNetwork: vi.fn().mockRejectedValue({ statusCode: 404, message: 'no such container' }),
|
||||
});
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.reason).toBe('not_in_docker');
|
||||
expect(lastDisable(svc)?.level).toBe('warn');
|
||||
});
|
||||
|
||||
it('records success as ok and re-enables the capability', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
// Force the capability off first to prove the success path flips it back on.
|
||||
capability.disableCapability('mesh_proxy_callback_bootstrap');
|
||||
mockDocker();
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
const status = svc.getDataPlaneStatus();
|
||||
expect(status.ok).toBe(true);
|
||||
expect(status.reason).toBe('ok');
|
||||
expect(status.subnet).toBe('10.42.0.0/24');
|
||||
expect(status.message).toBeNull();
|
||||
expect(capability.getActiveCapabilities()).toContain('mesh_proxy_callback_bootstrap');
|
||||
});
|
||||
|
||||
it('preserves the legacy networkSetupError getter on failure', async () => {
|
||||
process.env.SENCHO_MESH_SUBNET = '10.42.0.0/24';
|
||||
process.env.HOSTNAME = 'sencho';
|
||||
mockDocker({
|
||||
createNetwork: vi.fn().mockRejectedValue(
|
||||
Object.assign(new Error('Pool overlaps with other one on this address space'), { statusCode: 500 }),
|
||||
),
|
||||
});
|
||||
const svc = MeshService.getInstance();
|
||||
await callSetup(svc);
|
||||
expect(svc.getNetworkSetupError()).toMatch(/overlap/i);
|
||||
});
|
||||
});
|
||||
@@ -16,8 +16,9 @@ function actorFor(req: Request): string {
|
||||
meshRouter.get('/status', async (_req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmiral(_req, res)) return;
|
||||
try {
|
||||
const status = await MeshService.getInstance().getStatus();
|
||||
res.json({ nodes: status });
|
||||
const mesh = MeshService.getInstance();
|
||||
const status = await mesh.getStatus();
|
||||
res.json({ nodes: status, localDataPlane: mesh.getDataPlaneStatus() });
|
||||
} catch (err) {
|
||||
console.warn('[mesh] /status failed:', sanitizeForLog((err as Error).message));
|
||||
res.status(500).json({ error: 'Failed to load mesh status' });
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { getActiveCapabilities, getSenchoVersion } from '../services/CapabilityRegistry';
|
||||
import { MeshService } from '../services/MeshService';
|
||||
import SelfUpdateService from '../services/SelfUpdateService';
|
||||
|
||||
// Captured at boot. Exposed via /api/health and /api/meta so the Fleet update
|
||||
@@ -9,8 +10,19 @@ const processStartedAt = Date.now();
|
||||
export const metaRouter = Router();
|
||||
|
||||
// Public health endpoint (no auth). Used by Docker HEALTHCHECK and uptime monitors.
|
||||
// The `mesh.dataPlane` block reports whether `MeshService.setupMeshNetwork`
|
||||
// completed successfully; an `ok: false` value means cross-node mesh routing
|
||||
// is disabled on this node and the operator should consult the activity log
|
||||
// or set `SENCHO_MESH_SUBNET` to a free /24 and restart the container.
|
||||
metaRouter.get('/health', (_req: Request, res: Response): void => {
|
||||
res.json({ status: 'ok', uptime: process.uptime(), startedAt: processStartedAt });
|
||||
res.json({
|
||||
status: 'ok',
|
||||
uptime: process.uptime(),
|
||||
startedAt: processStartedAt,
|
||||
mesh: {
|
||||
dataPlane: MeshService.getInstance().getDataPlaneStatus(),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
// Public meta endpoint. Returns this instance's version and supported
|
||||
|
||||
@@ -14,6 +14,7 @@ import { NodeRegistry } from './NodeRegistry';
|
||||
import { PilotTunnelManager } from './PilotTunnelManager';
|
||||
import { MeshProxyTunnelDialer, type DialFailureCode } from './MeshProxyTunnelDialer';
|
||||
import { generateOverrideYaml, MeshAlias, SENCHO_MESH_NETWORK } from './MeshComposeOverride';
|
||||
import { disableCapability, enableCapability } from './CapabilityRegistry';
|
||||
import { lookupContainerIp } from '../mesh/containerLookup';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { isPathWithinBase, isValidStackName } from '../utils/validation';
|
||||
@@ -61,6 +62,29 @@ export function getSenchoIpFromSubnet(subnet: string): string {
|
||||
].join('.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Discriminator for why the mesh data plane is or is not healthy. Set by
|
||||
* `setupMeshNetwork` and exposed through `getDataPlaneStatus()` so
|
||||
* `/api/health` and `/api/meta` can surface the state without parsing the
|
||||
* raw error string.
|
||||
*/
|
||||
export type MeshDataPlaneReason =
|
||||
| 'ok'
|
||||
| 'not_started' // MeshService.start() has not finished setupMeshNetwork yet
|
||||
| 'subnet_invalid' // SENCHO_MESH_SUBNET did not parse
|
||||
| 'subnet_overlap' // Docker refused the IPAM pool, another network owns the CIDR
|
||||
| 'subnet_mismatch' // sencho_mesh already exists with a different subnet
|
||||
| 'ip_in_use' // another container squats <network>+2
|
||||
| 'attach_failed' // self-attach failed for any other reason
|
||||
| 'not_in_docker'; // HOSTNAME unset or self-container lookup returned 404
|
||||
|
||||
export interface MeshDataPlaneStatus {
|
||||
ok: boolean;
|
||||
reason: MeshDataPlaneReason;
|
||||
message: string | null;
|
||||
subnet: string;
|
||||
}
|
||||
|
||||
export type MeshActivitySource = 'pilot' | 'mesh';
|
||||
export type MeshActivityLevel = 'info' | 'warn' | 'error';
|
||||
export type MeshActivityType =
|
||||
@@ -228,6 +252,17 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
private senchoIp: string | null = null;
|
||||
private meshSubnet: string = DEFAULT_MESH_SUBNET;
|
||||
private networkSetupError: string | null = null;
|
||||
// Discriminator-typed mirror of networkSetupError. Both stay in sync via
|
||||
// `recordSetupFailure` / the setupMeshNetwork success path. The discriminator
|
||||
// is consumed by /api/health and the Routing tab; networkSetupError is
|
||||
// preserved for callers that already read the raw error string (optInStack,
|
||||
// applyLocalOverride, regenerateAllOverrides).
|
||||
private dataPlaneStatus: MeshDataPlaneStatus = {
|
||||
ok: false,
|
||||
reason: 'not_started',
|
||||
message: 'mesh data plane has not initialized yet',
|
||||
subnet: '',
|
||||
};
|
||||
// On a pilot node, central's DB id for this node (e.g. 14). Used by
|
||||
// handleAccept to decide same-node vs cross-node; the pilotAliasOverlay
|
||||
// carries nodeIds from central's perspective, so comparing against the
|
||||
@@ -335,9 +370,13 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
})();
|
||||
}, ALIAS_REFRESH_INTERVAL_MS);
|
||||
|
||||
const dataPlane = this.senchoIp ? 'ok' : `unavailable (${this.networkSetupError ?? 'unknown'})`;
|
||||
const dpReason = this.dataPlaneStatus.reason;
|
||||
const dataPlane = this.senchoIp ? 'ok' : `unavailable (${dpReason}: ${this.networkSetupError ?? 'unknown'})`;
|
||||
const summaryLevel: MeshActivityLevel = dpReason === 'ok'
|
||||
? 'info'
|
||||
: dpReason === 'not_in_docker' ? 'warn' : 'error';
|
||||
this.logActivity({
|
||||
source: 'mesh', level: this.senchoIp ? 'info' : 'warn', type: 'mesh.enable',
|
||||
source: 'mesh', level: summaryLevel, type: 'mesh.enable',
|
||||
message: `MeshService started (data plane ${dataPlane}, self nodeId ${this.selfCentralNodeId})`,
|
||||
});
|
||||
}
|
||||
@@ -429,6 +468,74 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
return this.networkSetupError;
|
||||
}
|
||||
|
||||
/**
|
||||
* Typed mirror of `networkSetupError` for consumers that need a
|
||||
* discriminator (e.g. `/api/health` and the Routing tab). Always returns
|
||||
* a value: `{ ok: true, reason: 'ok' }` once `setupMeshNetwork` completes
|
||||
* successfully, otherwise a typed failure shape.
|
||||
*/
|
||||
public getDataPlaneStatus(): MeshDataPlaneStatus {
|
||||
return this.dataPlaneStatus;
|
||||
}
|
||||
|
||||
/**
|
||||
* Single recording path for every mesh-setup failure. Keeps the legacy
|
||||
* `networkSetupError` string in sync, sets the typed `dataPlaneStatus`,
|
||||
* strips `mesh_proxy_callback_bootstrap` from advertised capabilities,
|
||||
* and emits a `mesh.disable` activity entry. Callers pass `level: 'warn'`
|
||||
* for expected conditions (`not_in_docker` in dev mode) and `'error'` for
|
||||
* real failures.
|
||||
*/
|
||||
private recordSetupFailure(
|
||||
reason: Exclude<MeshDataPlaneReason, 'ok' | 'not_started'>,
|
||||
err: unknown,
|
||||
level: MeshActivityLevel,
|
||||
// The subnet_invalid path fires before `this.meshSubnet` is assigned,
|
||||
// so callers must pass the value they tried explicitly; deriving from
|
||||
// the field would silently report DEFAULT_MESH_SUBNET instead of the
|
||||
// bad CIDR the operator actually configured.
|
||||
subnet: string,
|
||||
): void {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
this.networkSetupError = message;
|
||||
this.dataPlaneStatus = { ok: false, reason, message, subnet };
|
||||
this.senchoIp = null;
|
||||
disableCapability('mesh_proxy_callback_bootstrap');
|
||||
this.logActivity({
|
||||
source: 'mesh',
|
||||
level,
|
||||
type: 'mesh.disable',
|
||||
message: `mesh data plane unavailable (${reason}): ${sanitizeForLog(message)}`,
|
||||
details: { reason, subnet },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a throw from `ensureMeshNetwork` into a typed reason by matching
|
||||
* on the error message. Docker's "pool overlaps with other one on this
|
||||
* address space" surfaces as a 500 when another bridge owns the requested
|
||||
* CIDR; the subnet-mismatch error is thrown synchronously from
|
||||
* `ensureMeshNetwork` itself and contains the literal "exists with subnet".
|
||||
*/
|
||||
private classifyMeshNetworkError(err: unknown): 'subnet_overlap' | 'subnet_mismatch' | 'attach_failed' {
|
||||
const m = err instanceof Error ? err.message : String(err);
|
||||
if (/overlap/i.test(m)) return 'subnet_overlap';
|
||||
if (/exists with subnet/i.test(m)) return 'subnet_mismatch';
|
||||
return 'attach_failed';
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a throw from `ensureSelfAttached` (after its non-throwing
|
||||
* not-in-Docker paths). Docker's "Address already in use" /
|
||||
* "no available addresses" come back when another container squats
|
||||
* `<network>+2`; anything else is a generic attach failure.
|
||||
*/
|
||||
private classifySelfAttachError(err: unknown): 'ip_in_use' | 'attach_failed' {
|
||||
const m = err instanceof Error ? err.message : String(err);
|
||||
if (/already in use|no available addresses|address already/i.test(m)) return 'ip_in_use';
|
||||
return 'attach_failed';
|
||||
}
|
||||
|
||||
/**
|
||||
* Idempotent setup of the shared `sencho_mesh` Docker bridge network and
|
||||
* Sencho's static attachment to it. Called once at boot before alias
|
||||
@@ -446,31 +553,34 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
this.senchoIp = getSenchoIpFromSubnet(subnet);
|
||||
this.meshSubnet = subnet;
|
||||
} catch (err) {
|
||||
this.networkSetupError = (err as Error).message;
|
||||
console.warn('[Mesh]', this.networkSetupError);
|
||||
this.senchoIp = null;
|
||||
this.recordSetupFailure('subnet_invalid', err, 'error', subnet);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.ensureMeshNetwork(subnet);
|
||||
} catch (err) {
|
||||
this.networkSetupError = (err as Error).message;
|
||||
console.warn('[Mesh] mesh network setup failed:', sanitizeForLog(this.networkSetupError));
|
||||
this.senchoIp = null;
|
||||
this.recordSetupFailure(this.classifyMeshNetworkError(err), err, 'error', subnet);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.ensureSelfAttached();
|
||||
} catch (err) {
|
||||
this.networkSetupError = (err as Error).message;
|
||||
console.warn('[Mesh] self-attach failed:', sanitizeForLog(this.networkSetupError));
|
||||
this.senchoIp = null;
|
||||
this.recordSetupFailure(this.classifySelfAttachError(err), err, 'error', subnet);
|
||||
return;
|
||||
}
|
||||
|
||||
// `ensureSelfAttached` has non-throwing paths for the not-in-Docker
|
||||
// case (HOSTNAME unset / inspect 404). Those paths call
|
||||
// `recordSetupFailure` directly and leave `senchoIp` null, so a
|
||||
// null here means the data plane is intentionally disabled (dev
|
||||
// mode), not that the success path should run.
|
||||
if (!this.senchoIp) return;
|
||||
|
||||
this.networkSetupError = null;
|
||||
this.dataPlaneStatus = { ok: true, reason: 'ok', message: null, subnet };
|
||||
enableCapability('mesh_proxy_callback_bootstrap');
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -517,8 +627,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
if (!this.senchoIp) return;
|
||||
const hostname = process.env.HOSTNAME;
|
||||
if (!hostname) {
|
||||
console.log('[Mesh] HOSTNAME not set, mesh routing disabled (not running in Docker?)');
|
||||
this.senchoIp = null;
|
||||
this.recordSetupFailure(
|
||||
'not_in_docker',
|
||||
new Error('HOSTNAME unset; mesh routing disabled (not running in Docker?)'),
|
||||
'warn',
|
||||
this.meshSubnet,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const dc = DockerController.getInstance(NodeRegistry.getInstance().getDefaultNodeId());
|
||||
@@ -527,12 +641,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
|
||||
} catch (err) {
|
||||
const e = err as { statusCode?: number; message?: string };
|
||||
if (e?.statusCode === 404) {
|
||||
this.logActivity({
|
||||
source: 'mesh', level: 'warn', type: 'mesh.disable',
|
||||
message: 'self-container lookup failed; mesh routing disabled (not running in Docker?)',
|
||||
});
|
||||
console.warn('[Mesh] self-container lookup failed; mesh routing disabled (not running in Docker?)');
|
||||
this.senchoIp = null;
|
||||
this.recordSetupFailure(
|
||||
'not_in_docker',
|
||||
new Error('self-container lookup failed (404); mesh routing disabled (not running in Docker?)'),
|
||||
'warn',
|
||||
this.meshSubnet,
|
||||
);
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
|
||||
Reference in New Issue
Block a user