From 5e6e96e40555ebcc762f5c5547d7c70d9774701e Mon Sep 17 00:00:00 2001 From: Anso Date: Tue, 2 Jun 2026 16:12:13 -0400 Subject: [PATCH] feat(mesh): add developer-mode diagnostics to the mesh data plane (#1287) * feat(mesh): add developer-mode diagnostics to the mesh data plane Add a developer-mode-gated [Mesh:diag] diagnostic log to MeshService, following the existing diagnostic-logging pattern: gated on the shared developer_mode setting (off in production by default) and every value run through sanitizeForLog. The diagnostics cover the forwarder dispatch decision (which self-node resolution won, and whether the route is same-node or cross-node), the entry of the opt-in, opt-out, enable, and disable operations, and completion timing for opt-in, node disable, and the alias-cache refresh. This lets an operator trace a slow or stuck mesh operation from the logs without attaching a debugger. The calls sit only at per-operation, per-accept, and per-refresh cadences, never inside the per-frame byte-relay loops. Add tests covering the developer_mode on/off gating and a guard that a node's api token never reaches a diagnostic log or the activity buffer. * refactor(mesh): redact secret-shaped values in developer-mode diagnostics Run each [Mesh:diag] detail through redactSensitiveText before sanitizeForLog so a future caller cannot leak a Bearer token, JWT, or credentialed URL through a diagnostic line, regardless of which call site emits it. No current call site passes a secret; this is defense in depth. Use a JWT-shaped canary in the no-leak test so it guards the real credential class. --- .../src/__tests__/mesh-observability.test.ts | 106 ++++++++++++++++++ backend/src/services/MeshService.ts | 48 +++++++- 2 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 backend/src/__tests__/mesh-observability.test.ts diff --git a/backend/src/__tests__/mesh-observability.test.ts b/backend/src/__tests__/mesh-observability.test.ts new file mode 100644 index 00000000..3399c786 --- /dev/null +++ b/backend/src/__tests__/mesh-observability.test.ts @@ -0,0 +1,106 @@ +/** + * Developer-mode diagnostics for the mesh data plane. + * + * The `[Mesh:diag]` logs are gated on the shared `developer_mode` setting, so + * they must be silent in production (the default) and appear only when an + * operator turns developer mode on. They must also never carry a node's + * api_token: the cross-fleet inspect path runs that token in a Bearer header, + * and a diagnostic or error log that echoed it would leak a long-lived + * credential into the log surface. + */ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; + +let tmpDir: string; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let MeshService: typeof import('../services/MeshService').MeshService; + +function captureConsole() { + const lines: string[] = []; + const record = (...args: unknown[]): void => { + lines.push(args.map((a) => (typeof a === 'string' ? a : JSON.stringify(a))).join(' ')); + }; + const spies = [ + vi.spyOn(console, 'debug').mockImplementation(record), + vi.spyOn(console, 'log').mockImplementation(record), + vi.spyOn(console, 'warn').mockImplementation(record), + vi.spyOn(console, 'error').mockImplementation(record), + ]; + return { lines, restore: (): void => spies.forEach((s) => s.mockRestore()) }; +} + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); + ({ MeshService } = await import('../services/MeshService')); +}); + +beforeEach(() => { + DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_stacks').run(); + DatabaseService.getInstance().updateGlobalSetting('developer_mode', '0'); +}); + +afterEach(() => { + vi.restoreAllMocks(); + DatabaseService.getInstance().updateGlobalSetting('developer_mode', '0'); +}); + +afterAll(() => cleanupTestDb(tmpDir)); + +describe('mesh developer-mode diagnostics', () => { + it('emits no [Mesh:diag] logs when developer_mode is off', async () => { + const cap = captureConsole(); + await MeshService.getInstance().refreshAliasCache(); + cap.restore(); + expect(cap.lines.some((l) => l.includes('[Mesh:diag]'))).toBe(false); + }); + + it('emits the [Mesh:diag] refresh log when developer_mode is on', async () => { + DatabaseService.getInstance().updateGlobalSetting('developer_mode', '1'); + const cap = captureConsole(); + await MeshService.getInstance().refreshAliasCache(); + cap.restore(); + expect(cap.lines.some((l) => l.includes('[Mesh:diag] alias cache refreshed'))).toBe(true); + }); + + it('emits a [Mesh:diag] line from a second instrumented method (enable-for-node)', async () => { + const db = DatabaseService.getInstance(); + const localNodeId = db.getDefaultNode()?.id ?? 1; + db.updateGlobalSetting('developer_mode', '1'); + const cap = captureConsole(); + // enableForNode on the local node is a pure DB write plus the diag and + // activity log; the proxy-dial branch only runs for remote proxy nodes. + await MeshService.getInstance().enableForNode(localNodeId); + cap.restore(); + expect(cap.lines.some((l) => l.includes('[Mesh:diag] enable-for-node'))).toBe(true); + }); + + it('never leaks a remote node api token through diagnostics or logs', async () => { + // A JWT-shaped credential canary, so the assertion guards the actual + // class of secret (a Bearer/JWT node token) rather than an opaque string. + const SECRET = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJtZXNoLWNhbmFyeSJ9.bWVzaC1jYW5hcnlfc2ln'; + const db = DatabaseService.getInstance(); + const remoteId = db.addNode({ + name: 'obs-remote', + type: 'remote', + compose_dir: '', + is_default: false, + // Port 1 refuses immediately, so the inspect fetch fails fast without + // a real peer; the token still travels in the Bearer header. + api_url: 'http://127.0.0.1:1', + api_token: SECRET, + }); + db.insertMeshStack(remoteId, 'obs-stack', 'tester'); + db.updateGlobalSetting('developer_mode', '1'); + + const activity: unknown[] = []; + const unsub = MeshService.getInstance().subscribeActivity((e) => activity.push(e)); + const cap = captureConsole(); + await MeshService.getInstance().refreshAliasCache(); + cap.restore(); + unsub(); + + const haystack = `${cap.lines.join('\n')}\n${JSON.stringify(activity)}`; + expect(haystack).not.toContain(SECRET); + }); +}); diff --git a/backend/src/services/MeshService.ts b/backend/src/services/MeshService.ts index 03e62e25..0edc4360 100644 --- a/backend/src/services/MeshService.ts +++ b/backend/src/services/MeshService.ts @@ -16,7 +16,8 @@ import { MeshProxyTunnelDialer, type DialFailureCode } from './MeshProxyTunnelDi import { generateOverrideYaml, MeshAlias, SENCHO_MESH_NETWORK } from './MeshComposeOverride'; import { lookupContainerIp } from '../mesh/containerLookup'; import { STREAM_PENDING_DATA_MAX_BYTES } from '../pilot/protocol'; -import { sanitizeForLog } from '../utils/safeLog'; +import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog'; +import { isDebugEnabled } from '../utils/debug'; import { isPathWithinBase, isValidStackName } from '../utils/validation'; import { PORT as SENCHO_LISTEN_PORT } from '../helpers/constants'; import { assertPolicyGateAllows, buildSystemPolicyGateOptions } from '../helpers/policyGate'; @@ -1460,6 +1461,24 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { } } + /** + * Developer-mode diagnostic log for the mesh data plane. Off in production + * by default; gated on the shared `developer_mode` setting via + * isDebugEnabled(). Each value is run through redactSensitiveText and then + * sanitizeForLog, so a secret-shaped value (Bearer token, JWT, credentialed + * URL) is redacted and control characters are stripped before it reaches the + * log surface, even if a future caller passes one. Safe at per-operation, + * per-accept (once per new TCP connection), and per-60s-tick cadences; never + * call it from the per-frame relay loops in openSameNode / openCrossNode. + */ + private logDiag(message: string, details: Record = {}): void { + if (!isDebugEnabled()) return; + const cleaned = Object.fromEntries( + Object.entries(details).map(([key, value]) => [key, sanitizeForLog(redactSensitiveText(value))]), + ); + console.debug(`[Mesh:diag] ${message}`, cleaned); + } + public getActivity(filter?: { alias?: string; source?: MeshActivitySource; level?: MeshActivityLevel; limit?: number }): MeshActivityEvent[] { let out = this.activity; if (filter?.alias) out = out.filter((e) => e.alias === filter.alias); @@ -1477,6 +1496,8 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { // --- Opt-in / opt-out --- public async optInStack(nodeId: number, stackName: string, actor: string): Promise { + this.logDiag('opt-in start', { nodeId, stackName, actor }); + const t0 = Date.now(); if (!isValidStackName(stackName)) { throw new MeshError('denied', `invalid stack name: ${stackName}`); } @@ -1548,6 +1569,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { this.cascadeRecomposeAcrossFleet(nodeId, stackName, actor); this.triggerRedeploy(nodeId, stackName, actor); + this.logDiag('opt-in complete', { nodeId, stackName, services: services.length, ms: Date.now() - t0 }); this.logActivity({ source: 'mesh', level: 'info', type: 'opt_in', nodeId, message: `opt-in ${stackName}`, details: { actor }, @@ -1561,6 +1583,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { } public async optOutStack(nodeId: number, stackName: string, actor: string): Promise { + this.logDiag('opt-out start', { nodeId, stackName, actor }); if (!isValidStackName(stackName)) { throw new MeshError('denied', `invalid stack name: ${stackName}`); } @@ -1596,6 +1619,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { } public async enableForNode(nodeId: number): Promise { + this.logDiag('enable-for-node', { nodeId }); DatabaseService.getInstance().setNodeMeshEnabled(nodeId, true); this.logActivity({ source: 'mesh', level: 'info', type: 'mesh.enable', @@ -1617,6 +1641,8 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { nodeId: number, actor: string = 'system:mesh.disable', ): Promise { + this.logDiag('disable-for-node start', { nodeId, actor }); + const t0 = Date.now(); DatabaseService.getInstance().setNodeMeshEnabled(nodeId, false); const stacks = DatabaseService.getInstance().listMeshStacks(nodeId); for (const s of stacks) { @@ -1645,6 +1671,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { for (const s of stacks) { this.triggerRedeploy(nodeId, s.stack_name, actor); } + this.logDiag('disable-for-node complete', { nodeId, stacks: stacks.length, ms: Date.now() - t0 }); this.logActivity({ source: 'mesh', level: 'info', type: 'mesh.disable', nodeId, message: `mesh disabled on node ${nodeId}`, @@ -1969,6 +1996,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { // --- Alias aggregation --- public async refreshAliasCache(): Promise { + const t0 = Date.now(); const db = DatabaseService.getInstance(); const next = new Map(); const portMap = new Map(); @@ -2016,6 +2044,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { } this.aliasCache = next; this.aliasByPort = portMap; + this.logDiag('alias cache refreshed', { + stacks: stacks.length, + aliases: next.size, + ports: portMap.size, + ms: Date.now() - t0, + }); } public async listAliases(): Promise { @@ -2415,6 +2449,18 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { this.proxyTunnelSelfCentralNodeId ?? this.selfCentralNodeId ?? NodeRegistry.getInstance().getDefaultNodeId(); + let selfSource: string; + if (this.proxyTunnelSelfCentralNodeId != null) selfSource = 'proxy-tunnel'; + else if (this.selfCentralNodeId != null) selfSource = 'enroll-token'; + else selfSource = 'default-node'; + this.logDiag('forwarder dispatch', { + port, + alias: target.alias, + targetNodeId: target.nodeId, + selfNodeId, + selfSource, + path: target.nodeId === selfNodeId ? 'same-node' : 'cross-node', + }); if (target.nodeId === selfNodeId) { await this.openSameNode(target, src); } else {