gsadmin/sencho
1
0
Fork 0
mirror of https://github.com/Studio-Saelix/sencho.git synced 2026-07-26 11:49:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(mesh): add developer-mode diagnostics to the mesh data plane (#1287)

Browse Source 
* feat(mesh): add developer-mode diagnostics to the mesh data plane

Add a developer-mode-gated [Mesh:diag] diagnostic log to MeshService,
following the existing diagnostic-logging pattern: gated on the shared
developer_mode setting (off in production by default) and every value run
through sanitizeForLog. The diagnostics cover the forwarder dispatch
decision (which self-node resolution won, and whether the route is
same-node or cross-node), the entry of the opt-in, opt-out, enable, and
disable operations, and completion timing for opt-in, node disable, and
the alias-cache refresh. This lets an operator trace a slow or stuck mesh
operation from the logs without attaching a debugger. The calls sit only
at per-operation, per-accept, and per-refresh cadences, never inside the
per-frame byte-relay loops.

Add tests covering the developer_mode on/off gating and a guard that a
node's api token never reaches a diagnostic log or the activity buffer.

* refactor(mesh): redact secret-shaped values in developer-mode diagnostics

Run each [Mesh:diag] detail through redactSensitiveText before
sanitizeForLog so a future caller cannot leak a Bearer token, JWT, or
credentialed URL through a diagnostic line, regardless of which call site
emits it. No current call site passes a secret; this is defense in depth.
Use a JWT-shaped canary in the no-leak test so it guards the real credential
class.
This commit is contained in:
Anso
2026-06-02 16:12:13 -04:00
committed by GitHub
parent c6d1631afe
commit 5e6e96e405
2 changed files with 153 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/__tests__/mesh-observability.test.ts
+106
View File
@@ -0,0 +1,106 @@
/**
* Developer-mode diagnostics for the mesh data plane.
*
* The `[Mesh:diag]` logs are gated on the shared `developer_mode` setting, so
* they must be silent in production (the default) and appear only when an
* operator turns developer mode on. They must also never carry a node's
* api_token: the cross-fleet inspect path runs that token in a Bearer header,
* and a diagnostic or error log that echoed it would leak a long-lived
* credential into the log surface.
*/
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
let tmpDir: string;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let MeshService: typeof import('../services/MeshService').MeshService;
function captureConsole() {
const lines: string[] = [];
const record = (...args: unknown[]): void => {
lines.push(args.map((a) => (typeof a === 'string' ? a : JSON.stringify(a))).join(' '));
};
const spies = [
vi.spyOn(console, 'debug').mockImplementation(record),
vi.spyOn(console, 'log').mockImplementation(record),
vi.spyOn(console, 'warn').mockImplementation(record),
vi.spyOn(console, 'error').mockImplementation(record),
];
return { lines, restore: (): void => spies.forEach((s) => s.mockRestore()) };
}
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
({ MeshService } = await import('../services/MeshService'));
});
beforeEach(() => {
DatabaseService.getInstance().getDb().prepare('DELETE FROM mesh_stacks').run();
DatabaseService.getInstance().updateGlobalSetting('developer_mode', '0');
});
afterEach(() => {
vi.restoreAllMocks();
DatabaseService.getInstance().updateGlobalSetting('developer_mode', '0');
});
afterAll(() => cleanupTestDb(tmpDir));
describe('mesh developer-mode diagnostics', () => {
it('emits no [Mesh:diag] logs when developer_mode is off', async () => {
const cap = captureConsole();
await MeshService.getInstance().refreshAliasCache();
cap.restore();
expect(cap.lines.some((l) => l.includes('[Mesh:diag]'))).toBe(false);
});
it('emits the [Mesh:diag] refresh log when developer_mode is on', async () => {
DatabaseService.getInstance().updateGlobalSetting('developer_mode', '1');
const cap = captureConsole();
await MeshService.getInstance().refreshAliasCache();
cap.restore();
expect(cap.lines.some((l) => l.includes('[Mesh:diag] alias cache refreshed'))).toBe(true);
});
it('emits a [Mesh:diag] line from a second instrumented method (enable-for-node)', async () => {
const db = DatabaseService.getInstance();
const localNodeId = db.getDefaultNode()?.id ?? 1;
db.updateGlobalSetting('developer_mode', '1');
const cap = captureConsole();
// enableForNode on the local node is a pure DB write plus the diag and
// activity log; the proxy-dial branch only runs for remote proxy nodes.
await MeshService.getInstance().enableForNode(localNodeId);
cap.restore();
expect(cap.lines.some((l) => l.includes('[Mesh:diag] enable-for-node'))).toBe(true);
});
it('never leaks a remote node api token through diagnostics or logs', async () => {
// A JWT-shaped credential canary, so the assertion guards the actual
// class of secret (a Bearer/JWT node token) rather than an opaque string.
const SECRET = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJtZXNoLWNhbmFyeSJ9.bWVzaC1jYW5hcnlfc2ln';
const db = DatabaseService.getInstance();
const remoteId = db.addNode({
name: 'obs-remote',
type: 'remote',
compose_dir: '',
is_default: false,
// Port 1 refuses immediately, so the inspect fetch fails fast without
// a real peer; the token still travels in the Bearer header.
api_url: 'http://127.0.0.1:1',
api_token: SECRET,
});
db.insertMeshStack(remoteId, 'obs-stack', 'tester');
db.updateGlobalSetting('developer_mode', '1');
const activity: unknown[] = [];
const unsub = MeshService.getInstance().subscribeActivity((e) => activity.push(e));
const cap = captureConsole();
await MeshService.getInstance().refreshAliasCache();
cap.restore();
unsub();
const haystack = `${cap.lines.join('\n')}\n${JSON.stringify(activity)}`;
expect(haystack).not.toContain(SECRET);
});
});
backend/src/services/MeshService.ts
+47 -1
View File
@@ -16,7 +16,8 @@ import { MeshProxyTunnelDialer, type DialFailureCode } from './MeshProxyTunnelDi
import { generateOverrideYaml, MeshAlias, SENCHO_MESH_NETWORK } from './MeshComposeOverride';
import { lookupContainerIp } from '../mesh/containerLookup';
import { STREAM_PENDING_DATA_MAX_BYTES } from '../pilot/protocol';
import { sanitizeForLog } from '../utils/safeLog';
import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog';
import { isDebugEnabled } from '../utils/debug';
import { isPathWithinBase, isValidStackName } from '../utils/validation';
import { PORT as SENCHO_LISTEN_PORT } from '../helpers/constants';
import { assertPolicyGateAllows, buildSystemPolicyGateOptions } from '../helpers/policyGate';
@@ -1460,6 +1461,24 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
}
}
/**
* Developer-mode diagnostic log for the mesh data plane. Off in production
* by default; gated on the shared `developer_mode` setting via
* isDebugEnabled(). Each value is run through redactSensitiveText and then
* sanitizeForLog, so a secret-shaped value (Bearer token, JWT, credentialed
* URL) is redacted and control characters are stripped before it reaches the
* log surface, even if a future caller passes one. Safe at per-operation,
* per-accept (once per new TCP connection), and per-60s-tick cadences; never
* call it from the per-frame relay loops in openSameNode / openCrossNode.
*/
private logDiag(message: string, details: Record<string, unknown> = {}): void {
if (!isDebugEnabled()) return;
const cleaned = Object.fromEntries(
Object.entries(details).map(([key, value]) => [key, sanitizeForLog(redactSensitiveText(value))]),
);
console.debug(`[Mesh:diag] ${message}`, cleaned);
}
public getActivity(filter?: { alias?: string; source?: MeshActivitySource; level?: MeshActivityLevel; limit?: number }): MeshActivityEvent[] {
let out = this.activity;
if (filter?.alias) out = out.filter((e) => e.alias === filter.alias);
@@ -1477,6 +1496,8 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
// --- Opt-in / opt-out ---
public async optInStack(nodeId: number, stackName: string, actor: string): Promise<void> {
this.logDiag('opt-in start', { nodeId, stackName, actor });
const t0 = Date.now();
if (!isValidStackName(stackName)) {
throw new MeshError('denied', `invalid stack name: ${stackName}`);
}
@@ -1548,6 +1569,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
this.cascadeRecomposeAcrossFleet(nodeId, stackName, actor);
this.triggerRedeploy(nodeId, stackName, actor);
this.logDiag('opt-in complete', { nodeId, stackName, services: services.length, ms: Date.now() - t0 });
this.logActivity({
source: 'mesh', level: 'info', type: 'opt_in',
nodeId, message: `opt-in ${stackName}`, details: { actor },
@@ -1561,6 +1583,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
}
public async optOutStack(nodeId: number, stackName: string, actor: string): Promise<void> {
this.logDiag('opt-out start', { nodeId, stackName, actor });
if (!isValidStackName(stackName)) {
throw new MeshError('denied', `invalid stack name: ${stackName}`);
}
@@ -1596,6 +1619,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
}
public async enableForNode(nodeId: number): Promise<void> {
this.logDiag('enable-for-node', { nodeId });
DatabaseService.getInstance().setNodeMeshEnabled(nodeId, true);
this.logActivity({
source: 'mesh', level: 'info', type: 'mesh.enable',
@@ -1617,6 +1641,8 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
nodeId: number,
actor: string = 'system:mesh.disable',
): Promise<void> {
this.logDiag('disable-for-node start', { nodeId, actor });
const t0 = Date.now();
DatabaseService.getInstance().setNodeMeshEnabled(nodeId, false);
const stacks = DatabaseService.getInstance().listMeshStacks(nodeId);
for (const s of stacks) {
@@ -1645,6 +1671,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
for (const s of stacks) {
this.triggerRedeploy(nodeId, s.stack_name, actor);
}
this.logDiag('disable-for-node complete', { nodeId, stacks: stacks.length, ms: Date.now() - t0 });
this.logActivity({
source: 'mesh', level: 'info', type: 'mesh.disable',
nodeId, message: `mesh disabled on node ${nodeId}`,
@@ -1969,6 +1996,7 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
// --- Alias aggregation ---
public async refreshAliasCache(): Promise<void> {
const t0 = Date.now();
const db = DatabaseService.getInstance();
const next = new Map<string, MeshGlobalAlias>();
const portMap = new Map<number, MeshGlobalAlias>();
@@ -2016,6 +2044,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
}
this.aliasCache = next;
this.aliasByPort = portMap;
this.logDiag('alias cache refreshed', {
stacks: stacks.length,
aliases: next.size,
ports: portMap.size,
ms: Date.now() - t0,
});
}
public async listAliases(): Promise<MeshGlobalAlias[]> {
@@ -2415,6 +2449,18 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
this.proxyTunnelSelfCentralNodeId
?? this.selfCentralNodeId
?? NodeRegistry.getInstance().getDefaultNodeId();
let selfSource: string;
if (this.proxyTunnelSelfCentralNodeId != null) selfSource = 'proxy-tunnel';
else if (this.selfCentralNodeId != null) selfSource = 'enroll-token';
else selfSource = 'default-node';
this.logDiag('forwarder dispatch', {
port,
alias: target.alias,
targetNodeId: target.nodeId,
selfNodeId,
selfSource,
path: target.nodeId === selfNodeId ? 'same-node' : 'cross-node',
});
if (target.nodeId === selfNodeId) {
await this.openSameNode(target, src);
} else {