feat: add developer-mode startup and stack hydration timing (#1619)

* feat: add developer-mode startup and stack hydration timing

Instrument boot-to-list and detail hydration with commit-aligned milestones, truthful request stages, and destination/gateway debug duration logs so performance work is guided by measurements.

* fix: redact stack names and complete hydration request stages

Stop logging stack identifiers in containers debug timing, and record state_dispatch (plus detail fetch spans) so copied reports match the advertised stage breakdown.
This commit is contained in:
Anso
2026-07-14 17:24:25 -04:00
committed by GitHub
parent 4079cb9198
commit b70a529656
26 changed files with 2449 additions and 32 deletions
+102 -1
View File
@@ -4,7 +4,7 @@
* the entry-cap safety guard, and singleton identity.
*/
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { CacheService } from '../services/CacheService';
import { CacheService, type CacheFetchOutcome } from '../services/CacheService';
describe('CacheService', () => {
let cache: CacheService;
@@ -71,6 +71,107 @@ describe('CacheService', () => {
});
});
// ─── getOrFetchWithMeta: observational outcomes ──────────────────────
describe('getOrFetchWithMeta', () => {
it('reports "computed" when this caller runs the fetcher', async () => {
const fetcher = vi.fn().mockResolvedValue('fresh');
const res = await cache.getOrFetchWithMeta('ns:key', 60_000, fetcher);
expect(res).toEqual({ value: 'fresh', outcome: 'computed' });
expect(fetcher).toHaveBeenCalledTimes(1);
expect(cache.getStats().ns).toEqual({ hits: 0, misses: 1, stale: 0, size: 1 });
});
it('reports "hit" for a fresh entry without waiting or re-running the fetcher', async () => {
await cache.getOrFetchWithMeta('ns:key', 60_000, async () => 'cached');
const fetcher = vi.fn().mockResolvedValue('should-not-run');
const res = await cache.getOrFetchWithMeta('ns:key', 60_000, fetcher);
expect(res).toEqual({ value: 'cached', outcome: 'hit' });
expect(fetcher).not.toHaveBeenCalled();
// First call: 1 miss (computed). Second call: 1 hit.
expect(cache.getStats().ns).toMatchObject({ hits: 1, misses: 1, stale: 0 });
});
it('reports "inflight" for a caller that joins an existing in-flight promise', async () => {
let resolveFetch!: (value: string) => void;
const fetcher = vi.fn(() => new Promise<string>((resolve) => {
resolveFetch = resolve;
}));
const p1 = cache.getOrFetchWithMeta('ns:key', 60_000, fetcher);
const p2 = cache.getOrFetchWithMeta('ns:key', 60_000, fetcher);
const p3 = cache.getOrFetchWithMeta('ns:key', 60_000, fetcher);
resolveFetch('shared');
const [r1, r2, r3] = await Promise.all([p1, p2, p3]);
expect(r1).toEqual({ value: 'shared', outcome: 'computed' });
expect(r2).toEqual({ value: 'shared', outcome: 'inflight' });
expect(r3).toEqual({ value: 'shared', outcome: 'inflight' });
expect(fetcher).toHaveBeenCalledTimes(1);
// Every caller that did not hit a fresh entry recorded exactly one miss,
// including the two in-flight joins: 3 misses, no hits.
expect(cache.getStats().ns).toMatchObject({ hits: 0, misses: 3, stale: 0 });
});
it('reports "stale" when the fetcher rejects but a stale entry exists', async () => {
vi.useFakeTimers({ shouldAdvanceTime: true });
const fetcher = vi.fn()
.mockResolvedValueOnce('original')
.mockRejectedValueOnce(new Error('upstream down'));
const first = await cache.getOrFetchWithMeta('ns:key', 1_000, fetcher);
expect(first).toEqual({ value: 'original', outcome: 'computed' });
await vi.advanceTimersByTimeAsync(1_100);
const stale = await cache.getOrFetchWithMeta('ns:key', 1_000, fetcher);
expect(stale).toEqual({ value: 'original', outcome: 'stale' });
expect(cache.getStats().ns).toMatchObject({ stale: 1 });
});
it('propagates the error (no outcome) when the fetcher rejects with no stale entry', async () => {
const fetcher = vi.fn().mockRejectedValue(new Error('no fallback'));
await expect(cache.getOrFetchWithMeta('ns:key', 60_000, fetcher)).rejects.toThrow('no fallback');
});
it('narrows to the CacheFetchOutcome union', async () => {
const { outcome } = await cache.getOrFetchWithMeta('ns:key', 60_000, async () => 1);
const accepted: CacheFetchOutcome[] = ['hit', 'computed', 'inflight', 'stale'];
expect(accepted).toContain(outcome);
});
});
// ─── getOrFetch delegates to getOrFetchWithMeta (parity) ─────────────
describe('getOrFetch parity with getOrFetchWithMeta', () => {
it('returns only the value and records identical hit/miss counters', async () => {
const fetcher = vi.fn().mockResolvedValue('v');
const first = await cache.getOrFetch('ns:key', 60_000, fetcher);
const second = await cache.getOrFetch('ns:key', 60_000, fetcher);
expect(first).toBe('v');
expect(second).toBe('v');
expect(fetcher).toHaveBeenCalledTimes(1);
// One miss (compute) + one hit, matching the meta variant's accounting.
expect(cache.getStats().ns).toMatchObject({ hits: 1, misses: 1, stale: 0 });
});
it('records one miss per in-flight join, exactly as the meta variant', async () => {
let resolveFetch!: (value: string) => void;
const fetcher = vi.fn(() => new Promise<string>((resolve) => {
resolveFetch = resolve;
}));
const p1 = cache.getOrFetch('ns:key', 60_000, fetcher);
const p2 = cache.getOrFetch('ns:key', 60_000, fetcher);
resolveFetch('shared');
await Promise.all([p1, p2]);
expect(fetcher).toHaveBeenCalledTimes(1);
expect(cache.getStats().ns).toMatchObject({ hits: 0, misses: 2, stale: 0 });
});
});
// ─── inflight deduplication ──────────────────────────────────────────
describe('inflight deduplication', () => {
@@ -0,0 +1,208 @@
/**
* Developer-mode hydration timing on the instrumented GET routes.
*
* Locks down that each instrumented handler emits exactly one structured
* `console.debug` line under developer_mode, stays silent when it is off, and
* carries the documented fields (counts, cache outcome, docker subspan,
* sanitized stack name, elapsed, outcome). Also verifies the /nodes/:id/meta
* diagnostic was folded into a single `[Nodes:debug]` line with no leftover
* `[Nodes:diag]` output.
*/
import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
import request from 'supertest';
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
let tmpDir: string;
let app: import('express').Express;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let CacheService: typeof import('../services/CacheService').CacheService;
let DockerController: typeof import('../services/DockerController').default;
let adminCookie: string;
let nodeId: number;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
({ CacheService } = await import('../services/CacheService'));
DockerController = (await import('../services/DockerController')).default;
({ app } = await import('../index'));
adminCookie = await loginAsTestAdmin(app);
nodeId = DatabaseService.getInstance().getDefaultNode()!.id!;
});
afterAll(() => cleanupTestDb(tmpDir));
afterEach(() => {
DatabaseService.getInstance().updateGlobalSetting('developer_mode', '0');
});
function setDeveloperMode(on: boolean): void {
DatabaseService.getInstance().updateGlobalSetting('developer_mode', on ? '1' : '0');
}
/** Run `fn` while capturing every console.debug line, returned as strings. */
async function captureDebug(fn: () => Promise<void>): Promise<string[]> {
const spy = vi.spyOn(console, 'debug').mockImplementation(() => undefined);
let lines: string[];
try {
await fn();
lines = spy.mock.calls.map((args) => args.map((a) => (typeof a === 'string' ? a : String(a))).join(' '));
} finally {
spy.mockRestore();
}
return lines;
}
describe('[Stacks:debug] GET /api/stacks', () => {
it('stays silent when developer_mode is off', async () => {
setDeveloperMode(false);
const lines = await captureDebug(async () => {
await request(app).get('/api/stacks').set('Cookie', adminCookie);
});
expect(lines.filter((l) => l.startsWith('[Stacks:debug]'))).toHaveLength(0);
});
it('logs one line with nodeId, count, elapsed and outcome when on', async () => {
setDeveloperMode(true);
const lines = await captureDebug(async () => {
await request(app).get('/api/stacks').set('Cookie', adminCookie);
});
const stacks = lines.filter((l) => l.startsWith('[Stacks:debug]'));
expect(stacks).toHaveLength(1);
expect(stacks[0]).toContain('route=GET /');
expect(stacks[0]).toMatch(/nodeId=/);
expect(stacks[0]).toMatch(/count=\d+/);
expect(stacks[0]).toMatch(/elapsedMs=\d+/);
expect(stacks[0]).toMatch(/outcome=ok/);
});
});
describe('[Stacks:debug] GET /api/stacks/statuses', () => {
it('reports cache outcome computed then hit, timing docker only on the compute', async () => {
setDeveloperMode(true);
CacheService.getInstance().invalidateNamespace('stack-statuses');
// getInstance returns a fresh DockerController each call, so spy the shared
// prototype method rather than one instance.
const dockerSpy = vi.spyOn(DockerController.prototype, 'getBulkStackStatuses').mockResolvedValue({});
const firstLines = await captureDebug(async () => {
await request(app).get('/api/stacks/statuses').set('Cookie', adminCookie);
});
const secondLines = await captureDebug(async () => {
await request(app).get('/api/stacks/statuses').set('Cookie', adminCookie);
});
// Capture the call count before restore; mockRestore() clears the history.
const dockerCalls = dockerSpy.mock.calls.length;
dockerSpy.mockRestore();
const first = firstLines.find((l) => l.startsWith('[Stacks:debug]') && l.includes('route=GET /statuses'));
const second = secondLines.find((l) => l.startsWith('[Stacks:debug]') && l.includes('route=GET /statuses'));
expect(first).toMatch(/cacheOutcome=computed/);
expect(first).toMatch(/dockerMs=\d+/);
expect(second).toMatch(/cacheOutcome=hit/);
// No docker call on a cache hit, so the subspan is null rather than 0.
expect(second).toMatch(/dockerMs=null/);
// The compute ran the fetcher exactly once across both requests.
expect(dockerCalls).toBe(1);
});
});
describe('[Stacks:debug] GET /api/stacks/:stack/containers', () => {
it('logs the docker subspan and count without the stack name', async () => {
setDeveloperMode(true);
const dockerSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack').mockResolvedValue([]);
const lines = await captureDebug(async () => {
await request(app).get('/api/stacks/web/containers').set('Cookie', adminCookie);
});
dockerSpy.mockRestore();
const line = lines.find((l) => l.startsWith('[Stacks:debug]') && l.includes('/:stack/containers'));
expect(line).toBeDefined();
expect(line).toContain('route=GET /:stack/containers');
expect(line).not.toContain('web');
expect(line).not.toMatch(/\bstack=/);
expect(line).toMatch(/count=0/);
expect(line).toMatch(/dockerMs=\d+/);
expect(line).toMatch(/outcome=ok/);
});
});
describe('[Notifications:debug] GET /api/notifications', () => {
it('stays silent when developer_mode is off', async () => {
setDeveloperMode(false);
const lines = await captureDebug(async () => {
await request(app).get('/api/notifications').set('Cookie', adminCookie);
});
expect(lines.filter((l) => l.startsWith('[Notifications:debug]'))).toHaveLength(0);
});
it('logs count and elapsed when on', async () => {
setDeveloperMode(true);
const lines = await captureDebug(async () => {
await request(app).get('/api/notifications').set('Cookie', adminCookie);
});
const line = lines.find((l) => l.startsWith('[Notifications:debug]'));
expect(line).toBeDefined();
expect(line).toMatch(/count=\d+/);
expect(line).toMatch(/elapsedMs=\d+/);
expect(line).toMatch(/outcome=ok/);
});
});
describe('[Nodes:debug] GET /api/nodes', () => {
it('logs a gateway-owned count line when on', async () => {
setDeveloperMode(true);
const lines = await captureDebug(async () => {
await request(app).get('/api/nodes').set('Cookie', adminCookie);
});
const line = lines.find((l) => l.startsWith('[Nodes:debug]') && l.includes('route=GET /nodes'));
expect(line).toBeDefined();
expect(line).toMatch(/count=\d+/);
expect(line).toMatch(/outcome=ok/);
});
});
describe('[Nodes:debug] GET /api/nodes/:id/meta', () => {
it('folds the old [Nodes:diag] meta line into a single [Nodes:debug] timing line', async () => {
setDeveloperMode(true);
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
const debugSpy = vi.spyOn(console, 'debug').mockImplementation(() => undefined);
await request(app).get(`/api/nodes/${nodeId}/meta`).set('Cookie', adminCookie);
const logLines = logSpy.mock.calls.map((c) => String(c[0]));
const debugLines = debugSpy.mock.calls.map((c) => String(c[0]));
logSpy.mockRestore();
debugSpy.mockRestore();
expect(logLines.some((l) => l.includes('[Nodes:diag] meta'))).toBe(false);
const line = debugLines.find((l) => l.startsWith('[Nodes:debug]') && l.includes('/nodes/:id/meta'));
expect(line).toBeDefined();
expect(line).toContain('type=local');
expect(line).toMatch(new RegExp(`node=${nodeId}`));
expect(line).toMatch(/outcome=ok/);
});
});
describe('[ImageUpdates:debug] status and detail', () => {
it('logs a line for /status and a counted line for /detail', async () => {
setDeveloperMode(true);
const statusLines = await captureDebug(async () => {
await request(app).get('/api/image-updates/status').set('Cookie', adminCookie);
});
expect(
statusLines.find((l) => l.startsWith('[ImageUpdates:debug]') && l.includes('route=GET /status')),
).toBeDefined();
const detailLines = await captureDebug(async () => {
await request(app).get('/api/image-updates/detail').set('Cookie', adminCookie);
});
const detail = detailLines.find((l) => l.startsWith('[ImageUpdates:debug]') && l.includes('route=GET /detail'));
expect(detail).toBeDefined();
expect(detail).toMatch(/count=\d+/);
expect(detail).toMatch(/outcome=ok/);
});
});
+195
View File
@@ -0,0 +1,195 @@
/**
* Gateway hop timing for proxied critical hydration GETs ([Proxy:debug]).
*
* Verifies the exactly-once finalization contract:
* - a normal proxied request fires downstream finish then close but logs once
* (outcome ok, with upstreamStatus / ttfbMs / elapsedMs),
* - a client abort after headers finalizes as not-success (aborted/error),
* never ok, and still exactly once,
* - path templates never carry the real stack name or a query string,
* - nothing is logged when the gateway's developer_mode is off.
*
* The "remote" node is a loopback capture server; the gateway is exercised both
* via supertest (finish/close, templates, off) and via a raw client against a
* real listener (abort) so the downstream socket can be destroyed mid-body.
*/
import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
import http from 'http';
import type { AddressInfo } from 'net';
import request from 'supertest';
import jwt from 'jsonwebtoken';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
let tmpDir: string;
let app: import('express').Express;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let adminBearer: string;
let remoteNodeId: number;
let captureServer: http.Server;
let appServer: http.Server;
let appPort: number;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
({ app } = await import('../index'));
// authMiddleware resolves the role from the DB row, so a bearer for the
// seeded admin proxies with admin privileges (skips the cross-node RBAC probe).
adminBearer = `Bearer ${jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '5m' })}`;
// Loopback "remote": returns [] for GETs; hangs mid-body for the abort path.
captureServer = http.createServer((req, res) => {
if (req.url?.includes('/containers') && req.url.includes('hangstack')) {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.write('['); // partial body, intentionally never ended
return;
}
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end('[]');
});
await new Promise<void>((resolve) => captureServer.listen(0, '127.0.0.1', resolve));
const capturePort = (captureServer.address() as AddressInfo).port;
remoteNodeId = DatabaseService.getInstance().addNode({
name: 'timing-remote',
type: 'remote',
mode: 'proxy',
compose_dir: '/tmp',
is_default: false,
api_url: `http://127.0.0.1:${capturePort}`,
api_token: 'timing-node-token',
});
appServer = http.createServer(app);
await new Promise<void>((resolve) => appServer.listen(0, '127.0.0.1', resolve));
appPort = (appServer.address() as AddressInfo).port;
});
afterAll(async () => {
await new Promise<void>((resolve) => appServer.close(() => resolve()));
await new Promise<void>((resolve) => captureServer.close(() => resolve()));
cleanupTestDb(tmpDir);
});
afterEach(() => {
DatabaseService.getInstance().updateGlobalSetting('developer_mode', '0');
});
function setDeveloperMode(on: boolean): void {
DatabaseService.getInstance().updateGlobalSetting('developer_mode', on ? '1' : '0');
}
/** Run `fn` while capturing console.debug lines; returns lines before restore. */
async function captureDebug(fn: () => Promise<void>): Promise<string[]> {
const spy = vi.spyOn(console, 'debug').mockImplementation(() => undefined);
let lines: string[];
try {
await fn();
lines = spy.mock.calls.map((args) => args.map((a) => (typeof a === 'string' ? a : String(a))).join(' '));
} finally {
spy.mockRestore();
}
return lines;
}
const proxyLinesFrom = (lines: string[]): string[] => lines.filter((l) => l.startsWith('[Proxy:debug]'));
describe('[Proxy:debug] downstream finish/close finalization', () => {
it('logs exactly one line on a normal proxied GET (finish wins over close)', async () => {
setDeveloperMode(true);
const lines = await captureDebug(async () => {
const res = await request(app)
.get('/api/stacks')
.set('Authorization', adminBearer)
.set('x-node-id', String(remoteNodeId));
expect(res.status).toBe(200);
await new Promise((r) => setTimeout(r, 50));
});
const proxyLines = proxyLinesFrom(lines);
expect(proxyLines).toHaveLength(1);
expect(proxyLines[0]).toContain('route=/api/stacks');
expect(proxyLines[0]).toMatch(/outcome=ok/);
expect(proxyLines[0]).toMatch(/upstreamStatus=200/);
expect(proxyLines[0]).toMatch(/ttfbMs=\d+/);
expect(proxyLines[0]).toMatch(/elapsedMs=\d+/);
});
it('templates the path and never logs the real stack name or query string', async () => {
setDeveloperMode(true);
const lines = await captureDebug(async () => {
await request(app)
.get('/api/stacks/supersecretstack/containers?token=leak')
.set('Authorization', adminBearer)
.set('x-node-id', String(remoteNodeId));
await new Promise((r) => setTimeout(r, 50));
});
const proxyLines = proxyLinesFrom(lines);
expect(proxyLines).toHaveLength(1);
expect(proxyLines[0]).toContain('route=/api/stacks/:stack/containers');
expect(proxyLines[0]).not.toContain('supersecretstack');
expect(proxyLines[0]).not.toContain('token=leak');
});
it('logs nothing when the gateway developer_mode is off', async () => {
setDeveloperMode(false);
const lines = await captureDebug(async () => {
const res = await request(app)
.get('/api/stacks')
.set('Authorization', adminBearer)
.set('x-node-id', String(remoteNodeId));
expect(res.status).toBe(200);
await new Promise((r) => setTimeout(r, 50));
});
expect(proxyLinesFrom(lines)).toHaveLength(0);
});
});
describe('[Proxy:debug] client abort after headers', () => {
it('finalizes as not-success exactly once when the client aborts mid-body', async () => {
setDeveloperMode(true);
const debugSpy = vi.spyOn(console, 'debug').mockImplementation(() => undefined);
// The proxy logs a [Proxy] error on the aborted upstream; keep it quiet.
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
try {
await new Promise<void>((resolve) => {
const clientReq = http.request(
{
host: '127.0.0.1',
port: appPort,
path: '/api/stacks/hangstack/containers',
method: 'GET',
headers: { Authorization: adminBearer, 'x-node-id': String(remoteNodeId) },
},
(resp) => {
// Headers have arrived from the gateway (upstream status + ttfb are
// already captured). Abort before the body finishes.
resp.destroy();
clientReq.destroy();
resolve();
},
);
clientReq.on('error', () => resolve());
clientReq.end();
});
// Let the downstream 'close' / proxy 'error' fire and finalize.
await new Promise((r) => setTimeout(r, 200));
const proxyLines = debugSpy.mock.calls
.map((c) => String(c[0]))
.filter((l) => l.startsWith('[Proxy:debug]') && l.includes('/:stack/containers'));
expect(proxyLines).toHaveLength(1);
expect(proxyLines[0]).not.toMatch(/outcome=ok/);
expect(proxyLines[0]).toMatch(/outcome=(aborted|error)/);
} finally {
debugSpy.mockRestore();
errorSpy.mockRestore();
}
});
});