feat: add developer-mode startup and stack hydration timing (#1619)

* feat: add developer-mode startup and stack hydration timing

Instrument boot-to-list and detail hydration with commit-aligned milestones, truthful request stages, and destination/gateway debug duration logs so performance work is guided by measurements.

* fix: redact stack names and complete hydration request stages

Stop logging stack identifiers in containers debug timing, and record state_dispatch (plus detail fetch spans) so copied reports match the advertised stage breakdown.

backend/src/__tests__/cache-service.test.ts

@@ -4,7 +4,7 @@
  * the entry-cap safety guard, and singleton identity.
  */
 import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
-import { CacheService } from '../services/CacheService';
+import { CacheService, type CacheFetchOutcome } from '../services/CacheService';
 
 describe('CacheService', () => {
   let cache: CacheService;
@@ -71,6 +71,107 @@ describe('CacheService', () => {
     });
   });
 
+  // ─── getOrFetchWithMeta: observational outcomes ──────────────────────
+
+  describe('getOrFetchWithMeta', () => {
+    it('reports "computed" when this caller runs the fetcher', async () => {
+      const fetcher = vi.fn().mockResolvedValue('fresh');
+      const res = await cache.getOrFetchWithMeta('ns:key', 60_000, fetcher);
+      expect(res).toEqual({ value: 'fresh', outcome: 'computed' });
+      expect(fetcher).toHaveBeenCalledTimes(1);
+      expect(cache.getStats().ns).toEqual({ hits: 0, misses: 1, stale: 0, size: 1 });
+    });
+
+    it('reports "hit" for a fresh entry without waiting or re-running the fetcher', async () => {
+      await cache.getOrFetchWithMeta('ns:key', 60_000, async () => 'cached');
+      const fetcher = vi.fn().mockResolvedValue('should-not-run');
+      const res = await cache.getOrFetchWithMeta('ns:key', 60_000, fetcher);
+      expect(res).toEqual({ value: 'cached', outcome: 'hit' });
+      expect(fetcher).not.toHaveBeenCalled();
+      // First call: 1 miss (computed). Second call: 1 hit.
+      expect(cache.getStats().ns).toMatchObject({ hits: 1, misses: 1, stale: 0 });
+    });
+
+    it('reports "inflight" for a caller that joins an existing in-flight promise', async () => {
+      let resolveFetch!: (value: string) => void;
+      const fetcher = vi.fn(() => new Promise<string>((resolve) => {
+        resolveFetch = resolve;
+      }));
+
+      const p1 = cache.getOrFetchWithMeta('ns:key', 60_000, fetcher);
+      const p2 = cache.getOrFetchWithMeta('ns:key', 60_000, fetcher);
+      const p3 = cache.getOrFetchWithMeta('ns:key', 60_000, fetcher);
+
+      resolveFetch('shared');
+      const [r1, r2, r3] = await Promise.all([p1, p2, p3]);
+
+      expect(r1).toEqual({ value: 'shared', outcome: 'computed' });
+      expect(r2).toEqual({ value: 'shared', outcome: 'inflight' });
+      expect(r3).toEqual({ value: 'shared', outcome: 'inflight' });
+      expect(fetcher).toHaveBeenCalledTimes(1);
+      // Every caller that did not hit a fresh entry recorded exactly one miss,
+      // including the two in-flight joins: 3 misses, no hits.
+      expect(cache.getStats().ns).toMatchObject({ hits: 0, misses: 3, stale: 0 });
+    });
+
+    it('reports "stale" when the fetcher rejects but a stale entry exists', async () => {
+      vi.useFakeTimers({ shouldAdvanceTime: true });
+      const fetcher = vi.fn()
+        .mockResolvedValueOnce('original')
+        .mockRejectedValueOnce(new Error('upstream down'));
+
+      const first = await cache.getOrFetchWithMeta('ns:key', 1_000, fetcher);
+      expect(first).toEqual({ value: 'original', outcome: 'computed' });
+
+      await vi.advanceTimersByTimeAsync(1_100);
+
+      const stale = await cache.getOrFetchWithMeta('ns:key', 1_000, fetcher);
+      expect(stale).toEqual({ value: 'original', outcome: 'stale' });
+      expect(cache.getStats().ns).toMatchObject({ stale: 1 });
+    });
+
+    it('propagates the error (no outcome) when the fetcher rejects with no stale entry', async () => {
+      const fetcher = vi.fn().mockRejectedValue(new Error('no fallback'));
+      await expect(cache.getOrFetchWithMeta('ns:key', 60_000, fetcher)).rejects.toThrow('no fallback');
+    });
+
+    it('narrows to the CacheFetchOutcome union', async () => {
+      const { outcome } = await cache.getOrFetchWithMeta('ns:key', 60_000, async () => 1);
+      const accepted: CacheFetchOutcome[] = ['hit', 'computed', 'inflight', 'stale'];
+      expect(accepted).toContain(outcome);
+    });
+  });
+
+  // ─── getOrFetch delegates to getOrFetchWithMeta (parity) ─────────────
+
+  describe('getOrFetch parity with getOrFetchWithMeta', () => {
+    it('returns only the value and records identical hit/miss counters', async () => {
+      const fetcher = vi.fn().mockResolvedValue('v');
+      const first = await cache.getOrFetch('ns:key', 60_000, fetcher);
+      const second = await cache.getOrFetch('ns:key', 60_000, fetcher);
+      expect(first).toBe('v');
+      expect(second).toBe('v');
+      expect(fetcher).toHaveBeenCalledTimes(1);
+      // One miss (compute) + one hit, matching the meta variant's accounting.
+      expect(cache.getStats().ns).toMatchObject({ hits: 1, misses: 1, stale: 0 });
+    });
+
+    it('records one miss per in-flight join, exactly as the meta variant', async () => {
+      let resolveFetch!: (value: string) => void;
+      const fetcher = vi.fn(() => new Promise<string>((resolve) => {
+        resolveFetch = resolve;
+      }));
+
+      const p1 = cache.getOrFetch('ns:key', 60_000, fetcher);
+      const p2 = cache.getOrFetch('ns:key', 60_000, fetcher);
+      resolveFetch('shared');
+      await Promise.all([p1, p2]);
+
+      expect(fetcher).toHaveBeenCalledTimes(1);
+      expect(cache.getStats().ns).toMatchObject({ hits: 0, misses: 2, stale: 0 });
+    });
+  });
+
   // ─── inflight deduplication ──────────────────────────────────────────
 
   describe('inflight deduplication', () => {

backend/src/__tests__/hydration-timing-routes.test.ts

@@ -0,0 +1,208 @@
+/**
+ * Developer-mode hydration timing on the instrumented GET routes.
+ *
+ * Locks down that each instrumented handler emits exactly one structured
+ * `console.debug` line under developer_mode, stays silent when it is off, and
+ * carries the documented fields (counts, cache outcome, docker subspan,
+ * sanitized stack name, elapsed, outcome). Also verifies the /nodes/:id/meta
+ * diagnostic was folded into a single `[Nodes:debug]` line with no leftover
+ * `[Nodes:diag]` output.
+ */
+import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
+import request from 'supertest';
+import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
+
+let tmpDir: string;
+let app: import('express').Express;
+let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
+let CacheService: typeof import('../services/CacheService').CacheService;
+let DockerController: typeof import('../services/DockerController').default;
+let adminCookie: string;
+let nodeId: number;
+
+beforeAll(async () => {
+  tmpDir = await setupTestDb();
+  ({ DatabaseService } = await import('../services/DatabaseService'));
+  ({ CacheService } = await import('../services/CacheService'));
+  DockerController = (await import('../services/DockerController')).default;
+  ({ app } = await import('../index'));
+  adminCookie = await loginAsTestAdmin(app);
+  nodeId = DatabaseService.getInstance().getDefaultNode()!.id!;
+});
+
+afterAll(() => cleanupTestDb(tmpDir));
+
+afterEach(() => {
+  DatabaseService.getInstance().updateGlobalSetting('developer_mode', '0');
+});
+
+function setDeveloperMode(on: boolean): void {
+  DatabaseService.getInstance().updateGlobalSetting('developer_mode', on ? '1' : '0');
+}
+
+/** Run `fn` while capturing every console.debug line, returned as strings. */
+async function captureDebug(fn: () => Promise<void>): Promise<string[]> {
+  const spy = vi.spyOn(console, 'debug').mockImplementation(() => undefined);
+  let lines: string[];
+  try {
+    await fn();
+    lines = spy.mock.calls.map((args) => args.map((a) => (typeof a === 'string' ? a : String(a))).join(' '));
+  } finally {
+    spy.mockRestore();
+  }
+  return lines;
+}
+
+describe('[Stacks:debug] GET /api/stacks', () => {
+  it('stays silent when developer_mode is off', async () => {
+    setDeveloperMode(false);
+    const lines = await captureDebug(async () => {
+      await request(app).get('/api/stacks').set('Cookie', adminCookie);
+    });
+    expect(lines.filter((l) => l.startsWith('[Stacks:debug]'))).toHaveLength(0);
+  });
+
+  it('logs one line with nodeId, count, elapsed and outcome when on', async () => {
+    setDeveloperMode(true);
+    const lines = await captureDebug(async () => {
+      await request(app).get('/api/stacks').set('Cookie', adminCookie);
+    });
+    const stacks = lines.filter((l) => l.startsWith('[Stacks:debug]'));
+    expect(stacks).toHaveLength(1);
+    expect(stacks[0]).toContain('route=GET /');
+    expect(stacks[0]).toMatch(/nodeId=/);
+    expect(stacks[0]).toMatch(/count=\d+/);
+    expect(stacks[0]).toMatch(/elapsedMs=\d+/);
+    expect(stacks[0]).toMatch(/outcome=ok/);
+  });
+});
+
+describe('[Stacks:debug] GET /api/stacks/statuses', () => {
+  it('reports cache outcome computed then hit, timing docker only on the compute', async () => {
+    setDeveloperMode(true);
+    CacheService.getInstance().invalidateNamespace('stack-statuses');
+    // getInstance returns a fresh DockerController each call, so spy the shared
+    // prototype method rather than one instance.
+    const dockerSpy = vi.spyOn(DockerController.prototype, 'getBulkStackStatuses').mockResolvedValue({});
+
+    const firstLines = await captureDebug(async () => {
+      await request(app).get('/api/stacks/statuses').set('Cookie', adminCookie);
+    });
+    const secondLines = await captureDebug(async () => {
+      await request(app).get('/api/stacks/statuses').set('Cookie', adminCookie);
+    });
+
+    // Capture the call count before restore; mockRestore() clears the history.
+    const dockerCalls = dockerSpy.mock.calls.length;
+    dockerSpy.mockRestore();
+
+    const first = firstLines.find((l) => l.startsWith('[Stacks:debug]') && l.includes('route=GET /statuses'));
+    const second = secondLines.find((l) => l.startsWith('[Stacks:debug]') && l.includes('route=GET /statuses'));
+    expect(first).toMatch(/cacheOutcome=computed/);
+    expect(first).toMatch(/dockerMs=\d+/);
+    expect(second).toMatch(/cacheOutcome=hit/);
+    // No docker call on a cache hit, so the subspan is null rather than 0.
+    expect(second).toMatch(/dockerMs=null/);
+    // The compute ran the fetcher exactly once across both requests.
+    expect(dockerCalls).toBe(1);
+  });
+});
+
+describe('[Stacks:debug] GET /api/stacks/:stack/containers', () => {
+  it('logs the docker subspan and count without the stack name', async () => {
+    setDeveloperMode(true);
+    const dockerSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack').mockResolvedValue([]);
+
+    const lines = await captureDebug(async () => {
+      await request(app).get('/api/stacks/web/containers').set('Cookie', adminCookie);
+    });
+
+    dockerSpy.mockRestore();
+
+    const line = lines.find((l) => l.startsWith('[Stacks:debug]') && l.includes('/:stack/containers'));
+    expect(line).toBeDefined();
+    expect(line).toContain('route=GET /:stack/containers');
+    expect(line).not.toContain('web');
+    expect(line).not.toMatch(/\bstack=/);
+    expect(line).toMatch(/count=0/);
+    expect(line).toMatch(/dockerMs=\d+/);
+    expect(line).toMatch(/outcome=ok/);
+  });
+});
+
+describe('[Notifications:debug] GET /api/notifications', () => {
+  it('stays silent when developer_mode is off', async () => {
+    setDeveloperMode(false);
+    const lines = await captureDebug(async () => {
+      await request(app).get('/api/notifications').set('Cookie', adminCookie);
+    });
+    expect(lines.filter((l) => l.startsWith('[Notifications:debug]'))).toHaveLength(0);
+  });
+
+  it('logs count and elapsed when on', async () => {
+    setDeveloperMode(true);
+    const lines = await captureDebug(async () => {
+      await request(app).get('/api/notifications').set('Cookie', adminCookie);
+    });
+    const line = lines.find((l) => l.startsWith('[Notifications:debug]'));
+    expect(line).toBeDefined();
+    expect(line).toMatch(/count=\d+/);
+    expect(line).toMatch(/elapsedMs=\d+/);
+    expect(line).toMatch(/outcome=ok/);
+  });
+});
+
+describe('[Nodes:debug] GET /api/nodes', () => {
+  it('logs a gateway-owned count line when on', async () => {
+    setDeveloperMode(true);
+    const lines = await captureDebug(async () => {
+      await request(app).get('/api/nodes').set('Cookie', adminCookie);
+    });
+    const line = lines.find((l) => l.startsWith('[Nodes:debug]') && l.includes('route=GET /nodes'));
+    expect(line).toBeDefined();
+    expect(line).toMatch(/count=\d+/);
+    expect(line).toMatch(/outcome=ok/);
+  });
+});
+
+describe('[Nodes:debug] GET /api/nodes/:id/meta', () => {
+  it('folds the old [Nodes:diag] meta line into a single [Nodes:debug] timing line', async () => {
+    setDeveloperMode(true);
+    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
+    const debugSpy = vi.spyOn(console, 'debug').mockImplementation(() => undefined);
+
+    await request(app).get(`/api/nodes/${nodeId}/meta`).set('Cookie', adminCookie);
+
+    const logLines = logSpy.mock.calls.map((c) => String(c[0]));
+    const debugLines = debugSpy.mock.calls.map((c) => String(c[0]));
+    logSpy.mockRestore();
+    debugSpy.mockRestore();
+
+    expect(logLines.some((l) => l.includes('[Nodes:diag] meta'))).toBe(false);
+    const line = debugLines.find((l) => l.startsWith('[Nodes:debug]') && l.includes('/nodes/:id/meta'));
+    expect(line).toBeDefined();
+    expect(line).toContain('type=local');
+    expect(line).toMatch(new RegExp(`node=${nodeId}`));
+    expect(line).toMatch(/outcome=ok/);
+  });
+});
+
+describe('[ImageUpdates:debug] status and detail', () => {
+  it('logs a line for /status and a counted line for /detail', async () => {
+    setDeveloperMode(true);
+    const statusLines = await captureDebug(async () => {
+      await request(app).get('/api/image-updates/status').set('Cookie', adminCookie);
+    });
+    expect(
+      statusLines.find((l) => l.startsWith('[ImageUpdates:debug]') && l.includes('route=GET /status')),
+    ).toBeDefined();
+
+    const detailLines = await captureDebug(async () => {
+      await request(app).get('/api/image-updates/detail').set('Cookie', adminCookie);
+    });
+    const detail = detailLines.find((l) => l.startsWith('[ImageUpdates:debug]') && l.includes('route=GET /detail'));
+    expect(detail).toBeDefined();
+    expect(detail).toMatch(/count=\d+/);
+    expect(detail).toMatch(/outcome=ok/);
+  });
+});

backend/src/__tests__/proxy-timing.test.ts

@@ -0,0 +1,195 @@
+/**
+ * Gateway hop timing for proxied critical hydration GETs ([Proxy:debug]).
+ *
+ * Verifies the exactly-once finalization contract:
+ *   - a normal proxied request fires downstream finish then close but logs once
+ *     (outcome ok, with upstreamStatus / ttfbMs / elapsedMs),
+ *   - a client abort after headers finalizes as not-success (aborted/error),
+ *     never ok, and still exactly once,
+ *   - path templates never carry the real stack name or a query string,
+ *   - nothing is logged when the gateway's developer_mode is off.
+ *
+ * The "remote" node is a loopback capture server; the gateway is exercised both
+ * via supertest (finish/close, templates, off) and via a raw client against a
+ * real listener (abort) so the downstream socket can be destroyed mid-body.
+ */
+import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
+import http from 'http';
+import type { AddressInfo } from 'net';
+import request from 'supertest';
+import jwt from 'jsonwebtoken';
+import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
+
+let tmpDir: string;
+let app: import('express').Express;
+let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
+let adminBearer: string;
+let remoteNodeId: number;
+
+let captureServer: http.Server;
+let appServer: http.Server;
+let appPort: number;
+
+beforeAll(async () => {
+  tmpDir = await setupTestDb();
+  ({ DatabaseService } = await import('../services/DatabaseService'));
+  ({ app } = await import('../index'));
+
+  // authMiddleware resolves the role from the DB row, so a bearer for the
+  // seeded admin proxies with admin privileges (skips the cross-node RBAC probe).
+  adminBearer = `Bearer ${jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '5m' })}`;
+
+  // Loopback "remote": returns [] for GETs; hangs mid-body for the abort path.
+  captureServer = http.createServer((req, res) => {
+    if (req.url?.includes('/containers') && req.url.includes('hangstack')) {
+      res.writeHead(200, { 'Content-Type': 'application/json' });
+      res.write('['); // partial body, intentionally never ended
+      return;
+    }
+    res.writeHead(200, { 'Content-Type': 'application/json' });
+    res.end('[]');
+  });
+  await new Promise<void>((resolve) => captureServer.listen(0, '127.0.0.1', resolve));
+  const capturePort = (captureServer.address() as AddressInfo).port;
+
+  remoteNodeId = DatabaseService.getInstance().addNode({
+    name: 'timing-remote',
+    type: 'remote',
+    mode: 'proxy',
+    compose_dir: '/tmp',
+    is_default: false,
+    api_url: `http://127.0.0.1:${capturePort}`,
+    api_token: 'timing-node-token',
+  });
+
+  appServer = http.createServer(app);
+  await new Promise<void>((resolve) => appServer.listen(0, '127.0.0.1', resolve));
+  appPort = (appServer.address() as AddressInfo).port;
+});
+
+afterAll(async () => {
+  await new Promise<void>((resolve) => appServer.close(() => resolve()));
+  await new Promise<void>((resolve) => captureServer.close(() => resolve()));
+  cleanupTestDb(tmpDir);
+});
+
+afterEach(() => {
+  DatabaseService.getInstance().updateGlobalSetting('developer_mode', '0');
+});
+
+function setDeveloperMode(on: boolean): void {
+  DatabaseService.getInstance().updateGlobalSetting('developer_mode', on ? '1' : '0');
+}
+
+/** Run `fn` while capturing console.debug lines; returns lines before restore. */
+async function captureDebug(fn: () => Promise<void>): Promise<string[]> {
+  const spy = vi.spyOn(console, 'debug').mockImplementation(() => undefined);
+  let lines: string[];
+  try {
+    await fn();
+    lines = spy.mock.calls.map((args) => args.map((a) => (typeof a === 'string' ? a : String(a))).join(' '));
+  } finally {
+    spy.mockRestore();
+  }
+  return lines;
+}
+
+const proxyLinesFrom = (lines: string[]): string[] => lines.filter((l) => l.startsWith('[Proxy:debug]'));
+
+describe('[Proxy:debug] downstream finish/close finalization', () => {
+  it('logs exactly one line on a normal proxied GET (finish wins over close)', async () => {
+    setDeveloperMode(true);
+    const lines = await captureDebug(async () => {
+      const res = await request(app)
+        .get('/api/stacks')
+        .set('Authorization', adminBearer)
+        .set('x-node-id', String(remoteNodeId));
+      expect(res.status).toBe(200);
+      await new Promise((r) => setTimeout(r, 50));
+    });
+
+    const proxyLines = proxyLinesFrom(lines);
+    expect(proxyLines).toHaveLength(1);
+    expect(proxyLines[0]).toContain('route=/api/stacks');
+    expect(proxyLines[0]).toMatch(/outcome=ok/);
+    expect(proxyLines[0]).toMatch(/upstreamStatus=200/);
+    expect(proxyLines[0]).toMatch(/ttfbMs=\d+/);
+    expect(proxyLines[0]).toMatch(/elapsedMs=\d+/);
+  });
+
+  it('templates the path and never logs the real stack name or query string', async () => {
+    setDeveloperMode(true);
+    const lines = await captureDebug(async () => {
+      await request(app)
+        .get('/api/stacks/supersecretstack/containers?token=leak')
+        .set('Authorization', adminBearer)
+        .set('x-node-id', String(remoteNodeId));
+      await new Promise((r) => setTimeout(r, 50));
+    });
+
+    const proxyLines = proxyLinesFrom(lines);
+    expect(proxyLines).toHaveLength(1);
+    expect(proxyLines[0]).toContain('route=/api/stacks/:stack/containers');
+    expect(proxyLines[0]).not.toContain('supersecretstack');
+    expect(proxyLines[0]).not.toContain('token=leak');
+  });
+
+  it('logs nothing when the gateway developer_mode is off', async () => {
+    setDeveloperMode(false);
+    const lines = await captureDebug(async () => {
+      const res = await request(app)
+        .get('/api/stacks')
+        .set('Authorization', adminBearer)
+        .set('x-node-id', String(remoteNodeId));
+      expect(res.status).toBe(200);
+      await new Promise((r) => setTimeout(r, 50));
+    });
+    expect(proxyLinesFrom(lines)).toHaveLength(0);
+  });
+});
+
+describe('[Proxy:debug] client abort after headers', () => {
+  it('finalizes as not-success exactly once when the client aborts mid-body', async () => {
+    setDeveloperMode(true);
+    const debugSpy = vi.spyOn(console, 'debug').mockImplementation(() => undefined);
+    // The proxy logs a [Proxy] error on the aborted upstream; keep it quiet.
+    const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
+
+    try {
+      await new Promise<void>((resolve) => {
+        const clientReq = http.request(
+          {
+            host: '127.0.0.1',
+            port: appPort,
+            path: '/api/stacks/hangstack/containers',
+            method: 'GET',
+            headers: { Authorization: adminBearer, 'x-node-id': String(remoteNodeId) },
+          },
+          (resp) => {
+            // Headers have arrived from the gateway (upstream status + ttfb are
+            // already captured). Abort before the body finishes.
+            resp.destroy();
+            clientReq.destroy();
+            resolve();
+          },
+        );
+        clientReq.on('error', () => resolve());
+        clientReq.end();
+      });
+
+      // Let the downstream 'close' / proxy 'error' fire and finalize.
+      await new Promise((r) => setTimeout(r, 200));
+
+      const proxyLines = debugSpy.mock.calls
+        .map((c) => String(c[0]))
+        .filter((l) => l.startsWith('[Proxy:debug]') && l.includes('/:stack/containers'));
+
+      expect(proxyLines).toHaveLength(1);
+      expect(proxyLines[0]).not.toMatch(/outcome=ok/);
+      expect(proxyLines[0]).toMatch(/outcome=(aborted|error)/);
+    } finally {
+      debugSpy.mockRestore();
+      errorSpy.mockRestore();
+    }
+  });
+});

backend/src/proxy/remoteNodeProxy.ts

@@ -9,6 +9,67 @@ import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY } from '../services/CapabilityRegi
 import { getErrorMessage } from '../utils/errors';
 import { DatabaseService } from '../services/DatabaseService';
 import { redactSensitiveText } from '../utils/safeLog';
+import { isDebugEnabled } from '../utils/debug';
+import { logDebugTiming, templatizeHydrationPath } from '../utils/requestTiming';
+
+/**
+ * Per-request hop timing for the critical hydration GETs, kept off the Request
+ * type via a WeakMap so the entry is collected with the request. `logged`
+ * enforces exactly-once finalization across the downstream finish/close events
+ * and the proxy error handler.
+ */
+type ProxyTimingOutcome = 'ok' | 'non2xx' | 'aborted' | 'error';
+
+interface ProxyTiming {
+  startedAt: number;
+  template: string;
+  logged: boolean;
+  upstreamStatus?: number;
+  ttfbMs?: number;
+}
+
+const proxyTimings = new WeakMap<Request, ProxyTiming>();
+
+/**
+ * Arm hop timing for a request that is about to be proxied. No-op unless the
+ * gateway has developer_mode on and the path is a critical hydration GET, so
+ * non-instrumented traffic pays nothing. Templates never carry a real stack
+ * name or query string.
+ */
+function beginProxyTiming(req: Request, res: Response): void {
+  if (req.method !== 'GET' || !isDebugEnabled()) return;
+  const template = templatizeHydrationPath(`/api${req.path}`);
+  if (!template) return;
+
+  const timing: ProxyTiming = { startedAt: Date.now(), template, logged: false };
+  proxyTimings.set(req, timing);
+
+  // A completed response fires 'finish' then 'close'; the logged guard lets the
+  // finish result win. A 'close' with no prior 'finish' means the downstream
+  // client aborted before the body finished streaming.
+  res.once('finish', () => {
+    const status = timing.upstreamStatus ?? res.statusCode;
+    finalizeProxyTiming(req, status >= 200 && status < 300 ? 'ok' : 'non2xx');
+  });
+  res.once('close', () => {
+    finalizeProxyTiming(req, 'aborted');
+  });
+}
+
+/** Emit the single `[Proxy:debug]` line for a request, at most once. */
+function finalizeProxyTiming(req: Request, outcome: ProxyTimingOutcome): void {
+  const timing = proxyTimings.get(req);
+  if (!timing || timing.logged) return;
+  timing.logged = true;
+  logDebugTiming('[Proxy:debug]', {
+    route: timing.template,
+    nodeId: req.nodeId,
+    outcome,
+    upstreamStatus: timing.upstreamStatus ?? null,
+    ttfbMs: timing.ttfbMs ?? null,
+    elapsedMs: Date.now() - timing.startedAt,
+  });
+}
 
 /**
  * Build the remote-node HTTP proxy middleware. Mount once at `/api/` after
@@ -77,7 +138,7 @@ export function createRemoteProxyMiddleware(): RequestHandler {
         // intact and http-proxy's req.pipe(proxyReq) forwards the body
         // automatically.
       },
-      proxyRes: (proxyRes) => {
+      proxyRes: (proxyRes, req) => {
         // Mark every response forwarded from a remote node with a sentinel
         // header. The frontend (apiFetch / fetchForNode) checks this before
         // firing the global 'sencho-unauthorized' event: a 401 from a remote
@@ -85,8 +146,19 @@ export function createRemoteProxyMiddleware(): RequestHandler {
         // user's own session expired. Without this distinction, any node with
         // a bad token causes an immediate logout loop.
         proxyRes.headers['x-sencho-proxy'] = '1';
+        // Record upstream status and time-to-first-byte only; the log is
+        // finalized on the downstream finish/close so an abort mid-body is not
+        // mislabeled as success.
+        const timing = proxyTimings.get(req);
+        if (timing) {
+          timing.upstreamStatus = proxyRes.statusCode;
+          timing.ttfbMs = Date.now() - timing.startedAt;
+        }
       },
       error: (err, req, proxyRes) => {
+        // Finalize the hop timing with an error outcome before the existing
+        // 502 handling; the logged guard keeps the later finish/close a no-op.
+        finalizeProxyTiming(req, 'error');
         console.error('[Proxy] Remote node error:', getErrorMessage(err, 'unknown'));
         const path = req.originalUrl || req.url;
         if (req.method === 'POST' && /^\/api\/stacks\/[^/]+\/(?:deploy|update)(?:\?|$)/.test(path)) {
@@ -168,6 +240,7 @@ export function createRemoteProxyMiddleware(): RequestHandler {
       }
 
       req.proxyTarget = target;
+      beginProxyTiming(req, res);
       proxy(req, res, next);
     };
 

backend/src/routes/imageUpdates.ts

@@ -18,6 +18,7 @@ import { buildPolicyGateOptions } from '../helpers/policyGate';
 import { summarizeBlockReasons } from '../utils/policy-risk';
 import { isValidStackName } from '../utils/validation';
 import { sanitizeForLog } from '../utils/safeLog';
+import { logDebugTiming } from '../utils/requestTiming';
 import { getErrorMessage } from '../utils/errors';
 
 // Fleet aggregation cache: 2-minute TTL, shared across dashboard tabs.
@@ -41,12 +42,26 @@ imageUpdatesRouter.get('/', authMiddleware, (req: Request, res: Response): void
 // readiness view. Auth-only, matching GET /; the boolean GET / is left intact so
 // the cross-version fleet aggregation contract is unaffected.
 imageUpdatesRouter.get('/detail', authMiddleware, (req: Request, res: Response): void => {
+  const startedAt = Date.now();
+  let outcome: 'ok' | 'error' = 'ok';
+  let count = 0;
   try {
     const nodeId = req.nodeId ?? NodeRegistry.getInstance().getDefaultNodeId();
-    res.json(DatabaseService.getInstance().getStackUpdateDetail(nodeId));
+    const detail = DatabaseService.getInstance().getStackUpdateDetail(nodeId);
+    count = Object.keys(detail).length;
+    res.json(detail);
   } catch (error) {
+    outcome = 'error';
     console.error('Failed to fetch image update detail:', error);
     res.status(500).json({ error: 'Failed to fetch image update detail' });
+  } finally {
+    logDebugTiming('[ImageUpdates:debug]', {
+      route: 'GET /detail',
+      nodeId: req.nodeId,
+      count,
+      elapsedMs: Date.now() - startedAt,
+      outcome,
+    });
   }
 });
 
@@ -66,8 +81,23 @@ imageUpdatesRouter.post('/refresh', authMiddleware, (req: Request, res: Response
   }
 });
 
-imageUpdatesRouter.get('/status', authMiddleware, (_req: Request, res: Response): void => {
-  res.json(ImageUpdateService.getInstance().getStatus());
+imageUpdatesRouter.get('/status', authMiddleware, (req: Request, res: Response): void => {
+  const startedAt = Date.now();
+  let outcome: 'ok' | 'error' = 'ok';
+  try {
+    res.json(ImageUpdateService.getInstance().getStatus());
+  } catch (error) {
+    outcome = 'error';
+    console.error('Failed to fetch image update status:', error);
+    res.status(500).json({ error: 'Failed to fetch image update status' });
+  } finally {
+    logDebugTiming('[ImageUpdates:debug]', {
+      route: 'GET /status',
+      nodeId: req.nodeId,
+      elapsedMs: Date.now() - startedAt,
+      outcome,
+    });
+  }
 });
 
 /**

backend/src/routes/nodes.ts

@@ -22,6 +22,7 @@ import { getErrorMessage } from '../utils/errors';
 import { toPublicNode } from '../helpers/publicNode';
 import { isDebugEnabled } from '../utils/debug';
 import { sanitizeForLog } from '../utils/safeLog';
+import { logDebugTiming } from '../utils/requestTiming';
 
 const NODE_SCOPE_MESSAGE = 'API tokens cannot manage nodes.';
 const REMOTE_META_CACHE_TTL = 3 * 60 * 1000;
@@ -116,11 +117,25 @@ function mintPilotEnrollment(nodeId: number, req: Request): { token: string; exp
 export const nodesRouter = Router();
 
 nodesRouter.get('/', async (req: Request, res: Response) => {
+  const startedAt = Date.now();
+  let outcome: 'ok' | 'error' = 'ok';
+  let count = 0;
   try {
     const nodes = DatabaseService.getInstance().getNodes();
+    count = nodes.length;
     res.json(nodes.map(toPublicNode));
   } catch (error) {
+    outcome = 'error';
     res.status(500).json({ error: 'Failed to fetch nodes' });
+  } finally {
+    // Gateway-owned: /api/nodes is proxy-exempt, so this always runs on the
+    // control instance and gates on the gateway's developer_mode.
+    logDebugTiming('[Nodes:debug]', {
+      route: 'GET /nodes',
+      count,
+      elapsedMs: Date.now() - startedAt,
+      outcome,
+    });
   }
 });
 
@@ -577,14 +592,19 @@ nodesRouter.post('/:id/test', async (req: Request, res: Response) => {
 });
 
 nodesRouter.get('/:id/meta', authMiddleware, async (req: Request, res: Response) => {
+  const startedAt = Date.now();
+  let outcome: 'ok' | 'error' = 'ok';
+  let id = NaN;
+  let nodeType = 'unknown';
   try {
-    const id = parseInt(req.params.id as string);
+    id = parseInt(req.params.id as string);
     const node = DatabaseService.getInstance().getNode(id);
     if (!node) {
+      outcome = 'error';
       res.status(404).json({ error: 'Node not found' });
       return;
     }
-    if (isDebugEnabled()) console.log(`[Nodes:diag] meta node=${id} type=${node.type}`);
+    nodeType = node.type;
 
     if (node.type === 'local') {
       res.json({ version: getSenchoVersion(), capabilities: getActiveCapabilities() });
@@ -606,8 +626,19 @@ nodesRouter.get('/:id/meta', authMiddleware, async (req: Request, res: Response)
 
     res.json(meta);
   } catch (error: unknown) {
+    outcome = 'error';
     console.error('Failed to fetch node meta:', error);
     const message = getErrorMessage(error, 'Failed to fetch node metadata');
     res.status(500).json({ error: message });
+  } finally {
+    // Gateway-owned: the frontend fetches meta with localOnly, so this runs on
+    // the control instance and gates on the gateway's developer_mode.
+    logDebugTiming('[Nodes:debug]', {
+      route: 'GET /nodes/:id/meta',
+      node: id,
+      type: nodeType,
+      elapsedMs: Date.now() - startedAt,
+      outcome,
+    });
   }
 });

backend/src/routes/notifications.ts

@@ -16,6 +16,7 @@ import {
   syncSuppressionRuleToFleet,
 } from '../helpers/notificationSuppressionSync';
 import { isDebugEnabled } from '../utils/debug';
+import { logDebugTiming } from '../utils/requestTiming';
 import { getErrorMessage } from '../utils/errors';
 import { sanitizeForLog } from '../utils/safeLog';
 import { parseIntParam } from '../utils/parseIntParam';
@@ -183,14 +184,27 @@ function parseSuppressionRuleBody(
 export const notificationsRouter = Router();
 
 notificationsRouter.get('/', authMiddleware, async (req: Request, res: Response): Promise<void> => {
+  const startedAt = Date.now();
+  let outcome: 'ok' | 'error' = 'ok';
+  let count = 0;
   try {
     const nodeId = req.nodeId ?? 0;
     const category = typeof req.query.category === 'string' ? req.query.category : undefined;
     const history = DatabaseService.getInstance().getNotificationHistory(nodeId, 50, category);
+    count = history.length;
     res.json(history);
   } catch (error) {
+    outcome = 'error';
     console.error('Failed to fetch notifications:', error);
     res.status(500).json({ error: 'Failed to fetch notifications' });
+  } finally {
+    logDebugTiming('[Notifications:debug]', {
+      route: 'GET /',
+      nodeId: req.nodeId,
+      count,
+      elapsedMs: Date.now() - startedAt,
+      outcome,
+    });
   }
 });
 

backend/src/routes/stacks.ts

@@ -16,7 +16,7 @@ import { ComposeService, getComposeRollbackInfo } from '../services/ComposeServi
 import DockerController, { type BulkStackInfo } from '../services/DockerController';
 import { DatabaseService, type StackDossierFields } from '../services/DatabaseService';
 import { MeshService } from '../services/MeshService';
-import { CacheService } from '../services/CacheService';
+import { CacheService, type CacheFetchOutcome } from '../services/CacheService';
 import { UpdatePreviewService } from '../services/UpdatePreviewService';
 import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../services/GitSourceService';
 import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement';
@@ -47,6 +47,7 @@ import { normalizeBulkPaths, destWithinAnySource } from '../utils/bulkPaths';
 import { getErrorMessage } from '../utils/errors';
 import { isDebugEnabled } from '../utils/debug';
 import { sanitizeForLog } from '../utils/safeLog';
+import { logDebugTiming } from '../utils/requestTiming';
 import { sendGitSourceError } from '../utils/gitSourceHttp';
 import { buildPolicyGateOptions, runPolicyGate, triggerPostDeployScan, describePolicyBlock } from '../helpers/policyGate';
 import { parseComposePreview, type ComposePreview } from '../helpers/composePreview';
@@ -234,25 +235,45 @@ stacksRouter.param('stackName', (req, res, next, stackName) => {
 
 stacksRouter.get('/', async (req: Request, res: Response) => {
   if (!requirePermission(req, res, 'stack:read')) return;
+  const startedAt = Date.now();
+  let outcome: 'ok' | 'error' = 'ok';
+  let count = 0;
   try {
     const stacks = await FileSystemService.getInstance(req.nodeId).getStacks();
+    count = stacks.length;
     res.json(stacks);
   } catch (error) {
+    outcome = 'error';
     res.status(500).json({ error: 'Failed to fetch stacks' });
+  } finally {
+    logDebugTiming('[Stacks:debug]', {
+      route: 'GET /',
+      nodeId: req.nodeId,
+      count,
+      elapsedMs: Date.now() - startedAt,
+      outcome,
+    });
   }
 });
 
 stacksRouter.get('/statuses', async (req: Request, res: Response) => {
   if (!requirePermission(req, res, 'stack:read')) return;
+  const startedAt = Date.now();
+  let outcome: 'ok' | 'error' = 'ok';
+  let cacheOutcome: CacheFetchOutcome | null = null;
+  let dockerMs: number | null = null;
+  let count = 0;
   try {
-    const result = await CacheService.getInstance().getOrFetch(
+    const { value: result, outcome: fetchOutcome } = await CacheService.getInstance().getOrFetchWithMeta(
       `stack-statuses:${req.nodeId}`,
       STACK_STATUSES_CACHE_TTL_MS,
       async () => {
         const stacks = await FileSystemService.getInstance(req.nodeId).getStacks();
         const stackNames = stacks.map((s: string) => s.replace(/\.(yml|yaml)$/, ''));
         const dockerController = DockerController.getInstance(req.nodeId);
+        const dockerStartedAt = Date.now();
         const bulkInfo = await dockerController.getBulkStackStatuses(stackNames);
+        dockerMs = Date.now() - dockerStartedAt;
         const data: Record<string, BulkStackInfo> = {};
         for (const stack of stacks) {
           const name = stack.replace(/\.(yml|yaml)$/, '');
@@ -261,6 +282,8 @@ stacksRouter.get('/statuses', async (req: Request, res: Response) => {
         return data;
       },
     );
+    cacheOutcome = fetchOutcome;
+    count = Object.keys(result).length;
     // Git-source labels are computed live, outside the cache, so linking or
     // unlinking a stack's Git source is reflected immediately. The Docker
     // status portion keeps its short TTL; only the cheap source label is fresh.
@@ -285,8 +308,19 @@ stacksRouter.get('/statuses', async (req: Request, res: Response) => {
     }
     res.json(withSource);
   } catch (error) {
+    outcome = 'error';
     console.error('Failed to fetch stack statuses:', error);
     res.status(500).json({ error: 'Failed to fetch stack statuses' });
+  } finally {
+    logDebugTiming('[Stacks:debug]', {
+      route: 'GET /statuses',
+      nodeId: req.nodeId,
+      cacheOutcome,
+      count,
+      dockerMs,
+      elapsedMs: Date.now() - startedAt,
+      outcome,
+    });
   }
 });
 
@@ -1151,13 +1185,30 @@ stacksRouter.get('/:stackName/containers', async (req: Request, res: Response) =
     return;
   }
   if (!requirePermission(req, res, 'stack:read', 'stack', stackName)) return;
+  const startedAt = Date.now();
+  let outcome: 'ok' | 'error' = 'ok';
+  let dockerMs: number | null = null;
+  let count = 0;
   try {
     const dockerController = DockerController.getInstance(req.nodeId);
+    const dockerStartedAt = Date.now();
     const containers = await dockerController.getContainersByStack(stackName);
+    dockerMs = Date.now() - dockerStartedAt;
+    count = containers.length;
     res.json(containers);
   } catch (error) {
+    outcome = 'error';
     console.error('[Stacks] Failed to fetch containers for %s:', sanitizeForLog(stackName), error);
     res.status(500).json({ error: 'Failed to fetch containers' });
+  } finally {
+    logDebugTiming('[Stacks:debug]', {
+      route: 'GET /:stack/containers',
+      nodeId: req.nodeId,
+      count,
+      dockerMs,
+      elapsedMs: Date.now() - startedAt,
+      outcome,
+    });
   }
 });
 

backend/src/services/CacheService.ts

@@ -26,6 +26,24 @@ interface CacheEntry<T> {
   expiresAt: number;
 }
 
+/**
+ * Observational outcome of a single getOrFetchWithMeta call. Truthful under
+ * concurrency: an `inflight` join is never reported as a `hit`.
+ *
+ *   - hit:      fresh entry returned without waiting
+ *   - computed: this caller ran the fetcher and it succeeded
+ *   - inflight: joined an existing in-flight promise (this caller did not run
+ *               the fetcher)
+ *   - stale:    the fetcher this caller ran failed and a stale entry was
+ *               returned instead
+ */
+export type CacheFetchOutcome = 'hit' | 'computed' | 'inflight' | 'stale';
+
+export interface CacheFetchResult<T> {
+  value: T;
+  outcome: CacheFetchOutcome;
+}
+
 interface NamespaceStats {
   hits: number;
   misses: number;
@@ -61,26 +79,51 @@ export class CacheService {
    *
    * On fetcher rejection: if a stale entry exists, return it (counted as
    * `stale`); otherwise propagate the error.
+   *
+   * Thin wrapper over getOrFetchWithMeta that discards the outcome; all
+   * stats / TTL / inflight-dedup / stale-on-error semantics are identical.
    */
   public async getOrFetch<T>(
     key: string,
     ttlMs: number,
     fetcher: () => Promise<T>,
   ): Promise<T> {
+    const { value } = await this.getOrFetchWithMeta(key, ttlMs, fetcher);
+    return value;
+  }
+
+  /**
+   * Same behaviour as getOrFetch, but also reports how the value was obtained
+   * (see CacheFetchOutcome). Intended for diagnostic logging that must not
+   * mislabel an in-flight join as a cache hit. The stats counters, TTL, cap,
+   * and stale-on-error fallback are recorded exactly as getOrFetch does: one
+   * miss per in-flight join, one stale per failed computation.
+   */
+  public async getOrFetchWithMeta<T>(
+    key: string,
+    ttlMs: number,
+    fetcher: () => Promise<T>,
+  ): Promise<CacheFetchResult<T>> {
     const ns = namespaceOf(key);
     const now = Date.now();
     const existing = this.store.get(key) as CacheEntry<T> | undefined;
 
     if (existing && existing.expiresAt > now) {
       this.recordHit(ns);
-      return existing.value;
+      return { value: existing.value, outcome: 'hit' };
     }
 
     this.recordMiss(ns);
 
     const inflight = this.inflight.get(key) as Promise<T> | undefined;
-    if (inflight) return inflight;
+    if (inflight) {
+      const value = await inflight;
+      return { value, outcome: 'inflight' };
+    }
 
+    // This caller owns the computation; the closure records whether it ended
+    // as a fresh compute or a stale fallback, read after the promise settles.
+    let outcome: CacheFetchOutcome = 'computed';
     const promise = (async () => {
       try {
         const value = await fetcher();
@@ -89,6 +132,7 @@ export class CacheService {
       } catch (err) {
         if (existing) {
           this.recordStale(ns);
+          outcome = 'stale';
           return existing.value;
         }
         throw err;
@@ -98,7 +142,8 @@ export class CacheService {
     })();
 
     this.inflight.set(key, promise);
-    return promise;
+    const value = await promise;
+    return { value, outcome };
   }
 
   /**

backend/src/utils/requestTiming.ts

@@ -0,0 +1,60 @@
+import { isDebugEnabled } from './debug';
+import { sanitizeForLog } from './safeLog';
+
+/**
+ * Developer-mode hydration timing helpers.
+ *
+ * These emit one structured line per instrumented request via console.debug,
+ * gated on the same `developer_mode` flag as every other diagnostic log. The
+ * lines carry durations, counts, and outcomes only. Callers must never place
+ * a raw URL, query string, or token into `fields`; use templatizeHydrationPath
+ * for paths so a real stack name can never reach the log.
+ */
+
+/** Render one field value: numbers/booleans verbatim, everything else sanitized. */
+function formatFieldValue(value: unknown): string {
+  if (typeof value === 'number' || typeof value === 'boolean') return String(value);
+  if (value === null || value === undefined) return String(value);
+  return sanitizeForLog(value);
+}
+
+/**
+ * Emit a `<prefix> key=value ...` timing line, but only when developer_mode is
+ * enabled. String values are control-char stripped to prevent log injection.
+ */
+export function logDebugTiming(prefix: string, fields: Record<string, unknown>): void {
+  if (!isDebugEnabled()) return;
+  const rendered = Object.entries(fields)
+    .map(([key, value]) => `${key}=${formatFieldValue(value)}`)
+    .join(' ');
+  console.debug(`${prefix} ${rendered}`);
+}
+
+/**
+ * Critical hydration GET route templates, ordered most-specific-first so the
+ * bare `/api/stacks` collection never shadows a nested match. The templates
+ * are static strings, so a real stack name is never substituted back in.
+ */
+const HYDRATION_PATH_TEMPLATES: ReadonlyArray<{ pattern: RegExp; template: string }> = [
+  { pattern: /^\/api\/stacks\/statuses$/, template: '/api/stacks/statuses' },
+  { pattern: /^\/api\/stacks\/[^/]+\/containers$/, template: '/api/stacks/:stack/containers' },
+  { pattern: /^\/api\/stacks$/, template: '/api/stacks' },
+  { pattern: /^\/api\/image-updates\/status$/, template: '/api/image-updates/status' },
+  { pattern: /^\/api\/image-updates\/detail$/, template: '/api/image-updates/detail' },
+  { pattern: /^\/api\/notifications$/, template: '/api/notifications' },
+];
+
+/**
+ * Map a request path to a stable route template for logging. The query string
+ * is stripped before matching, and only the critical hydration GETs resolve;
+ * any other path returns null so callers skip logging. The returned value is
+ * always a fixed template (e.g. `/api/stacks/:stack/containers`), never a path
+ * segment taken from the request.
+ */
+export function templatizeHydrationPath(rawPath: string): string | null {
+  const pathname = rawPath.split('?')[0];
+  for (const { pattern, template } of HYDRATION_PATH_TEMPLATES) {
+    if (pattern.test(pathname)) return template;
+  }
+  return null;
+}