feat: add developer-mode startup and stack hydration timing (#1619)

* feat: add developer-mode startup and stack hydration timing

Instrument boot-to-list and detail hydration with commit-aligned milestones, truthful request stages, and destination/gateway debug duration logs so performance work is guided by measurements.

* fix: redact stack names and complete hydration request stages

Stop logging stack identifiers in containers debug timing, and record state_dispatch (plus detail fetch spans) so copied reports match the advertised stage breakdown.

frontend/src/components/EditorLayout/hooks/useNotifications.ts

@@ -1,5 +1,6 @@
 import { useEffect, useRef, useState } from 'react';
 import { apiFetch, fetchForNode } from '@/lib/api';
+import { beginSpan, endSpan, markMilestone } from '@/lib/hydrationTiming';
 import { toast } from '@/components/ui/toast-store';
 import type { Node } from '@/context/NodeContext';
 import type { NotificationItem } from '../../dashboard/types';
@@ -23,6 +24,10 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang
   onStateInvalidateRef.current = onStateInvalidate;
   const onImageUpdatesChangeRef = useRef(onImageUpdatesChange);
   onImageUpdatesChangeRef.current = onImageUpdatesChange;
+  // One-shot: the notifications_ready milestone reflects the first local settle.
+  // Its spans instrument only that first fetch so later polls do not pollute the
+  // report.
+  const notificationsReadyRef = useRef(false);
 
   const fetchNotifications = async () => {
     try {
@@ -31,18 +36,41 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang
       // Skip offline nodes: polling a removed/unreachable node only yields 502s.
       const remoteNodes = currentNodes.filter(n => n.type === 'remote' && n.status !== 'offline');
 
-      const [localResult, ...remoteNodeResults] = await Promise.allSettled([
-        apiFetch('/notifications', { localOnly: true } as Parameters<typeof apiFetch>[1]),
-        ...remoteNodes.map(n => fetchForNode('/notifications', n.id)),
-      ]);
+      // Launch every request concurrently, but settle the local one first so the
+      // notifications_ready milestone is not held back by the slowest remote.
+      const localPromise = apiFetch('/notifications', { localOnly: true } as Parameters<typeof apiFetch>[1]);
+      const remotePromises = remoteNodes.map(n => fetchForNode('/notifications', n.id));
 
       const all: NotificationItem[] = [];
 
-      if (localResult.status === 'fulfilled' && localResult.value.ok) {
-        const data = await localResult.value.json() as Omit<NotificationItem, 'nodeId' | 'nodeName'>[];
-        data.forEach(n => all.push({ ...n, nodeId: localNode?.id ?? -1, nodeName: localNode?.name ?? 'Local' }));
+      const instrument = !notificationsReadyRef.current;
+      const headersSpan = instrument ? beginSpan('fetch_headers', { background: true }) : null;
+      let bodySpan: ReturnType<typeof beginSpan> | null = null;
+      try {
+        const localRes = await localPromise;
+        if (headersSpan !== null) endSpan(headersSpan, { detail: { status: localRes.status } });
+        if (localRes.ok) {
+          bodySpan = instrument ? beginSpan('body_decode', { background: true }) : null;
+          const data = await localRes.json() as Omit<NotificationItem, 'nodeId' | 'nodeName'>[];
+          if (bodySpan !== null) endSpan(bodySpan);
+          bodySpan = null;
+          const dispatchSpan = instrument ? beginSpan('state_dispatch', { background: true }) : null;
+          data.forEach(n => all.push({ ...n, nodeId: localNode?.id ?? -1, nodeName: localNode?.name ?? 'Local' }));
+          if (dispatchSpan !== null) endSpan(dispatchSpan);
+        }
+      } catch (e) {
+        if (headersSpan !== null) endSpan(headersSpan, { outcome: 'error' });
+        if (bodySpan !== null) endSpan(bodySpan, { outcome: 'error' });
+        console.error('[Notifications] local fetch error:', e);
+      } finally {
+        if (!notificationsReadyRef.current) {
+          notificationsReadyRef.current = true;
+          markMilestone('notifications_ready');
+        }
       }
 
+      // Remotes settle in the background; they never gate the milestone above.
+      const remoteNodeResults = await Promise.allSettled(remotePromises);
       for (let i = 0; i < remoteNodes.length; i++) {
         const result = remoteNodeResults[i];
         if (result?.status === 'fulfilled' && result.value.ok) {

frontend/src/components/EditorLayout/hooks/useStackActions.ts

@@ -1,5 +1,14 @@
-import { useRef, useCallback, useEffect } from 'react';
+import { useRef, useCallback, useEffect, useState } from 'react';
 import { apiFetch, withDeploySession } from '@/lib/api';
+import {
+  newAttemptId,
+  abortAttempt,
+  beginSpan,
+  endSpan,
+  flushPendingCommit,
+  type PendingCommit,
+  type SpanHandle,
+} from '@/lib/hydrationTiming';
 import { toast } from '@/components/ui/toast-store';
 import { buildServiceUrl, openServiceUrl } from '@/lib/serviceUrl';
 import type { useEditorViewState } from './useEditorViewState';
@@ -268,6 +277,17 @@ export function useStackActions(options: UseStackActionsOptions) {
   // late responses never overwrite freshly-loaded state.
   const loadFileAbortRef = useRef<AbortController | null>(null);
 
+  // Hydration-timing: the current detail (loadFileCore) attempt, the file it is
+  // loading, and the commits waiting for React to observe committed state.
+  const detailAttemptRef = useRef<string | null>(null);
+  const detailFileRef = useRef<string | null>(null);
+  const detailVisiblePendingRef = useRef<PendingCommit | null>(null);
+  const detailContainersPendingRef = useRef<PendingCommit | null>(null);
+  const detailHydratedPendingRef = useRef<PendingCommit | null>(null);
+  // Bumped when arming detail_visible so same-file reloads (selectedFile and
+  // activeView unchanged) still re-run the commit effect.
+  const [detailVisibleEpoch, setDetailVisibleEpoch] = useState(0);
+
   useEffect(() => {
     return () => {
       if (checkUpdatesIntervalRef.current !== null) {
@@ -277,6 +297,28 @@ export function useStackActions(options: UseStackActionsOptions) {
     };
   }, []);
 
+  // Commit-aligned detail milestones. Each fires once React has committed the
+  // observed state for the owning attempt; commitMilestone no-ops for a
+  // superseded (node switch) or aborted (re-load) attempt so an interrupted
+  // load never records a success milestone.
+  useEffect(() => {
+    if (stackListState.selectedFile !== detailFileRef.current) return;
+    if (navState.activeView !== 'editor') return;
+    flushPendingCommit(detailVisiblePendingRef, 'detail_visible');
+  }, [stackListState.selectedFile, navState.activeView, detailVisibleEpoch]);
+
+  useEffect(() => {
+    if (stackListState.selectedFile !== detailFileRef.current) return;
+    flushPendingCommit(detailContainersPendingRef, 'detail_containers_ready');
+  }, [editorState.containers, stackListState.selectedFile]);
+
+  useEffect(() => {
+    // Wait for the load to settle so this reflects the fully hydrated detail.
+    if (editorState.isFileLoading) return;
+    if (stackListState.selectedFile !== detailFileRef.current) return;
+    flushPendingCommit(detailHydratedPendingRef, 'detail_hydrated');
+  }, [editorState.isFileLoading, stackListState.selectedFile, editorState.containers]);
+
   const isAbortError = (err: unknown): boolean =>
     err instanceof Error && err.name === 'AbortError';
 
@@ -476,16 +518,47 @@ export function useStackActions(options: UseStackActionsOptions) {
     }
   };
 
-  const loadContainerState = async (filename: string, signal?: AbortSignal) => {
+  const loadContainerState = async (
+    filename: string,
+    signal?: AbortSignal,
+    attemptId?: string,
+    proxied?: boolean,
+  ): Promise<number> => {
+    let headersSpan: SpanHandle | null = null;
+    let bodySpan: SpanHandle | null = null;
     try {
+      headersSpan = attemptId
+        ? beginSpan('fetch_headers', { attemptId, proxied })
+        : null;
       const containersRes = await apiFetch(`/stacks/${filename}/containers`, { signal });
-      if (signal?.aborted) return;
+      const hopProxied = containersRes.headers.get('x-sencho-proxy') === '1' || proxied === true;
+      if (headersSpan !== null) {
+        endSpan(headersSpan, { proxied: hopProxied, detail: { status: containersRes.status } });
+        headersSpan = null;
+      }
+      if (signal?.aborted) return 0;
+      bodySpan = attemptId
+        ? beginSpan('body_decode', { attemptId, proxied: hopProxied })
+        : null;
       const conts = await containersRes.json();
-      editorState.setContainers(Array.isArray(conts) ? conts : []);
+      if (bodySpan !== null) {
+        endSpan(bodySpan);
+        bodySpan = null;
+      }
+      const list = Array.isArray(conts) ? conts : [];
+      const dispatchSpan = attemptId
+        ? beginSpan('state_dispatch', { attemptId, proxied: hopProxied })
+        : null;
+      editorState.setContainers(list);
+      if (dispatchSpan !== null) endSpan(dispatchSpan);
+      return list.length;
     } catch (error) {
-      if (isAbortError(error)) return;
+      if (headersSpan !== null) endSpan(headersSpan, { outcome: 'error' });
+      if (bodySpan !== null) endSpan(bodySpan, { outcome: 'error' });
+      if (isAbortError(error)) return 0;
       console.error('Failed to load containers:', error);
       editorState.setContainers([]);
+      return 0;
     }
   };
 
@@ -520,29 +593,58 @@ export function useStackActions(options: UseStackActionsOptions) {
       return { ok: false };
     }
     loadFileAbortRef.current?.abort();
+    // Supersede the previous detail attempt so a late commit from an interrupted
+    // load can never record a success milestone.
+    if (detailAttemptRef.current) abortAttempt(detailAttemptRef.current);
+    detailVisiblePendingRef.current = null;
+    detailContainersPendingRef.current = null;
+    detailHydratedPendingRef.current = null;
     const controller = new AbortController();
     loadFileAbortRef.current = controller;
     const { signal } = controller;
+    const attemptId = newAttemptId();
+    detailAttemptRef.current = attemptId;
+    detailFileRef.current = filename;
 
     editorState.setIsFileLoading(true);
     editorState.setIsEditing(false);
     editorState.setEditingCompose(false);
     editorState.setActiveTab('compose');
+    let headersSpan: SpanHandle | null = null;
+    let bodySpan: SpanHandle | null = null;
     try {
+      headersSpan = beginSpan('fetch_headers', { attemptId });
       const res = await apiFetch(`/stacks/${filename}`, { signal });
-      if (signal.aborted) return { ok: false };
+      const proxied = res.headers.get('x-sencho-proxy') === '1';
+      endSpan(headersSpan, { proxied, detail: { status: res.status } });
+      headersSpan = null;
+      if (signal.aborted) { abortAttempt(attemptId); return { ok: false }; }
+      bodySpan = beginSpan('body_decode', { attemptId, proxied });
       const text = await res.text();
-      if (signal.aborted) return { ok: false };
+      endSpan(bodySpan);
+      bodySpan = null;
+      if (signal.aborted) { abortAttempt(attemptId); return { ok: false }; }
       if (!res.ok) {
         throw new Error(`Failed to load stack: ${res.status}`);
       }
+      const dispatchSpan = beginSpan('state_dispatch', { attemptId, proxied });
       stackListState.setSelectedFile(filename);
       navState.setActiveView('editor');
       editorState.setContent(text || '');
       editorState.setOriginalContent(text || '');
       editorState.setComposeEtag(res.headers.get('etag'));
+      endSpan(dispatchSpan);
+      detailVisiblePendingRef.current = { attemptId, token: filename, proxied };
+      setDetailVisibleEpoch((n) => n + 1);
       const envFiles = await loadEnvState(filename, signal);
-      await loadContainerState(filename, signal);
+      const containerCount = await loadContainerState(filename, signal, attemptId, proxied);
+      if (!signal.aborted) {
+        detailContainersPendingRef.current = {
+          attemptId,
+          token: `${filename}:${containerCount}`,
+          proxied,
+        };
+      }
       await loadBackupState(filename, signal);
       // Post-load auto-edit evaluates permission for the loaded target, not
       // the previously selected stack (selectedFile was just updated above).
@@ -551,9 +653,14 @@ export function useStackActions(options: UseStackActionsOptions) {
         editorState.setActiveTab('compose');
         editorState.setIsEditing(true);
       }
+      if (!signal.aborted) {
+        detailHydratedPendingRef.current = { attemptId, token: filename, proxied };
+      }
       return { ok: true, envFiles };
     } catch (error) {
-      if (isAbortError(error) || signal.aborted) return { ok: false };
+      if (headersSpan !== null) endSpan(headersSpan, { outcome: 'error' });
+      if (bodySpan !== null) endSpan(bodySpan, { outcome: 'error' });
+      if (isAbortError(error) || signal.aborted) { abortAttempt(attemptId); return { ok: false }; }
       console.error('Failed to load file:', error);
       toast.error(`Could not open "${filename.replace(/\.(ya?ml)$/, '')}". Check your connection and try again.`);
       stackListState.setSelectedFile(null);

frontend/src/components/EditorLayout/hooks/useStackListState.ts

@@ -1,5 +1,15 @@
 import { useState, useEffect, useRef, useMemo, useCallback } from 'react';
 import { apiFetch } from '@/lib/api';
+import {
+  newAttemptId,
+  abortAttempt,
+  beginSpan,
+  endSpan,
+  flushPendingCommit,
+  markMilestone,
+  type SpanHandle,
+  type PendingCommit,
+} from '@/lib/hydrationTiming';
 import { toast } from '@/components/ui/toast-store';
 import { useNodes } from '@/context/NodeContext';
 import { useImageUpdates } from '@/hooks/useImageUpdates';
@@ -85,6 +95,13 @@ export function useStackListState() {
   // state writes so a rapid node switch cannot leave a stale files/filesNodeId.
   const fetchSeqRef = useRef(0);
 
+  // Hydration-timing: the current foreground list attempt and the commits it is
+  // waiting for React to observe. Only foreground loads arm these; background
+  // refreshes still record diagnostic spans but do not re-commit the milestones.
+  const listAttemptRef = useRef<string | null>(null);
+  const listVisiblePendingRef = useRef<PendingCommit | null>(null);
+  const listHydratedPendingRef = useRef<PendingCommit | null>(null);
+
   // Per-stack terminal failure records driving the in-detail recovery panel.
   // In-memory only. Node scoping is enforced by the caller, which clears these
   // on active-node change (see EditorLayout's node-switch effect) so a repeated
@@ -196,6 +213,20 @@ export function useStackListState() {
     const mySeq = ++fetchSeqRef.current;
     const stale = () => fetchSeqRef.current !== mySeq;
 
+    // Supersede any in-flight list attempt so a late commit from an interrupted
+    // load cannot record list_visible / list_hydrated for a stale fetch.
+    if (listAttemptRef.current) abortAttempt(listAttemptRef.current);
+    listVisiblePendingRef.current = null;
+    listHydratedPendingRef.current = null;
+
+    const attemptId = newAttemptId();
+    listAttemptRef.current = attemptId;
+    // True once the list itself is committed, so the shared catch below can tell
+    // a list-fetch failure (nothing visible) from a status-path failure (list is
+    // visible, hydration errored).
+    let listSucceeded = false;
+    let proxied = false;
+
     if (!background) setIsLoading(true);
     setStacksLoadNodeId(fetchNodeId);
     if (!background || !hadSuccessfulListRef.current) {
@@ -203,9 +234,13 @@ export function useStackListState() {
       setStacksLoadError(null);
     }
 
+    const headersSpan = beginSpan('fetch_headers', { attemptId, background });
+    let bodySpan: SpanHandle | null = null;
     try {
       const res = await apiFetch('/stacks');
-      if (stale()) return [];
+      proxied = res.headers.get('x-sencho-proxy') === '1';
+      endSpan(headersSpan, { proxied, detail: { status: res.status } });
+      if (stale()) { abortAttempt(attemptId); return []; }
       if (!res.ok) {
         const message = `Could not load stacks (${res.status})`;
         if (background && hadSuccessfulListRef.current) {
@@ -218,26 +253,51 @@ export function useStackListState() {
         setStacksLoadError(message);
         return [];
       }
+      bodySpan = beginSpan('body_decode', { attemptId, background, proxied });
       const data = await res.json();
+      endSpan(bodySpan);
+      bodySpan = null;
       const fileList: string[] = Array.isArray(data) ? data : [];
+      const listDispatch = beginSpan('state_dispatch', { attemptId, background, proxied });
       setFiles(fileList);
       setFilesNodeId(fetchNodeId);
       hadSuccessfulListRef.current = true;
       setStacksLoadStatus('success');
       setStacksLoadError(null);
+      endSpan(listDispatch);
+      listSucceeded = true;
+      // Token folds node + count so an empty->empty commit still fires once per
+      // attempt even when the committed `files` is referentially equal.
+      const listToken = `${fetchNodeId}:${fileList.length}`;
+      if (!background) {
+        listVisiblePendingRef.current = { attemptId, token: listToken, proxied };
+      }
 
       // Fetch all stack statuses in a single bulk call. Only the current object
       // format can express `partial`; a node lacking the endpoint or returning
       // the legacy plain-string format is re-derived from per-stack containers
       // so a crashed container is not hidden behind a healthy sibling.
+      const statusHeaders = beginSpan('fetch_headers', { attemptId, background, proxied });
       const statusRes = await apiFetch('/stacks/statuses');
-      if (stale()) return fileList;
+      const statusProxied = statusRes.headers.get('x-sencho-proxy') === '1' || proxied;
+      endSpan(statusHeaders, { proxied: statusProxied, detail: { status: statusRes.status } });
+      if (stale()) { abortAttempt(attemptId); return fileList; }
       let bulkStatuses: Record<string, StackRowStatus> = {};
       const bulkPorts: Record<string, number | undefined> = {};
       const bulkSelf: Record<string, boolean> = {};
       const bulkCounts: StackCounts = {};
 
-      const raw: unknown = statusRes.ok ? await statusRes.json() : null;
+      let raw: unknown = null;
+      if (statusRes.ok) {
+        const statusBodySpan = beginSpan('body_decode', { attemptId, background, proxied: statusProxied });
+        try {
+          raw = await statusRes.json();
+          endSpan(statusBodySpan);
+        } catch (decodeErr) {
+          endSpan(statusBodySpan, { outcome: 'error' });
+          throw decodeErr;
+        }
+      }
       if (isBulkStatusObjectFormat(raw)) {
         for (const [key, val] of Object.entries(raw as Record<string, StackStatusInfo>)) {
           bulkStatuses[key] = val.status;
@@ -250,6 +310,7 @@ export function useStackListState() {
       } else {
         bulkStatuses = await deriveStatusesFromContainers(fileList);
       }
+      const statusDispatch = beginSpan('state_dispatch', { attemptId, background, proxied: statusProxied });
       setStackStatuses(prev => {
         const next: StackStatus = {};
         for (const file of fileList) {
@@ -265,12 +326,24 @@ export function useStackListState() {
       });
       setStackSelfFlags(bulkSelf);
       setStackCounts(bulkCounts);
+      endSpan(statusDispatch);
       refreshLabels();
+      if (!background) {
+        listHydratedPendingRef.current = { attemptId, token: listToken, proxied: statusProxied };
+      }
       return fileList;
     } catch (error) {
-      if (stale()) return [];
+      // endSpan is a no-op when the span was already closed (or never opened).
+      endSpan(headersSpan, { outcome: 'error' });
+      if (bodySpan !== null) endSpan(bodySpan, { outcome: 'error' });
+      if (stale()) { abortAttempt(attemptId); return []; }
       console.error('Failed to refresh stacks:', error);
       const message = error instanceof Error ? error.message : 'Failed to load stacks';
+      // The list committed but hydrating its statuses threw: record the list
+      // path as hydrated-with-error rather than leaving it hanging.
+      if (listSucceeded && !background) {
+        markMilestone('list_hydrated', { attemptId, outcome: 'error', proxied });
+      }
       if (background && hadSuccessfulListRef.current) {
         setStacksLoadError(message);
         return files;
@@ -290,6 +363,20 @@ export function useStackListState() {
   const refreshStacksRef = useRef(refreshStacks);
   useEffect(() => { refreshStacksRef.current = refreshStacks; });
 
+  // Commit-aligned list milestones: fire once React has actually committed the
+  // file list (list_visible) and the statuses (list_hydrated) for the owning
+  // attempt. commitMilestone no-ops for a superseded/aborted attempt, so a stale
+  // load can never complete a session it no longer owns. Empty lists still fire
+  // via the completion token.
+  useEffect(() => {
+    if (stacksLoadStatus !== 'success') return;
+    flushPendingCommit(listVisiblePendingRef, 'list_visible');
+  }, [files, filesNodeId, stacksLoadStatus]);
+
+  useEffect(() => {
+    flushPendingCommit(listHydratedPendingRef, 'list_hydrated');
+  }, [stackStatuses, filesNodeId]);
+
   const handleScanStacks = async () => {
     if (isScanning) return;
     setIsScanning(true);