feat(dashboard): surface metrics-stale indicator after sustained poll failure (#1213)

* feat(dashboard): surface metrics-paused indicator after sustained poll failure

useDashboardData previously failed silently when /stats or /system/stats
returned an error: stale data kept rendering and the last sync timestamp
quietly drifted. The operator could not tell whether the dashboard was just
slow or whether the Docker socket / metrics path had genuinely gone down.

Track consecutive failures per live-metrics endpoint. After three in a row
on either /stats or /system/stats (≈15 s at the 5 s poll cadence), expose a
metricsStale boolean on the hook result. HealthStatusBar renders a small
amber "metrics paused" chip beside the meta line when set. The indicator
clears on the first successful response when both endpoints are within the
threshold.

A unit test for the threshold logic is intentionally deferred to the Phase
4 E2E dashboard spec, which exercises the same path end-to-end by stopping
the Docker daemon and asserting the user-visible indicator.

* fix(dashboard): rename stale-metrics chip and cover the threshold with tests

Address two follow-up findings from independent review of the earlier
commit on this branch.

1. Rename the masthead chip from "metrics paused" to "metrics stale". The
   hook keeps polling on every cycle; the chip describes the freshness of
   the displayed numbers, not the polling cadence. The new wording matches
   the underlying `metricsStale` state variable.

2. Add a Vitest spec for the threshold logic. Captures the visibilityInterval
   callback at registration time and drives each polling cycle on demand,
   covering: three consecutive /stats failures trip the indicator and the
   next successful poll clears it; three consecutive /system/stats failures
   trip the indicator on the other endpoint; clearing requires both
   endpoints under threshold (a single endpoint recovering while the other
   is still failing keeps the indicator set).

The clarifying comment in useDashboardData notes that polling is unaffected
and only the data freshness is in scope, so future readers do not interpret
"stale" as "paused".

frontend/src/components/HomeDashboard.tsx

@@ -34,6 +34,7 @@ export default function HomeDashboard({ onNavigateToStack, onOpenSettingsSection
         activeNodeName={activeNodeName}
         nodeCount={data.nodeCount}
         lastSyncAt={data.lastSyncAt}
+        metricsStale={data.metricsStale}
       />
 
       <ResourceGauges

frontend/src/components/dashboard/HealthStatusBar.tsx

@@ -15,6 +15,8 @@ interface HealthStatusBarProps {
   activeNodeName: string;
   nodeCount: number;
   lastSyncAt: number | null;
+  /** True after several consecutive metrics polls have failed (Docker socket or metrics path down). */
+  metricsStale?: boolean;
 }
 
 interface HealthResult {
@@ -100,6 +102,7 @@ export function HealthStatusBar({
   activeNodeName,
   nodeCount,
   lastSyncAt,
+  metricsStale = false,
 }: HealthStatusBarProps) {
   const { level, reasons } = useMemo(
     () => deriveHealth(stats, systemStats, notifications),
@@ -134,6 +137,11 @@ export function HealthStatusBar({
             </span>
             <span className="font-mono text-[10px] leading-3 uppercase tracking-[0.18em] text-stat-subtitle">
               {metaLine}
+              {metricsStale ? (
+                <span className="ml-2 inline-flex items-center rounded-sm border border-warning/30 bg-warning/10 px-1.5 py-0.5 text-[10px] font-mono tracking-wide uppercase text-warning">
+                  metrics stale
+                </span>
+              ) : null}
             </span>
             {reasonsLine ? (
               <span className="font-mono text-[11px] text-stat-subtitle/90">

frontend/src/components/dashboard/__tests__/useDashboardData.metricsStale.test.tsx

@@ -0,0 +1,174 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { renderHook, act } from '@testing-library/react';
+
+const apiFetchMock = vi.fn();
+
+vi.mock('@/lib/api', () => ({
+  apiFetch: (...args: unknown[]) => apiFetchMock(...args),
+}));
+
+const useNodesMock = vi.fn();
+vi.mock('@/context/NodeContext', () => ({
+  useNodes: () => useNodesMock(),
+}));
+
+// Capture every visibilityInterval registration so each polling cycle can
+// be driven on demand from the test without leaning on real setInterval
+// timers (which fight with vi.useFakeTimers and the await-then-setState
+// sequence inside the polling callbacks).
+type PollEntry = { fn: () => void; intervalMs: number };
+let pollCallbacks: PollEntry[] = [];
+vi.mock('@/lib/utils', async () => {
+  const actual = await vi.importActual<typeof import('@/lib/utils')>('@/lib/utils');
+  return {
+    ...actual,
+    visibilityInterval: (fn: () => void, intervalMs: number) => {
+      pollCallbacks.push({ fn, intervalMs });
+      return () => { pollCallbacks = pollCallbacks.filter((c) => c.fn !== fn); };
+    },
+  };
+});
+
+import { useDashboardData } from '../useDashboardData';
+
+function okJson(payload: unknown): Response {
+  return new Response(JSON.stringify(payload), {
+    status: 200,
+    headers: { 'Content-Type': 'application/json' },
+  });
+}
+
+function failJson(): Response {
+  return new Response(JSON.stringify({ error: 'down' }), { status: 500 });
+}
+
+const STATS_PAYLOAD = { active: 0, managed: 0, unmanaged: 0, exited: 0, total: 0 };
+const SYS_PAYLOAD = {
+  cpu: { usage: '0', cores: 4 },
+  memory: { total: 0, used: 0, free: 0, usagePercent: '0' },
+  disk: null,
+};
+
+function setEndpointOutcome(endpoint: '/stats' | '/system/stats', ok: boolean): void {
+  apiFetchMock.mockImplementation((requested: string) => {
+    if (requested === '/stats') {
+      if (endpoint === '/stats' && !ok) return Promise.resolve(failJson());
+      return Promise.resolve(okJson(STATS_PAYLOAD));
+    }
+    if (requested === '/system/stats') {
+      if (endpoint === '/system/stats' && !ok) return Promise.resolve(failJson());
+      return Promise.resolve(okJson(SYS_PAYLOAD));
+    }
+    if (requested === '/stacks/statuses') return Promise.resolve(okJson({}));
+    if (requested === '/metrics/historical') return Promise.resolve(okJson([]));
+    return Promise.resolve(okJson(null));
+  });
+}
+
+// Drive the next /stats poll (it is the first 5 s interval registered).
+async function tickStats(): Promise<void> {
+  const stats = pollCallbacks.find((c) => c.intervalMs === 5000);
+  if (!stats) throw new Error('stats poll callback not registered');
+  await act(async () => {
+    stats.fn();
+    // Two microtask drains: one for the apiFetch promise, one for the await
+    // on res.json() inside fetchJson.
+    await Promise.resolve();
+    await Promise.resolve();
+  });
+}
+
+// Drive the next /system/stats poll (the second 5 s interval registered).
+async function tickSys(): Promise<void> {
+  const sys = pollCallbacks.filter((c) => c.intervalMs === 5000)[1];
+  if (!sys) throw new Error('system-stats poll callback not registered');
+  await act(async () => {
+    sys.fn();
+    await Promise.resolve();
+    await Promise.resolve();
+  });
+}
+
+beforeEach(() => {
+  pollCallbacks = [];
+  apiFetchMock.mockReset();
+  useNodesMock.mockReset();
+  useNodesMock.mockReturnValue({
+    activeNode: { id: 1, name: 'Local', type: 'local' },
+    nodes: [{ id: 1, name: 'Local', type: 'local' }],
+  });
+});
+
+afterEach(() => {
+  vi.clearAllMocks();
+});
+
+describe('useDashboardData metricsStale threshold', () => {
+  it('trips after three consecutive /stats failures and clears on the next success', async () => {
+    // Stats fails from the very first poll; system-stats keeps succeeding.
+    setEndpointOutcome('/stats', false);
+    const { result } = renderHook(() => useDashboardData());
+    // Drain mount-time fetches.
+    await act(async () => { await Promise.resolve(); await Promise.resolve(); });
+
+    // Mount fired one failed /stats poll (counter = 1) and one successful
+    // /system/stats poll. Two more failed stats polls cross the threshold.
+    expect(result.current.metricsStale).toBe(false);
+    await tickStats();
+    expect(result.current.metricsStale).toBe(false);
+    await tickStats();
+    expect(result.current.metricsStale).toBe(true);
+
+    // The first successful /stats poll resets the counter; with the
+    // /system/stats endpoint still below threshold, the indicator clears.
+    setEndpointOutcome('/stats', true);
+    await tickStats();
+    expect(result.current.metricsStale).toBe(false);
+  });
+
+  it('trips after three consecutive /system/stats failures as well', async () => {
+    setEndpointOutcome('/system/stats', false);
+    const { result } = renderHook(() => useDashboardData());
+    await act(async () => { await Promise.resolve(); await Promise.resolve(); });
+
+    expect(result.current.metricsStale).toBe(false);
+    await tickSys();
+    expect(result.current.metricsStale).toBe(false);
+    await tickSys();
+    expect(result.current.metricsStale).toBe(true);
+  });
+
+  it('keeps the indicator set when one endpoint recovers but the other is still failing', async () => {
+    // Both fail; trip on stats first, then recover stats only.
+    setEndpointOutcome('/stats', false);
+    const { result } = renderHook(() => useDashboardData());
+    await act(async () => { await Promise.resolve(); await Promise.resolve(); });
+
+    // Now break /system/stats too so its counter starts climbing.
+    apiFetchMock.mockImplementation((requested: string) => {
+      if (requested === '/stats') return Promise.resolve(failJson());
+      if (requested === '/system/stats') return Promise.resolve(failJson());
+      if (requested === '/stacks/statuses') return Promise.resolve(okJson({}));
+      if (requested === '/metrics/historical') return Promise.resolve(okJson([]));
+      return Promise.resolve(okJson(null));
+    });
+
+    // Three failing sys polls trip the indicator.
+    await tickSys();
+    await tickSys();
+    await tickSys();
+    expect(result.current.metricsStale).toBe(true);
+
+    // Stats recovers but /system/stats is still failing: indicator stays set
+    // because the sys counter is still above the threshold.
+    apiFetchMock.mockImplementation((requested: string) => {
+      if (requested === '/stats') return Promise.resolve(okJson(STATS_PAYLOAD));
+      if (requested === '/system/stats') return Promise.resolve(failJson());
+      if (requested === '/stacks/statuses') return Promise.resolve(okJson({}));
+      if (requested === '/metrics/historical') return Promise.resolve(okJson([]));
+      return Promise.resolve(okJson(null));
+    });
+    await tickStats();
+    expect(result.current.metricsStale).toBe(true);
+  });
+});

frontend/src/components/dashboard/types.ts

@@ -97,6 +97,8 @@ export interface DashboardData {
   stackCpuSeries: Record<string, StackCpuSeries>;
   cpuHistory: number[];
   netHistory: number[];
-  /** Anchor timestamp (ms) for the sparkline 10-minute window — the newest metric sample. */
+  /** Anchor timestamp (ms) for the sparkline 10-minute window: the newest metric sample. */
   historyEndAt: number | null;
+  /** True after several consecutive `/stats` or `/system/stats` polls have failed; surfaces a "metrics paused" indicator. */
+  metricsStale: boolean;
 }

frontend/src/components/dashboard/useDashboardData.ts

@@ -41,6 +41,14 @@ function bucketCpu(points: MetricPoint[], windowMs: number, buckets: number): nu
   return out;
 }
 
+// After three consecutive failures of the live metrics endpoints, surface a
+// "metrics stale" indicator so the operator knows the gauges are no longer
+// being refreshed (the Docker socket or the metrics service is unreachable)
+// rather than just slow. Polling continues; the indicator describes the
+// freshness of the visible numbers, not the polling cadence. The threshold
+// is chosen so a single transient hiccup does not trip the indicator.
+const METRICS_STALE_THRESHOLD = 3;
+
 export function useDashboardData(): DashboardData {
   const { activeNode, nodes } = useNodes();
   const nodeId = activeNode?.id;
@@ -50,12 +58,19 @@ export function useDashboardData(): DashboardData {
   const [metrics, setMetrics] = useState<MetricPoint[]>([]);
   const [stackStatuses, setStackStatuses] = useState<Record<string, StackStatusEntry>>({});
   const [lastSyncAt, setLastSyncAt] = useState<number | null>(null);
+  const [metricsStale, setMetricsStale] = useState(false);
 
   // Keep a ref to the latest nodeId so async callbacks don't write stale data
   // after a node switch has already triggered a new effect cycle.
   const nodeIdRef = useRef(nodeId);
   useEffect(() => { nodeIdRef.current = nodeId; }, [nodeId]);
 
+  // Consecutive failure counters per live-metrics endpoint. Either reaching
+  // METRICS_STALE_THRESHOLD trips the metricsStale indicator; the first
+  // successful response on the failing endpoint clears its own counter and,
+  // when both are within the threshold, clears the indicator.
+  const failureCountsRef = useRef({ stats: 0, sys: 0 });
+
   const fetchJson = useCallback(async <T>(endpoint: string, options?: { localOnly?: boolean }): Promise<T | null> => {
     try {
       const res = await apiFetch(endpoint, options);
@@ -66,39 +81,59 @@ export function useDashboardData(): DashboardData {
     }
   }, []);
 
+  const recordOutcome = useCallback((endpoint: 'stats' | 'sys', success: boolean) => {
+    const counts = failureCountsRef.current;
+    if (success) counts[endpoint] = 0;
+    else counts[endpoint] += 1;
+    const stale = counts.stats >= METRICS_STALE_THRESHOLD || counts.sys >= METRICS_STALE_THRESHOLD;
+    setMetricsStale(stale);
+  }, []);
+
   // Container stats: 5s polling, resets on node change
   useEffect(() => {
     setStats(DEFAULT_STATS); // eslint-disable-line react-hooks/set-state-in-effect
     setLastSyncAt(null);
+    failureCountsRef.current.stats = 0;
+    setMetricsStale(failureCountsRef.current.sys >= METRICS_STALE_THRESHOLD);
     const currentNodeId = nodeId;
     const fetchStats = async () => {
       if (nodeIdRef.current !== currentNodeId) return; // Stale effect
       const data = await fetchJson<Stats>('/stats');
-      if (data && nodeIdRef.current === currentNodeId) {
+      if (nodeIdRef.current !== currentNodeId) return;
+      if (data) {
         setStats(data);
         setLastSyncAt(Date.now());
+        recordOutcome('stats', true);
+      } else {
+        recordOutcome('stats', false);
       }
     };
     fetchStats();
     const cleanup = visibilityInterval(fetchStats, 5000);
     return cleanup;
-  }, [nodeId, fetchJson]);
+  }, [nodeId, fetchJson, recordOutcome]);
 
   // System stats: 5s polling, resets on node change
   useEffect(() => {
     setSystemStats(null); // eslint-disable-line react-hooks/set-state-in-effect
+    failureCountsRef.current.sys = 0;
+    setMetricsStale(failureCountsRef.current.stats >= METRICS_STALE_THRESHOLD);
     const currentNodeId = nodeId;
     const fetchSys = async () => {
       if (nodeIdRef.current !== currentNodeId) return;
       const data = await fetchJson<SystemStats>('/system/stats');
-      if (nodeIdRef.current === currentNodeId) {
+      if (nodeIdRef.current !== currentNodeId) return;
+      if (data) {
         setSystemStats(data);
+        recordOutcome('sys', true);
+      } else {
+        recordOutcome('sys', false);
       }
     };
     fetchSys();
     const cleanup = visibilityInterval(fetchSys, 5000);
     return cleanup;
-  }, [nodeId, fetchJson]);
+  }, [nodeId, fetchJson, recordOutcome]);
 
   // Historical metrics: 60s polling, resets on node change
   useEffect(() => {
@@ -274,5 +309,6 @@ export function useDashboardData(): DashboardData {
     cpuHistory,
     netHistory,
     historyEndAt,
+    metricsStale,
   };
 }