From 7c3ba3f24d9ee1cf1302c42cb6c33d659b4118e7 Mon Sep 17 00:00:00 2001 From: Anso Date: Mon, 25 May 2026 12:11:43 -0400 Subject: [PATCH] feat(dashboard): surface metrics-stale indicator after sustained poll failure (#1213) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(dashboard): surface metrics-paused indicator after sustained poll failure useDashboardData previously failed silently when /stats or /system/stats returned an error: stale data kept rendering and the last sync timestamp quietly drifted. The operator could not tell whether the dashboard was just slow or whether the Docker socket / metrics path had genuinely gone down. Track consecutive failures per live-metrics endpoint. After three in a row on either /stats or /system/stats (≈15 s at the 5 s poll cadence), expose a metricsStale boolean on the hook result. HealthStatusBar renders a small amber "metrics paused" chip beside the meta line when set. The indicator clears on the first successful response when both endpoints are within the threshold. A unit test for the threshold logic is intentionally deferred to the Phase 4 E2E dashboard spec, which exercises the same path end-to-end by stopping the Docker daemon and asserting the user-visible indicator. * fix(dashboard): rename stale-metrics chip and cover the threshold with tests Address two follow-up findings from independent review of the earlier commit on this branch. 1. Rename the masthead chip from "metrics paused" to "metrics stale". The hook keeps polling on every cycle; the chip describes the freshness of the displayed numbers, not the polling cadence. The new wording matches the underlying `metricsStale` state variable. 2. Add a Vitest spec for the threshold logic. Captures the visibilityInterval callback at registration time and drives each polling cycle on demand, covering: three consecutive /stats failures trip the indicator and the next successful poll clears it; three consecutive /system/stats failures trip the indicator on the other endpoint; clearing requires both endpoints under threshold (a single endpoint recovering while the other is still failing keeps the indicator set). The clarifying comment in useDashboardData notes that polling is unaffected and only the data freshness is in scope, so future readers do not interpret "stale" as "paused". --- frontend/src/components/HomeDashboard.tsx | 1 + .../components/dashboard/HealthStatusBar.tsx | 8 + .../useDashboardData.metricsStale.test.tsx | 174 ++++++++++++++++++ frontend/src/components/dashboard/types.ts | 4 +- .../components/dashboard/useDashboardData.ts | 44 ++++- 5 files changed, 226 insertions(+), 5 deletions(-) create mode 100644 frontend/src/components/dashboard/__tests__/useDashboardData.metricsStale.test.tsx diff --git a/frontend/src/components/HomeDashboard.tsx b/frontend/src/components/HomeDashboard.tsx index 3b1d1e2f..0906e085 100644 --- a/frontend/src/components/HomeDashboard.tsx +++ b/frontend/src/components/HomeDashboard.tsx @@ -34,6 +34,7 @@ export default function HomeDashboard({ onNavigateToStack, onOpenSettingsSection activeNodeName={activeNodeName} nodeCount={data.nodeCount} lastSyncAt={data.lastSyncAt} + metricsStale={data.metricsStale} /> deriveHealth(stats, systemStats, notifications), @@ -134,6 +137,11 @@ export function HealthStatusBar({ {metaLine} + {metricsStale ? ( + + metrics stale + + ) : null} {reasonsLine ? ( diff --git a/frontend/src/components/dashboard/__tests__/useDashboardData.metricsStale.test.tsx b/frontend/src/components/dashboard/__tests__/useDashboardData.metricsStale.test.tsx new file mode 100644 index 00000000..fd6fd129 --- /dev/null +++ b/frontend/src/components/dashboard/__tests__/useDashboardData.metricsStale.test.tsx @@ -0,0 +1,174 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { renderHook, act } from '@testing-library/react'; + +const apiFetchMock = vi.fn(); + +vi.mock('@/lib/api', () => ({ + apiFetch: (...args: unknown[]) => apiFetchMock(...args), +})); + +const useNodesMock = vi.fn(); +vi.mock('@/context/NodeContext', () => ({ + useNodes: () => useNodesMock(), +})); + +// Capture every visibilityInterval registration so each polling cycle can +// be driven on demand from the test without leaning on real setInterval +// timers (which fight with vi.useFakeTimers and the await-then-setState +// sequence inside the polling callbacks). +type PollEntry = { fn: () => void; intervalMs: number }; +let pollCallbacks: PollEntry[] = []; +vi.mock('@/lib/utils', async () => { + const actual = await vi.importActual('@/lib/utils'); + return { + ...actual, + visibilityInterval: (fn: () => void, intervalMs: number) => { + pollCallbacks.push({ fn, intervalMs }); + return () => { pollCallbacks = pollCallbacks.filter((c) => c.fn !== fn); }; + }, + }; +}); + +import { useDashboardData } from '../useDashboardData'; + +function okJson(payload: unknown): Response { + return new Response(JSON.stringify(payload), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); +} + +function failJson(): Response { + return new Response(JSON.stringify({ error: 'down' }), { status: 500 }); +} + +const STATS_PAYLOAD = { active: 0, managed: 0, unmanaged: 0, exited: 0, total: 0 }; +const SYS_PAYLOAD = { + cpu: { usage: '0', cores: 4 }, + memory: { total: 0, used: 0, free: 0, usagePercent: '0' }, + disk: null, +}; + +function setEndpointOutcome(endpoint: '/stats' | '/system/stats', ok: boolean): void { + apiFetchMock.mockImplementation((requested: string) => { + if (requested === '/stats') { + if (endpoint === '/stats' && !ok) return Promise.resolve(failJson()); + return Promise.resolve(okJson(STATS_PAYLOAD)); + } + if (requested === '/system/stats') { + if (endpoint === '/system/stats' && !ok) return Promise.resolve(failJson()); + return Promise.resolve(okJson(SYS_PAYLOAD)); + } + if (requested === '/stacks/statuses') return Promise.resolve(okJson({})); + if (requested === '/metrics/historical') return Promise.resolve(okJson([])); + return Promise.resolve(okJson(null)); + }); +} + +// Drive the next /stats poll (it is the first 5 s interval registered). +async function tickStats(): Promise { + const stats = pollCallbacks.find((c) => c.intervalMs === 5000); + if (!stats) throw new Error('stats poll callback not registered'); + await act(async () => { + stats.fn(); + // Two microtask drains: one for the apiFetch promise, one for the await + // on res.json() inside fetchJson. + await Promise.resolve(); + await Promise.resolve(); + }); +} + +// Drive the next /system/stats poll (the second 5 s interval registered). +async function tickSys(): Promise { + const sys = pollCallbacks.filter((c) => c.intervalMs === 5000)[1]; + if (!sys) throw new Error('system-stats poll callback not registered'); + await act(async () => { + sys.fn(); + await Promise.resolve(); + await Promise.resolve(); + }); +} + +beforeEach(() => { + pollCallbacks = []; + apiFetchMock.mockReset(); + useNodesMock.mockReset(); + useNodesMock.mockReturnValue({ + activeNode: { id: 1, name: 'Local', type: 'local' }, + nodes: [{ id: 1, name: 'Local', type: 'local' }], + }); +}); + +afterEach(() => { + vi.clearAllMocks(); +}); + +describe('useDashboardData metricsStale threshold', () => { + it('trips after three consecutive /stats failures and clears on the next success', async () => { + // Stats fails from the very first poll; system-stats keeps succeeding. + setEndpointOutcome('/stats', false); + const { result } = renderHook(() => useDashboardData()); + // Drain mount-time fetches. + await act(async () => { await Promise.resolve(); await Promise.resolve(); }); + + // Mount fired one failed /stats poll (counter = 1) and one successful + // /system/stats poll. Two more failed stats polls cross the threshold. + expect(result.current.metricsStale).toBe(false); + await tickStats(); + expect(result.current.metricsStale).toBe(false); + await tickStats(); + expect(result.current.metricsStale).toBe(true); + + // The first successful /stats poll resets the counter; with the + // /system/stats endpoint still below threshold, the indicator clears. + setEndpointOutcome('/stats', true); + await tickStats(); + expect(result.current.metricsStale).toBe(false); + }); + + it('trips after three consecutive /system/stats failures as well', async () => { + setEndpointOutcome('/system/stats', false); + const { result } = renderHook(() => useDashboardData()); + await act(async () => { await Promise.resolve(); await Promise.resolve(); }); + + expect(result.current.metricsStale).toBe(false); + await tickSys(); + expect(result.current.metricsStale).toBe(false); + await tickSys(); + expect(result.current.metricsStale).toBe(true); + }); + + it('keeps the indicator set when one endpoint recovers but the other is still failing', async () => { + // Both fail; trip on stats first, then recover stats only. + setEndpointOutcome('/stats', false); + const { result } = renderHook(() => useDashboardData()); + await act(async () => { await Promise.resolve(); await Promise.resolve(); }); + + // Now break /system/stats too so its counter starts climbing. + apiFetchMock.mockImplementation((requested: string) => { + if (requested === '/stats') return Promise.resolve(failJson()); + if (requested === '/system/stats') return Promise.resolve(failJson()); + if (requested === '/stacks/statuses') return Promise.resolve(okJson({})); + if (requested === '/metrics/historical') return Promise.resolve(okJson([])); + return Promise.resolve(okJson(null)); + }); + + // Three failing sys polls trip the indicator. + await tickSys(); + await tickSys(); + await tickSys(); + expect(result.current.metricsStale).toBe(true); + + // Stats recovers but /system/stats is still failing: indicator stays set + // because the sys counter is still above the threshold. + apiFetchMock.mockImplementation((requested: string) => { + if (requested === '/stats') return Promise.resolve(okJson(STATS_PAYLOAD)); + if (requested === '/system/stats') return Promise.resolve(failJson()); + if (requested === '/stacks/statuses') return Promise.resolve(okJson({})); + if (requested === '/metrics/historical') return Promise.resolve(okJson([])); + return Promise.resolve(okJson(null)); + }); + await tickStats(); + expect(result.current.metricsStale).toBe(true); + }); +}); diff --git a/frontend/src/components/dashboard/types.ts b/frontend/src/components/dashboard/types.ts index 5d67cda7..195ab68e 100644 --- a/frontend/src/components/dashboard/types.ts +++ b/frontend/src/components/dashboard/types.ts @@ -97,6 +97,8 @@ export interface DashboardData { stackCpuSeries: Record; cpuHistory: number[]; netHistory: number[]; - /** Anchor timestamp (ms) for the sparkline 10-minute window — the newest metric sample. */ + /** Anchor timestamp (ms) for the sparkline 10-minute window: the newest metric sample. */ historyEndAt: number | null; + /** True after several consecutive `/stats` or `/system/stats` polls have failed; surfaces a "metrics paused" indicator. */ + metricsStale: boolean; } diff --git a/frontend/src/components/dashboard/useDashboardData.ts b/frontend/src/components/dashboard/useDashboardData.ts index 08441214..e3caeb54 100644 --- a/frontend/src/components/dashboard/useDashboardData.ts +++ b/frontend/src/components/dashboard/useDashboardData.ts @@ -41,6 +41,14 @@ function bucketCpu(points: MetricPoint[], windowMs: number, buckets: number): nu return out; } +// After three consecutive failures of the live metrics endpoints, surface a +// "metrics stale" indicator so the operator knows the gauges are no longer +// being refreshed (the Docker socket or the metrics service is unreachable) +// rather than just slow. Polling continues; the indicator describes the +// freshness of the visible numbers, not the polling cadence. The threshold +// is chosen so a single transient hiccup does not trip the indicator. +const METRICS_STALE_THRESHOLD = 3; + export function useDashboardData(): DashboardData { const { activeNode, nodes } = useNodes(); const nodeId = activeNode?.id; @@ -50,12 +58,19 @@ export function useDashboardData(): DashboardData { const [metrics, setMetrics] = useState([]); const [stackStatuses, setStackStatuses] = useState>({}); const [lastSyncAt, setLastSyncAt] = useState(null); + const [metricsStale, setMetricsStale] = useState(false); // Keep a ref to the latest nodeId so async callbacks don't write stale data // after a node switch has already triggered a new effect cycle. const nodeIdRef = useRef(nodeId); useEffect(() => { nodeIdRef.current = nodeId; }, [nodeId]); + // Consecutive failure counters per live-metrics endpoint. Either reaching + // METRICS_STALE_THRESHOLD trips the metricsStale indicator; the first + // successful response on the failing endpoint clears its own counter and, + // when both are within the threshold, clears the indicator. + const failureCountsRef = useRef({ stats: 0, sys: 0 }); + const fetchJson = useCallback(async (endpoint: string, options?: { localOnly?: boolean }): Promise => { try { const res = await apiFetch(endpoint, options); @@ -66,39 +81,59 @@ export function useDashboardData(): DashboardData { } }, []); + const recordOutcome = useCallback((endpoint: 'stats' | 'sys', success: boolean) => { + const counts = failureCountsRef.current; + if (success) counts[endpoint] = 0; + else counts[endpoint] += 1; + const stale = counts.stats >= METRICS_STALE_THRESHOLD || counts.sys >= METRICS_STALE_THRESHOLD; + setMetricsStale(stale); + }, []); + // Container stats: 5s polling, resets on node change useEffect(() => { setStats(DEFAULT_STATS); // eslint-disable-line react-hooks/set-state-in-effect setLastSyncAt(null); + failureCountsRef.current.stats = 0; + setMetricsStale(failureCountsRef.current.sys >= METRICS_STALE_THRESHOLD); const currentNodeId = nodeId; const fetchStats = async () => { if (nodeIdRef.current !== currentNodeId) return; // Stale effect const data = await fetchJson('/stats'); - if (data && nodeIdRef.current === currentNodeId) { + if (nodeIdRef.current !== currentNodeId) return; + if (data) { setStats(data); setLastSyncAt(Date.now()); + recordOutcome('stats', true); + } else { + recordOutcome('stats', false); } }; fetchStats(); const cleanup = visibilityInterval(fetchStats, 5000); return cleanup; - }, [nodeId, fetchJson]); + }, [nodeId, fetchJson, recordOutcome]); // System stats: 5s polling, resets on node change useEffect(() => { setSystemStats(null); // eslint-disable-line react-hooks/set-state-in-effect + failureCountsRef.current.sys = 0; + setMetricsStale(failureCountsRef.current.stats >= METRICS_STALE_THRESHOLD); const currentNodeId = nodeId; const fetchSys = async () => { if (nodeIdRef.current !== currentNodeId) return; const data = await fetchJson('/system/stats'); - if (nodeIdRef.current === currentNodeId) { + if (nodeIdRef.current !== currentNodeId) return; + if (data) { setSystemStats(data); + recordOutcome('sys', true); + } else { + recordOutcome('sys', false); } }; fetchSys(); const cleanup = visibilityInterval(fetchSys, 5000); return cleanup; - }, [nodeId, fetchJson]); + }, [nodeId, fetchJson, recordOutcome]); // Historical metrics: 60s polling, resets on node change useEffect(() => { @@ -274,5 +309,6 @@ export function useDashboardData(): DashboardData { cpuHistory, netHistory, historyEndAt, + metricsStale, }; }