diff --git a/frontend/src/components/HomeDashboard.tsx b/frontend/src/components/HomeDashboard.tsx index 3b1d1e2f..0906e085 100644 --- a/frontend/src/components/HomeDashboard.tsx +++ b/frontend/src/components/HomeDashboard.tsx @@ -34,6 +34,7 @@ export default function HomeDashboard({ onNavigateToStack, onOpenSettingsSection activeNodeName={activeNodeName} nodeCount={data.nodeCount} lastSyncAt={data.lastSyncAt} + metricsStale={data.metricsStale} /> deriveHealth(stats, systemStats, notifications), @@ -134,6 +137,11 @@ export function HealthStatusBar({ {metaLine} + {metricsStale ? ( + + metrics stale + + ) : null} {reasonsLine ? ( diff --git a/frontend/src/components/dashboard/__tests__/useDashboardData.metricsStale.test.tsx b/frontend/src/components/dashboard/__tests__/useDashboardData.metricsStale.test.tsx new file mode 100644 index 00000000..fd6fd129 --- /dev/null +++ b/frontend/src/components/dashboard/__tests__/useDashboardData.metricsStale.test.tsx @@ -0,0 +1,174 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { renderHook, act } from '@testing-library/react'; + +const apiFetchMock = vi.fn(); + +vi.mock('@/lib/api', () => ({ + apiFetch: (...args: unknown[]) => apiFetchMock(...args), +})); + +const useNodesMock = vi.fn(); +vi.mock('@/context/NodeContext', () => ({ + useNodes: () => useNodesMock(), +})); + +// Capture every visibilityInterval registration so each polling cycle can +// be driven on demand from the test without leaning on real setInterval +// timers (which fight with vi.useFakeTimers and the await-then-setState +// sequence inside the polling callbacks). +type PollEntry = { fn: () => void; intervalMs: number }; +let pollCallbacks: PollEntry[] = []; +vi.mock('@/lib/utils', async () => { + const actual = await vi.importActual('@/lib/utils'); + return { + ...actual, + visibilityInterval: (fn: () => void, intervalMs: number) => { + pollCallbacks.push({ fn, intervalMs }); + return () => { pollCallbacks = pollCallbacks.filter((c) => c.fn !== fn); }; + }, + }; +}); + +import { useDashboardData } from '../useDashboardData'; + +function okJson(payload: unknown): Response { + return new Response(JSON.stringify(payload), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); +} + +function failJson(): Response { + return new Response(JSON.stringify({ error: 'down' }), { status: 500 }); +} + +const STATS_PAYLOAD = { active: 0, managed: 0, unmanaged: 0, exited: 0, total: 0 }; +const SYS_PAYLOAD = { + cpu: { usage: '0', cores: 4 }, + memory: { total: 0, used: 0, free: 0, usagePercent: '0' }, + disk: null, +}; + +function setEndpointOutcome(endpoint: '/stats' | '/system/stats', ok: boolean): void { + apiFetchMock.mockImplementation((requested: string) => { + if (requested === '/stats') { + if (endpoint === '/stats' && !ok) return Promise.resolve(failJson()); + return Promise.resolve(okJson(STATS_PAYLOAD)); + } + if (requested === '/system/stats') { + if (endpoint === '/system/stats' && !ok) return Promise.resolve(failJson()); + return Promise.resolve(okJson(SYS_PAYLOAD)); + } + if (requested === '/stacks/statuses') return Promise.resolve(okJson({})); + if (requested === '/metrics/historical') return Promise.resolve(okJson([])); + return Promise.resolve(okJson(null)); + }); +} + +// Drive the next /stats poll (it is the first 5 s interval registered). +async function tickStats(): Promise { + const stats = pollCallbacks.find((c) => c.intervalMs === 5000); + if (!stats) throw new Error('stats poll callback not registered'); + await act(async () => { + stats.fn(); + // Two microtask drains: one for the apiFetch promise, one for the await + // on res.json() inside fetchJson. + await Promise.resolve(); + await Promise.resolve(); + }); +} + +// Drive the next /system/stats poll (the second 5 s interval registered). +async function tickSys(): Promise { + const sys = pollCallbacks.filter((c) => c.intervalMs === 5000)[1]; + if (!sys) throw new Error('system-stats poll callback not registered'); + await act(async () => { + sys.fn(); + await Promise.resolve(); + await Promise.resolve(); + }); +} + +beforeEach(() => { + pollCallbacks = []; + apiFetchMock.mockReset(); + useNodesMock.mockReset(); + useNodesMock.mockReturnValue({ + activeNode: { id: 1, name: 'Local', type: 'local' }, + nodes: [{ id: 1, name: 'Local', type: 'local' }], + }); +}); + +afterEach(() => { + vi.clearAllMocks(); +}); + +describe('useDashboardData metricsStale threshold', () => { + it('trips after three consecutive /stats failures and clears on the next success', async () => { + // Stats fails from the very first poll; system-stats keeps succeeding. + setEndpointOutcome('/stats', false); + const { result } = renderHook(() => useDashboardData()); + // Drain mount-time fetches. + await act(async () => { await Promise.resolve(); await Promise.resolve(); }); + + // Mount fired one failed /stats poll (counter = 1) and one successful + // /system/stats poll. Two more failed stats polls cross the threshold. + expect(result.current.metricsStale).toBe(false); + await tickStats(); + expect(result.current.metricsStale).toBe(false); + await tickStats(); + expect(result.current.metricsStale).toBe(true); + + // The first successful /stats poll resets the counter; with the + // /system/stats endpoint still below threshold, the indicator clears. + setEndpointOutcome('/stats', true); + await tickStats(); + expect(result.current.metricsStale).toBe(false); + }); + + it('trips after three consecutive /system/stats failures as well', async () => { + setEndpointOutcome('/system/stats', false); + const { result } = renderHook(() => useDashboardData()); + await act(async () => { await Promise.resolve(); await Promise.resolve(); }); + + expect(result.current.metricsStale).toBe(false); + await tickSys(); + expect(result.current.metricsStale).toBe(false); + await tickSys(); + expect(result.current.metricsStale).toBe(true); + }); + + it('keeps the indicator set when one endpoint recovers but the other is still failing', async () => { + // Both fail; trip on stats first, then recover stats only. + setEndpointOutcome('/stats', false); + const { result } = renderHook(() => useDashboardData()); + await act(async () => { await Promise.resolve(); await Promise.resolve(); }); + + // Now break /system/stats too so its counter starts climbing. + apiFetchMock.mockImplementation((requested: string) => { + if (requested === '/stats') return Promise.resolve(failJson()); + if (requested === '/system/stats') return Promise.resolve(failJson()); + if (requested === '/stacks/statuses') return Promise.resolve(okJson({})); + if (requested === '/metrics/historical') return Promise.resolve(okJson([])); + return Promise.resolve(okJson(null)); + }); + + // Three failing sys polls trip the indicator. + await tickSys(); + await tickSys(); + await tickSys(); + expect(result.current.metricsStale).toBe(true); + + // Stats recovers but /system/stats is still failing: indicator stays set + // because the sys counter is still above the threshold. + apiFetchMock.mockImplementation((requested: string) => { + if (requested === '/stats') return Promise.resolve(okJson(STATS_PAYLOAD)); + if (requested === '/system/stats') return Promise.resolve(failJson()); + if (requested === '/stacks/statuses') return Promise.resolve(okJson({})); + if (requested === '/metrics/historical') return Promise.resolve(okJson([])); + return Promise.resolve(okJson(null)); + }); + await tickStats(); + expect(result.current.metricsStale).toBe(true); + }); +}); diff --git a/frontend/src/components/dashboard/types.ts b/frontend/src/components/dashboard/types.ts index 5d67cda7..195ab68e 100644 --- a/frontend/src/components/dashboard/types.ts +++ b/frontend/src/components/dashboard/types.ts @@ -97,6 +97,8 @@ export interface DashboardData { stackCpuSeries: Record; cpuHistory: number[]; netHistory: number[]; - /** Anchor timestamp (ms) for the sparkline 10-minute window — the newest metric sample. */ + /** Anchor timestamp (ms) for the sparkline 10-minute window: the newest metric sample. */ historyEndAt: number | null; + /** True after several consecutive `/stats` or `/system/stats` polls have failed; surfaces a "metrics paused" indicator. */ + metricsStale: boolean; } diff --git a/frontend/src/components/dashboard/useDashboardData.ts b/frontend/src/components/dashboard/useDashboardData.ts index 08441214..e3caeb54 100644 --- a/frontend/src/components/dashboard/useDashboardData.ts +++ b/frontend/src/components/dashboard/useDashboardData.ts @@ -41,6 +41,14 @@ function bucketCpu(points: MetricPoint[], windowMs: number, buckets: number): nu return out; } +// After three consecutive failures of the live metrics endpoints, surface a +// "metrics stale" indicator so the operator knows the gauges are no longer +// being refreshed (the Docker socket or the metrics service is unreachable) +// rather than just slow. Polling continues; the indicator describes the +// freshness of the visible numbers, not the polling cadence. The threshold +// is chosen so a single transient hiccup does not trip the indicator. +const METRICS_STALE_THRESHOLD = 3; + export function useDashboardData(): DashboardData { const { activeNode, nodes } = useNodes(); const nodeId = activeNode?.id; @@ -50,12 +58,19 @@ export function useDashboardData(): DashboardData { const [metrics, setMetrics] = useState([]); const [stackStatuses, setStackStatuses] = useState>({}); const [lastSyncAt, setLastSyncAt] = useState(null); + const [metricsStale, setMetricsStale] = useState(false); // Keep a ref to the latest nodeId so async callbacks don't write stale data // after a node switch has already triggered a new effect cycle. const nodeIdRef = useRef(nodeId); useEffect(() => { nodeIdRef.current = nodeId; }, [nodeId]); + // Consecutive failure counters per live-metrics endpoint. Either reaching + // METRICS_STALE_THRESHOLD trips the metricsStale indicator; the first + // successful response on the failing endpoint clears its own counter and, + // when both are within the threshold, clears the indicator. + const failureCountsRef = useRef({ stats: 0, sys: 0 }); + const fetchJson = useCallback(async (endpoint: string, options?: { localOnly?: boolean }): Promise => { try { const res = await apiFetch(endpoint, options); @@ -66,39 +81,59 @@ export function useDashboardData(): DashboardData { } }, []); + const recordOutcome = useCallback((endpoint: 'stats' | 'sys', success: boolean) => { + const counts = failureCountsRef.current; + if (success) counts[endpoint] = 0; + else counts[endpoint] += 1; + const stale = counts.stats >= METRICS_STALE_THRESHOLD || counts.sys >= METRICS_STALE_THRESHOLD; + setMetricsStale(stale); + }, []); + // Container stats: 5s polling, resets on node change useEffect(() => { setStats(DEFAULT_STATS); // eslint-disable-line react-hooks/set-state-in-effect setLastSyncAt(null); + failureCountsRef.current.stats = 0; + setMetricsStale(failureCountsRef.current.sys >= METRICS_STALE_THRESHOLD); const currentNodeId = nodeId; const fetchStats = async () => { if (nodeIdRef.current !== currentNodeId) return; // Stale effect const data = await fetchJson('/stats'); - if (data && nodeIdRef.current === currentNodeId) { + if (nodeIdRef.current !== currentNodeId) return; + if (data) { setStats(data); setLastSyncAt(Date.now()); + recordOutcome('stats', true); + } else { + recordOutcome('stats', false); } }; fetchStats(); const cleanup = visibilityInterval(fetchStats, 5000); return cleanup; - }, [nodeId, fetchJson]); + }, [nodeId, fetchJson, recordOutcome]); // System stats: 5s polling, resets on node change useEffect(() => { setSystemStats(null); // eslint-disable-line react-hooks/set-state-in-effect + failureCountsRef.current.sys = 0; + setMetricsStale(failureCountsRef.current.stats >= METRICS_STALE_THRESHOLD); const currentNodeId = nodeId; const fetchSys = async () => { if (nodeIdRef.current !== currentNodeId) return; const data = await fetchJson('/system/stats'); - if (nodeIdRef.current === currentNodeId) { + if (nodeIdRef.current !== currentNodeId) return; + if (data) { setSystemStats(data); + recordOutcome('sys', true); + } else { + recordOutcome('sys', false); } }; fetchSys(); const cleanup = visibilityInterval(fetchSys, 5000); return cleanup; - }, [nodeId, fetchJson]); + }, [nodeId, fetchJson, recordOutcome]); // Historical metrics: 60s polling, resets on node change useEffect(() => { @@ -274,5 +309,6 @@ export function useDashboardData(): DashboardData { cpuHistory, netHistory, historyEndAt, + metricsStale, }; }