diff --git a/backend/src/__tests__/docker-controller.test.ts b/backend/src/__tests__/docker-controller.test.ts index 111a016f..4f3b1b69 100644 --- a/backend/src/__tests__/docker-controller.test.ts +++ b/backend/src/__tests__/docker-controller.test.ts @@ -1179,3 +1179,125 @@ describe('DockerController - getDependencySnapshot', () => { expect(snap.containers[0].composeProject).toBeNull(); }); }); + +// --- getBulkStackStatuses uptime (runningSince from StartedAt) ----------------- + +describe('DockerController - getBulkStackStatuses uptime', () => { + beforeEach(() => { + // resolveProjectNameMap caches under a single key; flush so each test's + // stack names map cleanly to themselves (compose files do not exist here). + CacheService.getInstance().flush(); + }); + + const runningContainer = (id: string, project: string, created: number) => ({ + Id: id, + Names: [`/${id}`], + State: 'running', + Status: 'Up', + Image: 'nginx', + Created: created, + Labels: { 'com.docker.compose.project': project }, + }); + + it('uses StartedAt (not Created) for runningSince', async () => { + const created = 1000; // ancient creation time + const startedIso = '2026-06-09T12:00:00.000Z'; + const startedUnix = Math.floor(Date.parse(startedIso) / 1000); + + mockDocker.listContainers.mockResolvedValue([runningContainer('sa-c1', 'sa-stack', created)]); + mockDocker.getContainer.mockReturnValue({ + inspect: vi.fn().mockResolvedValue({ State: { StartedAt: startedIso } }), + }); + + const dc = DockerController.getInstance(1); + const result = await dc.getBulkStackStatuses(['sa-stack']); + + expect(result['sa-stack'].status).toBe('running'); + expect(result['sa-stack'].runningSince).toBe(startedUnix); + expect(result['sa-stack'].runningSince).not.toBe(created); + }); + + it('picks the oldest StartedAt across running containers in a stack', async () => { + const olderIso = '2026-06-09T10:00:00.000Z'; + const newerIso = '2026-06-09T11:30:00.000Z'; + const olderUnix = Math.floor(Date.parse(olderIso) / 1000); + + mockDocker.listContainers.mockResolvedValue([ + runningContainer('ow-c1', 'ow-stack', 5000), + runningContainer('ow-c2', 'ow-stack', 6000), + ]); + const byId: Record = { 'ow-c1': newerIso, 'ow-c2': olderIso }; + mockDocker.getContainer.mockImplementation((id: string) => ({ + inspect: vi.fn().mockResolvedValue({ State: { StartedAt: byId[id] } }), + })); + + const dc = DockerController.getInstance(1); + const result = await dc.getBulkStackStatuses(['ow-stack']); + + expect(result['ow-stack'].runningSince).toBe(olderUnix); + }); + + it('falls back to Created when inspect fails', async () => { + const created = 1234; + mockDocker.listContainers.mockResolvedValue([runningContainer('fb-c1', 'fb-stack', created)]); + mockDocker.getContainer.mockReturnValue({ + inspect: vi.fn().mockRejectedValue(new Error('inspect boom')), + }); + + const dc = DockerController.getInstance(1); + const result = await dc.getBulkStackStatuses(['fb-stack']); + + expect(result['fb-stack'].status).toBe('running'); + expect(result['fb-stack'].runningSince).toBe(created); + }); + + it('falls back to Created when StartedAt is the Docker zero-time', async () => { + const created = 4321; + mockDocker.listContainers.mockResolvedValue([runningContainer('zt-c1', 'zt-stack', created)]); + mockDocker.getContainer.mockReturnValue({ + inspect: vi.fn().mockResolvedValue({ State: { StartedAt: '0001-01-01T00:00:00Z' } }), + }); + + const dc = DockerController.getInstance(1); + const result = await dc.getBulkStackStatuses(['zt-stack']); + + expect(result['zt-stack'].runningSince).toBe(created); + }); + + it('derives uptime only from running containers, ignoring exited ones in the stack', async () => { + const startedIso = '2026-06-09T08:00:00.000Z'; + const startedUnix = Math.floor(Date.parse(startedIso) / 1000); + mockDocker.listContainers.mockResolvedValue([ + { ...runningContainer('me-run', 'me-stack', 7000) }, + { Id: 'me-exit', Names: ['/me-exit'], State: 'exited', Status: 'Exited (0)', Image: 'nginx', Created: 100, Labels: { 'com.docker.compose.project': 'me-stack' } }, + ]); + mockDocker.getContainer.mockReturnValue({ + inspect: vi.fn().mockResolvedValue({ State: { StartedAt: startedIso } }), + }); + + const dc = DockerController.getInstance(1); + const result = await dc.getBulkStackStatuses(['me-stack']); + + expect(result['me-stack'].status).toBe('running'); + // The exited container's ancient Created (100) must not drag uptime down. + expect(result['me-stack'].runningSince).toBe(startedUnix); + }); + + it('caches StartedAt so repeated calls do not re-inspect within the TTL', async () => { + const startedIso = '2026-06-09T09:00:00.000Z'; + mockDocker.listContainers.mockResolvedValue([runningContainer('ca-c1', 'ca-stack', 1000)]); + const inspect = vi.fn().mockResolvedValue({ State: { StartedAt: startedIso } }); + mockDocker.getContainer.mockReturnValue({ inspect }); + + // getInstance() hands out fresh instances per request, so a shared (static) + // cache is what must dedupe the inspect across separate calls. + const startedUnix = Math.floor(Date.parse(startedIso) / 1000); + await DockerController.getInstance(1).getBulkStackStatuses(['ca-stack']); + const second = await DockerController.getInstance(1).getBulkStackStatuses(['ca-stack']); + + expect(inspect).toHaveBeenCalledTimes(1); + // The cache must be read, not merely populated: the second call still + // resolves the real StartedAt rather than falling back to Created. + expect(second['ca-stack'].runningSince).toBe(startedUnix); + }); +}); diff --git a/backend/src/services/DockerController.ts b/backend/src/services/DockerController.ts index 12f2ff96..9b10a511 100644 --- a/backend/src/services/DockerController.ts +++ b/backend/src/services/DockerController.ts @@ -23,6 +23,10 @@ const COMPOSE_FILE_NAMES = ['compose.yaml', 'compose.yml', 'docker-compose.yaml' /** Cached mapping from compose `name:` field to stack directory name. TTL-based to avoid re-parsing YAML on every poll. */ const PROJECT_NAME_CACHE_TTL_MS = 60_000; const PROJECT_NAME_CACHE_KEY = 'project-name-map'; +/** How long a resolved container StartedAt stays fresh before re-inspecting. */ +const STARTED_AT_CACHE_TTL_MS = 20_000; +/** Cap on concurrent inspect() calls when resolving StartedAt in bulk. */ +const STARTED_AT_INSPECT_CONCURRENCY = 10; /** Common web-UI private ports, checked in priority order when detecting the main app port. */ const WEB_UI_PORTS = [32400, 8989, 7878, 9696, 5055, 8080, 80, 443, 3000, 9000]; @@ -32,7 +36,7 @@ const IGNORE_PORTS = [1900, 53, 22]; export interface BulkStackInfo { status: 'running' | 'exited' | 'unknown'; mainPort?: number; - /** Unix seconds of the oldest running container (approximates stack uptime). */ + /** Unix seconds of the oldest running container's last start (approximates stack uptime). */ runningSince?: number; } @@ -165,6 +169,12 @@ export interface CreateNetworkOptions { class DockerController { private static readonly SYSTEM_NETWORKS = new Set(['bridge', 'host', 'none']); + /** + * Cache of container last-start times (unix seconds), keyed by `${nodeId}:${containerId}`. + * Static because getInstance() hands out throwaway instances per request; the cache must + * outlive them so status polls do not re-inspect every container every few seconds. + */ + private static startedAtCache = new Map(); private docker: Docker; private nodeId: number; @@ -1108,6 +1118,11 @@ class DockerController { result[name] = { status: 'unknown' }; } + // Per stack, collect running container ids plus the oldest Created as a + // fallback. Uptime is resolved from StartedAt after the loop; Created is + // only used if an inspect fails, since it never moves on restart. + const runningByStack: Record = {}; + for (const container of allContainers as any[]) { const stackDir = DockerController.resolveContainerStack( container.Labels, projectToStack, knownStackSet, absDirToStack, resolvedBase, @@ -1118,16 +1133,11 @@ class DockerController { if (container.State === 'running') { result[stackDir].status = 'running'; - // Track the oldest running container's creation time as a proxy for - // stack uptime. Docker's listContainers payload exposes Created (unix - // seconds) but not StartedAt; for compose stacks the gap is small - // enough to treat as uptime without paying for a per-container inspect. + const acc = (runningByStack[stackDir] ??= { ids: [] }); + if (typeof container.Id === 'string') acc.ids.push(container.Id); const created = typeof container.Created === 'number' ? container.Created : undefined; - if (created !== undefined) { - const existing = result[stackDir].runningSince; - if (existing === undefined || created < existing) { - result[stackDir].runningSince = created; - } + if (created !== undefined && (acc.oldestCreated === undefined || created < acc.oldestCreated)) { + acc.oldestCreated = created; } // Detect main web port (first running container with a matchable port wins) @@ -1149,9 +1159,74 @@ class DockerController { } } + // Resolve real uptime: oldest StartedAt across each stack's running + // containers, falling back to the oldest Created when inspect is unavailable. + const allRunningIds = Object.values(runningByStack).flatMap(s => s.ids); + const startedAts = await this.getRunningStartedAts(allRunningIds); + for (const [stackDir, acc] of Object.entries(runningByStack)) { + let oldest: number | undefined; + for (const id of acc.ids) { + const started = startedAts.get(id); + if (started !== undefined && (oldest === undefined || started < oldest)) { + oldest = started; + } + } + result[stackDir].runningSince = oldest ?? acc.oldestCreated; + } + return result; } + /** + * Resolve each container's last start time (unix seconds) from State.StartedAt, + * which Docker exposes only via inspect() (listContainers carries Created, which + * does not move on restart). Results are cached briefly per node+container so + * steady-state status polls avoid re-inspecting. A container whose inspect fails + * (e.g. it vanished between listing and inspect) is omitted; callers fall back to + * Created for those rather than failing the whole batch. + */ + private async getRunningStartedAts(containerIds: string[]): Promise> { + const now = Date.now(); + const cache = DockerController.startedAtCache; + const out = new Map(); + const misses: string[] = []; + + for (const id of containerIds) { + const cached = cache.get(`${this.nodeId}:${id}`); + if (cached && now - cached.cachedAtMs < STARTED_AT_CACHE_TTL_MS) { + out.set(id, cached.startedAtSeconds); + } else { + misses.push(id); + } + } + + for (let i = 0; i < misses.length; i += STARTED_AT_INSPECT_CONCURRENCY) { + const chunk = misses.slice(i, i + STARTED_AT_INSPECT_CONCURRENCY); + await Promise.all(chunk.map(async (id) => { + try { + const info = await this.docker.getContainer(id).inspect(); + const startedAt = info.State?.StartedAt; + const seconds = startedAt ? Math.floor(Date.parse(startedAt) / 1000) : NaN; + // Skip the Docker zero-time (0001-01-01...) and unparseable values. + if (Number.isFinite(seconds) && seconds > 0) { + cache.set(`${this.nodeId}:${id}`, { startedAtSeconds: seconds, cachedAtMs: now }); + out.set(id, seconds); + } + } catch (err: unknown) { + console.warn('[DockerController] StartedAt inspect failed for %s: %s', sanitizeForLog(id), sanitizeForLog((err as Error)?.message ?? String(err))); + } + })); + } + + // Bound the cache: drop entries past their TTL so removed containers that + // will never be queried again do not accumulate. + for (const [key, entry] of cache) { + if (now - entry.cachedAtMs >= STARTED_AT_CACHE_TTL_MS) cache.delete(key); + } + + return out; + } + /** * Returns a map of host ports currently bound by running containers, * with ownership info (Sencho-managed stack name or external). diff --git a/frontend/src/components/dashboard/StackHealthTable.tsx b/frontend/src/components/dashboard/StackHealthTable.tsx index 1a6d0b28..b4991abd 100644 --- a/frontend/src/components/dashboard/StackHealthTable.tsx +++ b/frontend/src/components/dashboard/StackHealthTable.tsx @@ -3,6 +3,7 @@ import { Button } from '@/components/ui/button'; import { Sparkline } from '@/components/ui/sparkline'; import { ChevronLeft, ChevronRight, Layers } from 'lucide-react'; import type { StackStatusEntry, MetricPoint, StackCpuSeries } from './types'; +import { aggregateCurrentUsage } from './aggregateCurrentUsage'; interface StackHealthTableProps { stackStatuses: Record; @@ -80,28 +81,7 @@ export function StackHealthTable({ return () => clearInterval(id); }, []); - const stackAggregates = useMemo(() => { - const latestPerContainer: Record> = {}; - for (const m of metrics) { - if (!m.stack_name) continue; - if (!latestPerContainer[m.stack_name]) latestPerContainer[m.stack_name] = {}; - const existing = latestPerContainer[m.stack_name][m.container_id]; - if (!existing || m.timestamp > existing.timestamp) { - latestPerContainer[m.stack_name][m.container_id] = m; - } - } - const result: Record = {}; - for (const [stack, containers] of Object.entries(latestPerContainer)) { - let mem = 0; - let cpu = 0; - for (const m of Object.values(containers)) { - mem += m.memory_mb; - cpu += m.cpu_percent; - } - result[stack] = { mem, cpu }; - } - return result; - }, [metrics]); + const stackAggregates = useMemo(() => aggregateCurrentUsage(metrics), [metrics]); const rows = useMemo(() => { const list = Object.entries(stackStatuses).map(([file, entry]) => { diff --git a/frontend/src/components/dashboard/aggregateCurrentUsage.test.ts b/frontend/src/components/dashboard/aggregateCurrentUsage.test.ts new file mode 100644 index 00000000..bf5c1bab --- /dev/null +++ b/frontend/src/components/dashboard/aggregateCurrentUsage.test.ts @@ -0,0 +1,64 @@ +import { describe, it, expect } from 'vitest'; +import { aggregateCurrentUsage } from './aggregateCurrentUsage'; +import type { MetricPoint } from './types'; + +const point = (over: Partial): MetricPoint => ({ + container_id: 'c1', + stack_name: 'web', + timestamp: 0, + cpu_percent: 0, + memory_mb: 0, + net_rx_mb: 0, + net_tx_mb: 0, + ...over, +}); + +describe('aggregateCurrentUsage', () => { + it('returns an empty map when there are no metrics', () => { + expect(aggregateCurrentUsage([])).toEqual({}); + }); + + it('sums the latest sample per container into a per-stack total', () => { + const result = aggregateCurrentUsage([ + point({ container_id: 'a', timestamp: 1000, cpu_percent: 10, memory_mb: 100 }), + point({ container_id: 'a', timestamp: 2000, cpu_percent: 15, memory_mb: 150 }), // newer wins for a + point({ container_id: 'b', timestamp: 2000, cpu_percent: 5, memory_mb: 50 }), + ], 90_000); + + expect(result.web).toEqual({ cpu: 20, mem: 200 }); + }); + + it('excludes a container whose latest sample is stale relative to the freshest', () => { + const staleWindow = 90_000; + const result = aggregateCurrentUsage([ + // live container keeps reporting; its newest point is the freshest sample + point({ container_id: 'live', timestamp: 500_000, cpu_percent: 30, memory_mb: 300 }), + // stopped container's final reading lingers far behind the freshest sample + point({ container_id: 'stopped', timestamp: 500_000 - staleWindow - 1, cpu_percent: 99, memory_mb: 999 }), + ], staleWindow); + + // only the live container counts; the stale 99% / 999MB reading is dropped + expect(result.web).toEqual({ cpu: 30, mem: 300 }); + }); + + it('keeps a sample sitting exactly on the stale-window boundary', () => { + const staleWindow = 90_000; + const result = aggregateCurrentUsage([ + point({ container_id: 'live', timestamp: 500_000, cpu_percent: 10, memory_mb: 100 }), + // exactly freshest - staleWindow: the cutoff is `< cutoff`, so this is kept + point({ container_id: 'edge', timestamp: 500_000 - staleWindow, cpu_percent: 7, memory_mb: 70 }), + ], staleWindow); + + expect(result.web).toEqual({ cpu: 17, mem: 170 }); + }); + + it('omits a stack entirely when all its containers are stale', () => { + const result = aggregateCurrentUsage([ + point({ stack_name: 'alive', container_id: 'x', timestamp: 1_000_000, cpu_percent: 12, memory_mb: 120 }), + point({ stack_name: 'dead', container_id: 'y', timestamp: 1_000_000 - 200_000, cpu_percent: 80, memory_mb: 800 }), + ], 90_000); + + expect(result.alive).toEqual({ cpu: 12, mem: 120 }); + expect(result.dead).toBeUndefined(); + }); +}); diff --git a/frontend/src/components/dashboard/aggregateCurrentUsage.ts b/frontend/src/components/dashboard/aggregateCurrentUsage.ts new file mode 100644 index 00000000..b2c4844a --- /dev/null +++ b/frontend/src/components/dashboard/aggregateCurrentUsage.ts @@ -0,0 +1,57 @@ +import type { MetricPoint } from './types'; + +/** + * How far behind the freshest sample a container's latest point may be and + * still count toward current usage. Metrics are collected on a ~30s cadence, + * so 90s comfortably includes live containers while dropping the lingering + * final reading of one that stopped a couple of cycles ago. + */ +export const CURRENT_USAGE_STALE_WINDOW_MS = 90_000; + +export interface StackUsage { + mem: number; + cpu: number; +} + +/** + * Sum the latest CPU/memory sample per container into a per-stack total, + * ignoring containers whose most recent sample is stale. A stopped container + * stops producing samples, so its last reading would otherwise persist in the + * total until it aged out of the metrics window; gating on recency relative to + * the freshest sample (not wall-clock, so overall fetch lag does not matter) + * drops it once live containers report newer points. + */ +export function aggregateCurrentUsage( + metrics: MetricPoint[], + staleWindowMs: number = CURRENT_USAGE_STALE_WINDOW_MS, +): Record { + if (metrics.length === 0) return {}; + + let freshest = -Infinity; + const latestPerContainer: Record> = {}; + for (const m of metrics) { + if (m.timestamp > freshest) freshest = m.timestamp; + if (!m.stack_name) continue; + if (!latestPerContainer[m.stack_name]) latestPerContainer[m.stack_name] = {}; + const existing = latestPerContainer[m.stack_name][m.container_id]; + if (!existing || m.timestamp > existing.timestamp) { + latestPerContainer[m.stack_name][m.container_id] = m; + } + } + + const cutoff = freshest - staleWindowMs; + const result: Record = {}; + for (const [stack, containers] of Object.entries(latestPerContainer)) { + let mem = 0; + let cpu = 0; + let fresh = false; + for (const m of Object.values(containers)) { + if (m.timestamp < cutoff) continue; + mem += m.memory_mb; + cpu += m.cpu_percent; + fresh = true; + } + if (fresh) result[stack] = { mem, cpu }; + } + return result; +}