fix: base Stack health uptime on container start, not creation (#1341)

The dashboard Stack health UP column counted from each container's
Created timestamp, which never moves on stop/start or restart, so a
restarted container kept reporting its original age. Resolve uptime from
State.StartedAt (via a briefly cached inspect with bounded concurrency,
falling back to Created when inspect is unavailable) so it reflects the
real time since last start.

The current CPU and MEM columns separately summed the latest sample per
container with no recency filter, letting a recently stopped container's
final reading linger in the totals. Drop samples that trail the freshest
sample by more than the stale window so stopped containers leave the sum.
This commit is contained in:
Anso
2026-06-09 20:16:04 -04:00
committed by GitHub
parent 3dc8199907
commit e7895c889d
5 changed files with 330 additions and 32 deletions
@@ -1179,3 +1179,125 @@ describe('DockerController - getDependencySnapshot', () => {
expect(snap.containers[0].composeProject).toBeNull();
});
});
// --- getBulkStackStatuses uptime (runningSince from StartedAt) -----------------
describe('DockerController - getBulkStackStatuses uptime', () => {
beforeEach(() => {
// resolveProjectNameMap caches under a single key; flush so each test's
// stack names map cleanly to themselves (compose files do not exist here).
CacheService.getInstance().flush();
});
const runningContainer = (id: string, project: string, created: number) => ({
Id: id,
Names: [`/${id}`],
State: 'running',
Status: 'Up',
Image: 'nginx',
Created: created,
Labels: { 'com.docker.compose.project': project },
});
it('uses StartedAt (not Created) for runningSince', async () => {
const created = 1000; // ancient creation time
const startedIso = '2026-06-09T12:00:00.000Z';
const startedUnix = Math.floor(Date.parse(startedIso) / 1000);
mockDocker.listContainers.mockResolvedValue([runningContainer('sa-c1', 'sa-stack', created)]);
mockDocker.getContainer.mockReturnValue({
inspect: vi.fn().mockResolvedValue({ State: { StartedAt: startedIso } }),
});
const dc = DockerController.getInstance(1);
const result = await dc.getBulkStackStatuses(['sa-stack']);
expect(result['sa-stack'].status).toBe('running');
expect(result['sa-stack'].runningSince).toBe(startedUnix);
expect(result['sa-stack'].runningSince).not.toBe(created);
});
it('picks the oldest StartedAt across running containers in a stack', async () => {
const olderIso = '2026-06-09T10:00:00.000Z';
const newerIso = '2026-06-09T11:30:00.000Z';
const olderUnix = Math.floor(Date.parse(olderIso) / 1000);
mockDocker.listContainers.mockResolvedValue([
runningContainer('ow-c1', 'ow-stack', 5000),
runningContainer('ow-c2', 'ow-stack', 6000),
]);
const byId: Record<string, string> = { 'ow-c1': newerIso, 'ow-c2': olderIso };
mockDocker.getContainer.mockImplementation((id: string) => ({
inspect: vi.fn().mockResolvedValue({ State: { StartedAt: byId[id] } }),
}));
const dc = DockerController.getInstance(1);
const result = await dc.getBulkStackStatuses(['ow-stack']);
expect(result['ow-stack'].runningSince).toBe(olderUnix);
});
it('falls back to Created when inspect fails', async () => {
const created = 1234;
mockDocker.listContainers.mockResolvedValue([runningContainer('fb-c1', 'fb-stack', created)]);
mockDocker.getContainer.mockReturnValue({
inspect: vi.fn().mockRejectedValue(new Error('inspect boom')),
});
const dc = DockerController.getInstance(1);
const result = await dc.getBulkStackStatuses(['fb-stack']);
expect(result['fb-stack'].status).toBe('running');
expect(result['fb-stack'].runningSince).toBe(created);
});
it('falls back to Created when StartedAt is the Docker zero-time', async () => {
const created = 4321;
mockDocker.listContainers.mockResolvedValue([runningContainer('zt-c1', 'zt-stack', created)]);
mockDocker.getContainer.mockReturnValue({
inspect: vi.fn().mockResolvedValue({ State: { StartedAt: '0001-01-01T00:00:00Z' } }),
});
const dc = DockerController.getInstance(1);
const result = await dc.getBulkStackStatuses(['zt-stack']);
expect(result['zt-stack'].runningSince).toBe(created);
});
it('derives uptime only from running containers, ignoring exited ones in the stack', async () => {
const startedIso = '2026-06-09T08:00:00.000Z';
const startedUnix = Math.floor(Date.parse(startedIso) / 1000);
mockDocker.listContainers.mockResolvedValue([
{ ...runningContainer('me-run', 'me-stack', 7000) },
{ Id: 'me-exit', Names: ['/me-exit'], State: 'exited', Status: 'Exited (0)', Image: 'nginx', Created: 100, Labels: { 'com.docker.compose.project': 'me-stack' } },
]);
mockDocker.getContainer.mockReturnValue({
inspect: vi.fn().mockResolvedValue({ State: { StartedAt: startedIso } }),
});
const dc = DockerController.getInstance(1);
const result = await dc.getBulkStackStatuses(['me-stack']);
expect(result['me-stack'].status).toBe('running');
// The exited container's ancient Created (100) must not drag uptime down.
expect(result['me-stack'].runningSince).toBe(startedUnix);
});
it('caches StartedAt so repeated calls do not re-inspect within the TTL', async () => {
const startedIso = '2026-06-09T09:00:00.000Z';
mockDocker.listContainers.mockResolvedValue([runningContainer('ca-c1', 'ca-stack', 1000)]);
const inspect = vi.fn().mockResolvedValue({ State: { StartedAt: startedIso } });
mockDocker.getContainer.mockReturnValue({ inspect });
// getInstance() hands out fresh instances per request, so a shared (static)
// cache is what must dedupe the inspect across separate calls.
const startedUnix = Math.floor(Date.parse(startedIso) / 1000);
await DockerController.getInstance(1).getBulkStackStatuses(['ca-stack']);
const second = await DockerController.getInstance(1).getBulkStackStatuses(['ca-stack']);
expect(inspect).toHaveBeenCalledTimes(1);
// The cache must be read, not merely populated: the second call still
// resolves the real StartedAt rather than falling back to Created.
expect(second['ca-stack'].runningSince).toBe(startedUnix);
});
});
+85 -10
View File
@@ -23,6 +23,10 @@ const COMPOSE_FILE_NAMES = ['compose.yaml', 'compose.yml', 'docker-compose.yaml'
/** Cached mapping from compose `name:` field to stack directory name. TTL-based to avoid re-parsing YAML on every poll. */
const PROJECT_NAME_CACHE_TTL_MS = 60_000;
const PROJECT_NAME_CACHE_KEY = 'project-name-map';
/** How long a resolved container StartedAt stays fresh before re-inspecting. */
const STARTED_AT_CACHE_TTL_MS = 20_000;
/** Cap on concurrent inspect() calls when resolving StartedAt in bulk. */
const STARTED_AT_INSPECT_CONCURRENCY = 10;
/** Common web-UI private ports, checked in priority order when detecting the main app port. */
const WEB_UI_PORTS = [32400, 8989, 7878, 9696, 5055, 8080, 80, 443, 3000, 9000];
@@ -32,7 +36,7 @@ const IGNORE_PORTS = [1900, 53, 22];
export interface BulkStackInfo {
status: 'running' | 'exited' | 'unknown';
mainPort?: number;
/** Unix seconds of the oldest running container (approximates stack uptime). */
/** Unix seconds of the oldest running container's last start (approximates stack uptime). */
runningSince?: number;
}
@@ -165,6 +169,12 @@ export interface CreateNetworkOptions {
class DockerController {
private static readonly SYSTEM_NETWORKS = new Set(['bridge', 'host', 'none']);
/**
* Cache of container last-start times (unix seconds), keyed by `${nodeId}:${containerId}`.
* Static because getInstance() hands out throwaway instances per request; the cache must
* outlive them so status polls do not re-inspect every container every few seconds.
*/
private static startedAtCache = new Map<string, { startedAtSeconds: number; cachedAtMs: number }>();
private docker: Docker;
private nodeId: number;
@@ -1108,6 +1118,11 @@ class DockerController {
result[name] = { status: 'unknown' };
}
// Per stack, collect running container ids plus the oldest Created as a
// fallback. Uptime is resolved from StartedAt after the loop; Created is
// only used if an inspect fails, since it never moves on restart.
const runningByStack: Record<string, { ids: string[]; oldestCreated?: number }> = {};
for (const container of allContainers as any[]) {
const stackDir = DockerController.resolveContainerStack(
container.Labels, projectToStack, knownStackSet, absDirToStack, resolvedBase,
@@ -1118,16 +1133,11 @@ class DockerController {
if (container.State === 'running') {
result[stackDir].status = 'running';
// Track the oldest running container's creation time as a proxy for
// stack uptime. Docker's listContainers payload exposes Created (unix
// seconds) but not StartedAt; for compose stacks the gap is small
// enough to treat as uptime without paying for a per-container inspect.
const acc = (runningByStack[stackDir] ??= { ids: [] });
if (typeof container.Id === 'string') acc.ids.push(container.Id);
const created = typeof container.Created === 'number' ? container.Created : undefined;
if (created !== undefined) {
const existing = result[stackDir].runningSince;
if (existing === undefined || created < existing) {
result[stackDir].runningSince = created;
}
if (created !== undefined && (acc.oldestCreated === undefined || created < acc.oldestCreated)) {
acc.oldestCreated = created;
}
// Detect main web port (first running container with a matchable port wins)
@@ -1149,9 +1159,74 @@ class DockerController {
}
}
// Resolve real uptime: oldest StartedAt across each stack's running
// containers, falling back to the oldest Created when inspect is unavailable.
const allRunningIds = Object.values(runningByStack).flatMap(s => s.ids);
const startedAts = await this.getRunningStartedAts(allRunningIds);
for (const [stackDir, acc] of Object.entries(runningByStack)) {
let oldest: number | undefined;
for (const id of acc.ids) {
const started = startedAts.get(id);
if (started !== undefined && (oldest === undefined || started < oldest)) {
oldest = started;
}
}
result[stackDir].runningSince = oldest ?? acc.oldestCreated;
}
return result;
}
/**
* Resolve each container's last start time (unix seconds) from State.StartedAt,
* which Docker exposes only via inspect() (listContainers carries Created, which
* does not move on restart). Results are cached briefly per node+container so
* steady-state status polls avoid re-inspecting. A container whose inspect fails
* (e.g. it vanished between listing and inspect) is omitted; callers fall back to
* Created for those rather than failing the whole batch.
*/
private async getRunningStartedAts(containerIds: string[]): Promise<Map<string, number>> {
const now = Date.now();
const cache = DockerController.startedAtCache;
const out = new Map<string, number>();
const misses: string[] = [];
for (const id of containerIds) {
const cached = cache.get(`${this.nodeId}:${id}`);
if (cached && now - cached.cachedAtMs < STARTED_AT_CACHE_TTL_MS) {
out.set(id, cached.startedAtSeconds);
} else {
misses.push(id);
}
}
for (let i = 0; i < misses.length; i += STARTED_AT_INSPECT_CONCURRENCY) {
const chunk = misses.slice(i, i + STARTED_AT_INSPECT_CONCURRENCY);
await Promise.all(chunk.map(async (id) => {
try {
const info = await this.docker.getContainer(id).inspect();
const startedAt = info.State?.StartedAt;
const seconds = startedAt ? Math.floor(Date.parse(startedAt) / 1000) : NaN;
// Skip the Docker zero-time (0001-01-01...) and unparseable values.
if (Number.isFinite(seconds) && seconds > 0) {
cache.set(`${this.nodeId}:${id}`, { startedAtSeconds: seconds, cachedAtMs: now });
out.set(id, seconds);
}
} catch (err: unknown) {
console.warn('[DockerController] StartedAt inspect failed for %s: %s', sanitizeForLog(id), sanitizeForLog((err as Error)?.message ?? String(err)));
}
}));
}
// Bound the cache: drop entries past their TTL so removed containers that
// will never be queried again do not accumulate.
for (const [key, entry] of cache) {
if (now - entry.cachedAtMs >= STARTED_AT_CACHE_TTL_MS) cache.delete(key);
}
return out;
}
/**
* Returns a map of host ports currently bound by running containers,
* with ownership info (Sencho-managed stack name or external).
@@ -3,6 +3,7 @@ import { Button } from '@/components/ui/button';
import { Sparkline } from '@/components/ui/sparkline';
import { ChevronLeft, ChevronRight, Layers } from 'lucide-react';
import type { StackStatusEntry, MetricPoint, StackCpuSeries } from './types';
import { aggregateCurrentUsage } from './aggregateCurrentUsage';
interface StackHealthTableProps {
stackStatuses: Record<string, StackStatusEntry>;
@@ -80,28 +81,7 @@ export function StackHealthTable({
return () => clearInterval(id);
}, []);
const stackAggregates = useMemo(() => {
const latestPerContainer: Record<string, Record<string, MetricPoint>> = {};
for (const m of metrics) {
if (!m.stack_name) continue;
if (!latestPerContainer[m.stack_name]) latestPerContainer[m.stack_name] = {};
const existing = latestPerContainer[m.stack_name][m.container_id];
if (!existing || m.timestamp > existing.timestamp) {
latestPerContainer[m.stack_name][m.container_id] = m;
}
}
const result: Record<string, { mem: number; cpu: number }> = {};
for (const [stack, containers] of Object.entries(latestPerContainer)) {
let mem = 0;
let cpu = 0;
for (const m of Object.values(containers)) {
mem += m.memory_mb;
cpu += m.cpu_percent;
}
result[stack] = { mem, cpu };
}
return result;
}, [metrics]);
const stackAggregates = useMemo(() => aggregateCurrentUsage(metrics), [metrics]);
const rows = useMemo(() => {
const list = Object.entries(stackStatuses).map(([file, entry]) => {
@@ -0,0 +1,64 @@
import { describe, it, expect } from 'vitest';
import { aggregateCurrentUsage } from './aggregateCurrentUsage';
import type { MetricPoint } from './types';
const point = (over: Partial<MetricPoint>): MetricPoint => ({
container_id: 'c1',
stack_name: 'web',
timestamp: 0,
cpu_percent: 0,
memory_mb: 0,
net_rx_mb: 0,
net_tx_mb: 0,
...over,
});
describe('aggregateCurrentUsage', () => {
it('returns an empty map when there are no metrics', () => {
expect(aggregateCurrentUsage([])).toEqual({});
});
it('sums the latest sample per container into a per-stack total', () => {
const result = aggregateCurrentUsage([
point({ container_id: 'a', timestamp: 1000, cpu_percent: 10, memory_mb: 100 }),
point({ container_id: 'a', timestamp: 2000, cpu_percent: 15, memory_mb: 150 }), // newer wins for a
point({ container_id: 'b', timestamp: 2000, cpu_percent: 5, memory_mb: 50 }),
], 90_000);
expect(result.web).toEqual({ cpu: 20, mem: 200 });
});
it('excludes a container whose latest sample is stale relative to the freshest', () => {
const staleWindow = 90_000;
const result = aggregateCurrentUsage([
// live container keeps reporting; its newest point is the freshest sample
point({ container_id: 'live', timestamp: 500_000, cpu_percent: 30, memory_mb: 300 }),
// stopped container's final reading lingers far behind the freshest sample
point({ container_id: 'stopped', timestamp: 500_000 - staleWindow - 1, cpu_percent: 99, memory_mb: 999 }),
], staleWindow);
// only the live container counts; the stale 99% / 999MB reading is dropped
expect(result.web).toEqual({ cpu: 30, mem: 300 });
});
it('keeps a sample sitting exactly on the stale-window boundary', () => {
const staleWindow = 90_000;
const result = aggregateCurrentUsage([
point({ container_id: 'live', timestamp: 500_000, cpu_percent: 10, memory_mb: 100 }),
// exactly freshest - staleWindow: the cutoff is `< cutoff`, so this is kept
point({ container_id: 'edge', timestamp: 500_000 - staleWindow, cpu_percent: 7, memory_mb: 70 }),
], staleWindow);
expect(result.web).toEqual({ cpu: 17, mem: 170 });
});
it('omits a stack entirely when all its containers are stale', () => {
const result = aggregateCurrentUsage([
point({ stack_name: 'alive', container_id: 'x', timestamp: 1_000_000, cpu_percent: 12, memory_mb: 120 }),
point({ stack_name: 'dead', container_id: 'y', timestamp: 1_000_000 - 200_000, cpu_percent: 80, memory_mb: 800 }),
], 90_000);
expect(result.alive).toEqual({ cpu: 12, mem: 120 });
expect(result.dead).toBeUndefined();
});
});
@@ -0,0 +1,57 @@
import type { MetricPoint } from './types';
/**
* How far behind the freshest sample a container's latest point may be and
* still count toward current usage. Metrics are collected on a ~30s cadence,
* so 90s comfortably includes live containers while dropping the lingering
* final reading of one that stopped a couple of cycles ago.
*/
export const CURRENT_USAGE_STALE_WINDOW_MS = 90_000;
export interface StackUsage {
mem: number;
cpu: number;
}
/**
* Sum the latest CPU/memory sample per container into a per-stack total,
* ignoring containers whose most recent sample is stale. A stopped container
* stops producing samples, so its last reading would otherwise persist in the
* total until it aged out of the metrics window; gating on recency relative to
* the freshest sample (not wall-clock, so overall fetch lag does not matter)
* drops it once live containers report newer points.
*/
export function aggregateCurrentUsage(
metrics: MetricPoint[],
staleWindowMs: number = CURRENT_USAGE_STALE_WINDOW_MS,
): Record<string, StackUsage> {
if (metrics.length === 0) return {};
let freshest = -Infinity;
const latestPerContainer: Record<string, Record<string, MetricPoint>> = {};
for (const m of metrics) {
if (m.timestamp > freshest) freshest = m.timestamp;
if (!m.stack_name) continue;
if (!latestPerContainer[m.stack_name]) latestPerContainer[m.stack_name] = {};
const existing = latestPerContainer[m.stack_name][m.container_id];
if (!existing || m.timestamp > existing.timestamp) {
latestPerContainer[m.stack_name][m.container_id] = m;
}
}
const cutoff = freshest - staleWindowMs;
const result: Record<string, StackUsage> = {};
for (const [stack, containers] of Object.entries(latestPerContainer)) {
let mem = 0;
let cpu = 0;
let fresh = false;
for (const m of Object.values(containers)) {
if (m.timestamp < cutoff) continue;
mem += m.memory_mb;
cpu += m.cpu_percent;
fresh = true;
}
if (fresh) result[stack] = { mem, cpu };
}
return result;
}