feat(sidebar): surface partial status for multi-container stacks (#1426)

Bulk stack-status aggregation collapsed a stack to "running" as soon as any
container was up, so a multi-container stack with crashed containers showed a
green UP pill and the degradation was invisible from the sidebar.

Add a crash-aware "partial" state: a stack is partial when at least one
container is running and at least one has genuinely failed (exited with a
non-zero code, dead, or crash-looping). Cleanly finished one-shot containers
(exit 0) and clean restart-policy cycling do not count, so an app with a
completed init job stays UP. The exit code is read from the container Status
string, so no extra inspect calls are needed.

Render partial as an amber PT pill with a hover tooltip showing the
running/total count, fold it into the Down filter (needs-attention), and treat
it as running for context-menu lifecycle actions so operators keep
stop/restart/update. The dashboard stack-health table, cross-node search rows,
and the command palette all pick up the new state through the shared status
surfaces.
This commit is contained in:
Anso
2026-06-24 19:57:49 -04:00
committed by GitHub
parent 2eafee3594
commit bb4ddde35a
22 changed files with 423 additions and 40 deletions
+150 -1
View File
@@ -44,7 +44,7 @@ vi.mock('util', () => ({
promisify: () => vi.fn(),
}));
import DockerController, { selectMainWebPort } from '../services/DockerController';
import DockerController, { selectMainWebPort, parseExitCode, isContainerFailed } from '../services/DockerController';
import { CacheService } from '../services/CacheService';
beforeEach(() => {
@@ -1365,3 +1365,152 @@ describe('selectMainWebPort', () => {
expect(selectMainWebPort([])).toBeUndefined();
});
});
describe('parseExitCode', () => {
it('extracts the code from an exited Status string', () => {
expect(parseExitCode('Exited (0) 5 minutes ago')).toBe(0);
expect(parseExitCode('Exited (137) 2 minutes ago')).toBe(137);
});
it('reads the code from a restarting Status string', () => {
expect(parseExitCode('Restarting (1) 3 seconds ago')).toBe(1);
});
it('returns null when no parenthesized code is present', () => {
expect(parseExitCode('Up 3 hours')).toBeNull();
expect(parseExitCode('Up 2 hours (healthy)')).toBeNull();
expect(parseExitCode('Created')).toBeNull();
expect(parseExitCode(undefined)).toBeNull();
});
});
describe('isContainerFailed', () => {
it('treats a dead container as failed', () => {
expect(isContainerFailed('dead', 'Dead')).toBe(true);
});
it('treats a crash-looping restart as failed but a clean restart as not', () => {
expect(isContainerFailed('restarting', 'Restarting (1) 5 seconds ago')).toBe(true);
expect(isContainerFailed('restarting', 'Restarting (0) 2 seconds ago')).toBe(false);
});
it('treats a non-zero exit as failed and a clean exit as not failed', () => {
expect(isContainerFailed('exited', 'Exited (137) 2 minutes ago')).toBe(true);
expect(isContainerFailed('exited', 'Exited (0) 5 minutes ago')).toBe(false);
});
it('treats an exited container with an unreadable code as failed', () => {
expect(isContainerFailed('exited', 'Exited')).toBe(true);
});
it('does not treat running, created, or paused containers as failed', () => {
expect(isContainerFailed('running', 'Up 2 hours')).toBe(false);
expect(isContainerFailed('created', 'Created')).toBe(false);
expect(isContainerFailed('paused', 'Up 2 hours (Paused)')).toBe(false);
});
});
describe('DockerController - getBulkStackStatuses partial status', () => {
beforeEach(() => {
CacheService.getInstance().flush();
});
const container = (id: string, project: string, state: string, status: string) => ({
Id: id, Names: [`/${id}`], State: state, Status: status,
Image: 'nginx', Created: 1000, Labels: { 'com.docker.compose.project': project },
});
// Any running container triggers an inspect for uptime; stub a valid StartedAt.
const stubInspect = () => {
mockDocker.getContainer.mockReturnValue({
inspect: vi.fn().mockResolvedValue({ State: { StartedAt: '2026-06-09T12:00:00.000Z' } }),
});
};
it('keeps a stack UP when a one-shot container exits cleanly', async () => {
stubInspect();
mockDocker.listContainers.mockResolvedValue([
container('clean-run', 'clean-stack', 'running', 'Up 2 hours'),
container('clean-init', 'clean-stack', 'exited', 'Exited (0) 5 minutes ago'),
]);
const result = await DockerController.getInstance(1).getBulkStackStatuses(['clean-stack']);
expect(result['clean-stack'].status).toBe('running');
expect(result['clean-stack'].running).toBe(1);
expect(result['clean-stack'].total).toBe(2);
});
it('marks a stack partial when a container crashes alongside a running one', async () => {
stubInspect();
mockDocker.listContainers.mockResolvedValue([
container('crash-run', 'crash-stack', 'running', 'Up 2 hours'),
container('crash-exit', 'crash-stack', 'exited', 'Exited (137) 1 minute ago'),
]);
const result = await DockerController.getInstance(1).getBulkStackStatuses(['crash-stack']);
expect(result['crash-stack'].status).toBe('partial');
expect(result['crash-stack'].running).toBe(1);
expect(result['crash-stack'].total).toBe(2);
});
it('marks a stack partial for a dead or restart-looping container', async () => {
stubInspect();
mockDocker.listContainers.mockResolvedValue([
container('d-run', 'dead-stack', 'running', 'Up 2 hours'),
container('d-dead', 'dead-stack', 'dead', 'Dead'),
container('r-run', 'restart-stack', 'running', 'Up 2 hours'),
container('r-loop', 'restart-stack', 'restarting', 'Restarting (1) 5 seconds ago'),
]);
const result = await DockerController.getInstance(1).getBulkStackStatuses(['dead-stack', 'restart-stack']);
expect(result['dead-stack'].status).toBe('partial');
expect(result['restart-stack'].status).toBe('partial');
});
it('keeps a stack running when a sibling is paused rather than crashed', async () => {
stubInspect();
mockDocker.listContainers.mockResolvedValue([
container('p-run', 'paused-stack', 'running', 'Up 2 hours'),
container('p-paused', 'paused-stack', 'paused', 'Up 2 hours (Paused)'),
]);
const result = await DockerController.getInstance(1).getBulkStackStatuses(['paused-stack']);
expect(result['paused-stack'].status).toBe('running');
expect(result['paused-stack'].running).toBe(1);
expect(result['paused-stack'].total).toBe(2);
});
it('reports running when every container is up', async () => {
stubInspect();
mockDocker.listContainers.mockResolvedValue([
container('all-1', 'all-stack', 'running', 'Up 2 hours'),
container('all-2', 'all-stack', 'running', 'Up 1 hour'),
]);
const result = await DockerController.getInstance(1).getBulkStackStatuses(['all-stack']);
expect(result['all-stack'].status).toBe('running');
expect(result['all-stack'].running).toBe(2);
expect(result['all-stack'].total).toBe(2);
});
it('reports exited when no container is running, even if some crashed', async () => {
mockDocker.listContainers.mockResolvedValue([
container('down-1', 'down-stack', 'exited', 'Exited (1) 3 minutes ago'),
container('down-2', 'down-stack', 'exited', 'Exited (0) 3 minutes ago'),
]);
const result = await DockerController.getInstance(1).getBulkStackStatuses(['down-stack']);
expect(result['down-stack'].status).toBe('exited');
expect(result['down-stack'].running).toBe(0);
expect(result['down-stack'].total).toBe(2);
});
it('reports unknown for a stack with no containers', async () => {
mockDocker.listContainers.mockResolvedValue([]);
const result = await DockerController.getInstance(1).getBulkStackStatuses(['empty-stack']);
expect(result['empty-stack'].status).toBe('unknown');
expect(result['empty-stack'].running).toBeUndefined();
expect(result['empty-stack'].total).toBeUndefined();
});
});
+2 -2
View File
@@ -13,7 +13,7 @@ import { FileSystemService } from '../services/FileSystemService';
import { StackFileRootsService, STACK_SOURCE_ROOT_ID, stackSourceFileRoot, type StackFileRoot } from '../services/StackFileRootsService';
import { FileRootGateway } from '../services/FileRootGateway';
import { ComposeService, getComposeRollbackInfo } from '../services/ComposeService';
import DockerController from '../services/DockerController';
import DockerController, { type BulkStackInfo } from '../services/DockerController';
import { DatabaseService, type StackDossierFields } from '../services/DatabaseService';
import { MeshService } from '../services/MeshService';
import { CacheService } from '../services/CacheService';
@@ -240,7 +240,7 @@ stacksRouter.get('/statuses', async (req: Request, res: Response) => {
const stackNames = stacks.map((s: string) => s.replace(/\.(yml|yaml)$/, ''));
const dockerController = DockerController.getInstance(req.nodeId);
const bulkInfo = await dockerController.getBulkStackStatuses(stackNames);
const data: Record<string, { status: 'running' | 'exited' | 'unknown'; mainPort?: number; runningSince?: number }> = {};
const data: Record<string, BulkStackInfo> = {};
for (const stack of stacks) {
const name = stack.replace(/\.(yml|yaml)$/, '');
data[stack] = bulkInfo[name] ?? { status: 'unknown' };
+63 -4
View File
@@ -54,11 +54,45 @@ export function selectMainWebPort(
return chosen?.PublicPort;
}
/**
* Pull the exit code out of a Docker container Status string. listContainers
* exposes the code only inside Status (e.g. "Exited (137) 2 minutes ago"); the
* structured code would otherwise need a per-container inspect. Returns null when
* no parenthesized code is present (e.g. "Up 3 hours", "Created").
*/
export function parseExitCode(status: string | undefined): number | null {
if (!status) return null;
const match = /\((\d+)\)/.exec(status);
return match ? Number(match[1]) : null;
}
/**
* Whether a container represents a genuine failure (a crash) rather than a clean
* completion. A dead container always counts. An exited or restarting container
* counts only when it left with a non-zero exit code (read from its Status
* string), so a finished init job (exit 0) or a container cleanly cycling under a
* restart policy does not mark its stack as degraded, while a crash loop (e.g.
* "Restarting (1)") does. An exited or restarting container with an unreadable
* code is treated as failed, erring toward surfacing a crash.
*/
export function isContainerFailed(state: string, status: string | undefined): boolean {
if (state === 'dead') return true;
if (state === 'exited' || state === 'restarting') {
const code = parseExitCode(status);
return code === null ? true : code !== 0;
}
return false;
}
export interface BulkStackInfo {
status: 'running' | 'exited' | 'unknown';
status: 'running' | 'exited' | 'unknown' | 'partial';
mainPort?: number;
/** Unix seconds of the oldest running container's last start (approximates stack uptime). */
runningSince?: number;
/** Running container count for the stack (set when the stack has containers). */
running?: number;
/** Total container count for the stack; paired with `running` for the sidebar tooltip. */
total?: number;
}
export interface ClassifiedImage {
@@ -1144,6 +1178,14 @@ class DockerController {
// only used if an inspect fails, since it never moves on restart.
const runningByStack: Record<string, { ids: string[]; oldestCreated?: number }> = {};
// Per stack, tally running, genuinely-failed (crashed), and total containers
// so the status can distinguish a fully-up stack from one that is partially
// degraded (some running, some crashed).
const countsByStack: Record<string, { running: number; failed: number; total: number }> = {};
for (const name of stackNames) {
countsByStack[name] = { running: 0, failed: 0, total: 0 };
}
for (const container of allContainers as any[]) {
const stackDir = DockerController.resolveContainerStack(
container.Labels, projectToStack, knownStackSet, absDirToStack, resolvedBase,
@@ -1151,8 +1193,11 @@ class DockerController {
if (!stackDir || !result[stackDir]) continue;
const counts = countsByStack[stackDir];
counts.total += 1;
if (container.State === 'running') {
result[stackDir].status = 'running';
counts.running += 1;
const acc = (runningByStack[stackDir] ??= { ids: [] });
if (typeof container.Id === 'string') acc.ids.push(container.Id);
@@ -1168,11 +1213,25 @@ class DockerController {
);
if (mainPort) result[stackDir].mainPort = mainPort;
}
} else if (result[stackDir].status !== 'running') {
result[stackDir].status = 'exited';
} else if (isContainerFailed(container.State, container.Status)) {
counts.failed += 1;
}
}
// Classify each stack from its tallies. "partial" requires at least one
// running and at least one crashed container, so a stack with a cleanly
// finished one-shot container (exit 0) stays "running". A stack with no
// containers keeps its seeded "unknown".
for (const name of stackNames) {
const { running, failed, total } = countsByStack[name];
if (total === 0) continue;
if (running === 0) result[name].status = 'exited';
else if (failed > 0) result[name].status = 'partial';
else result[name].status = 'running';
result[name].running = running;
result[name].total = total;
}
// Resolve real uptime: oldest StartedAt across each stack's running
// containers, falling back to the oldest Created when inspect is unavailable.
const allRunningIds = Object.values(runningByStack).flatMap(s => s.ids);