mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-30 20:29:15 +00:00
feat(sidebar): surface partial status for multi-container stacks (#1426)
Bulk stack-status aggregation collapsed a stack to "running" as soon as any container was up, so a multi-container stack with crashed containers showed a green UP pill and the degradation was invisible from the sidebar. Add a crash-aware "partial" state: a stack is partial when at least one container is running and at least one has genuinely failed (exited with a non-zero code, dead, or crash-looping). Cleanly finished one-shot containers (exit 0) and clean restart-policy cycling do not count, so an app with a completed init job stays UP. The exit code is read from the container Status string, so no extra inspect calls are needed. Render partial as an amber PT pill with a hover tooltip showing the running/total count, fold it into the Down filter (needs-attention), and treat it as running for context-menu lifecycle actions so operators keep stop/restart/update. The dashboard stack-health table, cross-node search rows, and the command palette all pick up the new state through the shared status surfaces.
This commit is contained in:
@@ -44,7 +44,7 @@ vi.mock('util', () => ({
|
||||
promisify: () => vi.fn(),
|
||||
}));
|
||||
|
||||
import DockerController, { selectMainWebPort } from '../services/DockerController';
|
||||
import DockerController, { selectMainWebPort, parseExitCode, isContainerFailed } from '../services/DockerController';
|
||||
import { CacheService } from '../services/CacheService';
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -1365,3 +1365,152 @@ describe('selectMainWebPort', () => {
|
||||
expect(selectMainWebPort([])).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseExitCode', () => {
|
||||
it('extracts the code from an exited Status string', () => {
|
||||
expect(parseExitCode('Exited (0) 5 minutes ago')).toBe(0);
|
||||
expect(parseExitCode('Exited (137) 2 minutes ago')).toBe(137);
|
||||
});
|
||||
|
||||
it('reads the code from a restarting Status string', () => {
|
||||
expect(parseExitCode('Restarting (1) 3 seconds ago')).toBe(1);
|
||||
});
|
||||
|
||||
it('returns null when no parenthesized code is present', () => {
|
||||
expect(parseExitCode('Up 3 hours')).toBeNull();
|
||||
expect(parseExitCode('Up 2 hours (healthy)')).toBeNull();
|
||||
expect(parseExitCode('Created')).toBeNull();
|
||||
expect(parseExitCode(undefined)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('isContainerFailed', () => {
|
||||
it('treats a dead container as failed', () => {
|
||||
expect(isContainerFailed('dead', 'Dead')).toBe(true);
|
||||
});
|
||||
|
||||
it('treats a crash-looping restart as failed but a clean restart as not', () => {
|
||||
expect(isContainerFailed('restarting', 'Restarting (1) 5 seconds ago')).toBe(true);
|
||||
expect(isContainerFailed('restarting', 'Restarting (0) 2 seconds ago')).toBe(false);
|
||||
});
|
||||
|
||||
it('treats a non-zero exit as failed and a clean exit as not failed', () => {
|
||||
expect(isContainerFailed('exited', 'Exited (137) 2 minutes ago')).toBe(true);
|
||||
expect(isContainerFailed('exited', 'Exited (0) 5 minutes ago')).toBe(false);
|
||||
});
|
||||
|
||||
it('treats an exited container with an unreadable code as failed', () => {
|
||||
expect(isContainerFailed('exited', 'Exited')).toBe(true);
|
||||
});
|
||||
|
||||
it('does not treat running, created, or paused containers as failed', () => {
|
||||
expect(isContainerFailed('running', 'Up 2 hours')).toBe(false);
|
||||
expect(isContainerFailed('created', 'Created')).toBe(false);
|
||||
expect(isContainerFailed('paused', 'Up 2 hours (Paused)')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DockerController - getBulkStackStatuses partial status', () => {
|
||||
beforeEach(() => {
|
||||
CacheService.getInstance().flush();
|
||||
});
|
||||
|
||||
const container = (id: string, project: string, state: string, status: string) => ({
|
||||
Id: id, Names: [`/${id}`], State: state, Status: status,
|
||||
Image: 'nginx', Created: 1000, Labels: { 'com.docker.compose.project': project },
|
||||
});
|
||||
|
||||
// Any running container triggers an inspect for uptime; stub a valid StartedAt.
|
||||
const stubInspect = () => {
|
||||
mockDocker.getContainer.mockReturnValue({
|
||||
inspect: vi.fn().mockResolvedValue({ State: { StartedAt: '2026-06-09T12:00:00.000Z' } }),
|
||||
});
|
||||
};
|
||||
|
||||
it('keeps a stack UP when a one-shot container exits cleanly', async () => {
|
||||
stubInspect();
|
||||
mockDocker.listContainers.mockResolvedValue([
|
||||
container('clean-run', 'clean-stack', 'running', 'Up 2 hours'),
|
||||
container('clean-init', 'clean-stack', 'exited', 'Exited (0) 5 minutes ago'),
|
||||
]);
|
||||
|
||||
const result = await DockerController.getInstance(1).getBulkStackStatuses(['clean-stack']);
|
||||
expect(result['clean-stack'].status).toBe('running');
|
||||
expect(result['clean-stack'].running).toBe(1);
|
||||
expect(result['clean-stack'].total).toBe(2);
|
||||
});
|
||||
|
||||
it('marks a stack partial when a container crashes alongside a running one', async () => {
|
||||
stubInspect();
|
||||
mockDocker.listContainers.mockResolvedValue([
|
||||
container('crash-run', 'crash-stack', 'running', 'Up 2 hours'),
|
||||
container('crash-exit', 'crash-stack', 'exited', 'Exited (137) 1 minute ago'),
|
||||
]);
|
||||
|
||||
const result = await DockerController.getInstance(1).getBulkStackStatuses(['crash-stack']);
|
||||
expect(result['crash-stack'].status).toBe('partial');
|
||||
expect(result['crash-stack'].running).toBe(1);
|
||||
expect(result['crash-stack'].total).toBe(2);
|
||||
});
|
||||
|
||||
it('marks a stack partial for a dead or restart-looping container', async () => {
|
||||
stubInspect();
|
||||
mockDocker.listContainers.mockResolvedValue([
|
||||
container('d-run', 'dead-stack', 'running', 'Up 2 hours'),
|
||||
container('d-dead', 'dead-stack', 'dead', 'Dead'),
|
||||
container('r-run', 'restart-stack', 'running', 'Up 2 hours'),
|
||||
container('r-loop', 'restart-stack', 'restarting', 'Restarting (1) 5 seconds ago'),
|
||||
]);
|
||||
|
||||
const result = await DockerController.getInstance(1).getBulkStackStatuses(['dead-stack', 'restart-stack']);
|
||||
expect(result['dead-stack'].status).toBe('partial');
|
||||
expect(result['restart-stack'].status).toBe('partial');
|
||||
});
|
||||
|
||||
it('keeps a stack running when a sibling is paused rather than crashed', async () => {
|
||||
stubInspect();
|
||||
mockDocker.listContainers.mockResolvedValue([
|
||||
container('p-run', 'paused-stack', 'running', 'Up 2 hours'),
|
||||
container('p-paused', 'paused-stack', 'paused', 'Up 2 hours (Paused)'),
|
||||
]);
|
||||
|
||||
const result = await DockerController.getInstance(1).getBulkStackStatuses(['paused-stack']);
|
||||
expect(result['paused-stack'].status).toBe('running');
|
||||
expect(result['paused-stack'].running).toBe(1);
|
||||
expect(result['paused-stack'].total).toBe(2);
|
||||
});
|
||||
|
||||
it('reports running when every container is up', async () => {
|
||||
stubInspect();
|
||||
mockDocker.listContainers.mockResolvedValue([
|
||||
container('all-1', 'all-stack', 'running', 'Up 2 hours'),
|
||||
container('all-2', 'all-stack', 'running', 'Up 1 hour'),
|
||||
]);
|
||||
|
||||
const result = await DockerController.getInstance(1).getBulkStackStatuses(['all-stack']);
|
||||
expect(result['all-stack'].status).toBe('running');
|
||||
expect(result['all-stack'].running).toBe(2);
|
||||
expect(result['all-stack'].total).toBe(2);
|
||||
});
|
||||
|
||||
it('reports exited when no container is running, even if some crashed', async () => {
|
||||
mockDocker.listContainers.mockResolvedValue([
|
||||
container('down-1', 'down-stack', 'exited', 'Exited (1) 3 minutes ago'),
|
||||
container('down-2', 'down-stack', 'exited', 'Exited (0) 3 minutes ago'),
|
||||
]);
|
||||
|
||||
const result = await DockerController.getInstance(1).getBulkStackStatuses(['down-stack']);
|
||||
expect(result['down-stack'].status).toBe('exited');
|
||||
expect(result['down-stack'].running).toBe(0);
|
||||
expect(result['down-stack'].total).toBe(2);
|
||||
});
|
||||
|
||||
it('reports unknown for a stack with no containers', async () => {
|
||||
mockDocker.listContainers.mockResolvedValue([]);
|
||||
|
||||
const result = await DockerController.getInstance(1).getBulkStackStatuses(['empty-stack']);
|
||||
expect(result['empty-stack'].status).toBe('unknown');
|
||||
expect(result['empty-stack'].running).toBeUndefined();
|
||||
expect(result['empty-stack'].total).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -13,7 +13,7 @@ import { FileSystemService } from '../services/FileSystemService';
|
||||
import { StackFileRootsService, STACK_SOURCE_ROOT_ID, stackSourceFileRoot, type StackFileRoot } from '../services/StackFileRootsService';
|
||||
import { FileRootGateway } from '../services/FileRootGateway';
|
||||
import { ComposeService, getComposeRollbackInfo } from '../services/ComposeService';
|
||||
import DockerController from '../services/DockerController';
|
||||
import DockerController, { type BulkStackInfo } from '../services/DockerController';
|
||||
import { DatabaseService, type StackDossierFields } from '../services/DatabaseService';
|
||||
import { MeshService } from '../services/MeshService';
|
||||
import { CacheService } from '../services/CacheService';
|
||||
@@ -240,7 +240,7 @@ stacksRouter.get('/statuses', async (req: Request, res: Response) => {
|
||||
const stackNames = stacks.map((s: string) => s.replace(/\.(yml|yaml)$/, ''));
|
||||
const dockerController = DockerController.getInstance(req.nodeId);
|
||||
const bulkInfo = await dockerController.getBulkStackStatuses(stackNames);
|
||||
const data: Record<string, { status: 'running' | 'exited' | 'unknown'; mainPort?: number; runningSince?: number }> = {};
|
||||
const data: Record<string, BulkStackInfo> = {};
|
||||
for (const stack of stacks) {
|
||||
const name = stack.replace(/\.(yml|yaml)$/, '');
|
||||
data[stack] = bulkInfo[name] ?? { status: 'unknown' };
|
||||
|
||||
@@ -54,11 +54,45 @@ export function selectMainWebPort(
|
||||
return chosen?.PublicPort;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the exit code out of a Docker container Status string. listContainers
|
||||
* exposes the code only inside Status (e.g. "Exited (137) 2 minutes ago"); the
|
||||
* structured code would otherwise need a per-container inspect. Returns null when
|
||||
* no parenthesized code is present (e.g. "Up 3 hours", "Created").
|
||||
*/
|
||||
export function parseExitCode(status: string | undefined): number | null {
|
||||
if (!status) return null;
|
||||
const match = /\((\d+)\)/.exec(status);
|
||||
return match ? Number(match[1]) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a container represents a genuine failure (a crash) rather than a clean
|
||||
* completion. A dead container always counts. An exited or restarting container
|
||||
* counts only when it left with a non-zero exit code (read from its Status
|
||||
* string), so a finished init job (exit 0) or a container cleanly cycling under a
|
||||
* restart policy does not mark its stack as degraded, while a crash loop (e.g.
|
||||
* "Restarting (1)") does. An exited or restarting container with an unreadable
|
||||
* code is treated as failed, erring toward surfacing a crash.
|
||||
*/
|
||||
export function isContainerFailed(state: string, status: string | undefined): boolean {
|
||||
if (state === 'dead') return true;
|
||||
if (state === 'exited' || state === 'restarting') {
|
||||
const code = parseExitCode(status);
|
||||
return code === null ? true : code !== 0;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export interface BulkStackInfo {
|
||||
status: 'running' | 'exited' | 'unknown';
|
||||
status: 'running' | 'exited' | 'unknown' | 'partial';
|
||||
mainPort?: number;
|
||||
/** Unix seconds of the oldest running container's last start (approximates stack uptime). */
|
||||
runningSince?: number;
|
||||
/** Running container count for the stack (set when the stack has containers). */
|
||||
running?: number;
|
||||
/** Total container count for the stack; paired with `running` for the sidebar tooltip. */
|
||||
total?: number;
|
||||
}
|
||||
|
||||
export interface ClassifiedImage {
|
||||
@@ -1144,6 +1178,14 @@ class DockerController {
|
||||
// only used if an inspect fails, since it never moves on restart.
|
||||
const runningByStack: Record<string, { ids: string[]; oldestCreated?: number }> = {};
|
||||
|
||||
// Per stack, tally running, genuinely-failed (crashed), and total containers
|
||||
// so the status can distinguish a fully-up stack from one that is partially
|
||||
// degraded (some running, some crashed).
|
||||
const countsByStack: Record<string, { running: number; failed: number; total: number }> = {};
|
||||
for (const name of stackNames) {
|
||||
countsByStack[name] = { running: 0, failed: 0, total: 0 };
|
||||
}
|
||||
|
||||
for (const container of allContainers as any[]) {
|
||||
const stackDir = DockerController.resolveContainerStack(
|
||||
container.Labels, projectToStack, knownStackSet, absDirToStack, resolvedBase,
|
||||
@@ -1151,8 +1193,11 @@ class DockerController {
|
||||
|
||||
if (!stackDir || !result[stackDir]) continue;
|
||||
|
||||
const counts = countsByStack[stackDir];
|
||||
counts.total += 1;
|
||||
|
||||
if (container.State === 'running') {
|
||||
result[stackDir].status = 'running';
|
||||
counts.running += 1;
|
||||
|
||||
const acc = (runningByStack[stackDir] ??= { ids: [] });
|
||||
if (typeof container.Id === 'string') acc.ids.push(container.Id);
|
||||
@@ -1168,11 +1213,25 @@ class DockerController {
|
||||
);
|
||||
if (mainPort) result[stackDir].mainPort = mainPort;
|
||||
}
|
||||
} else if (result[stackDir].status !== 'running') {
|
||||
result[stackDir].status = 'exited';
|
||||
} else if (isContainerFailed(container.State, container.Status)) {
|
||||
counts.failed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Classify each stack from its tallies. "partial" requires at least one
|
||||
// running and at least one crashed container, so a stack with a cleanly
|
||||
// finished one-shot container (exit 0) stays "running". A stack with no
|
||||
// containers keeps its seeded "unknown".
|
||||
for (const name of stackNames) {
|
||||
const { running, failed, total } = countsByStack[name];
|
||||
if (total === 0) continue;
|
||||
if (running === 0) result[name].status = 'exited';
|
||||
else if (failed > 0) result[name].status = 'partial';
|
||||
else result[name].status = 'running';
|
||||
result[name].running = running;
|
||||
result[name].total = total;
|
||||
}
|
||||
|
||||
// Resolve real uptime: oldest StartedAt across each stack's running
|
||||
// containers, falling back to the oldest Created when inspect is unavailable.
|
||||
const allRunningIds = Object.values(runningByStack).flatMap(s => s.ids);
|
||||
|
||||
Reference in New Issue
Block a user