fix: recognize clean one-shot completions in health gate and drift (#1691)

* fix: recognize clean one-shot completions in health gate and drift

Treat exit 0 with restart policy no/absent as successful completion so
init and migration jobs no longer fail post-update observation or show as
service-missing, while long-running restart policies still fail closed.

* fix: ignore residual health on clean one-shots and honor deploy.restart_policy

Completed exit-0 jobs with no-restart intent no longer fail the health gate on leftover starting/unhealthy state, and Drift treats deploy.restart_policy with Compose precedence so any/on-failure services are not mistaken for one-shots.

* fix: require explicit Compose restart no for one-shot recognition

Docker inspect reports restart no for both intentional jobs and bare services that omit restart, so Health Gate and Drift now require declared restart:""no"" (or deploy.restart_policy condition none) and load Compose intent once per gate.
This commit is contained in:
Anso
2026-07-24 09:41:21 -04:00
committed by GitHub
parent 524cc56d2f
commit 79914fe750
22 changed files with 1015 additions and 69 deletions
@@ -24,7 +24,7 @@ function effSvc(over: Partial<EffService> = {}): EffService {
function container(over: Partial<DependencyContainer> = {}): DependencyContainer {
return {
id: 'c1', name: 'web1', service: 'web', composeProject: 'myapp', stack: 'myapp',
state: 'running', image: 'nginx:1.27', networks: [], volumes: [], ports: [], ...over,
state: 'running', exitCode: null, image: 'nginx:1.27', networks: [], volumes: [], ports: [], ...over,
};
}
@@ -32,7 +32,7 @@ function snap(partial: Partial<DependencySnapshot>): DependencySnapshot {
function container(p: Partial<DependencyContainer> & { id: string }): DependencyContainer {
return {
name: p.id, service: null, composeProject: null, stack: null,
state: 'running', image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
state: 'running', exitCode: null, image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
};
}
@@ -1316,6 +1316,7 @@ describe('DockerController - getDependencySnapshot', () => {
expect(c.networks).toEqual([{ name: 'web_frontend', id: 'net1', ip: '172.18.0.2' }]);
expect(c.volumes).toEqual(['web_data']); // bind mount dropped
expect(c.ports).toEqual([{ ip: '0.0.0.0', publishedPort: 8080, privatePort: 80, protocol: 'tcp' }]); // unpublished 9090 dropped
expect(c.exitCode).toBeNull();
expect(snap.networks.find((n) => n.name === 'bridge')?.isSystem).toBe(true);
const frontend = snap.networks.find((n) => n.name === 'web_frontend');
@@ -1338,6 +1339,23 @@ describe('DockerController - getDependencySnapshot', () => {
expect(snap.containers[0].stack).toBeNull();
expect(snap.containers[0].composeProject).toBeNull();
});
it('parses exitCode from list Status for exited containers', async () => {
mockDocker.listContainers.mockResolvedValue([
{ Id: 'a', Names: ['/job-0'], Image: 'busybox', State: 'exited', Status: 'Exited (0) 5 minutes ago', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
{ Id: 'b', Names: ['/crash-0'], Image: 'busybox', State: 'exited', Status: 'Exited (137) 1 minute ago', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
{ Id: 'c', Names: ['/up-0'], Image: 'busybox', State: 'running', Status: 'Up 3 hours', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
{ Id: 'd', Names: ['/odd-0'], Image: 'busybox', State: 'exited', Status: 'Exited', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
]);
mockDocker.listNetworks.mockResolvedValue([]);
mockDocker.listVolumes.mockResolvedValue({ Volumes: [] });
const snap = await DockerController.getInstance(1).getDependencySnapshot([]);
expect(snap.containers.find(c => c.id === 'a')?.exitCode).toBe(0);
expect(snap.containers.find(c => c.id === 'b')?.exitCode).toBe(137);
expect(snap.containers.find(c => c.id === 'c')?.exitCode).toBeNull();
expect(snap.containers.find(c => c.id === 'd')?.exitCode).toBeNull();
});
});
// --- getBulkStackStatuses uptime (runningSince from StartedAt) -----------------
+218 -1
View File
@@ -36,7 +36,7 @@ function declared(services: DeclaredService[], parseError?: string): DeclaredCom
function container(p: Partial<DependencyContainer> & { id: string }): DependencyContainer {
return {
name: p.id, service: null, composeProject: null, stack: 'app',
state: 'running', image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
state: 'running', exitCode: null, image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
};
}
@@ -108,6 +108,147 @@ describe('assembleStackDrift - status', () => {
expect(report.findings).toEqual([]);
});
it('does not emit service-missing for a clean one-shot beside a running service', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'migrate', image: 'migrate:1', restart: 'no' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }),
container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: 0 }),
],
});
expect(findingKinds(report)).not.toContain('service-missing');
expect(report.status).toBe('in-sync');
expect(report.hasContainers).toBe(true);
});
it('still emits service-missing when a one-shot exits non-zero', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'migrate', image: 'migrate:1', restart: 'no' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }),
container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: 1 }),
],
});
expect(findingKinds(report)).toContain('service-missing');
expect(report.findings.find(f => f.kind === 'service-missing')?.service).toBe('migrate');
});
it('still treats exited unless-stopped as missing even with exit 0', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([service({ name: 'web', restart: 'unless-stopped' })]),
containers: [container({ id: 'c1', service: 'web', state: 'exited', exitCode: 0 })],
});
expect(report.status).toBe('missing-runtime');
expect(report.hasContainers).toBe(false);
});
it('fails closed when exitCode is null even with restart no', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'migrate', image: 'migrate:1', restart: 'no' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }),
container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: null }),
],
});
expect(findingKinds(report)).toContain('service-missing');
expect(report.findings.find(f => f.kind === 'service-missing')?.service).toBe('migrate');
});
it('does not treat absent declared restart as a clean one-shot', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'daemon-default', image: 'daemon:1' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1' }),
container({ id: 'c2', service: 'daemon-default', image: 'daemon:1', state: 'exited', exitCode: 0 }),
],
});
expect(findingKinds(report)).toContain('service-missing');
expect(report.findings.find((f) => f.kind === 'service-missing')?.service).toBe('daemon-default');
});
it('all-one-shot stack with dedicated network is not missing-runtime but keeps network-missing and hasContainers false', () => {
const report = assembleStackDrift({
stack: 'app',
declared: {
services: [service({ name: 'migrate', restart: 'no', networks: ['jobs'] })],
networks: { jobs: { external: false } },
volumes: {},
projectName: 'app',
},
containers: [
container({
id: 'c1', service: 'migrate', state: 'exited', exitCode: 0,
networks: [{ name: 'app_jobs', id: 'j', ip: '' }],
}),
],
networks: [depNet('app_jobs')],
});
expect(report.status).not.toBe('missing-runtime');
expect(findingKinds(report)).not.toContain('service-missing');
expect(report.hasContainers).toBe(false);
expect(findingKinds(report)).toContain('network-missing');
expect(report.status).toBe('drifted');
});
it('all-one-shot stack without network findings is in-sync with hasContainers false', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([service({ name: 'migrate', restart: 'no' })]),
containers: [container({ id: 'c1', service: 'migrate', state: 'exited', exitCode: 0 })],
});
expect(report.status).toBe('in-sync');
expect(report.hasContainers).toBe(false);
expect(report.findings).toEqual([]);
});
it('emits service-missing when normalized restart is always (deploy any)', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'worker', image: 'worker:1', restart: 'always' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1' }),
container({ id: 'c2', service: 'worker', image: 'worker:1', state: 'exited', exitCode: 0 }),
],
});
expect(findingKinds(report)).toContain('service-missing');
expect(report.findings.find((f) => f.kind === 'service-missing')?.service).toBe('worker');
});
it('emits service-missing when normalized restart is on-failure', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'worker', image: 'worker:1', restart: 'on-failure' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1' }),
container({ id: 'c2', service: 'worker', image: 'worker:1', state: 'exited', exitCode: 0 }),
],
});
expect(findingKinds(report)).toContain('service-missing');
});
it('counts a restarting container as deployed', () => {
const report = assembleStackDrift({
stack: 'app',
@@ -456,6 +597,82 @@ describe('declaredFromEffectiveModel', () => {
expect(converted.services[0].ports).toEqual([{ hostIp: '127.0.0.1', publishedPort: 8080, protocol: 'tcp' }]);
});
it('preserves restart policy including no, unless-stopped, and absent', () => {
const withNo = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({ name: 'migrate', restart: 'no' })],
networks: {},
volumes: {},
});
expect(withNo.services[0].restart).toBe('no');
const withUnless = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({ name: 'web', restart: 'unless-stopped' })],
networks: {},
volumes: {},
});
expect(withUnless.services[0].restart).toBe('unless-stopped');
const absent = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({ name: 'job', restart: undefined })],
networks: {},
volumes: {},
});
expect(absent.services[0].restart).toBeNull();
});
it('normalizes deploy.restart_policy conditions with Compose precedence', () => {
const none = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({
name: 'migrate',
restart: 'unless-stopped',
deploy: { restart_policy: { condition: 'none' } },
})],
networks: {},
volumes: {},
});
expect(none.services[0].restart).toBe('no');
const any = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({
name: 'worker',
restart: 'no',
deploy: { restart_policy: { condition: 'any' } },
})],
networks: {},
volumes: {},
});
expect(any.services[0].restart).toBe('always');
const onFailure = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({
name: 'worker',
restart: undefined,
deploy: { restart_policy: { condition: 'on-failure' } },
})],
networks: {},
volumes: {},
});
expect(onFailure.services[0].restart).toBe('on-failure');
const defaultAny = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({
name: 'worker',
restart: 'no',
deploy: { restart_policy: {} },
})],
networks: {},
volumes: {},
});
expect(defaultAny.services[0].restart).toBe('always');
});
it('normalizes networks so drift matches the rendered model', () => {
const model: EffectiveModel = {
projectName: 'myapp',
+2 -2
View File
@@ -323,7 +323,7 @@ describe('DriftLedgerService.reconcileStack', () => {
// A running container on a different image than compose declares => image-mismatch.
const driftedContainer = (stack: string) => ({
id: `${stack}-c1`, name: `${stack}-web-1`, service: 'web', composeProject: stack, stack,
state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [],
state: 'running', exitCode: null, image: 'nginx:1.26', networks: [], volumes: [], ports: [],
});
beforeEach(() => {
@@ -377,7 +377,7 @@ describe('drift route (GET read-only, POST recheck persists)', () => {
getDependencySnapshot: vi.fn().mockResolvedValue({
containers: [{
id: 'c1', name: `${STACK}-web-1`, service: 'web', composeProject: STACK, stack: STACK,
state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [],
state: 'running', exitCode: null, image: 'nginx:1.26', networks: [], volumes: [], ports: [],
}],
networks: [], volumes: [],
}),
@@ -29,6 +29,7 @@ const { state } = vi.hoisted(() => ({
settings: {} as Record<string, string>,
listContainers: vi.fn(),
inspect: vi.fn(),
renderConfig: vi.fn(),
},
}));
@@ -68,6 +69,15 @@ vi.mock('../services/DockerController', () => ({
},
}));
vi.mock('../services/ComposeService', () => ({
getComposeCommandTimeoutMs: () => 30_000,
ComposeService: {
getInstance: () => ({
renderConfig: state.renderConfig,
}),
},
}));
// AutoHeal suppression is exercised elsewhere; here we only need the calls to
// not throw when the gate finalizes a service run.
vi.mock('../services/AutoHealService', () => ({
@@ -85,6 +95,8 @@ type Fixture = {
restartCount?: number;
startedAt?: string;
imageId?: string;
exitCode?: number | null;
restartPolicy?: string | null;
};
function setContainers(fixtures: Fixture[]): void {
@@ -100,16 +112,36 @@ function setContainers(fixtures: Fixture[]): void {
return Promise.resolve({
State: {
Status: f.state ?? 'running',
ExitCode: f.exitCode === undefined ? (f.state === 'exited' ? 1 : 0) : f.exitCode,
Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined,
StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z',
},
RestartCount: f.restartCount ?? 0,
Image: f.imageId ?? 'sha256:app',
HostConfig: { RestartPolicy: { Name: f.restartPolicy ?? 'unless-stopped' } },
Config: { Labels: { 'com.docker.compose.service': f.service } },
});
});
}
function setDeclaredRestarts(services: Record<string, string | undefined>): void {
const rendered = {
name: 'web',
services: Object.fromEntries(
Object.entries(services).map(([name, restart]) => [
name,
restart === undefined ? { image: `${name}:1` } : { image: `${name}:1`, restart },
]),
),
};
state.renderConfig.mockResolvedValue({
rendered: JSON.stringify(rendered),
stderr: '',
code: 0,
timedOut: false,
});
}
const svc = () => HealthGateService.getInstance();
async function ticks(n: number): Promise<void> {
@@ -137,6 +169,8 @@ beforeEach(() => {
state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' };
state.listContainers.mockReset();
state.inspect.mockReset();
state.renderConfig.mockReset();
setDeclaredRestarts({ app: 'unless-stopped', db: 'unless-stopped' });
svc().start();
});
@@ -216,7 +250,7 @@ describe('primary vs collateral attribution', () => {
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-app-1', service: 'app' },
{ id: 's1', name: 'web-db-1', service: 'db', state: 'exited' },
{ id: 's1', name: 'web-db-1', service: 'db', state: 'exited', exitCode: 1, restartPolicy: 'unless-stopped' },
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
@@ -224,6 +258,178 @@ describe('primary vs collateral attribution', () => {
expect(report.failureSource).toBe('collateral');
});
it('passes when a collateral one-shot exits 0 with restart no', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
const token = await prepareService([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{ id: 's1', name: 'web-migrate-1', service: 'migrate', restartPolicy: 'no' },
]);
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{ id: 's1', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
]);
await ticks(1);
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
await ticks(6);
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
});
it('passes a collateral one-shot with residual unhealthy health', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
const token = await prepareService([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{ id: 's1', name: 'web-migrate-1', service: 'migrate', restartPolicy: 'no' },
]);
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{
id: 's1', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0,
restartPolicy: 'no', health: 'unhealthy',
},
]);
await ticks(1);
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
await ticks(6);
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
});
it('fails when a collateral daemon with omitted restart exits 0', async () => {
setDeclaredRestarts({ app: 'unless-stopped', 'daemon-default': undefined });
const token = await prepareService([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{ id: 's1', name: 'web-daemon-1', service: 'daemon-default', restartPolicy: 'no' },
]);
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{
id: 's1', name: 'web-daemon-1', service: 'daemon-default',
state: 'exited', exitCode: 0, restartPolicy: 'no',
},
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('failed');
expect(report.failureSource).toBe('collateral');
expect(report.reason).toContain('exited during observation');
});
it('passes when the primary service is a completed one-shot', async () => {
setDeclaredRestarts({ job: 'no' });
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, restartPolicy: 'no', imageId: 'sha256:app' },
]);
await ticks(1);
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
await ticks(6);
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
});
it('passes a primary one-shot with residual unhealthy health', async () => {
setDeclaredRestarts({ job: 'no' });
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{
id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0,
restartPolicy: 'no', health: 'unhealthy', imageId: 'sha256:app',
},
]);
await ticks(1);
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
await ticks(6);
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
});
it('passes a primary one-shot with residual starting health', async () => {
setDeclaredRestarts({ job: 'no' });
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{
id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0,
restartPolicy: 'no', health: 'starting', imageId: 'sha256:app',
},
]);
await ticks(1);
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
await ticks(6);
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
});
it('fails when a primary one-shot exits with null exit code', async () => {
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: null, restartPolicy: 'no', imageId: 'sha256:app' },
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('failed');
expect(report.reason).toContain('exited during observation');
expect(report.failureSource).toBe('primary');
});
it('fails when a primary one-shot exits 0 under unless-stopped', async () => {
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'unless-stopped' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, restartPolicy: 'unless-stopped', imageId: 'sha256:app' },
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('failed');
expect(report.reason).toContain('exited during observation');
expect(report.failureSource).toBe('primary');
});
it('fails when a primary one-shot exits non-zero', async () => {
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 1, restartPolicy: 'no', imageId: 'sha256:app' },
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('failed');
expect(report.reason).toContain('exited during observation');
expect(report.failureSource).toBe('primary');
});
it('fails with failureSource collateral when a healthy sibling vanishes before the first poll', async () => {
// Sibling is healthy at prepare, then gone before arming. Seeding expected
// from the prepare baseline must still track it so the gate fails.
@@ -29,6 +29,7 @@ const { state } = vi.hoisted(() => ({
settings: {} as Record<string, string>,
listContainers: vi.fn(),
inspect: vi.fn(),
renderConfig: vi.fn(),
},
}));
@@ -80,6 +81,15 @@ vi.mock('../services/DockerController', () => ({
},
}));
vi.mock('../services/ComposeService', () => ({
getComposeCommandTimeoutMs: () => 30_000,
ComposeService: {
getInstance: () => ({
renderConfig: state.renderConfig,
}),
},
}));
import { HealthGateService } from '../services/HealthGateService';
type ContainerFixture = {
@@ -89,25 +99,57 @@ type ContainerFixture = {
health?: string | null;
restartCount?: number;
startedAt?: string;
exitCode?: number | null;
restartPolicy?: string | null;
service?: string;
imageId?: string;
};
/** Configure the docker mocks from a simple fixture list. */
function setContainers(fixtures: ContainerFixture[]): void {
state.listContainers.mockResolvedValue(fixtures.map(f => ({ Id: f.id, Names: [`/${f.name}`], State: f.state ?? 'running' })));
state.listContainers.mockResolvedValue(fixtures.map(f => ({
Id: f.id,
Names: [`/${f.name}`],
State: f.state ?? 'running',
Labels: f.service ? { 'com.docker.compose.service': f.service } : {},
})));
state.inspect.mockImplementation((id: string) => {
const f = fixtures.find(c => c.id === id);
if (!f) return Promise.reject(Object.assign(new Error('no such container'), { statusCode: 404 }));
return Promise.resolve({
State: {
Status: f.state ?? 'running',
ExitCode: f.exitCode === undefined ? (f.state === 'exited' ? 1 : 0) : f.exitCode,
Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined,
StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z',
},
RestartCount: f.restartCount ?? 0,
Image: f.imageId ?? 'sha256:img',
HostConfig: { RestartPolicy: { Name: f.restartPolicy ?? '' } },
Config: { Labels: f.service ? { 'com.docker.compose.service': f.service } : {} },
});
});
}
/** Declared Compose restart map used for one-shot recognition (not inspect). */
function setDeclaredRestarts(services: Record<string, string | undefined>): void {
const rendered = {
name: 'web',
services: Object.fromEntries(
Object.entries(services).map(([name, restart]) => [
name,
restart === undefined ? { image: `${name}:1` } : { image: `${name}:1`, restart },
]),
),
};
state.renderConfig.mockResolvedValue({
rendered: JSON.stringify(rendered),
stderr: '',
code: 0,
timedOut: false,
});
}
const svc = () => HealthGateService.getInstance();
const latest = (stack = 'web') => svc().getReport(0, stack);
@@ -125,7 +167,9 @@ beforeEach(() => {
state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' };
state.listContainers.mockReset();
state.inspect.mockReset();
setContainers([{ id: 'aaa', name: 'web-app-1' }]);
state.renderConfig.mockReset();
setDeclaredRestarts({ app: 'unless-stopped' });
setContainers([{ id: 'aaa', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }]);
svc().start();
});
@@ -149,7 +193,7 @@ describe('HealthGateService verdicts', () => {
it('fails fast when a container exits', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1); // baseline
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited' }]);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'unless-stopped' }]);
await ticks(1);
const report = latest();
expect(report.status).toBe('failed');
@@ -157,6 +201,167 @@ describe('HealthGateService verdicts', () => {
expect(state.activity.some(a => a.category === 'health_gate_failed')).toBe(true);
});
it('passes when a clean one-shot exits 0 with explicit declared restart no', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
]);
await ticks(1);
expect(latest().status).toBe('observing');
await ticks(6);
expect(latest().status).toBe('passed');
});
it('fails when a daemon with omitted Compose restart exits 0 (inspect also reports no)', async () => {
// QA P0: Docker HostConfig.RestartPolicy.Name is "no" for both omit and
// explicit restart:"no". Declared intent must decide, not inspect.
setDeclaredRestarts({ 'daemon-default': undefined });
setContainers([
{
id: 'daemon', name: 'web-daemon-default-1', service: 'daemon-default',
state: 'running', restartPolicy: 'no',
},
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{
id: 'daemon', name: 'web-daemon-default-1', service: 'daemon-default',
state: 'exited', exitCode: 0, restartPolicy: 'no',
},
]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('exited during observation');
});
it('passes a clean one-shot even when residual health is unhealthy', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{
id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0,
restartPolicy: 'no', health: 'unhealthy',
},
]);
await ticks(1);
expect(latest().status).toBe('observing');
await ticks(6);
expect(latest().status).toBe('passed');
});
it('passes a clean one-shot even when residual health is still starting', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{
id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0,
restartPolicy: 'no', health: 'starting',
},
]);
await ticks(1);
expect(latest().status).toBe('observing');
await ticks(6);
expect(latest().status).toBe('passed');
});
it('still fails unhealthy on a long-running container', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{
id: 'app', name: 'web-app-1', service: 'app', state: 'running',
restartPolicy: 'unless-stopped', health: 'unhealthy',
},
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('unhealthy');
});
it('still ends unknown when a long-running healthcheck is starting at window end', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{
id: 'app', name: 'web-app-1', service: 'app', state: 'running',
restartPolicy: 'unless-stopped', health: 'starting',
},
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
]);
await ticks(1);
expect(latest().status).toBe('observing');
await ticks(6);
expect(latest().status).toBe('unknown');
expect(latest().reason).toContain('still starting');
});
it('fails when exit 0 has unless-stopped restart policy', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'unless-stopped' }]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('exited during observation');
});
it('fails when exit 0 has always restart policy', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'always' }]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('exited during observation');
});
it('fails closed when exit code is null on an exited container', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: null, restartPolicy: 'no' }]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('exited during observation');
});
it('fails when a one-shot exits non-zero', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'no' }]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('exited during observation');
});
it('fails fast when a healthcheck reports unhealthy', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
@@ -142,7 +142,7 @@ describe('networking operator routes', () => {
it('blocks admin delete when network is attached', async () => {
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
getDependencySnapshot: vi.fn().mockResolvedValue({
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'nginx', networks: [{ name: 'orphan_net', id: NET_ID, ip: '' }], volumes: [], ports: [] }],
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'orphan_net', id: NET_ID, ip: '' }], volumes: [], ports: [] }],
networks: [{ id: NET_ID, name: 'orphan_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: null, stack: null }],
volumes: [],
}),
@@ -220,7 +220,7 @@ describe('evaluateNetworkDeleteGuard', () => {
it('blocks a network that still has an attached container', () => {
const snapshot = {
volumes: [],
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'img', networks: [{ name: 'app_net', id: 'n1', ip: '' }], volumes: [], ports: [] }],
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'img', networks: [{ name: 'app_net', id: 'n1', ip: '' }], volumes: [], ports: [] }],
networks: [{ id: 'n1', name: 'app_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK }],
};
expect(evaluateNetworkDeleteGuard('n1', snapshot, []).code).toBe('attached');
@@ -60,7 +60,7 @@ describe('networking summary', () => {
it('flags a stack with an undeclared runtime network as network drift', async () => {
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
getDependencySnapshot: vi.fn().mockResolvedValue({
containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'nginx', networks: [{ name: `${STACK}_default`, id: 'd', ip: '' }, { name: `${STACK}_rogue`, id: 'r', ip: '' }], volumes: [], ports: [] }],
containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: `${STACK}_default`, id: 'd', ip: '' }, { name: `${STACK}_rogue`, id: 'r', ip: '' }], volumes: [], ports: [] }],
networks: [
{ id: 'd', name: `${STACK}_default`, driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK },
{ id: 'r', name: `${STACK}_rogue`, driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK },
@@ -0,0 +1,92 @@
import { describe, expect, it } from 'vitest';
import {
isCleanOneShotCompletion,
isNoRestartPolicy,
normalizeComposeRestartIntent,
} from '../utils/oneShotCompletion';
describe('isNoRestartPolicy', () => {
it('accepts only explicit no', () => {
expect(isNoRestartPolicy('no')).toBe(true);
});
it('rejects absent, empty, and restarting policies', () => {
expect(isNoRestartPolicy(undefined)).toBe(false);
expect(isNoRestartPolicy(null)).toBe(false);
expect(isNoRestartPolicy('')).toBe(false);
expect(isNoRestartPolicy('unless-stopped')).toBe(false);
expect(isNoRestartPolicy('always')).toBe(false);
expect(isNoRestartPolicy('on-failure')).toBe(false);
});
});
describe('isCleanOneShotCompletion', () => {
const clean = {
state: 'exited',
exitCode: 0 as number | null,
restartPolicy: 'no' as string | null | undefined,
};
it('returns true only for exited + exit 0 + explicit restart no', () => {
expect(isCleanOneShotCompletion(clean)).toBe(true);
});
it('returns false for absent or empty declared restart', () => {
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: undefined })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: null })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: '' })).toBe(false);
});
it('returns false for non-exited states', () => {
expect(isCleanOneShotCompletion({ ...clean, state: 'running' })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, state: 'restarting' })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, state: 'created' })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, state: 'dead' })).toBe(false);
});
it('returns false for non-zero and null exit codes (fail closed)', () => {
expect(isCleanOneShotCompletion({ ...clean, exitCode: 1 })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, exitCode: 137 })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, exitCode: null })).toBe(false);
});
it('returns false for restarting policies even with exit 0', () => {
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'unless-stopped' })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'always' })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'on-failure' })).toBe(false);
});
});
describe('normalizeComposeRestartIntent', () => {
it('falls back to service restart when deploy.restart_policy is unset', () => {
expect(normalizeComposeRestartIntent('no')).toBe('no');
expect(normalizeComposeRestartIntent('unless-stopped')).toBe('unless-stopped');
expect(normalizeComposeRestartIntent(undefined)).toBeNull();
expect(normalizeComposeRestartIntent(null, {})).toBeNull();
expect(normalizeComposeRestartIntent('always', { replicas: 2 })).toBe('always');
});
it('maps deploy.restart_policy.condition with Compose defaults and precedence', () => {
expect(normalizeComposeRestartIntent('unless-stopped', {
restart_policy: { condition: 'none' },
})).toBe('no');
expect(normalizeComposeRestartIntent(null, {
restart_policy: { condition: 'any' },
})).toBe('always');
expect(normalizeComposeRestartIntent('no', {
restart_policy: { condition: 'on-failure' },
})).toBe('on-failure');
expect(normalizeComposeRestartIntent('no', {
restart_policy: {},
})).toBe('always');
});
it('fails closed on malformed restart_policy shapes', () => {
expect(normalizeComposeRestartIntent('no', { restart_policy: null })).toBe('always');
expect(normalizeComposeRestartIntent('no', { restart_policy: 'none' })).toBe('always');
expect(normalizeComposeRestartIntent('no', { restart_policy: [] })).toBe('always');
expect(normalizeComposeRestartIntent('no', {
restart_policy: { condition: 'weird' },
})).toBe('always');
});
});
@@ -212,7 +212,11 @@ describe('hygiene rules', () => {
});
it('flags a missing restart policy and healthcheck', () => {
const bare = model([svc({ restart: undefined, hasHealthcheck: false })]);
expect(ids(runRules(ctx({ model: bare })), 'no-restart-policy')).toHaveLength(1);
const restartFindings = ids(runRules(ctx({ model: bare })), 'no-restart-policy');
expect(restartFindings).toHaveLength(1);
expect(restartFindings[0].remediation).toMatch(/one-shot|init jobs/i);
expect(restartFindings[0].remediation).toMatch(/restart: "no"/);
expect(restartFindings[0].remediation).toMatch(/unless-stopped/);
expect(ids(runRules(ctx({ model: bare })), 'no-healthcheck')).toHaveLength(1);
const withDeployRestart = model([svc({ restart: undefined, deploy: { restart_policy: { condition: 'any' } }})]);
expect(ids(runRules(ctx({ model: withDeployRestart })), 'no-restart-policy')).toHaveLength(0);
@@ -13,7 +13,7 @@ describe('sanitizeNetworkInspect connected containers', () => {
networks: [{ id: 'net1', name: 'app_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: 'app', stack: 'app' }],
volumes: [],
containers: [{
id: 'c1', name: 'app-web-1', service: 'web', composeProject: 'app', stack: 'app', state: 'running', image: 'nginx',
id: 'c1', name: 'app-web-1', service: 'web', composeProject: 'app', stack: 'app', state: 'running', exitCode: null, image: 'nginx',
networks: [{ name: 'app_net', id: 'net1', ip: '172.20.0.5/16' }], volumes: [], ports: [],
}],
};