fix: recognize clean one-shot completions in health gate and drift (#1691)

* fix: recognize clean one-shot completions in health gate and drift

Treat exit 0 with restart policy no/absent as successful completion so
init and migration jobs no longer fail post-update observation or show as
service-missing, while long-running restart policies still fail closed.

* fix: ignore residual health on clean one-shots and honor deploy.restart_policy

Completed exit-0 jobs with no-restart intent no longer fail the health gate on leftover starting/unhealthy state, and Drift treats deploy.restart_policy with Compose precedence so any/on-failure services are not mistaken for one-shots.

* fix: require explicit Compose restart no for one-shot recognition

Docker inspect reports restart no for both intentional jobs and bare services that omit restart, so Health Gate and Drift now require declared restart:""no"" (or deploy.restart_policy condition none) and load Compose intent once per gate.
This commit is contained in:
Anso
2026-07-24 09:41:21 -04:00
committed by GitHub
parent 524cc56d2f
commit 79914fe750
22 changed files with 1015 additions and 69 deletions
@@ -24,7 +24,7 @@ function effSvc(over: Partial<EffService> = {}): EffService {
function container(over: Partial<DependencyContainer> = {}): DependencyContainer {
return {
id: 'c1', name: 'web1', service: 'web', composeProject: 'myapp', stack: 'myapp',
state: 'running', image: 'nginx:1.27', networks: [], volumes: [], ports: [], ...over,
state: 'running', exitCode: null, image: 'nginx:1.27', networks: [], volumes: [], ports: [], ...over,
};
}
@@ -32,7 +32,7 @@ function snap(partial: Partial<DependencySnapshot>): DependencySnapshot {
function container(p: Partial<DependencyContainer> & { id: string }): DependencyContainer {
return {
name: p.id, service: null, composeProject: null, stack: null,
state: 'running', image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
state: 'running', exitCode: null, image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
};
}
@@ -1316,6 +1316,7 @@ describe('DockerController - getDependencySnapshot', () => {
expect(c.networks).toEqual([{ name: 'web_frontend', id: 'net1', ip: '172.18.0.2' }]);
expect(c.volumes).toEqual(['web_data']); // bind mount dropped
expect(c.ports).toEqual([{ ip: '0.0.0.0', publishedPort: 8080, privatePort: 80, protocol: 'tcp' }]); // unpublished 9090 dropped
expect(c.exitCode).toBeNull();
expect(snap.networks.find((n) => n.name === 'bridge')?.isSystem).toBe(true);
const frontend = snap.networks.find((n) => n.name === 'web_frontend');
@@ -1338,6 +1339,23 @@ describe('DockerController - getDependencySnapshot', () => {
expect(snap.containers[0].stack).toBeNull();
expect(snap.containers[0].composeProject).toBeNull();
});
it('parses exitCode from list Status for exited containers', async () => {
mockDocker.listContainers.mockResolvedValue([
{ Id: 'a', Names: ['/job-0'], Image: 'busybox', State: 'exited', Status: 'Exited (0) 5 minutes ago', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
{ Id: 'b', Names: ['/crash-0'], Image: 'busybox', State: 'exited', Status: 'Exited (137) 1 minute ago', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
{ Id: 'c', Names: ['/up-0'], Image: 'busybox', State: 'running', Status: 'Up 3 hours', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
{ Id: 'd', Names: ['/odd-0'], Image: 'busybox', State: 'exited', Status: 'Exited', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
]);
mockDocker.listNetworks.mockResolvedValue([]);
mockDocker.listVolumes.mockResolvedValue({ Volumes: [] });
const snap = await DockerController.getInstance(1).getDependencySnapshot([]);
expect(snap.containers.find(c => c.id === 'a')?.exitCode).toBe(0);
expect(snap.containers.find(c => c.id === 'b')?.exitCode).toBe(137);
expect(snap.containers.find(c => c.id === 'c')?.exitCode).toBeNull();
expect(snap.containers.find(c => c.id === 'd')?.exitCode).toBeNull();
});
});
// --- getBulkStackStatuses uptime (runningSince from StartedAt) -----------------
+218 -1
View File
@@ -36,7 +36,7 @@ function declared(services: DeclaredService[], parseError?: string): DeclaredCom
function container(p: Partial<DependencyContainer> & { id: string }): DependencyContainer {
return {
name: p.id, service: null, composeProject: null, stack: 'app',
state: 'running', image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
state: 'running', exitCode: null, image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
};
}
@@ -108,6 +108,147 @@ describe('assembleStackDrift - status', () => {
expect(report.findings).toEqual([]);
});
it('does not emit service-missing for a clean one-shot beside a running service', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'migrate', image: 'migrate:1', restart: 'no' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }),
container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: 0 }),
],
});
expect(findingKinds(report)).not.toContain('service-missing');
expect(report.status).toBe('in-sync');
expect(report.hasContainers).toBe(true);
});
it('still emits service-missing when a one-shot exits non-zero', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'migrate', image: 'migrate:1', restart: 'no' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }),
container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: 1 }),
],
});
expect(findingKinds(report)).toContain('service-missing');
expect(report.findings.find(f => f.kind === 'service-missing')?.service).toBe('migrate');
});
it('still treats exited unless-stopped as missing even with exit 0', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([service({ name: 'web', restart: 'unless-stopped' })]),
containers: [container({ id: 'c1', service: 'web', state: 'exited', exitCode: 0 })],
});
expect(report.status).toBe('missing-runtime');
expect(report.hasContainers).toBe(false);
});
it('fails closed when exitCode is null even with restart no', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'migrate', image: 'migrate:1', restart: 'no' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }),
container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: null }),
],
});
expect(findingKinds(report)).toContain('service-missing');
expect(report.findings.find(f => f.kind === 'service-missing')?.service).toBe('migrate');
});
it('does not treat absent declared restart as a clean one-shot', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'daemon-default', image: 'daemon:1' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1' }),
container({ id: 'c2', service: 'daemon-default', image: 'daemon:1', state: 'exited', exitCode: 0 }),
],
});
expect(findingKinds(report)).toContain('service-missing');
expect(report.findings.find((f) => f.kind === 'service-missing')?.service).toBe('daemon-default');
});
it('all-one-shot stack with dedicated network is not missing-runtime but keeps network-missing and hasContainers false', () => {
const report = assembleStackDrift({
stack: 'app',
declared: {
services: [service({ name: 'migrate', restart: 'no', networks: ['jobs'] })],
networks: { jobs: { external: false } },
volumes: {},
projectName: 'app',
},
containers: [
container({
id: 'c1', service: 'migrate', state: 'exited', exitCode: 0,
networks: [{ name: 'app_jobs', id: 'j', ip: '' }],
}),
],
networks: [depNet('app_jobs')],
});
expect(report.status).not.toBe('missing-runtime');
expect(findingKinds(report)).not.toContain('service-missing');
expect(report.hasContainers).toBe(false);
expect(findingKinds(report)).toContain('network-missing');
expect(report.status).toBe('drifted');
});
it('all-one-shot stack without network findings is in-sync with hasContainers false', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([service({ name: 'migrate', restart: 'no' })]),
containers: [container({ id: 'c1', service: 'migrate', state: 'exited', exitCode: 0 })],
});
expect(report.status).toBe('in-sync');
expect(report.hasContainers).toBe(false);
expect(report.findings).toEqual([]);
});
it('emits service-missing when normalized restart is always (deploy any)', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'worker', image: 'worker:1', restart: 'always' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1' }),
container({ id: 'c2', service: 'worker', image: 'worker:1', state: 'exited', exitCode: 0 }),
],
});
expect(findingKinds(report)).toContain('service-missing');
expect(report.findings.find((f) => f.kind === 'service-missing')?.service).toBe('worker');
});
it('emits service-missing when normalized restart is on-failure', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'worker', image: 'worker:1', restart: 'on-failure' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1' }),
container({ id: 'c2', service: 'worker', image: 'worker:1', state: 'exited', exitCode: 0 }),
],
});
expect(findingKinds(report)).toContain('service-missing');
});
it('counts a restarting container as deployed', () => {
const report = assembleStackDrift({
stack: 'app',
@@ -456,6 +597,82 @@ describe('declaredFromEffectiveModel', () => {
expect(converted.services[0].ports).toEqual([{ hostIp: '127.0.0.1', publishedPort: 8080, protocol: 'tcp' }]);
});
it('preserves restart policy including no, unless-stopped, and absent', () => {
const withNo = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({ name: 'migrate', restart: 'no' })],
networks: {},
volumes: {},
});
expect(withNo.services[0].restart).toBe('no');
const withUnless = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({ name: 'web', restart: 'unless-stopped' })],
networks: {},
volumes: {},
});
expect(withUnless.services[0].restart).toBe('unless-stopped');
const absent = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({ name: 'job', restart: undefined })],
networks: {},
volumes: {},
});
expect(absent.services[0].restart).toBeNull();
});
it('normalizes deploy.restart_policy conditions with Compose precedence', () => {
const none = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({
name: 'migrate',
restart: 'unless-stopped',
deploy: { restart_policy: { condition: 'none' } },
})],
networks: {},
volumes: {},
});
expect(none.services[0].restart).toBe('no');
const any = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({
name: 'worker',
restart: 'no',
deploy: { restart_policy: { condition: 'any' } },
})],
networks: {},
volumes: {},
});
expect(any.services[0].restart).toBe('always');
const onFailure = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({
name: 'worker',
restart: undefined,
deploy: { restart_policy: { condition: 'on-failure' } },
})],
networks: {},
volumes: {},
});
expect(onFailure.services[0].restart).toBe('on-failure');
const defaultAny = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({
name: 'worker',
restart: 'no',
deploy: { restart_policy: {} },
})],
networks: {},
volumes: {},
});
expect(defaultAny.services[0].restart).toBe('always');
});
it('normalizes networks so drift matches the rendered model', () => {
const model: EffectiveModel = {
projectName: 'myapp',
+2 -2
View File
@@ -323,7 +323,7 @@ describe('DriftLedgerService.reconcileStack', () => {
// A running container on a different image than compose declares => image-mismatch.
const driftedContainer = (stack: string) => ({
id: `${stack}-c1`, name: `${stack}-web-1`, service: 'web', composeProject: stack, stack,
state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [],
state: 'running', exitCode: null, image: 'nginx:1.26', networks: [], volumes: [], ports: [],
});
beforeEach(() => {
@@ -377,7 +377,7 @@ describe('drift route (GET read-only, POST recheck persists)', () => {
getDependencySnapshot: vi.fn().mockResolvedValue({
containers: [{
id: 'c1', name: `${STACK}-web-1`, service: 'web', composeProject: STACK, stack: STACK,
state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [],
state: 'running', exitCode: null, image: 'nginx:1.26', networks: [], volumes: [], ports: [],
}],
networks: [], volumes: [],
}),
@@ -29,6 +29,7 @@ const { state } = vi.hoisted(() => ({
settings: {} as Record<string, string>,
listContainers: vi.fn(),
inspect: vi.fn(),
renderConfig: vi.fn(),
},
}));
@@ -68,6 +69,15 @@ vi.mock('../services/DockerController', () => ({
},
}));
vi.mock('../services/ComposeService', () => ({
getComposeCommandTimeoutMs: () => 30_000,
ComposeService: {
getInstance: () => ({
renderConfig: state.renderConfig,
}),
},
}));
// AutoHeal suppression is exercised elsewhere; here we only need the calls to
// not throw when the gate finalizes a service run.
vi.mock('../services/AutoHealService', () => ({
@@ -85,6 +95,8 @@ type Fixture = {
restartCount?: number;
startedAt?: string;
imageId?: string;
exitCode?: number | null;
restartPolicy?: string | null;
};
function setContainers(fixtures: Fixture[]): void {
@@ -100,16 +112,36 @@ function setContainers(fixtures: Fixture[]): void {
return Promise.resolve({
State: {
Status: f.state ?? 'running',
ExitCode: f.exitCode === undefined ? (f.state === 'exited' ? 1 : 0) : f.exitCode,
Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined,
StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z',
},
RestartCount: f.restartCount ?? 0,
Image: f.imageId ?? 'sha256:app',
HostConfig: { RestartPolicy: { Name: f.restartPolicy ?? 'unless-stopped' } },
Config: { Labels: { 'com.docker.compose.service': f.service } },
});
});
}
function setDeclaredRestarts(services: Record<string, string | undefined>): void {
const rendered = {
name: 'web',
services: Object.fromEntries(
Object.entries(services).map(([name, restart]) => [
name,
restart === undefined ? { image: `${name}:1` } : { image: `${name}:1`, restart },
]),
),
};
state.renderConfig.mockResolvedValue({
rendered: JSON.stringify(rendered),
stderr: '',
code: 0,
timedOut: false,
});
}
const svc = () => HealthGateService.getInstance();
async function ticks(n: number): Promise<void> {
@@ -137,6 +169,8 @@ beforeEach(() => {
state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' };
state.listContainers.mockReset();
state.inspect.mockReset();
state.renderConfig.mockReset();
setDeclaredRestarts({ app: 'unless-stopped', db: 'unless-stopped' });
svc().start();
});
@@ -216,7 +250,7 @@ describe('primary vs collateral attribution', () => {
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-app-1', service: 'app' },
{ id: 's1', name: 'web-db-1', service: 'db', state: 'exited' },
{ id: 's1', name: 'web-db-1', service: 'db', state: 'exited', exitCode: 1, restartPolicy: 'unless-stopped' },
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
@@ -224,6 +258,178 @@ describe('primary vs collateral attribution', () => {
expect(report.failureSource).toBe('collateral');
});
it('passes when a collateral one-shot exits 0 with restart no', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
const token = await prepareService([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{ id: 's1', name: 'web-migrate-1', service: 'migrate', restartPolicy: 'no' },
]);
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{ id: 's1', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
]);
await ticks(1);
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
await ticks(6);
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
});
it('passes a collateral one-shot with residual unhealthy health', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
const token = await prepareService([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{ id: 's1', name: 'web-migrate-1', service: 'migrate', restartPolicy: 'no' },
]);
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{
id: 's1', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0,
restartPolicy: 'no', health: 'unhealthy',
},
]);
await ticks(1);
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
await ticks(6);
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
});
it('fails when a collateral daemon with omitted restart exits 0', async () => {
setDeclaredRestarts({ app: 'unless-stopped', 'daemon-default': undefined });
const token = await prepareService([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{ id: 's1', name: 'web-daemon-1', service: 'daemon-default', restartPolicy: 'no' },
]);
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{
id: 's1', name: 'web-daemon-1', service: 'daemon-default',
state: 'exited', exitCode: 0, restartPolicy: 'no',
},
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('failed');
expect(report.failureSource).toBe('collateral');
expect(report.reason).toContain('exited during observation');
});
it('passes when the primary service is a completed one-shot', async () => {
setDeclaredRestarts({ job: 'no' });
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, restartPolicy: 'no', imageId: 'sha256:app' },
]);
await ticks(1);
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
await ticks(6);
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
});
it('passes a primary one-shot with residual unhealthy health', async () => {
setDeclaredRestarts({ job: 'no' });
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{
id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0,
restartPolicy: 'no', health: 'unhealthy', imageId: 'sha256:app',
},
]);
await ticks(1);
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
await ticks(6);
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
});
it('passes a primary one-shot with residual starting health', async () => {
setDeclaredRestarts({ job: 'no' });
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{
id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0,
restartPolicy: 'no', health: 'starting', imageId: 'sha256:app',
},
]);
await ticks(1);
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
await ticks(6);
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
});
it('fails when a primary one-shot exits with null exit code', async () => {
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: null, restartPolicy: 'no', imageId: 'sha256:app' },
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('failed');
expect(report.reason).toContain('exited during observation');
expect(report.failureSource).toBe('primary');
});
it('fails when a primary one-shot exits 0 under unless-stopped', async () => {
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'unless-stopped' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, restartPolicy: 'unless-stopped', imageId: 'sha256:app' },
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('failed');
expect(report.reason).toContain('exited during observation');
expect(report.failureSource).toBe('primary');
});
it('fails when a primary one-shot exits non-zero', async () => {
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 1, restartPolicy: 'no', imageId: 'sha256:app' },
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('failed');
expect(report.reason).toContain('exited during observation');
expect(report.failureSource).toBe('primary');
});
it('fails with failureSource collateral when a healthy sibling vanishes before the first poll', async () => {
// Sibling is healthy at prepare, then gone before arming. Seeding expected
// from the prepare baseline must still track it so the gate fails.
@@ -29,6 +29,7 @@ const { state } = vi.hoisted(() => ({
settings: {} as Record<string, string>,
listContainers: vi.fn(),
inspect: vi.fn(),
renderConfig: vi.fn(),
},
}));
@@ -80,6 +81,15 @@ vi.mock('../services/DockerController', () => ({
},
}));
vi.mock('../services/ComposeService', () => ({
getComposeCommandTimeoutMs: () => 30_000,
ComposeService: {
getInstance: () => ({
renderConfig: state.renderConfig,
}),
},
}));
import { HealthGateService } from '../services/HealthGateService';
type ContainerFixture = {
@@ -89,25 +99,57 @@ type ContainerFixture = {
health?: string | null;
restartCount?: number;
startedAt?: string;
exitCode?: number | null;
restartPolicy?: string | null;
service?: string;
imageId?: string;
};
/** Configure the docker mocks from a simple fixture list. */
function setContainers(fixtures: ContainerFixture[]): void {
state.listContainers.mockResolvedValue(fixtures.map(f => ({ Id: f.id, Names: [`/${f.name}`], State: f.state ?? 'running' })));
state.listContainers.mockResolvedValue(fixtures.map(f => ({
Id: f.id,
Names: [`/${f.name}`],
State: f.state ?? 'running',
Labels: f.service ? { 'com.docker.compose.service': f.service } : {},
})));
state.inspect.mockImplementation((id: string) => {
const f = fixtures.find(c => c.id === id);
if (!f) return Promise.reject(Object.assign(new Error('no such container'), { statusCode: 404 }));
return Promise.resolve({
State: {
Status: f.state ?? 'running',
ExitCode: f.exitCode === undefined ? (f.state === 'exited' ? 1 : 0) : f.exitCode,
Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined,
StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z',
},
RestartCount: f.restartCount ?? 0,
Image: f.imageId ?? 'sha256:img',
HostConfig: { RestartPolicy: { Name: f.restartPolicy ?? '' } },
Config: { Labels: f.service ? { 'com.docker.compose.service': f.service } : {} },
});
});
}
/** Declared Compose restart map used for one-shot recognition (not inspect). */
function setDeclaredRestarts(services: Record<string, string | undefined>): void {
const rendered = {
name: 'web',
services: Object.fromEntries(
Object.entries(services).map(([name, restart]) => [
name,
restart === undefined ? { image: `${name}:1` } : { image: `${name}:1`, restart },
]),
),
};
state.renderConfig.mockResolvedValue({
rendered: JSON.stringify(rendered),
stderr: '',
code: 0,
timedOut: false,
});
}
const svc = () => HealthGateService.getInstance();
const latest = (stack = 'web') => svc().getReport(0, stack);
@@ -125,7 +167,9 @@ beforeEach(() => {
state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' };
state.listContainers.mockReset();
state.inspect.mockReset();
setContainers([{ id: 'aaa', name: 'web-app-1' }]);
state.renderConfig.mockReset();
setDeclaredRestarts({ app: 'unless-stopped' });
setContainers([{ id: 'aaa', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }]);
svc().start();
});
@@ -149,7 +193,7 @@ describe('HealthGateService verdicts', () => {
it('fails fast when a container exits', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1); // baseline
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited' }]);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'unless-stopped' }]);
await ticks(1);
const report = latest();
expect(report.status).toBe('failed');
@@ -157,6 +201,167 @@ describe('HealthGateService verdicts', () => {
expect(state.activity.some(a => a.category === 'health_gate_failed')).toBe(true);
});
it('passes when a clean one-shot exits 0 with explicit declared restart no', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
]);
await ticks(1);
expect(latest().status).toBe('observing');
await ticks(6);
expect(latest().status).toBe('passed');
});
it('fails when a daemon with omitted Compose restart exits 0 (inspect also reports no)', async () => {
// QA P0: Docker HostConfig.RestartPolicy.Name is "no" for both omit and
// explicit restart:"no". Declared intent must decide, not inspect.
setDeclaredRestarts({ 'daemon-default': undefined });
setContainers([
{
id: 'daemon', name: 'web-daemon-default-1', service: 'daemon-default',
state: 'running', restartPolicy: 'no',
},
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{
id: 'daemon', name: 'web-daemon-default-1', service: 'daemon-default',
state: 'exited', exitCode: 0, restartPolicy: 'no',
},
]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('exited during observation');
});
it('passes a clean one-shot even when residual health is unhealthy', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{
id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0,
restartPolicy: 'no', health: 'unhealthy',
},
]);
await ticks(1);
expect(latest().status).toBe('observing');
await ticks(6);
expect(latest().status).toBe('passed');
});
it('passes a clean one-shot even when residual health is still starting', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{
id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0,
restartPolicy: 'no', health: 'starting',
},
]);
await ticks(1);
expect(latest().status).toBe('observing');
await ticks(6);
expect(latest().status).toBe('passed');
});
it('still fails unhealthy on a long-running container', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{
id: 'app', name: 'web-app-1', service: 'app', state: 'running',
restartPolicy: 'unless-stopped', health: 'unhealthy',
},
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('unhealthy');
});
it('still ends unknown when a long-running healthcheck is starting at window end', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{
id: 'app', name: 'web-app-1', service: 'app', state: 'running',
restartPolicy: 'unless-stopped', health: 'starting',
},
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
]);
await ticks(1);
expect(latest().status).toBe('observing');
await ticks(6);
expect(latest().status).toBe('unknown');
expect(latest().reason).toContain('still starting');
});
it('fails when exit 0 has unless-stopped restart policy', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'unless-stopped' }]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('exited during observation');
});
it('fails when exit 0 has always restart policy', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'always' }]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('exited during observation');
});
it('fails closed when exit code is null on an exited container', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: null, restartPolicy: 'no' }]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('exited during observation');
});
it('fails when a one-shot exits non-zero', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'no' }]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('exited during observation');
});
it('fails fast when a healthcheck reports unhealthy', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
@@ -142,7 +142,7 @@ describe('networking operator routes', () => {
it('blocks admin delete when network is attached', async () => {
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
getDependencySnapshot: vi.fn().mockResolvedValue({
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'nginx', networks: [{ name: 'orphan_net', id: NET_ID, ip: '' }], volumes: [], ports: [] }],
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'orphan_net', id: NET_ID, ip: '' }], volumes: [], ports: [] }],
networks: [{ id: NET_ID, name: 'orphan_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: null, stack: null }],
volumes: [],
}),
@@ -220,7 +220,7 @@ describe('evaluateNetworkDeleteGuard', () => {
it('blocks a network that still has an attached container', () => {
const snapshot = {
volumes: [],
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'img', networks: [{ name: 'app_net', id: 'n1', ip: '' }], volumes: [], ports: [] }],
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'img', networks: [{ name: 'app_net', id: 'n1', ip: '' }], volumes: [], ports: [] }],
networks: [{ id: 'n1', name: 'app_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK }],
};
expect(evaluateNetworkDeleteGuard('n1', snapshot, []).code).toBe('attached');
@@ -60,7 +60,7 @@ describe('networking summary', () => {
it('flags a stack with an undeclared runtime network as network drift', async () => {
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
getDependencySnapshot: vi.fn().mockResolvedValue({
containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'nginx', networks: [{ name: `${STACK}_default`, id: 'd', ip: '' }, { name: `${STACK}_rogue`, id: 'r', ip: '' }], volumes: [], ports: [] }],
containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: `${STACK}_default`, id: 'd', ip: '' }, { name: `${STACK}_rogue`, id: 'r', ip: '' }], volumes: [], ports: [] }],
networks: [
{ id: 'd', name: `${STACK}_default`, driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK },
{ id: 'r', name: `${STACK}_rogue`, driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK },
@@ -0,0 +1,92 @@
import { describe, expect, it } from 'vitest';
import {
isCleanOneShotCompletion,
isNoRestartPolicy,
normalizeComposeRestartIntent,
} from '../utils/oneShotCompletion';
describe('isNoRestartPolicy', () => {
it('accepts only explicit no', () => {
expect(isNoRestartPolicy('no')).toBe(true);
});
it('rejects absent, empty, and restarting policies', () => {
expect(isNoRestartPolicy(undefined)).toBe(false);
expect(isNoRestartPolicy(null)).toBe(false);
expect(isNoRestartPolicy('')).toBe(false);
expect(isNoRestartPolicy('unless-stopped')).toBe(false);
expect(isNoRestartPolicy('always')).toBe(false);
expect(isNoRestartPolicy('on-failure')).toBe(false);
});
});
describe('isCleanOneShotCompletion', () => {
const clean = {
state: 'exited',
exitCode: 0 as number | null,
restartPolicy: 'no' as string | null | undefined,
};
it('returns true only for exited + exit 0 + explicit restart no', () => {
expect(isCleanOneShotCompletion(clean)).toBe(true);
});
it('returns false for absent or empty declared restart', () => {
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: undefined })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: null })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: '' })).toBe(false);
});
it('returns false for non-exited states', () => {
expect(isCleanOneShotCompletion({ ...clean, state: 'running' })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, state: 'restarting' })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, state: 'created' })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, state: 'dead' })).toBe(false);
});
it('returns false for non-zero and null exit codes (fail closed)', () => {
expect(isCleanOneShotCompletion({ ...clean, exitCode: 1 })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, exitCode: 137 })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, exitCode: null })).toBe(false);
});
it('returns false for restarting policies even with exit 0', () => {
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'unless-stopped' })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'always' })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'on-failure' })).toBe(false);
});
});
describe('normalizeComposeRestartIntent', () => {
it('falls back to service restart when deploy.restart_policy is unset', () => {
expect(normalizeComposeRestartIntent('no')).toBe('no');
expect(normalizeComposeRestartIntent('unless-stopped')).toBe('unless-stopped');
expect(normalizeComposeRestartIntent(undefined)).toBeNull();
expect(normalizeComposeRestartIntent(null, {})).toBeNull();
expect(normalizeComposeRestartIntent('always', { replicas: 2 })).toBe('always');
});
it('maps deploy.restart_policy.condition with Compose defaults and precedence', () => {
expect(normalizeComposeRestartIntent('unless-stopped', {
restart_policy: { condition: 'none' },
})).toBe('no');
expect(normalizeComposeRestartIntent(null, {
restart_policy: { condition: 'any' },
})).toBe('always');
expect(normalizeComposeRestartIntent('no', {
restart_policy: { condition: 'on-failure' },
})).toBe('on-failure');
expect(normalizeComposeRestartIntent('no', {
restart_policy: {},
})).toBe('always');
});
it('fails closed on malformed restart_policy shapes', () => {
expect(normalizeComposeRestartIntent('no', { restart_policy: null })).toBe('always');
expect(normalizeComposeRestartIntent('no', { restart_policy: 'none' })).toBe('always');
expect(normalizeComposeRestartIntent('no', { restart_policy: [] })).toBe('always');
expect(normalizeComposeRestartIntent('no', {
restart_policy: { condition: 'weird' },
})).toBe('always');
});
});
@@ -212,7 +212,11 @@ describe('hygiene rules', () => {
});
it('flags a missing restart policy and healthcheck', () => {
const bare = model([svc({ restart: undefined, hasHealthcheck: false })]);
expect(ids(runRules(ctx({ model: bare })), 'no-restart-policy')).toHaveLength(1);
const restartFindings = ids(runRules(ctx({ model: bare })), 'no-restart-policy');
expect(restartFindings).toHaveLength(1);
expect(restartFindings[0].remediation).toMatch(/one-shot|init jobs/i);
expect(restartFindings[0].remediation).toMatch(/restart: "no"/);
expect(restartFindings[0].remediation).toMatch(/unless-stopped/);
expect(ids(runRules(ctx({ model: bare })), 'no-healthcheck')).toHaveLength(1);
const withDeployRestart = model([svc({ restart: undefined, deploy: { restart_policy: { condition: 'any' } }})]);
expect(ids(runRules(ctx({ model: withDeployRestart })), 'no-restart-policy')).toHaveLength(0);
@@ -13,7 +13,7 @@ describe('sanitizeNetworkInspect connected containers', () => {
networks: [{ id: 'net1', name: 'app_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: 'app', stack: 'app' }],
volumes: [],
containers: [{
id: 'c1', name: 'app-web-1', service: 'web', composeProject: 'app', stack: 'app', state: 'running', image: 'nginx',
id: 'c1', name: 'app-web-1', service: 'web', composeProject: 'app', stack: 'app', state: 'running', exitCode: null, image: 'nginx',
networks: [{ name: 'app_net', id: 'net1', ip: '172.20.0.5/16' }], volumes: [], ports: [],
}],
};
@@ -26,6 +26,8 @@ export interface DeclaredService {
image?: string;
/** Service `network_mode:` as declared (e.g. `host`), or undefined. */
networkMode?: string;
/** Service `restart` from raw YAML parse. The effective-model path may set this via normalizeComposeRestartIntent (including deploy.restart_policy). Null when unset. */
restart?: string | null;
}
/** A top-level networks:/volumes: entry. */
@@ -200,6 +202,7 @@ export function parseComposeDependencies(content: string): DeclaredCompose {
ports,
image: asString(svc.image),
networkMode: asString(svc.network_mode),
restart: asString(svc.restart) ?? null,
});
}
@@ -1,5 +1,6 @@
import type { EffectiveModel, EffResource } from '../services/preflight/effectiveModel';
import type { DeclaredCompose, DeclaredPort, DeclaredResource, DeclaredService } from './composeDependencyParse';
import { normalizeComposeRestartIntent } from '../utils/oneShotCompletion';
/** Map one rendered network/volume entry to the declared shape drift expects. */
function toDeclaredResource(key: string, res: EffResource, projectName: string): DeclaredResource {
@@ -41,6 +42,7 @@ export function declaredFromEffectiveModel(model: EffectiveModel): DeclaredCompo
),
image: s.image,
networkMode: s.networkMode,
restart: normalizeComposeRestartIntent(s.restart, s.deploy),
}));
return {
+7
View File
@@ -210,6 +210,12 @@ export interface DependencyContainer {
/** Resolved Sencho stack, or null when the container is not Sencho-managed. */
stack: string | null;
state: string;
/**
* Exit code from list Status when a parenthesized code is present
* (e.g. "Exited (0) …", "Restarting (1) …"); null when none (e.g. "Up …", bare "Exited").
* Required so Drift can fail closed on unknown codes.
*/
exitCode: number | null;
image: string;
networks: { name: string; id: string; ip: string }[];
/** Named-volume sources mounted by the container (bind mounts excluded). */
@@ -1833,6 +1839,7 @@ class DockerController {
composeProject: c.Labels?.['com.docker.compose.project'] ?? null,
stack: DockerController.resolveContainerStack(c.Labels, projectToStack, knownSet, absDirToStack, resolvedBase),
state: c.State ?? 'unknown',
exitCode: parseExitCode(typeof c.Status === 'string' ? c.Status : undefined),
image: c.Image ?? '',
networks,
volumes,
+41 -25
View File
@@ -8,6 +8,7 @@ import { parseEffectiveModel } from './preflight/effectiveModel';
import { compareStackNetworks, fromDeclaredCompose } from './network/normalize';
import { sanitizeForLog, redactSensitiveText } from '../utils/safeLog';
import { getErrorMessage } from '../utils/errors';
import { isCleanOneShotCompletion } from '../utils/oneShotCompletion';
const MAX_RENDER_ERROR = 600;
@@ -178,16 +179,17 @@ function networkDriftFindings(
}
/**
* Pure diff step (no Docker / FS access) so it is directly unit-testable. Only
* running containers are compared, since a stopped container publishes no ports
* and is not "deployed": a declared service with no running container is
* service-missing, a running container with no matching service is
* service-undeclared, and image / port differences are checked only for services
* present on both sides so a missing/undeclared service is not double-reported.
* Pure diff step (no Docker / FS access) so it is directly unit-testable.
* Running/restarting containers drive image/port comparison. Clean one-shot
* completions (exit 0 + explicit declared restart "no", including normalized
* `deploy.restart_policy.condition: none`) satisfy service presence without
* counting as hasContainers. Omitting restart does not qualify. Network
* comparison still uses only running/restarting attachments.
*/
export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftReport {
const { stack, declared, containers, parseError } = input;
const networks = input.networks ?? [];
// Public contract: at least one running/restarting container (not "satisfied").
const hasContainers = containers.some((c) => RUNNING_STATES.has(c.state));
// A parse failure means the declared model is untrustworthy: report drift
@@ -196,30 +198,44 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe
return { stack, status: 'drifted', hasComposeFile: false, hasContainers, findings: [], parseError };
}
const runtimeByService = new Map<string, RuntimeService>();
for (const c of containers) {
if (!RUNNING_STATES.has(c.state)) continue;
const name = c.service ?? c.name;
const agg = runtimeByService.get(name) ?? { images: new Set<string>(), ports: new Set<string>() };
if (c.image) agg.images.add(normalizeImageRef(c.image));
for (const p of c.ports) agg.ports.add(portKey(p.publishedPort, p.protocol));
runtimeByService.set(name, agg);
}
// Nothing running: the stack is defined on disk but not deployed. One status
// conveys this; per-service findings would just be noise.
if (!hasContainers) {
return { stack, status: 'missing-runtime', hasComposeFile: true, hasContainers: false, findings: [] };
}
const declaredByName = new Map<string, DeclaredService>();
for (const svc of declared.services) declaredByName.set(svc.name, svc);
const runtimeByService = new Map<string, RuntimeService>();
const oneShotSatisfied = new Set<string>();
for (const c of containers) {
const name = c.service ?? c.name;
if (RUNNING_STATES.has(c.state)) {
const agg = runtimeByService.get(name) ?? { images: new Set<string>(), ports: new Set<string>() };
if (c.image) agg.images.add(normalizeImageRef(c.image));
for (const p of c.ports) agg.ports.add(portKey(p.publishedPort, p.protocol));
runtimeByService.set(name, agg);
continue;
}
const declaredRestart = declaredByName.get(name)?.restart;
if (isCleanOneShotCompletion({
state: c.state,
exitCode: c.exitCode,
restartPolicy: declaredRestart,
})) {
oneShotSatisfied.add(name);
}
}
const servicePresent = (serviceName: string): boolean =>
runtimeByService.has(serviceName) || oneShotSatisfied.has(serviceName);
// Nothing running and no declared service satisfied by a clean one-shot: the
// stack is defined on disk but not deployed. One status conveys this.
if (!hasContainers && !declared.services.some((svc) => servicePresent(svc.name))) {
return { stack, status: 'missing-runtime', hasComposeFile: true, hasContainers: false, findings: [] };
}
const findings: StackDriftFinding[] = [];
// Declared service with no running container.
// Declared service with no running container and no clean one-shot completion.
for (const svc of declared.services) {
if (!runtimeByService.has(svc.name)) {
if (!servicePresent(svc.name)) {
findings.push({
kind: 'service-missing',
service: svc.name,
@@ -239,7 +255,7 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe
}
}
// Image / port divergence for services present on both sides.
// Image / port divergence for services present on both sides (running only).
for (const [name, svc] of declaredByName) {
const runtime = runtimeByService.get(name);
if (!runtime) continue;
+112 -15
View File
@@ -5,8 +5,11 @@ import { AutoHealService } from './AutoHealService';
import { sanitizeForLog } from '../utils/safeLog';
import { getErrorMessage } from '../utils/errors';
import { withTimeout } from '../utils/withTimeout';
import { isCleanOneShotCompletion } from '../utils/oneShotCompletion';
import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompose';
import { parseEffectiveModel } from './preflight/effectiveModel';
import type { HealthGateContainer, HealthGateReport } from './updateGuard/types';
import { getComposeCommandTimeoutMs } from './ComposeService';
import { ComposeService, getComposeCommandTimeoutMs } from './ComposeService';
const POLL_INTERVAL_MS = 5_000;
// A prepared-but-never-begun token (prepare called, mutation then failed before
@@ -47,6 +50,10 @@ interface ObservedContainer {
service: string | null;
/** Image id the container is running (service gates check convergence on it). */
imageId: string;
/** Inspect State.ExitCode; null when unavailable. */
exitCode: number | null;
/** HostConfig.RestartPolicy.Name (report/debug only; not used for one-shot intent). */
restartPolicy: string | null;
}
interface ActiveGate {
@@ -89,6 +96,13 @@ interface ActiveGate {
collateralBaselineByName: Map<string, ObservedContainer>;
/** Role of each expected container (service gates), for failure attribution. */
roleByName: Map<string, GateRole>;
/**
* Declared Compose restart intent by service name (normalized). Loaded once
* per gate; null until the first load attempt completes. Used for one-shot
* recognition instead of Docker inspect (which cannot distinguish omit vs
* explicit `restart: "no"`).
*/
declaredRestartByService: Map<string, string | null> | null;
}
/** A pre-mutation baseline container captured at prepare time. */
@@ -101,6 +115,8 @@ interface PreparedBaseline {
restartCount: number;
startedAt: string | null;
imageId: string;
exitCode: number | null;
restartPolicy: string | null;
}
function isRegressionEligibleSibling(baseline: PreparedBaseline): boolean {
@@ -118,9 +134,35 @@ function observedFromPreparedBaseline(baseline: PreparedBaseline): ObservedConta
health: baseline.health,
service: baseline.service,
imageId: baseline.imageId,
exitCode: baseline.exitCode,
restartPolicy: baseline.restartPolicy,
};
}
/** Running, or a clean one-shot exit (exit 0 + explicit declared restart "no"). */
function isObservedContainerSatisfied(
gate: ActiveGate,
current: ObservedContainer | undefined,
): boolean {
if (!current) return false;
if (current.state === 'running') return true;
return isDeclaredCleanOneShot(gate, current);
}
/**
* One-shot recognition from declared Compose intent only. Unlabeled containers
* and services missing from the effective model fail closed.
*/
function isDeclaredCleanOneShot(gate: ActiveGate, current: ObservedContainer): boolean {
const map = gate.declaredRestartByService;
if (!map || !current.service || !map.has(current.service)) return false;
return isCleanOneShotCompletion({
state: current.state,
exitCode: current.exitCode,
restartPolicy: map.get(current.service),
});
}
/** An in-memory prepare snapshot awaiting attachExpectedImage + beginPrepared. */
interface PreparedGate {
token: string;
@@ -329,6 +371,7 @@ export class HealthGateService {
collateralEligibleNames: new Set(),
collateralBaselineByName: new Map(),
roleByName: new Map(),
declaredRestartByService: null,
};
this.active.set(key, gate);
this.scheduleNextPoll(gate);
@@ -488,6 +531,7 @@ export class HealthGateService {
prep.collateralBaseline.filter(b => prep.collateralEligibleNames.has(b.name)),
),
roleByName: new Map(),
declaredRestartByService: null,
};
this.active.set(key, gate);
this.scheduleNextPoll(gate);
@@ -587,6 +631,9 @@ export class HealthGateService {
if (gate.finalized || this.active.get(key) !== gate) return;
gate.consecutivePollErrors = 0;
await this.ensureDeclaredRestartMap(gate);
if (gate.finalized || this.active.get(key) !== gate) return;
const elapsedMs = Date.now() - gate.startedAt;
if (gate.expected === null) {
@@ -629,12 +676,16 @@ export class HealthGateService {
}
gate.missingLastPoll.delete(name);
if (current.state === 'exited' && baseline.restarts === current.restarts) {
// An exit with no restart attempt is terminal for the window.
const cleanOneShot = isDeclaredCleanOneShot(gate, current);
// An exit with no restart attempt is terminal for the window, unless
// this is an expected one-shot (exit 0 + explicit declared restart "no").
if (current.state === 'exited' && baseline.restarts === current.restarts && !cleanOneShot) {
this.finalize(gate, 'failed', `container ${name} exited during observation`, summary);
return;
}
if (current.health === 'unhealthy') {
// Residual Docker health on a completed one-shot is not a gate failure;
// long-running containers still fail on unhealthy.
if (!cleanOneShot && current.health === 'unhealthy') {
this.finalize(gate, 'failed', `container ${name} reported unhealthy`, summary);
return;
}
@@ -657,14 +708,19 @@ export class HealthGateService {
if (elapsedMs < gate.windowSeconds * 1000) return;
// Window complete: pass requires everything running and healthy wherever a
// healthcheck exists. A health state still 'starting' is not a pass.
const stillStarting = observed.filter(c => c.health === 'starting');
// Window complete: pass requires everything running (or a clean one-shot
// completion) and healthy wherever a healthcheck exists. A health state
// still 'starting' is not a pass (except residual health on a clean one-shot).
const stillStarting = observed.filter(
c => c.health === 'starting' && !isDeclaredCleanOneShot(gate, c),
);
if (stillStarting.length > 0) {
this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary);
return;
}
const notRunning = [...gate.expected.keys()].filter(name => byName.get(name)?.state !== 'running');
const notRunning = [...gate.expected.keys()].filter(
name => !isObservedContainerSatisfied(gate, byName.get(name)),
);
if (notRunning.length > 0) {
this.finalize(gate, 'failed', `not running at the end of the window: ${notRunning.join(', ')}`, summary);
return;
@@ -700,6 +756,9 @@ export class HealthGateService {
if (gate.finalized || this.active.get(key) !== gate) return;
gate.consecutivePollErrors = 0;
await this.ensureDeclaredRestartMap(gate);
if (gate.finalized || this.active.get(key) !== gate) return;
const elapsedMs = Date.now() - gate.startedAt;
const serviceName = gate.serviceName ?? '';
@@ -781,11 +840,12 @@ export class HealthGateService {
}
gate.missingLastPoll.delete(name);
if (current.state === 'exited' && baseline.restarts === current.restarts) {
const cleanOneShot = isDeclaredCleanOneShot(gate, current);
if (current.state === 'exited' && baseline.restarts === current.restarts && !cleanOneShot) {
this.finalize(gate, 'failed', `${noun} ${name} exited during observation`, summary, role);
return;
}
if (current.health === 'unhealthy') {
if (!cleanOneShot && current.health === 'unhealthy') {
this.finalize(gate, 'failed', `${noun} ${name} reported unhealthy`, summary, role);
return;
}
@@ -808,24 +868,28 @@ export class HealthGateService {
if (elapsedMs < gate.windowSeconds * 1000) return;
const stillStarting = observed.filter(
c => c.health === 'starting' && (c.service === serviceName || gate.collateralEligibleNames.has(c.name)),
c => c.health === 'starting'
&& !isDeclaredCleanOneShot(gate, c)
&& (c.service === serviceName || gate.collateralEligibleNames.has(c.name)),
);
if (stillStarting.length > 0) {
this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary);
return;
}
const runningPrimary = observed.filter(c => c.service === serviceName && c.state === 'running');
if (runningPrimary.length !== gate.expectedReplicas) {
const satisfiedPrimary = observed.filter(
c => c.service === serviceName && isObservedContainerSatisfied(gate, c),
);
if (satisfiedPrimary.length !== gate.expectedReplicas) {
this.finalize(
gate, 'failed',
`service ${serviceName} has ${runningPrimary.length} running replica(s), expected ${gate.expectedReplicas}`,
`service ${serviceName} has ${satisfiedPrimary.length} satisfied replica(s), expected ${gate.expectedReplicas}`,
summary, 'primary',
);
return;
}
const collateralNotRunning = [...gate.expected.keys()]
.filter(name => gate.roleByName.get(name) === 'collateral')
.filter(name => byName.get(name)?.state !== 'running');
.filter(name => !isObservedContainerSatisfied(gate, byName.get(name)));
if (collateralNotRunning.length > 0) {
this.finalize(gate, 'failed', `sibling(s) not running at the end of the window: ${collateralNotRunning.join(', ')}`, summary, 'collateral');
return;
@@ -855,6 +919,35 @@ export class HealthGateService {
return this.listStackContainers(gate.nodeId, gate.stackName);
}
/**
* Load declared Compose restart intent once per gate. Fail closed to an empty
* map on render/parse errors so inspect "no" cannot false-qualify one-shots.
*/
private async ensureDeclaredRestartMap(gate: ActiveGate): Promise<void> {
if (gate.declaredRestartByService !== null) return;
gate.declaredRestartByService = new Map();
try {
const result = await ComposeService.getInstance(gate.nodeId).renderConfig(gate.stackName);
if (result.rendered === null) {
console.warn(
'[HealthGate] declared restart map unavailable for %s (compose render failed)',
sanitizeForLog(gate.stackName),
);
return;
}
const model = parseEffectiveModel(JSON.parse(result.rendered), gate.stackName);
const declared = declaredFromEffectiveModel(model);
gate.declaredRestartByService = new Map(
declared.services.map((s) => [s.name, s.restart ?? null]),
);
} catch (error) {
console.warn(
'[HealthGate] declared restart map load failed for %s:',
sanitizeForLog(gate.stackName), getErrorMessage(error, 'unknown'),
);
}
}
/** List and inspect a stack's containers into the gate's observation shape. */
private async listStackContainers(nodeId: number, stackName: string): Promise<ObservedContainer[]> {
const docker = DockerController.getInstance(nodeId).getDocker();
@@ -877,6 +970,8 @@ export class HealthGateService {
health: inspect.State?.Health?.Status ?? null,
service: labels['com.docker.compose.service'] ?? null,
imageId: inspect.Image ?? '',
exitCode: typeof inspect.State?.ExitCode === 'number' ? inspect.State.ExitCode : null,
restartPolicy: inspect.HostConfig?.RestartPolicy?.Name || null,
};
} catch (e: unknown) {
// Removed between list and inspect; the missing-container logic will
@@ -903,6 +998,8 @@ export class HealthGateService {
restartCount: c.restartCount,
startedAt: c.startedAt,
imageId: c.imageId,
exitCode: c.exitCode,
restartPolicy: c.restartPolicy,
}));
}
@@ -56,7 +56,7 @@ export interface EffService {
networkMode?: string;
restart?: string;
hasHealthcheck: boolean;
/** Raw deploy block (read for key presence only, never values; undefined = none). */
/** Raw deploy block (preflight uses key presence; Drift also reads restart_policy.condition). Undefined = none. */
deploy?: Record<string, unknown>;
containerName?: string;
user?: string;
+1 -1
View File
@@ -363,7 +363,7 @@ const noRestartPolicy: PreflightRule = {
message: `Service "${s.name}" has no restart policy, so it will not come back after a crash or host reboot.`,
sourcePath: s.name,
service: s.name,
remediation: 'Add restart: unless-stopped.',
remediation: 'For long-running services, add restart: unless-stopped. For one-shot or init jobs that should finish and stay stopped, restart: "no" is appropriate.',
}));
},
};
+77
View File
@@ -0,0 +1,77 @@
/**
* Helpers for clean one-shot completion and Compose restart-intent normalization.
* Used by Health Gate and Drift service-presence only; does not change bulk
* status, Auto-Heal, or atomic-deploy helpers.
*/
/**
* True only for an explicit Compose `restart: "no"` (after normalization).
* Absent / null / empty do not qualify: Docker reports HostConfig restart
* "no" for both intentional one-shots and bare long-running services that
* omit `restart:`, so consumers must pass declared Compose intent, not inspect.
*/
export function isNoRestartPolicy(policy: string | null | undefined): boolean {
return policy === 'no';
}
/**
* Normalize Compose restart intent for Drift / declared-model consumers.
*
* Compose precedence: when `deploy.restart_policy` is set, its `condition`
* wins; otherwise the service-level `restart` field is used. Conditions map to
* Docker-like policy names so {@link isNoRestartPolicy} stays the single gate:
* `none` `no`, `any` `always`, `on-failure` `on-failure`. Missing
* condition defaults to `any` (Compose default). Unknown shapes fail closed.
* Absent service restart (no deploy policy) stays null and is not one-shot eligible.
*/
export function normalizeComposeRestartIntent(
serviceRestart: string | null | undefined,
deploy?: Record<string, unknown> | null,
): string | null {
if (deploy && Object.prototype.hasOwnProperty.call(deploy, 'restart_policy')) {
const policy = deploy['restart_policy'];
if (policy === null || typeof policy !== 'object' || Array.isArray(policy)) {
return 'always';
}
const condition = (policy as Record<string, unknown>).condition;
// Missing, empty, or non-string → Compose default `any` → always.
if (typeof condition !== 'string' || condition === '') {
return 'always';
}
switch (condition.toLowerCase()) {
case 'none':
return 'no';
case 'on-failure':
return 'on-failure';
case 'any':
return 'always';
default:
return 'always';
}
}
if (serviceRestart == null || serviceRestart === '') return null;
return serviceRestart;
}
export interface OneShotCompletionInput {
state: string;
/** Exact Docker exit code; null means unknown (fail closed). */
exitCode: number | null;
/**
* Declared Compose restart intent after normalization (`"no"` for explicit
* one-shots / `deploy.restart_policy.condition: none`). Do not pass Docker
* inspect HostConfig values here.
*/
restartPolicy: string | null | undefined;
}
/**
* True only for an exited container with exit code exactly 0 and explicit
* declared restart `"no"`. Null/absent restart, null exit codes, and restarting
* policies never qualify.
*/
export function isCleanOneShotCompletion(input: OneShotCompletionInput): boolean {
return input.state === 'exited'
&& input.exitCode === 0
&& isNoRestartPolicy(input.restartPolicy);
}