mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-07-26 11:49:16 +00:00
fix: recognize clean one-shot completions in health gate and drift (#1691)
* fix: recognize clean one-shot completions in health gate and drift Treat exit 0 with restart policy no/absent as successful completion so init and migration jobs no longer fail post-update observation or show as service-missing, while long-running restart policies still fail closed. * fix: ignore residual health on clean one-shots and honor deploy.restart_policy Completed exit-0 jobs with no-restart intent no longer fail the health gate on leftover starting/unhealthy state, and Drift treats deploy.restart_policy with Compose precedence so any/on-failure services are not mistaken for one-shots. * fix: require explicit Compose restart no for one-shot recognition Docker inspect reports restart no for both intentional jobs and bare services that omit restart, so Health Gate and Drift now require declared restart:""no"" (or deploy.restart_policy condition none) and load Compose intent once per gate.
This commit is contained in:
@@ -24,7 +24,7 @@ function effSvc(over: Partial<EffService> = {}): EffService {
|
||||
function container(over: Partial<DependencyContainer> = {}): DependencyContainer {
|
||||
return {
|
||||
id: 'c1', name: 'web1', service: 'web', composeProject: 'myapp', stack: 'myapp',
|
||||
state: 'running', image: 'nginx:1.27', networks: [], volumes: [], ports: [], ...over,
|
||||
state: 'running', exitCode: null, image: 'nginx:1.27', networks: [], volumes: [], ports: [], ...over,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ function snap(partial: Partial<DependencySnapshot>): DependencySnapshot {
|
||||
function container(p: Partial<DependencyContainer> & { id: string }): DependencyContainer {
|
||||
return {
|
||||
name: p.id, service: null, composeProject: null, stack: null,
|
||||
state: 'running', image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
|
||||
state: 'running', exitCode: null, image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1316,6 +1316,7 @@ describe('DockerController - getDependencySnapshot', () => {
|
||||
expect(c.networks).toEqual([{ name: 'web_frontend', id: 'net1', ip: '172.18.0.2' }]);
|
||||
expect(c.volumes).toEqual(['web_data']); // bind mount dropped
|
||||
expect(c.ports).toEqual([{ ip: '0.0.0.0', publishedPort: 8080, privatePort: 80, protocol: 'tcp' }]); // unpublished 9090 dropped
|
||||
expect(c.exitCode).toBeNull();
|
||||
|
||||
expect(snap.networks.find((n) => n.name === 'bridge')?.isSystem).toBe(true);
|
||||
const frontend = snap.networks.find((n) => n.name === 'web_frontend');
|
||||
@@ -1338,6 +1339,23 @@ describe('DockerController - getDependencySnapshot', () => {
|
||||
expect(snap.containers[0].stack).toBeNull();
|
||||
expect(snap.containers[0].composeProject).toBeNull();
|
||||
});
|
||||
|
||||
it('parses exitCode from list Status for exited containers', async () => {
|
||||
mockDocker.listContainers.mockResolvedValue([
|
||||
{ Id: 'a', Names: ['/job-0'], Image: 'busybox', State: 'exited', Status: 'Exited (0) 5 minutes ago', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
|
||||
{ Id: 'b', Names: ['/crash-0'], Image: 'busybox', State: 'exited', Status: 'Exited (137) 1 minute ago', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
|
||||
{ Id: 'c', Names: ['/up-0'], Image: 'busybox', State: 'running', Status: 'Up 3 hours', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
|
||||
{ Id: 'd', Names: ['/odd-0'], Image: 'busybox', State: 'exited', Status: 'Exited', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
|
||||
]);
|
||||
mockDocker.listNetworks.mockResolvedValue([]);
|
||||
mockDocker.listVolumes.mockResolvedValue({ Volumes: [] });
|
||||
|
||||
const snap = await DockerController.getInstance(1).getDependencySnapshot([]);
|
||||
expect(snap.containers.find(c => c.id === 'a')?.exitCode).toBe(0);
|
||||
expect(snap.containers.find(c => c.id === 'b')?.exitCode).toBe(137);
|
||||
expect(snap.containers.find(c => c.id === 'c')?.exitCode).toBeNull();
|
||||
expect(snap.containers.find(c => c.id === 'd')?.exitCode).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// --- getBulkStackStatuses uptime (runningSince from StartedAt) -----------------
|
||||
|
||||
@@ -36,7 +36,7 @@ function declared(services: DeclaredService[], parseError?: string): DeclaredCom
|
||||
function container(p: Partial<DependencyContainer> & { id: string }): DependencyContainer {
|
||||
return {
|
||||
name: p.id, service: null, composeProject: null, stack: 'app',
|
||||
state: 'running', image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
|
||||
state: 'running', exitCode: null, image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -108,6 +108,147 @@ describe('assembleStackDrift - status', () => {
|
||||
expect(report.findings).toEqual([]);
|
||||
});
|
||||
|
||||
it('does not emit service-missing for a clean one-shot beside a running service', () => {
|
||||
const report = assembleStackDrift({
|
||||
stack: 'app',
|
||||
declared: declared([
|
||||
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
|
||||
service({ name: 'migrate', image: 'migrate:1', restart: 'no' }),
|
||||
]),
|
||||
containers: [
|
||||
container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }),
|
||||
container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: 0 }),
|
||||
],
|
||||
});
|
||||
expect(findingKinds(report)).not.toContain('service-missing');
|
||||
expect(report.status).toBe('in-sync');
|
||||
expect(report.hasContainers).toBe(true);
|
||||
});
|
||||
|
||||
it('still emits service-missing when a one-shot exits non-zero', () => {
|
||||
const report = assembleStackDrift({
|
||||
stack: 'app',
|
||||
declared: declared([
|
||||
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
|
||||
service({ name: 'migrate', image: 'migrate:1', restart: 'no' }),
|
||||
]),
|
||||
containers: [
|
||||
container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }),
|
||||
container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: 1 }),
|
||||
],
|
||||
});
|
||||
expect(findingKinds(report)).toContain('service-missing');
|
||||
expect(report.findings.find(f => f.kind === 'service-missing')?.service).toBe('migrate');
|
||||
});
|
||||
|
||||
it('still treats exited unless-stopped as missing even with exit 0', () => {
|
||||
const report = assembleStackDrift({
|
||||
stack: 'app',
|
||||
declared: declared([service({ name: 'web', restart: 'unless-stopped' })]),
|
||||
containers: [container({ id: 'c1', service: 'web', state: 'exited', exitCode: 0 })],
|
||||
});
|
||||
expect(report.status).toBe('missing-runtime');
|
||||
expect(report.hasContainers).toBe(false);
|
||||
});
|
||||
|
||||
it('fails closed when exitCode is null even with restart no', () => {
|
||||
const report = assembleStackDrift({
|
||||
stack: 'app',
|
||||
declared: declared([
|
||||
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
|
||||
service({ name: 'migrate', image: 'migrate:1', restart: 'no' }),
|
||||
]),
|
||||
containers: [
|
||||
container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }),
|
||||
container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: null }),
|
||||
],
|
||||
});
|
||||
expect(findingKinds(report)).toContain('service-missing');
|
||||
expect(report.findings.find(f => f.kind === 'service-missing')?.service).toBe('migrate');
|
||||
});
|
||||
|
||||
it('does not treat absent declared restart as a clean one-shot', () => {
|
||||
const report = assembleStackDrift({
|
||||
stack: 'app',
|
||||
declared: declared([
|
||||
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
|
||||
service({ name: 'daemon-default', image: 'daemon:1' }),
|
||||
]),
|
||||
containers: [
|
||||
container({ id: 'c1', service: 'app', image: 'app:1' }),
|
||||
container({ id: 'c2', service: 'daemon-default', image: 'daemon:1', state: 'exited', exitCode: 0 }),
|
||||
],
|
||||
});
|
||||
expect(findingKinds(report)).toContain('service-missing');
|
||||
expect(report.findings.find((f) => f.kind === 'service-missing')?.service).toBe('daemon-default');
|
||||
});
|
||||
|
||||
it('all-one-shot stack with dedicated network is not missing-runtime but keeps network-missing and hasContainers false', () => {
|
||||
const report = assembleStackDrift({
|
||||
stack: 'app',
|
||||
declared: {
|
||||
services: [service({ name: 'migrate', restart: 'no', networks: ['jobs'] })],
|
||||
networks: { jobs: { external: false } },
|
||||
volumes: {},
|
||||
projectName: 'app',
|
||||
},
|
||||
containers: [
|
||||
container({
|
||||
id: 'c1', service: 'migrate', state: 'exited', exitCode: 0,
|
||||
networks: [{ name: 'app_jobs', id: 'j', ip: '' }],
|
||||
}),
|
||||
],
|
||||
networks: [depNet('app_jobs')],
|
||||
});
|
||||
expect(report.status).not.toBe('missing-runtime');
|
||||
expect(findingKinds(report)).not.toContain('service-missing');
|
||||
expect(report.hasContainers).toBe(false);
|
||||
expect(findingKinds(report)).toContain('network-missing');
|
||||
expect(report.status).toBe('drifted');
|
||||
});
|
||||
|
||||
it('all-one-shot stack without network findings is in-sync with hasContainers false', () => {
|
||||
const report = assembleStackDrift({
|
||||
stack: 'app',
|
||||
declared: declared([service({ name: 'migrate', restart: 'no' })]),
|
||||
containers: [container({ id: 'c1', service: 'migrate', state: 'exited', exitCode: 0 })],
|
||||
});
|
||||
expect(report.status).toBe('in-sync');
|
||||
expect(report.hasContainers).toBe(false);
|
||||
expect(report.findings).toEqual([]);
|
||||
});
|
||||
|
||||
it('emits service-missing when normalized restart is always (deploy any)', () => {
|
||||
const report = assembleStackDrift({
|
||||
stack: 'app',
|
||||
declared: declared([
|
||||
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
|
||||
service({ name: 'worker', image: 'worker:1', restart: 'always' }),
|
||||
]),
|
||||
containers: [
|
||||
container({ id: 'c1', service: 'app', image: 'app:1' }),
|
||||
container({ id: 'c2', service: 'worker', image: 'worker:1', state: 'exited', exitCode: 0 }),
|
||||
],
|
||||
});
|
||||
expect(findingKinds(report)).toContain('service-missing');
|
||||
expect(report.findings.find((f) => f.kind === 'service-missing')?.service).toBe('worker');
|
||||
});
|
||||
|
||||
it('emits service-missing when normalized restart is on-failure', () => {
|
||||
const report = assembleStackDrift({
|
||||
stack: 'app',
|
||||
declared: declared([
|
||||
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
|
||||
service({ name: 'worker', image: 'worker:1', restart: 'on-failure' }),
|
||||
]),
|
||||
containers: [
|
||||
container({ id: 'c1', service: 'app', image: 'app:1' }),
|
||||
container({ id: 'c2', service: 'worker', image: 'worker:1', state: 'exited', exitCode: 0 }),
|
||||
],
|
||||
});
|
||||
expect(findingKinds(report)).toContain('service-missing');
|
||||
});
|
||||
|
||||
it('counts a restarting container as deployed', () => {
|
||||
const report = assembleStackDrift({
|
||||
stack: 'app',
|
||||
@@ -456,6 +597,82 @@ describe('declaredFromEffectiveModel', () => {
|
||||
expect(converted.services[0].ports).toEqual([{ hostIp: '127.0.0.1', publishedPort: 8080, protocol: 'tcp' }]);
|
||||
});
|
||||
|
||||
it('preserves restart policy including no, unless-stopped, and absent', () => {
|
||||
const withNo = declaredFromEffectiveModel({
|
||||
projectName: 'app',
|
||||
services: [effSvc({ name: 'migrate', restart: 'no' })],
|
||||
networks: {},
|
||||
volumes: {},
|
||||
});
|
||||
expect(withNo.services[0].restart).toBe('no');
|
||||
|
||||
const withUnless = declaredFromEffectiveModel({
|
||||
projectName: 'app',
|
||||
services: [effSvc({ name: 'web', restart: 'unless-stopped' })],
|
||||
networks: {},
|
||||
volumes: {},
|
||||
});
|
||||
expect(withUnless.services[0].restart).toBe('unless-stopped');
|
||||
|
||||
const absent = declaredFromEffectiveModel({
|
||||
projectName: 'app',
|
||||
services: [effSvc({ name: 'job', restart: undefined })],
|
||||
networks: {},
|
||||
volumes: {},
|
||||
});
|
||||
expect(absent.services[0].restart).toBeNull();
|
||||
});
|
||||
|
||||
it('normalizes deploy.restart_policy conditions with Compose precedence', () => {
|
||||
const none = declaredFromEffectiveModel({
|
||||
projectName: 'app',
|
||||
services: [effSvc({
|
||||
name: 'migrate',
|
||||
restart: 'unless-stopped',
|
||||
deploy: { restart_policy: { condition: 'none' } },
|
||||
})],
|
||||
networks: {},
|
||||
volumes: {},
|
||||
});
|
||||
expect(none.services[0].restart).toBe('no');
|
||||
|
||||
const any = declaredFromEffectiveModel({
|
||||
projectName: 'app',
|
||||
services: [effSvc({
|
||||
name: 'worker',
|
||||
restart: 'no',
|
||||
deploy: { restart_policy: { condition: 'any' } },
|
||||
})],
|
||||
networks: {},
|
||||
volumes: {},
|
||||
});
|
||||
expect(any.services[0].restart).toBe('always');
|
||||
|
||||
const onFailure = declaredFromEffectiveModel({
|
||||
projectName: 'app',
|
||||
services: [effSvc({
|
||||
name: 'worker',
|
||||
restart: undefined,
|
||||
deploy: { restart_policy: { condition: 'on-failure' } },
|
||||
})],
|
||||
networks: {},
|
||||
volumes: {},
|
||||
});
|
||||
expect(onFailure.services[0].restart).toBe('on-failure');
|
||||
|
||||
const defaultAny = declaredFromEffectiveModel({
|
||||
projectName: 'app',
|
||||
services: [effSvc({
|
||||
name: 'worker',
|
||||
restart: 'no',
|
||||
deploy: { restart_policy: {} },
|
||||
})],
|
||||
networks: {},
|
||||
volumes: {},
|
||||
});
|
||||
expect(defaultAny.services[0].restart).toBe('always');
|
||||
});
|
||||
|
||||
it('normalizes networks so drift matches the rendered model', () => {
|
||||
const model: EffectiveModel = {
|
||||
projectName: 'myapp',
|
||||
|
||||
@@ -323,7 +323,7 @@ describe('DriftLedgerService.reconcileStack', () => {
|
||||
// A running container on a different image than compose declares => image-mismatch.
|
||||
const driftedContainer = (stack: string) => ({
|
||||
id: `${stack}-c1`, name: `${stack}-web-1`, service: 'web', composeProject: stack, stack,
|
||||
state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [],
|
||||
state: 'running', exitCode: null, image: 'nginx:1.26', networks: [], volumes: [], ports: [],
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -377,7 +377,7 @@ describe('drift route (GET read-only, POST recheck persists)', () => {
|
||||
getDependencySnapshot: vi.fn().mockResolvedValue({
|
||||
containers: [{
|
||||
id: 'c1', name: `${STACK}-web-1`, service: 'web', composeProject: STACK, stack: STACK,
|
||||
state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [],
|
||||
state: 'running', exitCode: null, image: 'nginx:1.26', networks: [], volumes: [], ports: [],
|
||||
}],
|
||||
networks: [], volumes: [],
|
||||
}),
|
||||
|
||||
@@ -29,6 +29,7 @@ const { state } = vi.hoisted(() => ({
|
||||
settings: {} as Record<string, string>,
|
||||
listContainers: vi.fn(),
|
||||
inspect: vi.fn(),
|
||||
renderConfig: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
@@ -68,6 +69,15 @@ vi.mock('../services/DockerController', () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/ComposeService', () => ({
|
||||
getComposeCommandTimeoutMs: () => 30_000,
|
||||
ComposeService: {
|
||||
getInstance: () => ({
|
||||
renderConfig: state.renderConfig,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
// AutoHeal suppression is exercised elsewhere; here we only need the calls to
|
||||
// not throw when the gate finalizes a service run.
|
||||
vi.mock('../services/AutoHealService', () => ({
|
||||
@@ -85,6 +95,8 @@ type Fixture = {
|
||||
restartCount?: number;
|
||||
startedAt?: string;
|
||||
imageId?: string;
|
||||
exitCode?: number | null;
|
||||
restartPolicy?: string | null;
|
||||
};
|
||||
|
||||
function setContainers(fixtures: Fixture[]): void {
|
||||
@@ -100,16 +112,36 @@ function setContainers(fixtures: Fixture[]): void {
|
||||
return Promise.resolve({
|
||||
State: {
|
||||
Status: f.state ?? 'running',
|
||||
ExitCode: f.exitCode === undefined ? (f.state === 'exited' ? 1 : 0) : f.exitCode,
|
||||
Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined,
|
||||
StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z',
|
||||
},
|
||||
RestartCount: f.restartCount ?? 0,
|
||||
Image: f.imageId ?? 'sha256:app',
|
||||
HostConfig: { RestartPolicy: { Name: f.restartPolicy ?? 'unless-stopped' } },
|
||||
Config: { Labels: { 'com.docker.compose.service': f.service } },
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function setDeclaredRestarts(services: Record<string, string | undefined>): void {
|
||||
const rendered = {
|
||||
name: 'web',
|
||||
services: Object.fromEntries(
|
||||
Object.entries(services).map(([name, restart]) => [
|
||||
name,
|
||||
restart === undefined ? { image: `${name}:1` } : { image: `${name}:1`, restart },
|
||||
]),
|
||||
),
|
||||
};
|
||||
state.renderConfig.mockResolvedValue({
|
||||
rendered: JSON.stringify(rendered),
|
||||
stderr: '',
|
||||
code: 0,
|
||||
timedOut: false,
|
||||
});
|
||||
}
|
||||
|
||||
const svc = () => HealthGateService.getInstance();
|
||||
|
||||
async function ticks(n: number): Promise<void> {
|
||||
@@ -137,6 +169,8 @@ beforeEach(() => {
|
||||
state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' };
|
||||
state.listContainers.mockReset();
|
||||
state.inspect.mockReset();
|
||||
state.renderConfig.mockReset();
|
||||
setDeclaredRestarts({ app: 'unless-stopped', db: 'unless-stopped' });
|
||||
svc().start();
|
||||
});
|
||||
|
||||
@@ -216,7 +250,7 @@ describe('primary vs collateral attribution', () => {
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{ id: 'p1', name: 'web-app-1', service: 'app' },
|
||||
{ id: 's1', name: 'web-db-1', service: 'db', state: 'exited' },
|
||||
{ id: 's1', name: 'web-db-1', service: 'db', state: 'exited', exitCode: 1, restartPolicy: 'unless-stopped' },
|
||||
]);
|
||||
await ticks(1);
|
||||
const report = svc().getReport(0, 'web', runId!);
|
||||
@@ -224,6 +258,178 @@ describe('primary vs collateral attribution', () => {
|
||||
expect(report.failureSource).toBe('collateral');
|
||||
});
|
||||
|
||||
it('passes when a collateral one-shot exits 0 with restart no', async () => {
|
||||
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
|
||||
const token = await prepareService([
|
||||
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
|
||||
{ id: 's1', name: 'web-migrate-1', service: 'migrate', restartPolicy: 'no' },
|
||||
]);
|
||||
svc().attachExpectedImage(token, 'sha256:app');
|
||||
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
|
||||
{ id: 's1', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
|
||||
]);
|
||||
await ticks(1);
|
||||
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
|
||||
await ticks(6);
|
||||
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
|
||||
});
|
||||
|
||||
it('passes a collateral one-shot with residual unhealthy health', async () => {
|
||||
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
|
||||
const token = await prepareService([
|
||||
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
|
||||
{ id: 's1', name: 'web-migrate-1', service: 'migrate', restartPolicy: 'no' },
|
||||
]);
|
||||
svc().attachExpectedImage(token, 'sha256:app');
|
||||
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
|
||||
{
|
||||
id: 's1', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0,
|
||||
restartPolicy: 'no', health: 'unhealthy',
|
||||
},
|
||||
]);
|
||||
await ticks(1);
|
||||
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
|
||||
await ticks(6);
|
||||
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
|
||||
});
|
||||
|
||||
it('fails when a collateral daemon with omitted restart exits 0', async () => {
|
||||
setDeclaredRestarts({ app: 'unless-stopped', 'daemon-default': undefined });
|
||||
const token = await prepareService([
|
||||
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
|
||||
{ id: 's1', name: 'web-daemon-1', service: 'daemon-default', restartPolicy: 'no' },
|
||||
]);
|
||||
svc().attachExpectedImage(token, 'sha256:app');
|
||||
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
|
||||
{
|
||||
id: 's1', name: 'web-daemon-1', service: 'daemon-default',
|
||||
state: 'exited', exitCode: 0, restartPolicy: 'no',
|
||||
},
|
||||
]);
|
||||
await ticks(1);
|
||||
const report = svc().getReport(0, 'web', runId!);
|
||||
expect(report.status).toBe('failed');
|
||||
expect(report.failureSource).toBe('collateral');
|
||||
expect(report.reason).toContain('exited during observation');
|
||||
});
|
||||
|
||||
it('passes when the primary service is a completed one-shot', async () => {
|
||||
setDeclaredRestarts({ job: 'no' });
|
||||
const token = await prepareService([
|
||||
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
|
||||
], { serviceName: 'job', expectedReplicas: 1 });
|
||||
svc().attachExpectedImage(token, 'sha256:app');
|
||||
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, restartPolicy: 'no', imageId: 'sha256:app' },
|
||||
]);
|
||||
await ticks(1);
|
||||
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
|
||||
await ticks(6);
|
||||
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
|
||||
});
|
||||
|
||||
it('passes a primary one-shot with residual unhealthy health', async () => {
|
||||
setDeclaredRestarts({ job: 'no' });
|
||||
const token = await prepareService([
|
||||
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
|
||||
], { serviceName: 'job', expectedReplicas: 1 });
|
||||
svc().attachExpectedImage(token, 'sha256:app');
|
||||
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{
|
||||
id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0,
|
||||
restartPolicy: 'no', health: 'unhealthy', imageId: 'sha256:app',
|
||||
},
|
||||
]);
|
||||
await ticks(1);
|
||||
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
|
||||
await ticks(6);
|
||||
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
|
||||
});
|
||||
|
||||
it('passes a primary one-shot with residual starting health', async () => {
|
||||
setDeclaredRestarts({ job: 'no' });
|
||||
const token = await prepareService([
|
||||
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
|
||||
], { serviceName: 'job', expectedReplicas: 1 });
|
||||
svc().attachExpectedImage(token, 'sha256:app');
|
||||
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{
|
||||
id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0,
|
||||
restartPolicy: 'no', health: 'starting', imageId: 'sha256:app',
|
||||
},
|
||||
]);
|
||||
await ticks(1);
|
||||
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
|
||||
await ticks(6);
|
||||
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
|
||||
});
|
||||
|
||||
it('fails when a primary one-shot exits with null exit code', async () => {
|
||||
const token = await prepareService([
|
||||
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
|
||||
], { serviceName: 'job', expectedReplicas: 1 });
|
||||
svc().attachExpectedImage(token, 'sha256:app');
|
||||
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: null, restartPolicy: 'no', imageId: 'sha256:app' },
|
||||
]);
|
||||
await ticks(1);
|
||||
const report = svc().getReport(0, 'web', runId!);
|
||||
expect(report.status).toBe('failed');
|
||||
expect(report.reason).toContain('exited during observation');
|
||||
expect(report.failureSource).toBe('primary');
|
||||
});
|
||||
|
||||
it('fails when a primary one-shot exits 0 under unless-stopped', async () => {
|
||||
const token = await prepareService([
|
||||
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'unless-stopped' },
|
||||
], { serviceName: 'job', expectedReplicas: 1 });
|
||||
svc().attachExpectedImage(token, 'sha256:app');
|
||||
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, restartPolicy: 'unless-stopped', imageId: 'sha256:app' },
|
||||
]);
|
||||
await ticks(1);
|
||||
const report = svc().getReport(0, 'web', runId!);
|
||||
expect(report.status).toBe('failed');
|
||||
expect(report.reason).toContain('exited during observation');
|
||||
expect(report.failureSource).toBe('primary');
|
||||
});
|
||||
|
||||
it('fails when a primary one-shot exits non-zero', async () => {
|
||||
const token = await prepareService([
|
||||
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
|
||||
], { serviceName: 'job', expectedReplicas: 1 });
|
||||
svc().attachExpectedImage(token, 'sha256:app');
|
||||
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 1, restartPolicy: 'no', imageId: 'sha256:app' },
|
||||
]);
|
||||
await ticks(1);
|
||||
const report = svc().getReport(0, 'web', runId!);
|
||||
expect(report.status).toBe('failed');
|
||||
expect(report.reason).toContain('exited during observation');
|
||||
expect(report.failureSource).toBe('primary');
|
||||
});
|
||||
|
||||
it('fails with failureSource collateral when a healthy sibling vanishes before the first poll', async () => {
|
||||
// Sibling is healthy at prepare, then gone before arming. Seeding expected
|
||||
// from the prepare baseline must still track it so the gate fails.
|
||||
|
||||
@@ -29,6 +29,7 @@ const { state } = vi.hoisted(() => ({
|
||||
settings: {} as Record<string, string>,
|
||||
listContainers: vi.fn(),
|
||||
inspect: vi.fn(),
|
||||
renderConfig: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
@@ -80,6 +81,15 @@ vi.mock('../services/DockerController', () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/ComposeService', () => ({
|
||||
getComposeCommandTimeoutMs: () => 30_000,
|
||||
ComposeService: {
|
||||
getInstance: () => ({
|
||||
renderConfig: state.renderConfig,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
import { HealthGateService } from '../services/HealthGateService';
|
||||
|
||||
type ContainerFixture = {
|
||||
@@ -89,25 +99,57 @@ type ContainerFixture = {
|
||||
health?: string | null;
|
||||
restartCount?: number;
|
||||
startedAt?: string;
|
||||
exitCode?: number | null;
|
||||
restartPolicy?: string | null;
|
||||
service?: string;
|
||||
imageId?: string;
|
||||
};
|
||||
|
||||
/** Configure the docker mocks from a simple fixture list. */
|
||||
function setContainers(fixtures: ContainerFixture[]): void {
|
||||
state.listContainers.mockResolvedValue(fixtures.map(f => ({ Id: f.id, Names: [`/${f.name}`], State: f.state ?? 'running' })));
|
||||
state.listContainers.mockResolvedValue(fixtures.map(f => ({
|
||||
Id: f.id,
|
||||
Names: [`/${f.name}`],
|
||||
State: f.state ?? 'running',
|
||||
Labels: f.service ? { 'com.docker.compose.service': f.service } : {},
|
||||
})));
|
||||
state.inspect.mockImplementation((id: string) => {
|
||||
const f = fixtures.find(c => c.id === id);
|
||||
if (!f) return Promise.reject(Object.assign(new Error('no such container'), { statusCode: 404 }));
|
||||
return Promise.resolve({
|
||||
State: {
|
||||
Status: f.state ?? 'running',
|
||||
ExitCode: f.exitCode === undefined ? (f.state === 'exited' ? 1 : 0) : f.exitCode,
|
||||
Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined,
|
||||
StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z',
|
||||
},
|
||||
RestartCount: f.restartCount ?? 0,
|
||||
Image: f.imageId ?? 'sha256:img',
|
||||
HostConfig: { RestartPolicy: { Name: f.restartPolicy ?? '' } },
|
||||
Config: { Labels: f.service ? { 'com.docker.compose.service': f.service } : {} },
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** Declared Compose restart map used for one-shot recognition (not inspect). */
|
||||
function setDeclaredRestarts(services: Record<string, string | undefined>): void {
|
||||
const rendered = {
|
||||
name: 'web',
|
||||
services: Object.fromEntries(
|
||||
Object.entries(services).map(([name, restart]) => [
|
||||
name,
|
||||
restart === undefined ? { image: `${name}:1` } : { image: `${name}:1`, restart },
|
||||
]),
|
||||
),
|
||||
};
|
||||
state.renderConfig.mockResolvedValue({
|
||||
rendered: JSON.stringify(rendered),
|
||||
stderr: '',
|
||||
code: 0,
|
||||
timedOut: false,
|
||||
});
|
||||
}
|
||||
|
||||
const svc = () => HealthGateService.getInstance();
|
||||
|
||||
const latest = (stack = 'web') => svc().getReport(0, stack);
|
||||
@@ -125,7 +167,9 @@ beforeEach(() => {
|
||||
state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' };
|
||||
state.listContainers.mockReset();
|
||||
state.inspect.mockReset();
|
||||
setContainers([{ id: 'aaa', name: 'web-app-1' }]);
|
||||
state.renderConfig.mockReset();
|
||||
setDeclaredRestarts({ app: 'unless-stopped' });
|
||||
setContainers([{ id: 'aaa', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }]);
|
||||
svc().start();
|
||||
});
|
||||
|
||||
@@ -149,7 +193,7 @@ describe('HealthGateService verdicts', () => {
|
||||
it('fails fast when a container exits', async () => {
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1); // baseline
|
||||
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited' }]);
|
||||
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'unless-stopped' }]);
|
||||
await ticks(1);
|
||||
const report = latest();
|
||||
expect(report.status).toBe('failed');
|
||||
@@ -157,6 +201,167 @@ describe('HealthGateService verdicts', () => {
|
||||
expect(state.activity.some(a => a.category === 'health_gate_failed')).toBe(true);
|
||||
});
|
||||
|
||||
it('passes when a clean one-shot exits 0 with explicit declared restart no', async () => {
|
||||
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
|
||||
setContainers([
|
||||
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
|
||||
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
|
||||
]);
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
|
||||
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
|
||||
]);
|
||||
await ticks(1);
|
||||
expect(latest().status).toBe('observing');
|
||||
await ticks(6);
|
||||
expect(latest().status).toBe('passed');
|
||||
});
|
||||
|
||||
it('fails when a daemon with omitted Compose restart exits 0 (inspect also reports no)', async () => {
|
||||
// QA P0: Docker HostConfig.RestartPolicy.Name is "no" for both omit and
|
||||
// explicit restart:"no". Declared intent must decide, not inspect.
|
||||
setDeclaredRestarts({ 'daemon-default': undefined });
|
||||
setContainers([
|
||||
{
|
||||
id: 'daemon', name: 'web-daemon-default-1', service: 'daemon-default',
|
||||
state: 'running', restartPolicy: 'no',
|
||||
},
|
||||
]);
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{
|
||||
id: 'daemon', name: 'web-daemon-default-1', service: 'daemon-default',
|
||||
state: 'exited', exitCode: 0, restartPolicy: 'no',
|
||||
},
|
||||
]);
|
||||
await ticks(1);
|
||||
expect(latest().status).toBe('failed');
|
||||
expect(latest().reason).toContain('exited during observation');
|
||||
});
|
||||
|
||||
it('passes a clean one-shot even when residual health is unhealthy', async () => {
|
||||
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
|
||||
setContainers([
|
||||
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
|
||||
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
|
||||
]);
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
|
||||
{
|
||||
id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0,
|
||||
restartPolicy: 'no', health: 'unhealthy',
|
||||
},
|
||||
]);
|
||||
await ticks(1);
|
||||
expect(latest().status).toBe('observing');
|
||||
await ticks(6);
|
||||
expect(latest().status).toBe('passed');
|
||||
});
|
||||
|
||||
it('passes a clean one-shot even when residual health is still starting', async () => {
|
||||
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
|
||||
setContainers([
|
||||
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
|
||||
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
|
||||
]);
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
|
||||
{
|
||||
id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0,
|
||||
restartPolicy: 'no', health: 'starting',
|
||||
},
|
||||
]);
|
||||
await ticks(1);
|
||||
expect(latest().status).toBe('observing');
|
||||
await ticks(6);
|
||||
expect(latest().status).toBe('passed');
|
||||
});
|
||||
|
||||
it('still fails unhealthy on a long-running container', async () => {
|
||||
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
|
||||
setContainers([
|
||||
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
|
||||
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
|
||||
]);
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{
|
||||
id: 'app', name: 'web-app-1', service: 'app', state: 'running',
|
||||
restartPolicy: 'unless-stopped', health: 'unhealthy',
|
||||
},
|
||||
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
|
||||
]);
|
||||
await ticks(1);
|
||||
expect(latest().status).toBe('failed');
|
||||
expect(latest().reason).toContain('unhealthy');
|
||||
});
|
||||
|
||||
it('still ends unknown when a long-running healthcheck is starting at window end', async () => {
|
||||
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
|
||||
setContainers([
|
||||
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
|
||||
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
|
||||
]);
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{
|
||||
id: 'app', name: 'web-app-1', service: 'app', state: 'running',
|
||||
restartPolicy: 'unless-stopped', health: 'starting',
|
||||
},
|
||||
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
|
||||
]);
|
||||
await ticks(1);
|
||||
expect(latest().status).toBe('observing');
|
||||
await ticks(6);
|
||||
expect(latest().status).toBe('unknown');
|
||||
expect(latest().reason).toContain('still starting');
|
||||
});
|
||||
|
||||
it('fails when exit 0 has unless-stopped restart policy', async () => {
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'unless-stopped' }]);
|
||||
await ticks(1);
|
||||
expect(latest().status).toBe('failed');
|
||||
expect(latest().reason).toContain('exited during observation');
|
||||
});
|
||||
|
||||
it('fails when exit 0 has always restart policy', async () => {
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'always' }]);
|
||||
await ticks(1);
|
||||
expect(latest().status).toBe('failed');
|
||||
expect(latest().reason).toContain('exited during observation');
|
||||
});
|
||||
|
||||
it('fails closed when exit code is null on an exited container', async () => {
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: null, restartPolicy: 'no' }]);
|
||||
await ticks(1);
|
||||
expect(latest().status).toBe('failed');
|
||||
expect(latest().reason).toContain('exited during observation');
|
||||
});
|
||||
|
||||
it('fails when a one-shot exits non-zero', async () => {
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'no' }]);
|
||||
await ticks(1);
|
||||
expect(latest().status).toBe('failed');
|
||||
expect(latest().reason).toContain('exited during observation');
|
||||
});
|
||||
|
||||
it('fails fast when a healthcheck reports unhealthy', async () => {
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
|
||||
@@ -142,7 +142,7 @@ describe('networking operator routes', () => {
|
||||
it('blocks admin delete when network is attached', async () => {
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getDependencySnapshot: vi.fn().mockResolvedValue({
|
||||
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'nginx', networks: [{ name: 'orphan_net', id: NET_ID, ip: '' }], volumes: [], ports: [] }],
|
||||
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'orphan_net', id: NET_ID, ip: '' }], volumes: [], ports: [] }],
|
||||
networks: [{ id: NET_ID, name: 'orphan_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: null, stack: null }],
|
||||
volumes: [],
|
||||
}),
|
||||
@@ -220,7 +220,7 @@ describe('evaluateNetworkDeleteGuard', () => {
|
||||
it('blocks a network that still has an attached container', () => {
|
||||
const snapshot = {
|
||||
volumes: [],
|
||||
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'img', networks: [{ name: 'app_net', id: 'n1', ip: '' }], volumes: [], ports: [] }],
|
||||
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'img', networks: [{ name: 'app_net', id: 'n1', ip: '' }], volumes: [], ports: [] }],
|
||||
networks: [{ id: 'n1', name: 'app_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK }],
|
||||
};
|
||||
expect(evaluateNetworkDeleteGuard('n1', snapshot, []).code).toBe('attached');
|
||||
|
||||
@@ -60,7 +60,7 @@ describe('networking summary', () => {
|
||||
it('flags a stack with an undeclared runtime network as network drift', async () => {
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getDependencySnapshot: vi.fn().mockResolvedValue({
|
||||
containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'nginx', networks: [{ name: `${STACK}_default`, id: 'd', ip: '' }, { name: `${STACK}_rogue`, id: 'r', ip: '' }], volumes: [], ports: [] }],
|
||||
containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: `${STACK}_default`, id: 'd', ip: '' }, { name: `${STACK}_rogue`, id: 'r', ip: '' }], volumes: [], ports: [] }],
|
||||
networks: [
|
||||
{ id: 'd', name: `${STACK}_default`, driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK },
|
||||
{ id: 'r', name: `${STACK}_rogue`, driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK },
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
isCleanOneShotCompletion,
|
||||
isNoRestartPolicy,
|
||||
normalizeComposeRestartIntent,
|
||||
} from '../utils/oneShotCompletion';
|
||||
|
||||
describe('isNoRestartPolicy', () => {
|
||||
it('accepts only explicit no', () => {
|
||||
expect(isNoRestartPolicy('no')).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects absent, empty, and restarting policies', () => {
|
||||
expect(isNoRestartPolicy(undefined)).toBe(false);
|
||||
expect(isNoRestartPolicy(null)).toBe(false);
|
||||
expect(isNoRestartPolicy('')).toBe(false);
|
||||
expect(isNoRestartPolicy('unless-stopped')).toBe(false);
|
||||
expect(isNoRestartPolicy('always')).toBe(false);
|
||||
expect(isNoRestartPolicy('on-failure')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isCleanOneShotCompletion', () => {
|
||||
const clean = {
|
||||
state: 'exited',
|
||||
exitCode: 0 as number | null,
|
||||
restartPolicy: 'no' as string | null | undefined,
|
||||
};
|
||||
|
||||
it('returns true only for exited + exit 0 + explicit restart no', () => {
|
||||
expect(isCleanOneShotCompletion(clean)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false for absent or empty declared restart', () => {
|
||||
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: undefined })).toBe(false);
|
||||
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: null })).toBe(false);
|
||||
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: '' })).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for non-exited states', () => {
|
||||
expect(isCleanOneShotCompletion({ ...clean, state: 'running' })).toBe(false);
|
||||
expect(isCleanOneShotCompletion({ ...clean, state: 'restarting' })).toBe(false);
|
||||
expect(isCleanOneShotCompletion({ ...clean, state: 'created' })).toBe(false);
|
||||
expect(isCleanOneShotCompletion({ ...clean, state: 'dead' })).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for non-zero and null exit codes (fail closed)', () => {
|
||||
expect(isCleanOneShotCompletion({ ...clean, exitCode: 1 })).toBe(false);
|
||||
expect(isCleanOneShotCompletion({ ...clean, exitCode: 137 })).toBe(false);
|
||||
expect(isCleanOneShotCompletion({ ...clean, exitCode: null })).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for restarting policies even with exit 0', () => {
|
||||
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'unless-stopped' })).toBe(false);
|
||||
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'always' })).toBe(false);
|
||||
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'on-failure' })).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeComposeRestartIntent', () => {
|
||||
it('falls back to service restart when deploy.restart_policy is unset', () => {
|
||||
expect(normalizeComposeRestartIntent('no')).toBe('no');
|
||||
expect(normalizeComposeRestartIntent('unless-stopped')).toBe('unless-stopped');
|
||||
expect(normalizeComposeRestartIntent(undefined)).toBeNull();
|
||||
expect(normalizeComposeRestartIntent(null, {})).toBeNull();
|
||||
expect(normalizeComposeRestartIntent('always', { replicas: 2 })).toBe('always');
|
||||
});
|
||||
|
||||
it('maps deploy.restart_policy.condition with Compose defaults and precedence', () => {
|
||||
expect(normalizeComposeRestartIntent('unless-stopped', {
|
||||
restart_policy: { condition: 'none' },
|
||||
})).toBe('no');
|
||||
expect(normalizeComposeRestartIntent(null, {
|
||||
restart_policy: { condition: 'any' },
|
||||
})).toBe('always');
|
||||
expect(normalizeComposeRestartIntent('no', {
|
||||
restart_policy: { condition: 'on-failure' },
|
||||
})).toBe('on-failure');
|
||||
expect(normalizeComposeRestartIntent('no', {
|
||||
restart_policy: {},
|
||||
})).toBe('always');
|
||||
});
|
||||
|
||||
it('fails closed on malformed restart_policy shapes', () => {
|
||||
expect(normalizeComposeRestartIntent('no', { restart_policy: null })).toBe('always');
|
||||
expect(normalizeComposeRestartIntent('no', { restart_policy: 'none' })).toBe('always');
|
||||
expect(normalizeComposeRestartIntent('no', { restart_policy: [] })).toBe('always');
|
||||
expect(normalizeComposeRestartIntent('no', {
|
||||
restart_policy: { condition: 'weird' },
|
||||
})).toBe('always');
|
||||
});
|
||||
});
|
||||
@@ -212,7 +212,11 @@ describe('hygiene rules', () => {
|
||||
});
|
||||
it('flags a missing restart policy and healthcheck', () => {
|
||||
const bare = model([svc({ restart: undefined, hasHealthcheck: false })]);
|
||||
expect(ids(runRules(ctx({ model: bare })), 'no-restart-policy')).toHaveLength(1);
|
||||
const restartFindings = ids(runRules(ctx({ model: bare })), 'no-restart-policy');
|
||||
expect(restartFindings).toHaveLength(1);
|
||||
expect(restartFindings[0].remediation).toMatch(/one-shot|init jobs/i);
|
||||
expect(restartFindings[0].remediation).toMatch(/restart: "no"/);
|
||||
expect(restartFindings[0].remediation).toMatch(/unless-stopped/);
|
||||
expect(ids(runRules(ctx({ model: bare })), 'no-healthcheck')).toHaveLength(1);
|
||||
const withDeployRestart = model([svc({ restart: undefined, deploy: { restart_policy: { condition: 'any' } }})]);
|
||||
expect(ids(runRules(ctx({ model: withDeployRestart })), 'no-restart-policy')).toHaveLength(0);
|
||||
|
||||
@@ -13,7 +13,7 @@ describe('sanitizeNetworkInspect connected containers', () => {
|
||||
networks: [{ id: 'net1', name: 'app_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: 'app', stack: 'app' }],
|
||||
volumes: [],
|
||||
containers: [{
|
||||
id: 'c1', name: 'app-web-1', service: 'web', composeProject: 'app', stack: 'app', state: 'running', image: 'nginx',
|
||||
id: 'c1', name: 'app-web-1', service: 'web', composeProject: 'app', stack: 'app', state: 'running', exitCode: null, image: 'nginx',
|
||||
networks: [{ name: 'app_net', id: 'net1', ip: '172.20.0.5/16' }], volumes: [], ports: [],
|
||||
}],
|
||||
};
|
||||
|
||||
@@ -26,6 +26,8 @@ export interface DeclaredService {
|
||||
image?: string;
|
||||
/** Service `network_mode:` as declared (e.g. `host`), or undefined. */
|
||||
networkMode?: string;
|
||||
/** Service `restart` from raw YAML parse. The effective-model path may set this via normalizeComposeRestartIntent (including deploy.restart_policy). Null when unset. */
|
||||
restart?: string | null;
|
||||
}
|
||||
|
||||
/** A top-level networks:/volumes: entry. */
|
||||
@@ -200,6 +202,7 @@ export function parseComposeDependencies(content: string): DeclaredCompose {
|
||||
ports,
|
||||
image: asString(svc.image),
|
||||
networkMode: asString(svc.network_mode),
|
||||
restart: asString(svc.restart) ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { EffectiveModel, EffResource } from '../services/preflight/effectiveModel';
|
||||
import type { DeclaredCompose, DeclaredPort, DeclaredResource, DeclaredService } from './composeDependencyParse';
|
||||
import { normalizeComposeRestartIntent } from '../utils/oneShotCompletion';
|
||||
|
||||
/** Map one rendered network/volume entry to the declared shape drift expects. */
|
||||
function toDeclaredResource(key: string, res: EffResource, projectName: string): DeclaredResource {
|
||||
@@ -41,6 +42,7 @@ export function declaredFromEffectiveModel(model: EffectiveModel): DeclaredCompo
|
||||
),
|
||||
image: s.image,
|
||||
networkMode: s.networkMode,
|
||||
restart: normalizeComposeRestartIntent(s.restart, s.deploy),
|
||||
}));
|
||||
|
||||
return {
|
||||
|
||||
@@ -210,6 +210,12 @@ export interface DependencyContainer {
|
||||
/** Resolved Sencho stack, or null when the container is not Sencho-managed. */
|
||||
stack: string | null;
|
||||
state: string;
|
||||
/**
|
||||
* Exit code from list Status when a parenthesized code is present
|
||||
* (e.g. "Exited (0) …", "Restarting (1) …"); null when none (e.g. "Up …", bare "Exited").
|
||||
* Required so Drift can fail closed on unknown codes.
|
||||
*/
|
||||
exitCode: number | null;
|
||||
image: string;
|
||||
networks: { name: string; id: string; ip: string }[];
|
||||
/** Named-volume sources mounted by the container (bind mounts excluded). */
|
||||
@@ -1833,6 +1839,7 @@ class DockerController {
|
||||
composeProject: c.Labels?.['com.docker.compose.project'] ?? null,
|
||||
stack: DockerController.resolveContainerStack(c.Labels, projectToStack, knownSet, absDirToStack, resolvedBase),
|
||||
state: c.State ?? 'unknown',
|
||||
exitCode: parseExitCode(typeof c.Status === 'string' ? c.Status : undefined),
|
||||
image: c.Image ?? '',
|
||||
networks,
|
||||
volumes,
|
||||
|
||||
@@ -8,6 +8,7 @@ import { parseEffectiveModel } from './preflight/effectiveModel';
|
||||
import { compareStackNetworks, fromDeclaredCompose } from './network/normalize';
|
||||
import { sanitizeForLog, redactSensitiveText } from '../utils/safeLog';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { isCleanOneShotCompletion } from '../utils/oneShotCompletion';
|
||||
|
||||
const MAX_RENDER_ERROR = 600;
|
||||
|
||||
@@ -178,16 +179,17 @@ function networkDriftFindings(
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure diff step (no Docker / FS access) so it is directly unit-testable. Only
|
||||
* running containers are compared, since a stopped container publishes no ports
|
||||
* and is not "deployed": a declared service with no running container is
|
||||
* service-missing, a running container with no matching service is
|
||||
* service-undeclared, and image / port differences are checked only for services
|
||||
* present on both sides so a missing/undeclared service is not double-reported.
|
||||
* Pure diff step (no Docker / FS access) so it is directly unit-testable.
|
||||
* Running/restarting containers drive image/port comparison. Clean one-shot
|
||||
* completions (exit 0 + explicit declared restart "no", including normalized
|
||||
* `deploy.restart_policy.condition: none`) satisfy service presence without
|
||||
* counting as hasContainers. Omitting restart does not qualify. Network
|
||||
* comparison still uses only running/restarting attachments.
|
||||
*/
|
||||
export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftReport {
|
||||
const { stack, declared, containers, parseError } = input;
|
||||
const networks = input.networks ?? [];
|
||||
// Public contract: at least one running/restarting container (not "satisfied").
|
||||
const hasContainers = containers.some((c) => RUNNING_STATES.has(c.state));
|
||||
|
||||
// A parse failure means the declared model is untrustworthy: report drift
|
||||
@@ -196,30 +198,44 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe
|
||||
return { stack, status: 'drifted', hasComposeFile: false, hasContainers, findings: [], parseError };
|
||||
}
|
||||
|
||||
const runtimeByService = new Map<string, RuntimeService>();
|
||||
for (const c of containers) {
|
||||
if (!RUNNING_STATES.has(c.state)) continue;
|
||||
const name = c.service ?? c.name;
|
||||
const agg = runtimeByService.get(name) ?? { images: new Set<string>(), ports: new Set<string>() };
|
||||
if (c.image) agg.images.add(normalizeImageRef(c.image));
|
||||
for (const p of c.ports) agg.ports.add(portKey(p.publishedPort, p.protocol));
|
||||
runtimeByService.set(name, agg);
|
||||
}
|
||||
|
||||
// Nothing running: the stack is defined on disk but not deployed. One status
|
||||
// conveys this; per-service findings would just be noise.
|
||||
if (!hasContainers) {
|
||||
return { stack, status: 'missing-runtime', hasComposeFile: true, hasContainers: false, findings: [] };
|
||||
}
|
||||
|
||||
const declaredByName = new Map<string, DeclaredService>();
|
||||
for (const svc of declared.services) declaredByName.set(svc.name, svc);
|
||||
|
||||
const runtimeByService = new Map<string, RuntimeService>();
|
||||
const oneShotSatisfied = new Set<string>();
|
||||
for (const c of containers) {
|
||||
const name = c.service ?? c.name;
|
||||
if (RUNNING_STATES.has(c.state)) {
|
||||
const agg = runtimeByService.get(name) ?? { images: new Set<string>(), ports: new Set<string>() };
|
||||
if (c.image) agg.images.add(normalizeImageRef(c.image));
|
||||
for (const p of c.ports) agg.ports.add(portKey(p.publishedPort, p.protocol));
|
||||
runtimeByService.set(name, agg);
|
||||
continue;
|
||||
}
|
||||
const declaredRestart = declaredByName.get(name)?.restart;
|
||||
if (isCleanOneShotCompletion({
|
||||
state: c.state,
|
||||
exitCode: c.exitCode,
|
||||
restartPolicy: declaredRestart,
|
||||
})) {
|
||||
oneShotSatisfied.add(name);
|
||||
}
|
||||
}
|
||||
|
||||
const servicePresent = (serviceName: string): boolean =>
|
||||
runtimeByService.has(serviceName) || oneShotSatisfied.has(serviceName);
|
||||
|
||||
// Nothing running and no declared service satisfied by a clean one-shot: the
|
||||
// stack is defined on disk but not deployed. One status conveys this.
|
||||
if (!hasContainers && !declared.services.some((svc) => servicePresent(svc.name))) {
|
||||
return { stack, status: 'missing-runtime', hasComposeFile: true, hasContainers: false, findings: [] };
|
||||
}
|
||||
|
||||
const findings: StackDriftFinding[] = [];
|
||||
|
||||
// Declared service with no running container.
|
||||
// Declared service with no running container and no clean one-shot completion.
|
||||
for (const svc of declared.services) {
|
||||
if (!runtimeByService.has(svc.name)) {
|
||||
if (!servicePresent(svc.name)) {
|
||||
findings.push({
|
||||
kind: 'service-missing',
|
||||
service: svc.name,
|
||||
@@ -239,7 +255,7 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe
|
||||
}
|
||||
}
|
||||
|
||||
// Image / port divergence for services present on both sides.
|
||||
// Image / port divergence for services present on both sides (running only).
|
||||
for (const [name, svc] of declaredByName) {
|
||||
const runtime = runtimeByService.get(name);
|
||||
if (!runtime) continue;
|
||||
|
||||
@@ -5,8 +5,11 @@ import { AutoHealService } from './AutoHealService';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { withTimeout } from '../utils/withTimeout';
|
||||
import { isCleanOneShotCompletion } from '../utils/oneShotCompletion';
|
||||
import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompose';
|
||||
import { parseEffectiveModel } from './preflight/effectiveModel';
|
||||
import type { HealthGateContainer, HealthGateReport } from './updateGuard/types';
|
||||
import { getComposeCommandTimeoutMs } from './ComposeService';
|
||||
import { ComposeService, getComposeCommandTimeoutMs } from './ComposeService';
|
||||
|
||||
const POLL_INTERVAL_MS = 5_000;
|
||||
// A prepared-but-never-begun token (prepare called, mutation then failed before
|
||||
@@ -47,6 +50,10 @@ interface ObservedContainer {
|
||||
service: string | null;
|
||||
/** Image id the container is running (service gates check convergence on it). */
|
||||
imageId: string;
|
||||
/** Inspect State.ExitCode; null when unavailable. */
|
||||
exitCode: number | null;
|
||||
/** HostConfig.RestartPolicy.Name (report/debug only; not used for one-shot intent). */
|
||||
restartPolicy: string | null;
|
||||
}
|
||||
|
||||
interface ActiveGate {
|
||||
@@ -89,6 +96,13 @@ interface ActiveGate {
|
||||
collateralBaselineByName: Map<string, ObservedContainer>;
|
||||
/** Role of each expected container (service gates), for failure attribution. */
|
||||
roleByName: Map<string, GateRole>;
|
||||
/**
|
||||
* Declared Compose restart intent by service name (normalized). Loaded once
|
||||
* per gate; null until the first load attempt completes. Used for one-shot
|
||||
* recognition instead of Docker inspect (which cannot distinguish omit vs
|
||||
* explicit `restart: "no"`).
|
||||
*/
|
||||
declaredRestartByService: Map<string, string | null> | null;
|
||||
}
|
||||
|
||||
/** A pre-mutation baseline container captured at prepare time. */
|
||||
@@ -101,6 +115,8 @@ interface PreparedBaseline {
|
||||
restartCount: number;
|
||||
startedAt: string | null;
|
||||
imageId: string;
|
||||
exitCode: number | null;
|
||||
restartPolicy: string | null;
|
||||
}
|
||||
|
||||
function isRegressionEligibleSibling(baseline: PreparedBaseline): boolean {
|
||||
@@ -118,9 +134,35 @@ function observedFromPreparedBaseline(baseline: PreparedBaseline): ObservedConta
|
||||
health: baseline.health,
|
||||
service: baseline.service,
|
||||
imageId: baseline.imageId,
|
||||
exitCode: baseline.exitCode,
|
||||
restartPolicy: baseline.restartPolicy,
|
||||
};
|
||||
}
|
||||
|
||||
/** Running, or a clean one-shot exit (exit 0 + explicit declared restart "no"). */
|
||||
function isObservedContainerSatisfied(
|
||||
gate: ActiveGate,
|
||||
current: ObservedContainer | undefined,
|
||||
): boolean {
|
||||
if (!current) return false;
|
||||
if (current.state === 'running') return true;
|
||||
return isDeclaredCleanOneShot(gate, current);
|
||||
}
|
||||
|
||||
/**
|
||||
* One-shot recognition from declared Compose intent only. Unlabeled containers
|
||||
* and services missing from the effective model fail closed.
|
||||
*/
|
||||
function isDeclaredCleanOneShot(gate: ActiveGate, current: ObservedContainer): boolean {
|
||||
const map = gate.declaredRestartByService;
|
||||
if (!map || !current.service || !map.has(current.service)) return false;
|
||||
return isCleanOneShotCompletion({
|
||||
state: current.state,
|
||||
exitCode: current.exitCode,
|
||||
restartPolicy: map.get(current.service),
|
||||
});
|
||||
}
|
||||
|
||||
/** An in-memory prepare snapshot awaiting attachExpectedImage + beginPrepared. */
|
||||
interface PreparedGate {
|
||||
token: string;
|
||||
@@ -329,6 +371,7 @@ export class HealthGateService {
|
||||
collateralEligibleNames: new Set(),
|
||||
collateralBaselineByName: new Map(),
|
||||
roleByName: new Map(),
|
||||
declaredRestartByService: null,
|
||||
};
|
||||
this.active.set(key, gate);
|
||||
this.scheduleNextPoll(gate);
|
||||
@@ -488,6 +531,7 @@ export class HealthGateService {
|
||||
prep.collateralBaseline.filter(b => prep.collateralEligibleNames.has(b.name)),
|
||||
),
|
||||
roleByName: new Map(),
|
||||
declaredRestartByService: null,
|
||||
};
|
||||
this.active.set(key, gate);
|
||||
this.scheduleNextPoll(gate);
|
||||
@@ -587,6 +631,9 @@ export class HealthGateService {
|
||||
if (gate.finalized || this.active.get(key) !== gate) return;
|
||||
gate.consecutivePollErrors = 0;
|
||||
|
||||
await this.ensureDeclaredRestartMap(gate);
|
||||
if (gate.finalized || this.active.get(key) !== gate) return;
|
||||
|
||||
const elapsedMs = Date.now() - gate.startedAt;
|
||||
|
||||
if (gate.expected === null) {
|
||||
@@ -629,12 +676,16 @@ export class HealthGateService {
|
||||
}
|
||||
gate.missingLastPoll.delete(name);
|
||||
|
||||
if (current.state === 'exited' && baseline.restarts === current.restarts) {
|
||||
// An exit with no restart attempt is terminal for the window.
|
||||
const cleanOneShot = isDeclaredCleanOneShot(gate, current);
|
||||
// An exit with no restart attempt is terminal for the window, unless
|
||||
// this is an expected one-shot (exit 0 + explicit declared restart "no").
|
||||
if (current.state === 'exited' && baseline.restarts === current.restarts && !cleanOneShot) {
|
||||
this.finalize(gate, 'failed', `container ${name} exited during observation`, summary);
|
||||
return;
|
||||
}
|
||||
if (current.health === 'unhealthy') {
|
||||
// Residual Docker health on a completed one-shot is not a gate failure;
|
||||
// long-running containers still fail on unhealthy.
|
||||
if (!cleanOneShot && current.health === 'unhealthy') {
|
||||
this.finalize(gate, 'failed', `container ${name} reported unhealthy`, summary);
|
||||
return;
|
||||
}
|
||||
@@ -657,14 +708,19 @@ export class HealthGateService {
|
||||
|
||||
if (elapsedMs < gate.windowSeconds * 1000) return;
|
||||
|
||||
// Window complete: pass requires everything running and healthy wherever a
|
||||
// healthcheck exists. A health state still 'starting' is not a pass.
|
||||
const stillStarting = observed.filter(c => c.health === 'starting');
|
||||
// Window complete: pass requires everything running (or a clean one-shot
|
||||
// completion) and healthy wherever a healthcheck exists. A health state
|
||||
// still 'starting' is not a pass (except residual health on a clean one-shot).
|
||||
const stillStarting = observed.filter(
|
||||
c => c.health === 'starting' && !isDeclaredCleanOneShot(gate, c),
|
||||
);
|
||||
if (stillStarting.length > 0) {
|
||||
this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary);
|
||||
return;
|
||||
}
|
||||
const notRunning = [...gate.expected.keys()].filter(name => byName.get(name)?.state !== 'running');
|
||||
const notRunning = [...gate.expected.keys()].filter(
|
||||
name => !isObservedContainerSatisfied(gate, byName.get(name)),
|
||||
);
|
||||
if (notRunning.length > 0) {
|
||||
this.finalize(gate, 'failed', `not running at the end of the window: ${notRunning.join(', ')}`, summary);
|
||||
return;
|
||||
@@ -700,6 +756,9 @@ export class HealthGateService {
|
||||
if (gate.finalized || this.active.get(key) !== gate) return;
|
||||
gate.consecutivePollErrors = 0;
|
||||
|
||||
await this.ensureDeclaredRestartMap(gate);
|
||||
if (gate.finalized || this.active.get(key) !== gate) return;
|
||||
|
||||
const elapsedMs = Date.now() - gate.startedAt;
|
||||
const serviceName = gate.serviceName ?? '';
|
||||
|
||||
@@ -781,11 +840,12 @@ export class HealthGateService {
|
||||
}
|
||||
gate.missingLastPoll.delete(name);
|
||||
|
||||
if (current.state === 'exited' && baseline.restarts === current.restarts) {
|
||||
const cleanOneShot = isDeclaredCleanOneShot(gate, current);
|
||||
if (current.state === 'exited' && baseline.restarts === current.restarts && !cleanOneShot) {
|
||||
this.finalize(gate, 'failed', `${noun} ${name} exited during observation`, summary, role);
|
||||
return;
|
||||
}
|
||||
if (current.health === 'unhealthy') {
|
||||
if (!cleanOneShot && current.health === 'unhealthy') {
|
||||
this.finalize(gate, 'failed', `${noun} ${name} reported unhealthy`, summary, role);
|
||||
return;
|
||||
}
|
||||
@@ -808,24 +868,28 @@ export class HealthGateService {
|
||||
if (elapsedMs < gate.windowSeconds * 1000) return;
|
||||
|
||||
const stillStarting = observed.filter(
|
||||
c => c.health === 'starting' && (c.service === serviceName || gate.collateralEligibleNames.has(c.name)),
|
||||
c => c.health === 'starting'
|
||||
&& !isDeclaredCleanOneShot(gate, c)
|
||||
&& (c.service === serviceName || gate.collateralEligibleNames.has(c.name)),
|
||||
);
|
||||
if (stillStarting.length > 0) {
|
||||
this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary);
|
||||
return;
|
||||
}
|
||||
const runningPrimary = observed.filter(c => c.service === serviceName && c.state === 'running');
|
||||
if (runningPrimary.length !== gate.expectedReplicas) {
|
||||
const satisfiedPrimary = observed.filter(
|
||||
c => c.service === serviceName && isObservedContainerSatisfied(gate, c),
|
||||
);
|
||||
if (satisfiedPrimary.length !== gate.expectedReplicas) {
|
||||
this.finalize(
|
||||
gate, 'failed',
|
||||
`service ${serviceName} has ${runningPrimary.length} running replica(s), expected ${gate.expectedReplicas}`,
|
||||
`service ${serviceName} has ${satisfiedPrimary.length} satisfied replica(s), expected ${gate.expectedReplicas}`,
|
||||
summary, 'primary',
|
||||
);
|
||||
return;
|
||||
}
|
||||
const collateralNotRunning = [...gate.expected.keys()]
|
||||
.filter(name => gate.roleByName.get(name) === 'collateral')
|
||||
.filter(name => byName.get(name)?.state !== 'running');
|
||||
.filter(name => !isObservedContainerSatisfied(gate, byName.get(name)));
|
||||
if (collateralNotRunning.length > 0) {
|
||||
this.finalize(gate, 'failed', `sibling(s) not running at the end of the window: ${collateralNotRunning.join(', ')}`, summary, 'collateral');
|
||||
return;
|
||||
@@ -855,6 +919,35 @@ export class HealthGateService {
|
||||
return this.listStackContainers(gate.nodeId, gate.stackName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load declared Compose restart intent once per gate. Fail closed to an empty
|
||||
* map on render/parse errors so inspect "no" cannot false-qualify one-shots.
|
||||
*/
|
||||
private async ensureDeclaredRestartMap(gate: ActiveGate): Promise<void> {
|
||||
if (gate.declaredRestartByService !== null) return;
|
||||
gate.declaredRestartByService = new Map();
|
||||
try {
|
||||
const result = await ComposeService.getInstance(gate.nodeId).renderConfig(gate.stackName);
|
||||
if (result.rendered === null) {
|
||||
console.warn(
|
||||
'[HealthGate] declared restart map unavailable for %s (compose render failed)',
|
||||
sanitizeForLog(gate.stackName),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const model = parseEffectiveModel(JSON.parse(result.rendered), gate.stackName);
|
||||
const declared = declaredFromEffectiveModel(model);
|
||||
gate.declaredRestartByService = new Map(
|
||||
declared.services.map((s) => [s.name, s.restart ?? null]),
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[HealthGate] declared restart map load failed for %s:',
|
||||
sanitizeForLog(gate.stackName), getErrorMessage(error, 'unknown'),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** List and inspect a stack's containers into the gate's observation shape. */
|
||||
private async listStackContainers(nodeId: number, stackName: string): Promise<ObservedContainer[]> {
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
@@ -877,6 +970,8 @@ export class HealthGateService {
|
||||
health: inspect.State?.Health?.Status ?? null,
|
||||
service: labels['com.docker.compose.service'] ?? null,
|
||||
imageId: inspect.Image ?? '',
|
||||
exitCode: typeof inspect.State?.ExitCode === 'number' ? inspect.State.ExitCode : null,
|
||||
restartPolicy: inspect.HostConfig?.RestartPolicy?.Name || null,
|
||||
};
|
||||
} catch (e: unknown) {
|
||||
// Removed between list and inspect; the missing-container logic will
|
||||
@@ -903,6 +998,8 @@ export class HealthGateService {
|
||||
restartCount: c.restartCount,
|
||||
startedAt: c.startedAt,
|
||||
imageId: c.imageId,
|
||||
exitCode: c.exitCode,
|
||||
restartPolicy: c.restartPolicy,
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ export interface EffService {
|
||||
networkMode?: string;
|
||||
restart?: string;
|
||||
hasHealthcheck: boolean;
|
||||
/** Raw deploy block (read for key presence only, never values; undefined = none). */
|
||||
/** Raw deploy block (preflight uses key presence; Drift also reads restart_policy.condition). Undefined = none. */
|
||||
deploy?: Record<string, unknown>;
|
||||
containerName?: string;
|
||||
user?: string;
|
||||
|
||||
@@ -363,7 +363,7 @@ const noRestartPolicy: PreflightRule = {
|
||||
message: `Service "${s.name}" has no restart policy, so it will not come back after a crash or host reboot.`,
|
||||
sourcePath: s.name,
|
||||
service: s.name,
|
||||
remediation: 'Add restart: unless-stopped.',
|
||||
remediation: 'For long-running services, add restart: unless-stopped. For one-shot or init jobs that should finish and stay stopped, restart: "no" is appropriate.',
|
||||
}));
|
||||
},
|
||||
};
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
/**
|
||||
* Helpers for clean one-shot completion and Compose restart-intent normalization.
|
||||
* Used by Health Gate and Drift service-presence only; does not change bulk
|
||||
* status, Auto-Heal, or atomic-deploy helpers.
|
||||
*/
|
||||
|
||||
/**
|
||||
* True only for an explicit Compose `restart: "no"` (after normalization).
|
||||
* Absent / null / empty do not qualify: Docker reports HostConfig restart
|
||||
* "no" for both intentional one-shots and bare long-running services that
|
||||
* omit `restart:`, so consumers must pass declared Compose intent, not inspect.
|
||||
*/
|
||||
export function isNoRestartPolicy(policy: string | null | undefined): boolean {
|
||||
return policy === 'no';
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize Compose restart intent for Drift / declared-model consumers.
|
||||
*
|
||||
* Compose precedence: when `deploy.restart_policy` is set, its `condition`
|
||||
* wins; otherwise the service-level `restart` field is used. Conditions map to
|
||||
* Docker-like policy names so {@link isNoRestartPolicy} stays the single gate:
|
||||
* `none` → `no`, `any` → `always`, `on-failure` → `on-failure`. Missing
|
||||
* condition defaults to `any` (Compose default). Unknown shapes fail closed.
|
||||
* Absent service restart (no deploy policy) stays null and is not one-shot eligible.
|
||||
*/
|
||||
export function normalizeComposeRestartIntent(
|
||||
serviceRestart: string | null | undefined,
|
||||
deploy?: Record<string, unknown> | null,
|
||||
): string | null {
|
||||
if (deploy && Object.prototype.hasOwnProperty.call(deploy, 'restart_policy')) {
|
||||
const policy = deploy['restart_policy'];
|
||||
if (policy === null || typeof policy !== 'object' || Array.isArray(policy)) {
|
||||
return 'always';
|
||||
}
|
||||
const condition = (policy as Record<string, unknown>).condition;
|
||||
// Missing, empty, or non-string → Compose default `any` → always.
|
||||
if (typeof condition !== 'string' || condition === '') {
|
||||
return 'always';
|
||||
}
|
||||
switch (condition.toLowerCase()) {
|
||||
case 'none':
|
||||
return 'no';
|
||||
case 'on-failure':
|
||||
return 'on-failure';
|
||||
case 'any':
|
||||
return 'always';
|
||||
default:
|
||||
return 'always';
|
||||
}
|
||||
}
|
||||
if (serviceRestart == null || serviceRestart === '') return null;
|
||||
return serviceRestart;
|
||||
}
|
||||
|
||||
export interface OneShotCompletionInput {
|
||||
state: string;
|
||||
/** Exact Docker exit code; null means unknown (fail closed). */
|
||||
exitCode: number | null;
|
||||
/**
|
||||
* Declared Compose restart intent after normalization (`"no"` for explicit
|
||||
* one-shots / `deploy.restart_policy.condition: none`). Do not pass Docker
|
||||
* inspect HostConfig values here.
|
||||
*/
|
||||
restartPolicy: string | null | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* True only for an exited container with exit code exactly 0 and explicit
|
||||
* declared restart `"no"`. Null/absent restart, null exit codes, and restarting
|
||||
* policies never qualify.
|
||||
*/
|
||||
export function isCleanOneShotCompletion(input: OneShotCompletionInput): boolean {
|
||||
return input.state === 'exited'
|
||||
&& input.exitCode === 0
|
||||
&& isNoRestartPolicy(input.restartPolicy);
|
||||
}
|
||||
Reference in New Issue
Block a user