From 79914fe7503923edebe64e9f006b4fbe5b7d189f Mon Sep 17 00:00:00 2001 From: Anso Date: Fri, 24 Jul 2026 09:41:21 -0400 Subject: [PATCH] fix: recognize clean one-shot completions in health gate and drift (#1691) * fix: recognize clean one-shot completions in health gate and drift Treat exit 0 with restart policy no/absent as successful completion so init and migration jobs no longer fail post-update observation or show as service-missing, while long-running restart policies still fail closed. * fix: ignore residual health on clean one-shots and honor deploy.restart_policy Completed exit-0 jobs with no-restart intent no longer fail the health gate on leftover starting/unhealthy state, and Drift treats deploy.restart_policy with Compose precedence so any/on-failure services are not mistaken for one-shots. * fix: require explicit Compose restart no for one-shot recognition Docker inspect reports restart no for both intentional jobs and bare services that omit restart, so Health Gate and Drift now require declared restart:""no"" (or deploy.restart_policy condition none) and load Compose intent once per gate. --- .../compose-network-inspector.test.ts | 2 +- .../dependency-graph-builder.test.ts | 2 +- .../src/__tests__/docker-controller.test.ts | 18 ++ backend/src/__tests__/drift-detection.test.ts | 219 +++++++++++++++++- backend/src/__tests__/drift-ledger.test.ts | 4 +- .../src/__tests__/health-gate-prepare.test.ts | 208 ++++++++++++++++- .../src/__tests__/health-gate-service.test.ts | 211 ++++++++++++++++- .../src/__tests__/networking-operator.test.ts | 4 +- .../src/__tests__/networking-summary.test.ts | 2 +- .../src/__tests__/one-shot-completion.test.ts | 92 ++++++++ backend/src/__tests__/preflight-rules.test.ts | 6 +- .../sanitize-network-inspect.test.ts | 2 +- backend/src/helpers/composeDependencyParse.ts | 3 + .../src/helpers/effectiveToDeclaredCompose.ts | 2 + backend/src/services/DockerController.ts | 7 + backend/src/services/DriftDetectionService.ts | 66 ++++-- backend/src/services/HealthGateService.ts | 127 ++++++++-- .../src/services/preflight/effectiveModel.ts | 2 +- backend/src/services/preflight/rules.ts | 2 +- backend/src/utils/oneShotCompletion.ts | 77 ++++++ docs/features/health-gated-updates.mdx | 16 +- docs/features/stack-drift.mdx | 12 +- 22 files changed, 1015 insertions(+), 69 deletions(-) create mode 100644 backend/src/__tests__/one-shot-completion.test.ts create mode 100644 backend/src/utils/oneShotCompletion.ts diff --git a/backend/src/__tests__/compose-network-inspector.test.ts b/backend/src/__tests__/compose-network-inspector.test.ts index d8d0d062..035d1599 100644 --- a/backend/src/__tests__/compose-network-inspector.test.ts +++ b/backend/src/__tests__/compose-network-inspector.test.ts @@ -24,7 +24,7 @@ function effSvc(over: Partial = {}): EffService { function container(over: Partial = {}): DependencyContainer { return { id: 'c1', name: 'web1', service: 'web', composeProject: 'myapp', stack: 'myapp', - state: 'running', image: 'nginx:1.27', networks: [], volumes: [], ports: [], ...over, + state: 'running', exitCode: null, image: 'nginx:1.27', networks: [], volumes: [], ports: [], ...over, }; } diff --git a/backend/src/__tests__/dependency-graph-builder.test.ts b/backend/src/__tests__/dependency-graph-builder.test.ts index 7ec5982e..3bc0f3a7 100644 --- a/backend/src/__tests__/dependency-graph-builder.test.ts +++ b/backend/src/__tests__/dependency-graph-builder.test.ts @@ -32,7 +32,7 @@ function snap(partial: Partial): DependencySnapshot { function container(p: Partial & { id: string }): DependencyContainer { return { name: p.id, service: null, composeProject: null, stack: null, - state: 'running', image: 'img:latest', networks: [], volumes: [], ports: [], ...p, + state: 'running', exitCode: null, image: 'img:latest', networks: [], volumes: [], ports: [], ...p, }; } diff --git a/backend/src/__tests__/docker-controller.test.ts b/backend/src/__tests__/docker-controller.test.ts index 30cc4e99..e55e7082 100644 --- a/backend/src/__tests__/docker-controller.test.ts +++ b/backend/src/__tests__/docker-controller.test.ts @@ -1316,6 +1316,7 @@ describe('DockerController - getDependencySnapshot', () => { expect(c.networks).toEqual([{ name: 'web_frontend', id: 'net1', ip: '172.18.0.2' }]); expect(c.volumes).toEqual(['web_data']); // bind mount dropped expect(c.ports).toEqual([{ ip: '0.0.0.0', publishedPort: 8080, privatePort: 80, protocol: 'tcp' }]); // unpublished 9090 dropped + expect(c.exitCode).toBeNull(); expect(snap.networks.find((n) => n.name === 'bridge')?.isSystem).toBe(true); const frontend = snap.networks.find((n) => n.name === 'web_frontend'); @@ -1338,6 +1339,23 @@ describe('DockerController - getDependencySnapshot', () => { expect(snap.containers[0].stack).toBeNull(); expect(snap.containers[0].composeProject).toBeNull(); }); + + it('parses exitCode from list Status for exited containers', async () => { + mockDocker.listContainers.mockResolvedValue([ + { Id: 'a', Names: ['/job-0'], Image: 'busybox', State: 'exited', Status: 'Exited (0) 5 minutes ago', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] }, + { Id: 'b', Names: ['/crash-0'], Image: 'busybox', State: 'exited', Status: 'Exited (137) 1 minute ago', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] }, + { Id: 'c', Names: ['/up-0'], Image: 'busybox', State: 'running', Status: 'Up 3 hours', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] }, + { Id: 'd', Names: ['/odd-0'], Image: 'busybox', State: 'exited', Status: 'Exited', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] }, + ]); + mockDocker.listNetworks.mockResolvedValue([]); + mockDocker.listVolumes.mockResolvedValue({ Volumes: [] }); + + const snap = await DockerController.getInstance(1).getDependencySnapshot([]); + expect(snap.containers.find(c => c.id === 'a')?.exitCode).toBe(0); + expect(snap.containers.find(c => c.id === 'b')?.exitCode).toBe(137); + expect(snap.containers.find(c => c.id === 'c')?.exitCode).toBeNull(); + expect(snap.containers.find(c => c.id === 'd')?.exitCode).toBeNull(); + }); }); // --- getBulkStackStatuses uptime (runningSince from StartedAt) ----------------- diff --git a/backend/src/__tests__/drift-detection.test.ts b/backend/src/__tests__/drift-detection.test.ts index e927c303..efb4694a 100644 --- a/backend/src/__tests__/drift-detection.test.ts +++ b/backend/src/__tests__/drift-detection.test.ts @@ -36,7 +36,7 @@ function declared(services: DeclaredService[], parseError?: string): DeclaredCom function container(p: Partial & { id: string }): DependencyContainer { return { name: p.id, service: null, composeProject: null, stack: 'app', - state: 'running', image: 'img:latest', networks: [], volumes: [], ports: [], ...p, + state: 'running', exitCode: null, image: 'img:latest', networks: [], volumes: [], ports: [], ...p, }; } @@ -108,6 +108,147 @@ describe('assembleStackDrift - status', () => { expect(report.findings).toEqual([]); }); + it('does not emit service-missing for a clean one-shot beside a running service', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'migrate', image: 'migrate:1', restart: 'no' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }), + container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: 0 }), + ], + }); + expect(findingKinds(report)).not.toContain('service-missing'); + expect(report.status).toBe('in-sync'); + expect(report.hasContainers).toBe(true); + }); + + it('still emits service-missing when a one-shot exits non-zero', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'migrate', image: 'migrate:1', restart: 'no' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }), + container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: 1 }), + ], + }); + expect(findingKinds(report)).toContain('service-missing'); + expect(report.findings.find(f => f.kind === 'service-missing')?.service).toBe('migrate'); + }); + + it('still treats exited unless-stopped as missing even with exit 0', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([service({ name: 'web', restart: 'unless-stopped' })]), + containers: [container({ id: 'c1', service: 'web', state: 'exited', exitCode: 0 })], + }); + expect(report.status).toBe('missing-runtime'); + expect(report.hasContainers).toBe(false); + }); + + it('fails closed when exitCode is null even with restart no', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'migrate', image: 'migrate:1', restart: 'no' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }), + container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: null }), + ], + }); + expect(findingKinds(report)).toContain('service-missing'); + expect(report.findings.find(f => f.kind === 'service-missing')?.service).toBe('migrate'); + }); + + it('does not treat absent declared restart as a clean one-shot', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'daemon-default', image: 'daemon:1' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1' }), + container({ id: 'c2', service: 'daemon-default', image: 'daemon:1', state: 'exited', exitCode: 0 }), + ], + }); + expect(findingKinds(report)).toContain('service-missing'); + expect(report.findings.find((f) => f.kind === 'service-missing')?.service).toBe('daemon-default'); + }); + + it('all-one-shot stack with dedicated network is not missing-runtime but keeps network-missing and hasContainers false', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: { + services: [service({ name: 'migrate', restart: 'no', networks: ['jobs'] })], + networks: { jobs: { external: false } }, + volumes: {}, + projectName: 'app', + }, + containers: [ + container({ + id: 'c1', service: 'migrate', state: 'exited', exitCode: 0, + networks: [{ name: 'app_jobs', id: 'j', ip: '' }], + }), + ], + networks: [depNet('app_jobs')], + }); + expect(report.status).not.toBe('missing-runtime'); + expect(findingKinds(report)).not.toContain('service-missing'); + expect(report.hasContainers).toBe(false); + expect(findingKinds(report)).toContain('network-missing'); + expect(report.status).toBe('drifted'); + }); + + it('all-one-shot stack without network findings is in-sync with hasContainers false', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([service({ name: 'migrate', restart: 'no' })]), + containers: [container({ id: 'c1', service: 'migrate', state: 'exited', exitCode: 0 })], + }); + expect(report.status).toBe('in-sync'); + expect(report.hasContainers).toBe(false); + expect(report.findings).toEqual([]); + }); + + it('emits service-missing when normalized restart is always (deploy any)', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'worker', image: 'worker:1', restart: 'always' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1' }), + container({ id: 'c2', service: 'worker', image: 'worker:1', state: 'exited', exitCode: 0 }), + ], + }); + expect(findingKinds(report)).toContain('service-missing'); + expect(report.findings.find((f) => f.kind === 'service-missing')?.service).toBe('worker'); + }); + + it('emits service-missing when normalized restart is on-failure', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'worker', image: 'worker:1', restart: 'on-failure' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1' }), + container({ id: 'c2', service: 'worker', image: 'worker:1', state: 'exited', exitCode: 0 }), + ], + }); + expect(findingKinds(report)).toContain('service-missing'); + }); + it('counts a restarting container as deployed', () => { const report = assembleStackDrift({ stack: 'app', @@ -456,6 +597,82 @@ describe('declaredFromEffectiveModel', () => { expect(converted.services[0].ports).toEqual([{ hostIp: '127.0.0.1', publishedPort: 8080, protocol: 'tcp' }]); }); + it('preserves restart policy including no, unless-stopped, and absent', () => { + const withNo = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ name: 'migrate', restart: 'no' })], + networks: {}, + volumes: {}, + }); + expect(withNo.services[0].restart).toBe('no'); + + const withUnless = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ name: 'web', restart: 'unless-stopped' })], + networks: {}, + volumes: {}, + }); + expect(withUnless.services[0].restart).toBe('unless-stopped'); + + const absent = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ name: 'job', restart: undefined })], + networks: {}, + volumes: {}, + }); + expect(absent.services[0].restart).toBeNull(); + }); + + it('normalizes deploy.restart_policy conditions with Compose precedence', () => { + const none = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ + name: 'migrate', + restart: 'unless-stopped', + deploy: { restart_policy: { condition: 'none' } }, + })], + networks: {}, + volumes: {}, + }); + expect(none.services[0].restart).toBe('no'); + + const any = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ + name: 'worker', + restart: 'no', + deploy: { restart_policy: { condition: 'any' } }, + })], + networks: {}, + volumes: {}, + }); + expect(any.services[0].restart).toBe('always'); + + const onFailure = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ + name: 'worker', + restart: undefined, + deploy: { restart_policy: { condition: 'on-failure' } }, + })], + networks: {}, + volumes: {}, + }); + expect(onFailure.services[0].restart).toBe('on-failure'); + + const defaultAny = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ + name: 'worker', + restart: 'no', + deploy: { restart_policy: {} }, + })], + networks: {}, + volumes: {}, + }); + expect(defaultAny.services[0].restart).toBe('always'); + }); + it('normalizes networks so drift matches the rendered model', () => { const model: EffectiveModel = { projectName: 'myapp', diff --git a/backend/src/__tests__/drift-ledger.test.ts b/backend/src/__tests__/drift-ledger.test.ts index f81fbcd2..1bb4b7f6 100644 --- a/backend/src/__tests__/drift-ledger.test.ts +++ b/backend/src/__tests__/drift-ledger.test.ts @@ -323,7 +323,7 @@ describe('DriftLedgerService.reconcileStack', () => { // A running container on a different image than compose declares => image-mismatch. const driftedContainer = (stack: string) => ({ id: `${stack}-c1`, name: `${stack}-web-1`, service: 'web', composeProject: stack, stack, - state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [], + state: 'running', exitCode: null, image: 'nginx:1.26', networks: [], volumes: [], ports: [], }); beforeEach(() => { @@ -377,7 +377,7 @@ describe('drift route (GET read-only, POST recheck persists)', () => { getDependencySnapshot: vi.fn().mockResolvedValue({ containers: [{ id: 'c1', name: `${STACK}-web-1`, service: 'web', composeProject: STACK, stack: STACK, - state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [], + state: 'running', exitCode: null, image: 'nginx:1.26', networks: [], volumes: [], ports: [], }], networks: [], volumes: [], }), diff --git a/backend/src/__tests__/health-gate-prepare.test.ts b/backend/src/__tests__/health-gate-prepare.test.ts index 258be3e2..3a4a3cd4 100644 --- a/backend/src/__tests__/health-gate-prepare.test.ts +++ b/backend/src/__tests__/health-gate-prepare.test.ts @@ -29,6 +29,7 @@ const { state } = vi.hoisted(() => ({ settings: {} as Record, listContainers: vi.fn(), inspect: vi.fn(), + renderConfig: vi.fn(), }, })); @@ -68,6 +69,15 @@ vi.mock('../services/DockerController', () => ({ }, })); +vi.mock('../services/ComposeService', () => ({ + getComposeCommandTimeoutMs: () => 30_000, + ComposeService: { + getInstance: () => ({ + renderConfig: state.renderConfig, + }), + }, +})); + // AutoHeal suppression is exercised elsewhere; here we only need the calls to // not throw when the gate finalizes a service run. vi.mock('../services/AutoHealService', () => ({ @@ -85,6 +95,8 @@ type Fixture = { restartCount?: number; startedAt?: string; imageId?: string; + exitCode?: number | null; + restartPolicy?: string | null; }; function setContainers(fixtures: Fixture[]): void { @@ -100,16 +112,36 @@ function setContainers(fixtures: Fixture[]): void { return Promise.resolve({ State: { Status: f.state ?? 'running', + ExitCode: f.exitCode === undefined ? (f.state === 'exited' ? 1 : 0) : f.exitCode, Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined, StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z', }, RestartCount: f.restartCount ?? 0, Image: f.imageId ?? 'sha256:app', + HostConfig: { RestartPolicy: { Name: f.restartPolicy ?? 'unless-stopped' } }, Config: { Labels: { 'com.docker.compose.service': f.service } }, }); }); } +function setDeclaredRestarts(services: Record): void { + const rendered = { + name: 'web', + services: Object.fromEntries( + Object.entries(services).map(([name, restart]) => [ + name, + restart === undefined ? { image: `${name}:1` } : { image: `${name}:1`, restart }, + ]), + ), + }; + state.renderConfig.mockResolvedValue({ + rendered: JSON.stringify(rendered), + stderr: '', + code: 0, + timedOut: false, + }); +} + const svc = () => HealthGateService.getInstance(); async function ticks(n: number): Promise { @@ -137,6 +169,8 @@ beforeEach(() => { state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' }; state.listContainers.mockReset(); state.inspect.mockReset(); + state.renderConfig.mockReset(); + setDeclaredRestarts({ app: 'unless-stopped', db: 'unless-stopped' }); svc().start(); }); @@ -216,7 +250,7 @@ describe('primary vs collateral attribution', () => { await ticks(1); setContainers([ { id: 'p1', name: 'web-app-1', service: 'app' }, - { id: 's1', name: 'web-db-1', service: 'db', state: 'exited' }, + { id: 's1', name: 'web-db-1', service: 'db', state: 'exited', exitCode: 1, restartPolicy: 'unless-stopped' }, ]); await ticks(1); const report = svc().getReport(0, 'web', runId!); @@ -224,6 +258,178 @@ describe('primary vs collateral attribution', () => { expect(report.failureSource).toBe('collateral'); }); + it('passes when a collateral one-shot exits 0 with restart no', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + const token = await prepareService([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { id: 's1', name: 'web-migrate-1', service: 'migrate', restartPolicy: 'no' }, + ]); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { id: 's1', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' }, + ]); + await ticks(1); + expect(svc().getReport(0, 'web', runId!).status).toBe('observing'); + await ticks(6); + expect(svc().getReport(0, 'web', runId!).status).toBe('passed'); + }); + + it('passes a collateral one-shot with residual unhealthy health', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + const token = await prepareService([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { id: 's1', name: 'web-migrate-1', service: 'migrate', restartPolicy: 'no' }, + ]); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { + id: 's1', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, + restartPolicy: 'no', health: 'unhealthy', + }, + ]); + await ticks(1); + expect(svc().getReport(0, 'web', runId!).status).toBe('observing'); + await ticks(6); + expect(svc().getReport(0, 'web', runId!).status).toBe('passed'); + }); + + it('fails when a collateral daemon with omitted restart exits 0', async () => { + setDeclaredRestarts({ app: 'unless-stopped', 'daemon-default': undefined }); + const token = await prepareService([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { id: 's1', name: 'web-daemon-1', service: 'daemon-default', restartPolicy: 'no' }, + ]); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { + id: 's1', name: 'web-daemon-1', service: 'daemon-default', + state: 'exited', exitCode: 0, restartPolicy: 'no', + }, + ]); + await ticks(1); + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('failed'); + expect(report.failureSource).toBe('collateral'); + expect(report.reason).toContain('exited during observation'); + }); + + it('passes when the primary service is a completed one-shot', async () => { + setDeclaredRestarts({ job: 'no' }); + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, restartPolicy: 'no', imageId: 'sha256:app' }, + ]); + await ticks(1); + expect(svc().getReport(0, 'web', runId!).status).toBe('observing'); + await ticks(6); + expect(svc().getReport(0, 'web', runId!).status).toBe('passed'); + }); + + it('passes a primary one-shot with residual unhealthy health', async () => { + setDeclaredRestarts({ job: 'no' }); + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { + id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, + restartPolicy: 'no', health: 'unhealthy', imageId: 'sha256:app', + }, + ]); + await ticks(1); + expect(svc().getReport(0, 'web', runId!).status).toBe('observing'); + await ticks(6); + expect(svc().getReport(0, 'web', runId!).status).toBe('passed'); + }); + + it('passes a primary one-shot with residual starting health', async () => { + setDeclaredRestarts({ job: 'no' }); + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { + id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, + restartPolicy: 'no', health: 'starting', imageId: 'sha256:app', + }, + ]); + await ticks(1); + expect(svc().getReport(0, 'web', runId!).status).toBe('observing'); + await ticks(6); + expect(svc().getReport(0, 'web', runId!).status).toBe('passed'); + }); + + it('fails when a primary one-shot exits with null exit code', async () => { + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: null, restartPolicy: 'no', imageId: 'sha256:app' }, + ]); + await ticks(1); + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('failed'); + expect(report.reason).toContain('exited during observation'); + expect(report.failureSource).toBe('primary'); + }); + + it('fails when a primary one-shot exits 0 under unless-stopped', async () => { + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'unless-stopped' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, restartPolicy: 'unless-stopped', imageId: 'sha256:app' }, + ]); + await ticks(1); + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('failed'); + expect(report.reason).toContain('exited during observation'); + expect(report.failureSource).toBe('primary'); + }); + + it('fails when a primary one-shot exits non-zero', async () => { + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 1, restartPolicy: 'no', imageId: 'sha256:app' }, + ]); + await ticks(1); + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('failed'); + expect(report.reason).toContain('exited during observation'); + expect(report.failureSource).toBe('primary'); + }); + it('fails with failureSource collateral when a healthy sibling vanishes before the first poll', async () => { // Sibling is healthy at prepare, then gone before arming. Seeding expected // from the prepare baseline must still track it so the gate fails. diff --git a/backend/src/__tests__/health-gate-service.test.ts b/backend/src/__tests__/health-gate-service.test.ts index 161bc570..0ba4458c 100644 --- a/backend/src/__tests__/health-gate-service.test.ts +++ b/backend/src/__tests__/health-gate-service.test.ts @@ -29,6 +29,7 @@ const { state } = vi.hoisted(() => ({ settings: {} as Record, listContainers: vi.fn(), inspect: vi.fn(), + renderConfig: vi.fn(), }, })); @@ -80,6 +81,15 @@ vi.mock('../services/DockerController', () => ({ }, })); +vi.mock('../services/ComposeService', () => ({ + getComposeCommandTimeoutMs: () => 30_000, + ComposeService: { + getInstance: () => ({ + renderConfig: state.renderConfig, + }), + }, +})); + import { HealthGateService } from '../services/HealthGateService'; type ContainerFixture = { @@ -89,25 +99,57 @@ type ContainerFixture = { health?: string | null; restartCount?: number; startedAt?: string; + exitCode?: number | null; + restartPolicy?: string | null; + service?: string; + imageId?: string; }; /** Configure the docker mocks from a simple fixture list. */ function setContainers(fixtures: ContainerFixture[]): void { - state.listContainers.mockResolvedValue(fixtures.map(f => ({ Id: f.id, Names: [`/${f.name}`], State: f.state ?? 'running' }))); + state.listContainers.mockResolvedValue(fixtures.map(f => ({ + Id: f.id, + Names: [`/${f.name}`], + State: f.state ?? 'running', + Labels: f.service ? { 'com.docker.compose.service': f.service } : {}, + }))); state.inspect.mockImplementation((id: string) => { const f = fixtures.find(c => c.id === id); if (!f) return Promise.reject(Object.assign(new Error('no such container'), { statusCode: 404 })); return Promise.resolve({ State: { Status: f.state ?? 'running', + ExitCode: f.exitCode === undefined ? (f.state === 'exited' ? 1 : 0) : f.exitCode, Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined, StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z', }, RestartCount: f.restartCount ?? 0, + Image: f.imageId ?? 'sha256:img', + HostConfig: { RestartPolicy: { Name: f.restartPolicy ?? '' } }, + Config: { Labels: f.service ? { 'com.docker.compose.service': f.service } : {} }, }); }); } +/** Declared Compose restart map used for one-shot recognition (not inspect). */ +function setDeclaredRestarts(services: Record): void { + const rendered = { + name: 'web', + services: Object.fromEntries( + Object.entries(services).map(([name, restart]) => [ + name, + restart === undefined ? { image: `${name}:1` } : { image: `${name}:1`, restart }, + ]), + ), + }; + state.renderConfig.mockResolvedValue({ + rendered: JSON.stringify(rendered), + stderr: '', + code: 0, + timedOut: false, + }); +} + const svc = () => HealthGateService.getInstance(); const latest = (stack = 'web') => svc().getReport(0, stack); @@ -125,7 +167,9 @@ beforeEach(() => { state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' }; state.listContainers.mockReset(); state.inspect.mockReset(); - setContainers([{ id: 'aaa', name: 'web-app-1' }]); + state.renderConfig.mockReset(); + setDeclaredRestarts({ app: 'unless-stopped' }); + setContainers([{ id: 'aaa', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }]); svc().start(); }); @@ -149,7 +193,7 @@ describe('HealthGateService verdicts', () => { it('fails fast when a container exits', async () => { svc().beginStack(0, 'web', 'update', 'tester'); await ticks(1); // baseline - setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited' }]); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'unless-stopped' }]); await ticks(1); const report = latest(); expect(report.status).toBe('failed'); @@ -157,6 +201,167 @@ describe('HealthGateService verdicts', () => { expect(state.activity.some(a => a.category === 'health_gate_failed')).toBe(true); }); + it('passes when a clean one-shot exits 0 with explicit declared restart no', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' }, + ]); + await ticks(1); + expect(latest().status).toBe('observing'); + await ticks(6); + expect(latest().status).toBe('passed'); + }); + + it('fails when a daemon with omitted Compose restart exits 0 (inspect also reports no)', async () => { + // QA P0: Docker HostConfig.RestartPolicy.Name is "no" for both omit and + // explicit restart:"no". Declared intent must decide, not inspect. + setDeclaredRestarts({ 'daemon-default': undefined }); + setContainers([ + { + id: 'daemon', name: 'web-daemon-default-1', service: 'daemon-default', + state: 'running', restartPolicy: 'no', + }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { + id: 'daemon', name: 'web-daemon-default-1', service: 'daemon-default', + state: 'exited', exitCode: 0, restartPolicy: 'no', + }, + ]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('exited during observation'); + }); + + it('passes a clean one-shot even when residual health is unhealthy', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { + id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, + restartPolicy: 'no', health: 'unhealthy', + }, + ]); + await ticks(1); + expect(latest().status).toBe('observing'); + await ticks(6); + expect(latest().status).toBe('passed'); + }); + + it('passes a clean one-shot even when residual health is still starting', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { + id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, + restartPolicy: 'no', health: 'starting', + }, + ]); + await ticks(1); + expect(latest().status).toBe('observing'); + await ticks(6); + expect(latest().status).toBe('passed'); + }); + + it('still fails unhealthy on a long-running container', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { + id: 'app', name: 'web-app-1', service: 'app', state: 'running', + restartPolicy: 'unless-stopped', health: 'unhealthy', + }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' }, + ]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('unhealthy'); + }); + + it('still ends unknown when a long-running healthcheck is starting at window end', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { + id: 'app', name: 'web-app-1', service: 'app', state: 'running', + restartPolicy: 'unless-stopped', health: 'starting', + }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' }, + ]); + await ticks(1); + expect(latest().status).toBe('observing'); + await ticks(6); + expect(latest().status).toBe('unknown'); + expect(latest().reason).toContain('still starting'); + }); + + it('fails when exit 0 has unless-stopped restart policy', async () => { + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'unless-stopped' }]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('exited during observation'); + }); + + it('fails when exit 0 has always restart policy', async () => { + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'always' }]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('exited during observation'); + }); + + it('fails closed when exit code is null on an exited container', async () => { + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: null, restartPolicy: 'no' }]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('exited during observation'); + }); + + it('fails when a one-shot exits non-zero', async () => { + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'no' }]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('exited during observation'); + }); + it('fails fast when a healthcheck reports unhealthy', async () => { svc().beginStack(0, 'web', 'update', 'tester'); await ticks(1); diff --git a/backend/src/__tests__/networking-operator.test.ts b/backend/src/__tests__/networking-operator.test.ts index 466fcb95..7d2a6f6d 100644 --- a/backend/src/__tests__/networking-operator.test.ts +++ b/backend/src/__tests__/networking-operator.test.ts @@ -142,7 +142,7 @@ describe('networking operator routes', () => { it('blocks admin delete when network is attached', async () => { vi.spyOn(DockerController, 'getInstance').mockReturnValue({ getDependencySnapshot: vi.fn().mockResolvedValue({ - containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'nginx', networks: [{ name: 'orphan_net', id: NET_ID, ip: '' }], volumes: [], ports: [] }], + containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'orphan_net', id: NET_ID, ip: '' }], volumes: [], ports: [] }], networks: [{ id: NET_ID, name: 'orphan_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: null, stack: null }], volumes: [], }), @@ -220,7 +220,7 @@ describe('evaluateNetworkDeleteGuard', () => { it('blocks a network that still has an attached container', () => { const snapshot = { volumes: [], - containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'img', networks: [{ name: 'app_net', id: 'n1', ip: '' }], volumes: [], ports: [] }], + containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'img', networks: [{ name: 'app_net', id: 'n1', ip: '' }], volumes: [], ports: [] }], networks: [{ id: 'n1', name: 'app_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK }], }; expect(evaluateNetworkDeleteGuard('n1', snapshot, []).code).toBe('attached'); diff --git a/backend/src/__tests__/networking-summary.test.ts b/backend/src/__tests__/networking-summary.test.ts index c1ca54dc..caa3d011 100644 --- a/backend/src/__tests__/networking-summary.test.ts +++ b/backend/src/__tests__/networking-summary.test.ts @@ -60,7 +60,7 @@ describe('networking summary', () => { it('flags a stack with an undeclared runtime network as network drift', async () => { vi.spyOn(DockerController, 'getInstance').mockReturnValue({ getDependencySnapshot: vi.fn().mockResolvedValue({ - containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'nginx', networks: [{ name: `${STACK}_default`, id: 'd', ip: '' }, { name: `${STACK}_rogue`, id: 'r', ip: '' }], volumes: [], ports: [] }], + containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: `${STACK}_default`, id: 'd', ip: '' }, { name: `${STACK}_rogue`, id: 'r', ip: '' }], volumes: [], ports: [] }], networks: [ { id: 'd', name: `${STACK}_default`, driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK }, { id: 'r', name: `${STACK}_rogue`, driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK }, diff --git a/backend/src/__tests__/one-shot-completion.test.ts b/backend/src/__tests__/one-shot-completion.test.ts new file mode 100644 index 00000000..0faddd26 --- /dev/null +++ b/backend/src/__tests__/one-shot-completion.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from 'vitest'; +import { + isCleanOneShotCompletion, + isNoRestartPolicy, + normalizeComposeRestartIntent, +} from '../utils/oneShotCompletion'; + +describe('isNoRestartPolicy', () => { + it('accepts only explicit no', () => { + expect(isNoRestartPolicy('no')).toBe(true); + }); + + it('rejects absent, empty, and restarting policies', () => { + expect(isNoRestartPolicy(undefined)).toBe(false); + expect(isNoRestartPolicy(null)).toBe(false); + expect(isNoRestartPolicy('')).toBe(false); + expect(isNoRestartPolicy('unless-stopped')).toBe(false); + expect(isNoRestartPolicy('always')).toBe(false); + expect(isNoRestartPolicy('on-failure')).toBe(false); + }); +}); + +describe('isCleanOneShotCompletion', () => { + const clean = { + state: 'exited', + exitCode: 0 as number | null, + restartPolicy: 'no' as string | null | undefined, + }; + + it('returns true only for exited + exit 0 + explicit restart no', () => { + expect(isCleanOneShotCompletion(clean)).toBe(true); + }); + + it('returns false for absent or empty declared restart', () => { + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: undefined })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: null })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: '' })).toBe(false); + }); + + it('returns false for non-exited states', () => { + expect(isCleanOneShotCompletion({ ...clean, state: 'running' })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, state: 'restarting' })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, state: 'created' })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, state: 'dead' })).toBe(false); + }); + + it('returns false for non-zero and null exit codes (fail closed)', () => { + expect(isCleanOneShotCompletion({ ...clean, exitCode: 1 })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, exitCode: 137 })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, exitCode: null })).toBe(false); + }); + + it('returns false for restarting policies even with exit 0', () => { + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'unless-stopped' })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'always' })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'on-failure' })).toBe(false); + }); +}); + +describe('normalizeComposeRestartIntent', () => { + it('falls back to service restart when deploy.restart_policy is unset', () => { + expect(normalizeComposeRestartIntent('no')).toBe('no'); + expect(normalizeComposeRestartIntent('unless-stopped')).toBe('unless-stopped'); + expect(normalizeComposeRestartIntent(undefined)).toBeNull(); + expect(normalizeComposeRestartIntent(null, {})).toBeNull(); + expect(normalizeComposeRestartIntent('always', { replicas: 2 })).toBe('always'); + }); + + it('maps deploy.restart_policy.condition with Compose defaults and precedence', () => { + expect(normalizeComposeRestartIntent('unless-stopped', { + restart_policy: { condition: 'none' }, + })).toBe('no'); + expect(normalizeComposeRestartIntent(null, { + restart_policy: { condition: 'any' }, + })).toBe('always'); + expect(normalizeComposeRestartIntent('no', { + restart_policy: { condition: 'on-failure' }, + })).toBe('on-failure'); + expect(normalizeComposeRestartIntent('no', { + restart_policy: {}, + })).toBe('always'); + }); + + it('fails closed on malformed restart_policy shapes', () => { + expect(normalizeComposeRestartIntent('no', { restart_policy: null })).toBe('always'); + expect(normalizeComposeRestartIntent('no', { restart_policy: 'none' })).toBe('always'); + expect(normalizeComposeRestartIntent('no', { restart_policy: [] })).toBe('always'); + expect(normalizeComposeRestartIntent('no', { + restart_policy: { condition: 'weird' }, + })).toBe('always'); + }); +}); diff --git a/backend/src/__tests__/preflight-rules.test.ts b/backend/src/__tests__/preflight-rules.test.ts index aaab42cd..b6ffb191 100644 --- a/backend/src/__tests__/preflight-rules.test.ts +++ b/backend/src/__tests__/preflight-rules.test.ts @@ -212,7 +212,11 @@ describe('hygiene rules', () => { }); it('flags a missing restart policy and healthcheck', () => { const bare = model([svc({ restart: undefined, hasHealthcheck: false })]); - expect(ids(runRules(ctx({ model: bare })), 'no-restart-policy')).toHaveLength(1); + const restartFindings = ids(runRules(ctx({ model: bare })), 'no-restart-policy'); + expect(restartFindings).toHaveLength(1); + expect(restartFindings[0].remediation).toMatch(/one-shot|init jobs/i); + expect(restartFindings[0].remediation).toMatch(/restart: "no"/); + expect(restartFindings[0].remediation).toMatch(/unless-stopped/); expect(ids(runRules(ctx({ model: bare })), 'no-healthcheck')).toHaveLength(1); const withDeployRestart = model([svc({ restart: undefined, deploy: { restart_policy: { condition: 'any' } }})]); expect(ids(runRules(ctx({ model: withDeployRestart })), 'no-restart-policy')).toHaveLength(0); diff --git a/backend/src/__tests__/sanitize-network-inspect.test.ts b/backend/src/__tests__/sanitize-network-inspect.test.ts index 316f5934..7358e52f 100644 --- a/backend/src/__tests__/sanitize-network-inspect.test.ts +++ b/backend/src/__tests__/sanitize-network-inspect.test.ts @@ -13,7 +13,7 @@ describe('sanitizeNetworkInspect connected containers', () => { networks: [{ id: 'net1', name: 'app_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: 'app', stack: 'app' }], volumes: [], containers: [{ - id: 'c1', name: 'app-web-1', service: 'web', composeProject: 'app', stack: 'app', state: 'running', image: 'nginx', + id: 'c1', name: 'app-web-1', service: 'web', composeProject: 'app', stack: 'app', state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'app_net', id: 'net1', ip: '172.20.0.5/16' }], volumes: [], ports: [], }], }; diff --git a/backend/src/helpers/composeDependencyParse.ts b/backend/src/helpers/composeDependencyParse.ts index f8059fd0..e73f94ab 100644 --- a/backend/src/helpers/composeDependencyParse.ts +++ b/backend/src/helpers/composeDependencyParse.ts @@ -26,6 +26,8 @@ export interface DeclaredService { image?: string; /** Service `network_mode:` as declared (e.g. `host`), or undefined. */ networkMode?: string; + /** Service `restart` from raw YAML parse. The effective-model path may set this via normalizeComposeRestartIntent (including deploy.restart_policy). Null when unset. */ + restart?: string | null; } /** A top-level networks:/volumes: entry. */ @@ -200,6 +202,7 @@ export function parseComposeDependencies(content: string): DeclaredCompose { ports, image: asString(svc.image), networkMode: asString(svc.network_mode), + restart: asString(svc.restart) ?? null, }); } diff --git a/backend/src/helpers/effectiveToDeclaredCompose.ts b/backend/src/helpers/effectiveToDeclaredCompose.ts index 64325c07..bf73690d 100644 --- a/backend/src/helpers/effectiveToDeclaredCompose.ts +++ b/backend/src/helpers/effectiveToDeclaredCompose.ts @@ -1,5 +1,6 @@ import type { EffectiveModel, EffResource } from '../services/preflight/effectiveModel'; import type { DeclaredCompose, DeclaredPort, DeclaredResource, DeclaredService } from './composeDependencyParse'; +import { normalizeComposeRestartIntent } from '../utils/oneShotCompletion'; /** Map one rendered network/volume entry to the declared shape drift expects. */ function toDeclaredResource(key: string, res: EffResource, projectName: string): DeclaredResource { @@ -41,6 +42,7 @@ export function declaredFromEffectiveModel(model: EffectiveModel): DeclaredCompo ), image: s.image, networkMode: s.networkMode, + restart: normalizeComposeRestartIntent(s.restart, s.deploy), })); return { diff --git a/backend/src/services/DockerController.ts b/backend/src/services/DockerController.ts index 8ee42120..7c46a91a 100644 --- a/backend/src/services/DockerController.ts +++ b/backend/src/services/DockerController.ts @@ -210,6 +210,12 @@ export interface DependencyContainer { /** Resolved Sencho stack, or null when the container is not Sencho-managed. */ stack: string | null; state: string; + /** + * Exit code from list Status when a parenthesized code is present + * (e.g. "Exited (0) …", "Restarting (1) …"); null when none (e.g. "Up …", bare "Exited"). + * Required so Drift can fail closed on unknown codes. + */ + exitCode: number | null; image: string; networks: { name: string; id: string; ip: string }[]; /** Named-volume sources mounted by the container (bind mounts excluded). */ @@ -1833,6 +1839,7 @@ class DockerController { composeProject: c.Labels?.['com.docker.compose.project'] ?? null, stack: DockerController.resolveContainerStack(c.Labels, projectToStack, knownSet, absDirToStack, resolvedBase), state: c.State ?? 'unknown', + exitCode: parseExitCode(typeof c.Status === 'string' ? c.Status : undefined), image: c.Image ?? '', networks, volumes, diff --git a/backend/src/services/DriftDetectionService.ts b/backend/src/services/DriftDetectionService.ts index 92fdab29..3151b4d2 100644 --- a/backend/src/services/DriftDetectionService.ts +++ b/backend/src/services/DriftDetectionService.ts @@ -8,6 +8,7 @@ import { parseEffectiveModel } from './preflight/effectiveModel'; import { compareStackNetworks, fromDeclaredCompose } from './network/normalize'; import { sanitizeForLog, redactSensitiveText } from '../utils/safeLog'; import { getErrorMessage } from '../utils/errors'; +import { isCleanOneShotCompletion } from '../utils/oneShotCompletion'; const MAX_RENDER_ERROR = 600; @@ -178,16 +179,17 @@ function networkDriftFindings( } /** - * Pure diff step (no Docker / FS access) so it is directly unit-testable. Only - * running containers are compared, since a stopped container publishes no ports - * and is not "deployed": a declared service with no running container is - * service-missing, a running container with no matching service is - * service-undeclared, and image / port differences are checked only for services - * present on both sides so a missing/undeclared service is not double-reported. + * Pure diff step (no Docker / FS access) so it is directly unit-testable. + * Running/restarting containers drive image/port comparison. Clean one-shot + * completions (exit 0 + explicit declared restart "no", including normalized + * `deploy.restart_policy.condition: none`) satisfy service presence without + * counting as hasContainers. Omitting restart does not qualify. Network + * comparison still uses only running/restarting attachments. */ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftReport { const { stack, declared, containers, parseError } = input; const networks = input.networks ?? []; + // Public contract: at least one running/restarting container (not "satisfied"). const hasContainers = containers.some((c) => RUNNING_STATES.has(c.state)); // A parse failure means the declared model is untrustworthy: report drift @@ -196,30 +198,44 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe return { stack, status: 'drifted', hasComposeFile: false, hasContainers, findings: [], parseError }; } - const runtimeByService = new Map(); - for (const c of containers) { - if (!RUNNING_STATES.has(c.state)) continue; - const name = c.service ?? c.name; - const agg = runtimeByService.get(name) ?? { images: new Set(), ports: new Set() }; - if (c.image) agg.images.add(normalizeImageRef(c.image)); - for (const p of c.ports) agg.ports.add(portKey(p.publishedPort, p.protocol)); - runtimeByService.set(name, agg); - } - - // Nothing running: the stack is defined on disk but not deployed. One status - // conveys this; per-service findings would just be noise. - if (!hasContainers) { - return { stack, status: 'missing-runtime', hasComposeFile: true, hasContainers: false, findings: [] }; - } - const declaredByName = new Map(); for (const svc of declared.services) declaredByName.set(svc.name, svc); + const runtimeByService = new Map(); + const oneShotSatisfied = new Set(); + for (const c of containers) { + const name = c.service ?? c.name; + if (RUNNING_STATES.has(c.state)) { + const agg = runtimeByService.get(name) ?? { images: new Set(), ports: new Set() }; + if (c.image) agg.images.add(normalizeImageRef(c.image)); + for (const p of c.ports) agg.ports.add(portKey(p.publishedPort, p.protocol)); + runtimeByService.set(name, agg); + continue; + } + const declaredRestart = declaredByName.get(name)?.restart; + if (isCleanOneShotCompletion({ + state: c.state, + exitCode: c.exitCode, + restartPolicy: declaredRestart, + })) { + oneShotSatisfied.add(name); + } + } + + const servicePresent = (serviceName: string): boolean => + runtimeByService.has(serviceName) || oneShotSatisfied.has(serviceName); + + // Nothing running and no declared service satisfied by a clean one-shot: the + // stack is defined on disk but not deployed. One status conveys this. + if (!hasContainers && !declared.services.some((svc) => servicePresent(svc.name))) { + return { stack, status: 'missing-runtime', hasComposeFile: true, hasContainers: false, findings: [] }; + } + const findings: StackDriftFinding[] = []; - // Declared service with no running container. + // Declared service with no running container and no clean one-shot completion. for (const svc of declared.services) { - if (!runtimeByService.has(svc.name)) { + if (!servicePresent(svc.name)) { findings.push({ kind: 'service-missing', service: svc.name, @@ -239,7 +255,7 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe } } - // Image / port divergence for services present on both sides. + // Image / port divergence for services present on both sides (running only). for (const [name, svc] of declaredByName) { const runtime = runtimeByService.get(name); if (!runtime) continue; diff --git a/backend/src/services/HealthGateService.ts b/backend/src/services/HealthGateService.ts index 0e51f1ce..a5c413df 100644 --- a/backend/src/services/HealthGateService.ts +++ b/backend/src/services/HealthGateService.ts @@ -5,8 +5,11 @@ import { AutoHealService } from './AutoHealService'; import { sanitizeForLog } from '../utils/safeLog'; import { getErrorMessage } from '../utils/errors'; import { withTimeout } from '../utils/withTimeout'; +import { isCleanOneShotCompletion } from '../utils/oneShotCompletion'; +import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompose'; +import { parseEffectiveModel } from './preflight/effectiveModel'; import type { HealthGateContainer, HealthGateReport } from './updateGuard/types'; -import { getComposeCommandTimeoutMs } from './ComposeService'; +import { ComposeService, getComposeCommandTimeoutMs } from './ComposeService'; const POLL_INTERVAL_MS = 5_000; // A prepared-but-never-begun token (prepare called, mutation then failed before @@ -47,6 +50,10 @@ interface ObservedContainer { service: string | null; /** Image id the container is running (service gates check convergence on it). */ imageId: string; + /** Inspect State.ExitCode; null when unavailable. */ + exitCode: number | null; + /** HostConfig.RestartPolicy.Name (report/debug only; not used for one-shot intent). */ + restartPolicy: string | null; } interface ActiveGate { @@ -89,6 +96,13 @@ interface ActiveGate { collateralBaselineByName: Map; /** Role of each expected container (service gates), for failure attribution. */ roleByName: Map; + /** + * Declared Compose restart intent by service name (normalized). Loaded once + * per gate; null until the first load attempt completes. Used for one-shot + * recognition instead of Docker inspect (which cannot distinguish omit vs + * explicit `restart: "no"`). + */ + declaredRestartByService: Map | null; } /** A pre-mutation baseline container captured at prepare time. */ @@ -101,6 +115,8 @@ interface PreparedBaseline { restartCount: number; startedAt: string | null; imageId: string; + exitCode: number | null; + restartPolicy: string | null; } function isRegressionEligibleSibling(baseline: PreparedBaseline): boolean { @@ -118,9 +134,35 @@ function observedFromPreparedBaseline(baseline: PreparedBaseline): ObservedConta health: baseline.health, service: baseline.service, imageId: baseline.imageId, + exitCode: baseline.exitCode, + restartPolicy: baseline.restartPolicy, }; } +/** Running, or a clean one-shot exit (exit 0 + explicit declared restart "no"). */ +function isObservedContainerSatisfied( + gate: ActiveGate, + current: ObservedContainer | undefined, +): boolean { + if (!current) return false; + if (current.state === 'running') return true; + return isDeclaredCleanOneShot(gate, current); +} + +/** + * One-shot recognition from declared Compose intent only. Unlabeled containers + * and services missing from the effective model fail closed. + */ +function isDeclaredCleanOneShot(gate: ActiveGate, current: ObservedContainer): boolean { + const map = gate.declaredRestartByService; + if (!map || !current.service || !map.has(current.service)) return false; + return isCleanOneShotCompletion({ + state: current.state, + exitCode: current.exitCode, + restartPolicy: map.get(current.service), + }); +} + /** An in-memory prepare snapshot awaiting attachExpectedImage + beginPrepared. */ interface PreparedGate { token: string; @@ -329,6 +371,7 @@ export class HealthGateService { collateralEligibleNames: new Set(), collateralBaselineByName: new Map(), roleByName: new Map(), + declaredRestartByService: null, }; this.active.set(key, gate); this.scheduleNextPoll(gate); @@ -488,6 +531,7 @@ export class HealthGateService { prep.collateralBaseline.filter(b => prep.collateralEligibleNames.has(b.name)), ), roleByName: new Map(), + declaredRestartByService: null, }; this.active.set(key, gate); this.scheduleNextPoll(gate); @@ -587,6 +631,9 @@ export class HealthGateService { if (gate.finalized || this.active.get(key) !== gate) return; gate.consecutivePollErrors = 0; + await this.ensureDeclaredRestartMap(gate); + if (gate.finalized || this.active.get(key) !== gate) return; + const elapsedMs = Date.now() - gate.startedAt; if (gate.expected === null) { @@ -629,12 +676,16 @@ export class HealthGateService { } gate.missingLastPoll.delete(name); - if (current.state === 'exited' && baseline.restarts === current.restarts) { - // An exit with no restart attempt is terminal for the window. + const cleanOneShot = isDeclaredCleanOneShot(gate, current); + // An exit with no restart attempt is terminal for the window, unless + // this is an expected one-shot (exit 0 + explicit declared restart "no"). + if (current.state === 'exited' && baseline.restarts === current.restarts && !cleanOneShot) { this.finalize(gate, 'failed', `container ${name} exited during observation`, summary); return; } - if (current.health === 'unhealthy') { + // Residual Docker health on a completed one-shot is not a gate failure; + // long-running containers still fail on unhealthy. + if (!cleanOneShot && current.health === 'unhealthy') { this.finalize(gate, 'failed', `container ${name} reported unhealthy`, summary); return; } @@ -657,14 +708,19 @@ export class HealthGateService { if (elapsedMs < gate.windowSeconds * 1000) return; - // Window complete: pass requires everything running and healthy wherever a - // healthcheck exists. A health state still 'starting' is not a pass. - const stillStarting = observed.filter(c => c.health === 'starting'); + // Window complete: pass requires everything running (or a clean one-shot + // completion) and healthy wherever a healthcheck exists. A health state + // still 'starting' is not a pass (except residual health on a clean one-shot). + const stillStarting = observed.filter( + c => c.health === 'starting' && !isDeclaredCleanOneShot(gate, c), + ); if (stillStarting.length > 0) { this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary); return; } - const notRunning = [...gate.expected.keys()].filter(name => byName.get(name)?.state !== 'running'); + const notRunning = [...gate.expected.keys()].filter( + name => !isObservedContainerSatisfied(gate, byName.get(name)), + ); if (notRunning.length > 0) { this.finalize(gate, 'failed', `not running at the end of the window: ${notRunning.join(', ')}`, summary); return; @@ -700,6 +756,9 @@ export class HealthGateService { if (gate.finalized || this.active.get(key) !== gate) return; gate.consecutivePollErrors = 0; + await this.ensureDeclaredRestartMap(gate); + if (gate.finalized || this.active.get(key) !== gate) return; + const elapsedMs = Date.now() - gate.startedAt; const serviceName = gate.serviceName ?? ''; @@ -781,11 +840,12 @@ export class HealthGateService { } gate.missingLastPoll.delete(name); - if (current.state === 'exited' && baseline.restarts === current.restarts) { + const cleanOneShot = isDeclaredCleanOneShot(gate, current); + if (current.state === 'exited' && baseline.restarts === current.restarts && !cleanOneShot) { this.finalize(gate, 'failed', `${noun} ${name} exited during observation`, summary, role); return; } - if (current.health === 'unhealthy') { + if (!cleanOneShot && current.health === 'unhealthy') { this.finalize(gate, 'failed', `${noun} ${name} reported unhealthy`, summary, role); return; } @@ -808,24 +868,28 @@ export class HealthGateService { if (elapsedMs < gate.windowSeconds * 1000) return; const stillStarting = observed.filter( - c => c.health === 'starting' && (c.service === serviceName || gate.collateralEligibleNames.has(c.name)), + c => c.health === 'starting' + && !isDeclaredCleanOneShot(gate, c) + && (c.service === serviceName || gate.collateralEligibleNames.has(c.name)), ); if (stillStarting.length > 0) { this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary); return; } - const runningPrimary = observed.filter(c => c.service === serviceName && c.state === 'running'); - if (runningPrimary.length !== gate.expectedReplicas) { + const satisfiedPrimary = observed.filter( + c => c.service === serviceName && isObservedContainerSatisfied(gate, c), + ); + if (satisfiedPrimary.length !== gate.expectedReplicas) { this.finalize( gate, 'failed', - `service ${serviceName} has ${runningPrimary.length} running replica(s), expected ${gate.expectedReplicas}`, + `service ${serviceName} has ${satisfiedPrimary.length} satisfied replica(s), expected ${gate.expectedReplicas}`, summary, 'primary', ); return; } const collateralNotRunning = [...gate.expected.keys()] .filter(name => gate.roleByName.get(name) === 'collateral') - .filter(name => byName.get(name)?.state !== 'running'); + .filter(name => !isObservedContainerSatisfied(gate, byName.get(name))); if (collateralNotRunning.length > 0) { this.finalize(gate, 'failed', `sibling(s) not running at the end of the window: ${collateralNotRunning.join(', ')}`, summary, 'collateral'); return; @@ -855,6 +919,35 @@ export class HealthGateService { return this.listStackContainers(gate.nodeId, gate.stackName); } + /** + * Load declared Compose restart intent once per gate. Fail closed to an empty + * map on render/parse errors so inspect "no" cannot false-qualify one-shots. + */ + private async ensureDeclaredRestartMap(gate: ActiveGate): Promise { + if (gate.declaredRestartByService !== null) return; + gate.declaredRestartByService = new Map(); + try { + const result = await ComposeService.getInstance(gate.nodeId).renderConfig(gate.stackName); + if (result.rendered === null) { + console.warn( + '[HealthGate] declared restart map unavailable for %s (compose render failed)', + sanitizeForLog(gate.stackName), + ); + return; + } + const model = parseEffectiveModel(JSON.parse(result.rendered), gate.stackName); + const declared = declaredFromEffectiveModel(model); + gate.declaredRestartByService = new Map( + declared.services.map((s) => [s.name, s.restart ?? null]), + ); + } catch (error) { + console.warn( + '[HealthGate] declared restart map load failed for %s:', + sanitizeForLog(gate.stackName), getErrorMessage(error, 'unknown'), + ); + } + } + /** List and inspect a stack's containers into the gate's observation shape. */ private async listStackContainers(nodeId: number, stackName: string): Promise { const docker = DockerController.getInstance(nodeId).getDocker(); @@ -877,6 +970,8 @@ export class HealthGateService { health: inspect.State?.Health?.Status ?? null, service: labels['com.docker.compose.service'] ?? null, imageId: inspect.Image ?? '', + exitCode: typeof inspect.State?.ExitCode === 'number' ? inspect.State.ExitCode : null, + restartPolicy: inspect.HostConfig?.RestartPolicy?.Name || null, }; } catch (e: unknown) { // Removed between list and inspect; the missing-container logic will @@ -903,6 +998,8 @@ export class HealthGateService { restartCount: c.restartCount, startedAt: c.startedAt, imageId: c.imageId, + exitCode: c.exitCode, + restartPolicy: c.restartPolicy, })); } diff --git a/backend/src/services/preflight/effectiveModel.ts b/backend/src/services/preflight/effectiveModel.ts index 44d6f41e..e5bf0efd 100644 --- a/backend/src/services/preflight/effectiveModel.ts +++ b/backend/src/services/preflight/effectiveModel.ts @@ -56,7 +56,7 @@ export interface EffService { networkMode?: string; restart?: string; hasHealthcheck: boolean; - /** Raw deploy block (read for key presence only, never values; undefined = none). */ + /** Raw deploy block (preflight uses key presence; Drift also reads restart_policy.condition). Undefined = none. */ deploy?: Record; containerName?: string; user?: string; diff --git a/backend/src/services/preflight/rules.ts b/backend/src/services/preflight/rules.ts index e58ca8e3..2e39aef1 100644 --- a/backend/src/services/preflight/rules.ts +++ b/backend/src/services/preflight/rules.ts @@ -363,7 +363,7 @@ const noRestartPolicy: PreflightRule = { message: `Service "${s.name}" has no restart policy, so it will not come back after a crash or host reboot.`, sourcePath: s.name, service: s.name, - remediation: 'Add restart: unless-stopped.', + remediation: 'For long-running services, add restart: unless-stopped. For one-shot or init jobs that should finish and stay stopped, restart: "no" is appropriate.', })); }, }; diff --git a/backend/src/utils/oneShotCompletion.ts b/backend/src/utils/oneShotCompletion.ts new file mode 100644 index 00000000..4c5206bb --- /dev/null +++ b/backend/src/utils/oneShotCompletion.ts @@ -0,0 +1,77 @@ +/** + * Helpers for clean one-shot completion and Compose restart-intent normalization. + * Used by Health Gate and Drift service-presence only; does not change bulk + * status, Auto-Heal, or atomic-deploy helpers. + */ + +/** + * True only for an explicit Compose `restart: "no"` (after normalization). + * Absent / null / empty do not qualify: Docker reports HostConfig restart + * "no" for both intentional one-shots and bare long-running services that + * omit `restart:`, so consumers must pass declared Compose intent, not inspect. + */ +export function isNoRestartPolicy(policy: string | null | undefined): boolean { + return policy === 'no'; +} + +/** + * Normalize Compose restart intent for Drift / declared-model consumers. + * + * Compose precedence: when `deploy.restart_policy` is set, its `condition` + * wins; otherwise the service-level `restart` field is used. Conditions map to + * Docker-like policy names so {@link isNoRestartPolicy} stays the single gate: + * `none` → `no`, `any` → `always`, `on-failure` → `on-failure`. Missing + * condition defaults to `any` (Compose default). Unknown shapes fail closed. + * Absent service restart (no deploy policy) stays null and is not one-shot eligible. + */ +export function normalizeComposeRestartIntent( + serviceRestart: string | null | undefined, + deploy?: Record | null, +): string | null { + if (deploy && Object.prototype.hasOwnProperty.call(deploy, 'restart_policy')) { + const policy = deploy['restart_policy']; + if (policy === null || typeof policy !== 'object' || Array.isArray(policy)) { + return 'always'; + } + const condition = (policy as Record).condition; + // Missing, empty, or non-string → Compose default `any` → always. + if (typeof condition !== 'string' || condition === '') { + return 'always'; + } + switch (condition.toLowerCase()) { + case 'none': + return 'no'; + case 'on-failure': + return 'on-failure'; + case 'any': + return 'always'; + default: + return 'always'; + } + } + if (serviceRestart == null || serviceRestart === '') return null; + return serviceRestart; +} + +export interface OneShotCompletionInput { + state: string; + /** Exact Docker exit code; null means unknown (fail closed). */ + exitCode: number | null; + /** + * Declared Compose restart intent after normalization (`"no"` for explicit + * one-shots / `deploy.restart_policy.condition: none`). Do not pass Docker + * inspect HostConfig values here. + */ + restartPolicy: string | null | undefined; +} + +/** + * True only for an exited container with exit code exactly 0 and explicit + * declared restart `"no"`. Null/absent restart, null exit codes, and restarting + * policies never qualify. + */ +export function isCleanOneShotCompletion(input: OneShotCompletionInput): boolean { + return input.state === 'exited' + && input.exitCode === 0 + && isNoRestartPolicy(input.restartPolicy); +} diff --git a/docs/features/health-gated-updates.mdx b/docs/features/health-gated-updates.mdx index 8abafdae..be9c27fa 100644 --- a/docs/features/health-gated-updates.mdx +++ b/docs/features/health-gated-updates.mdx @@ -49,11 +49,13 @@ The 3-second crash probe from [atomic deployments](/features/atomic-deployments) During the window, the gate checks every container that came up with the stack: -- all expected containers stay **running**, -- Docker healthchecks report **healthy** wherever a service defines one, -- no container **exits**, **disappears**, or enters a **restart loop**. +- all expected containers stay **running** (or finish cleanly as a one-shot; see below), +- Docker healthchecks report **healthy** on long-running containers wherever a service defines one (residual health on a completed one-shot is ignored), +- no container **crashes** (non-zero exit), **disappears**, or enters a **restart loop**. -A clear failure ends the observation immediately. Passing requires the full window, and a healthcheck still in its start period when the window ends records the verdict as unknown rather than claiming success. +A one-shot or init service that finishes with exit code `0` and explicitly declares `restart: "no"` (or `deploy.restart_policy.condition: none`) counts as a successful completion, not a gate failure. Omitting `restart:` does not qualify: Compose defaults to no-restart, and Docker reports the same inspect value for both intentional jobs and bare long-running services. Residual Docker health state on a completed one-shot (`starting` or `unhealthy` from an inherited healthcheck) does not fail or leave the gate unknown. A clean exit under `unless-stopped`, `always`, or `on-failure`, or with no declared restart, still fails the gate. An exit whose code cannot be read also fails the gate. + +A clear failure ends the observation immediately. Passing requires the full window, and a healthcheck still in its start period on a long-running container when the window ends records the verdict as unknown rather than claiming success. After a **service-scoped** update or restore, the gate still watches the whole project, but it attributes failures: a problem on the updated service is a primary failure; a previously healthy sibling that regresses during the window is a collateral failure. Siblings that were already unhealthy before the update do not fail the gate on their own. @@ -78,7 +80,7 @@ The gate also runs for updates you did not click: scheduled image updates, webho Open **Settings > Infrastructure > Stacks > Deploy Guardrails** on the node you want to configure: - **Observe health after updates** turns the gate on or off for that node. On by default; turning it off only stops the observation and its timeline events, never the update itself. -- **Observation window** sets how long containers are watched, from 15 to 600 seconds. The default is 90 seconds; raise it for stacks that take a while to settle, since a healthcheck still starting at the end of the window records an unknown verdict instead of a pass. +- **Observation window** sets how long containers are watched, from 15 to 600 seconds. The default is 90 seconds; raise it for stacks that take a while to settle, since a long-running healthcheck still starting at the end of the window records an unknown verdict instead of a pass. Stacks settings page showing the Deploy Guardrails subsection, with the Observe health after updates toggle set to On and the Observation window field set to 90 seconds @@ -110,10 +112,10 @@ When a deploy or update fails, Sencho classifies the failure from the compose ou Unknown means one of the verdict-affecting signals could not be verified, most often because Docker or the node did not answer in time. The update path is never blocked by an unknown verdict; check the node's connection if it persists, and proceed when you are confident in the stack's state. - The gate fails on the first clear problem it observes: a container exit, an unhealthy healthcheck, a restart loop, or a container disappearing mid-window. Check the named container's logs from the stack page. If the container legitimately restarts during startup (a migration step, for example), raise the observation window under **Settings > Infrastructure > Stacks > Deploy Guardrails** so the gate watches past the settle period. + The gate fails on the first clear problem it observes: a non-zero exit, an unexpected clean exit of a service that is not an explicit one-shot, an unhealthy healthcheck on a long-running container, a restart loop, or a container disappearing mid-window. A one-shot that exits `0` with explicit `restart: "no"` (or `deploy.restart_policy.condition: none`) does not fail the gate, including when residual Docker health on that completed container is still starting or unhealthy. Check the named container's logs from the stack page. If the container legitimately restarts during startup (a migration step that should stay up, for example), raise the observation window under **Settings > Infrastructure > Stacks > Deploy Guardrails** so the gate watches past the settle period. - Unknown means the observation could not finish: Sencho restarted mid-window, a newer update superseded the observation, Docker became unreachable, a healthcheck was still starting when the window ended, or no containers appeared to observe. The stack timeline records the reason. Check the containers directly; an unknown verdict makes no claim either way. + Unknown means the observation could not finish: Sencho restarted mid-window, a newer update superseded the observation, Docker became unreachable, a long-running healthcheck was still starting when the window ended, or no containers appeared to observe. Residual starting health on a completed one-shot does not produce unknown. The stack timeline records the reason. Check the containers directly; an unknown verdict makes no claim either way. The modal holds its auto-close while the gate observes so the verdict is not lost. You can close it at any time; the observation continues server-side and the verdict lands on the stack timeline. If the gate result repeatedly cannot be retrieved, the modal gives up with an unknown verdict instead of waiting forever. diff --git a/docs/features/stack-drift.mdx b/docs/features/stack-drift.mdx index ae7c5897..2297f106 100644 --- a/docs/features/stack-drift.mdx +++ b/docs/features/stack-drift.mdx @@ -28,9 +28,9 @@ The status badge at the top of the tab summarizes the comparison between the eff | Status | Meaning | |--------|---------| -| **In sync** | Running containers match the effective model: same services, images, and published ports. | -| **Drifted** | Something running differs from the model. Specific findings are listed below the badge. | -| **Not running** | The stack has no running containers. The file is present but nothing is up. | +| **In sync** | Declared services match runtime: same running services, images, and published ports, or a declared one-shot finished cleanly (`Exited (0)` with explicit `restart: "no"` or `deploy.restart_policy.condition: none`). Omitting `restart:` does not count as a one-shot. | +| **Not running** | The stack has no running containers, and no declared service is satisfied by a finished one-shot. The file is present but nothing is up. | +| **Drifted** | Something differs from the model. Specific findings are listed below the badge. | | **Unreachable** | Docker could not be reached on the active node, so drift cannot be assessed. | ## Since your last deploy @@ -62,10 +62,10 @@ When a stack is drifted, each reason is listed in the **Findings** section again |---------|---------------| | **Image** | A running container uses a different image than the Compose file declares. Expected and actual values are shown side by side. | | **Ports** | The published ports of a service differ from what the Compose file declares. | -| **Service missing** | The Compose file declares a service, but no running container matches it. | +| **Service missing** | The Compose file declares a service, but no running container matches it and no clean one-shot completion satisfies it. | | **Undeclared service** | A container is running for the stack but no matching service exists in the Compose file. | | **Network undeclared** | A running service is attached to a network that is not declared in the Compose file. | -| **Network missing** | A network declared in the Compose file is not used by any running service or is absent from the runtime. | +| **Network missing** | A network declared in the Compose file is not used by any running service or is absent from the runtime. Network usage still counts only running or restarting containers, so a finished one-shot does not keep a dedicated Compose network marked as in use. | ### Image comparison @@ -157,7 +157,7 @@ Viewing the tab requires read access to the stack; any role that can see a stack - That is expected. Drift compares the file on disk against what is actually running, so a stack with no running containers reports **Not running** even when you stopped it intentionally. Deploy it to return it to **In sync**. + That is expected for long-running services. Drift compares the file on disk against what is actually running, so a stack with no running containers and no finished one-shot jobs reports **Not running** even when you stopped it intentionally. Deploy it to return it to **In sync**. A stack made only of one-shot jobs that already exited `0` with explicit `restart: "no"` (or `deploy.restart_policy.condition: none`) can show **In sync** (or **Drifted** for other findings such as unused networks) with no containers currently running. Click **re-check** after the deploy finishes. During a rolling update, replicas can briefly run different images, and the report captures that moment. Once every container is on the declared image, the finding clears.