/** * Digest drift on a real Blueprint deployment, and what each policy does about it. * * A tag is a promise, not an identity, so the drift this exercises is created * the way it happens in the field: the compose pins an approved image digest, * the running containers are then replaced with a different image, and Sencho * must report the per-service digest difference, show both digests on the Drift * tab, and leave the approved identity where it was instead of adopting whatever * is actually running. * * Two policies are covered against that same drift, because the interesting part * is what happens next. Observe reports it and leaves the row drifted, which is * what the first test asserts. Enforce has to take that same drifted row and put * it back: the second test watches the row return to Active with no drift * summary and the workload running the approved digest again, which is the whole * cycle rather than only its detection. It switches the policy on *after* the * drift is recorded, so the repair lands on a pass later than the one that * recorded it, and the pass that picks a drifted row back up is genuinely * exercised instead of being raced. * * The approved digest is discovered from `nginx:alpine` and the digest the * workload is moved to is discovered from `nginx:1.27`, both of them real * upstream tags of one repository. A locally invented repository cannot work * here, for a reason that showed up as a CI-only failure: the observation * filters the running image's repo digests by the authored repository, so an * invented repository never yields an observed digest at all, and a reference no * registry knows can only be satisfied by whatever the local Docker version * feels like doing with it. Pinning a real repository by digest keeps the * approved side resolvable from the local content store without a registry call, * and keeps the moved side a real digest difference rather than an unresolved * one. * * Needs the Docker CLI and Compose plugin, because making a running workload * diverge is a Docker operation. A machine with no `docker` executable at all * skips the spec outside CI only: in CI the runner is required to have it, and a * silent skip there would hide a broken environment. A Docker that is installed * but unusable, including a missing Compose plugin, fails loudly instead of * skipping, so a broken environment never reads as a pass. */ import { execFileSync } from 'node:child_process'; import { writeFileSync, rmSync } from 'node:fs'; import { test, expect, type Page } from '@playwright/test'; import { loginAs } from './helpers'; const DOCKER_TIMEOUT_MS = 60_000; const REQUEST_TIMEOUT_MS = 15_000; const DIGEST_RE = /^sha256:[0-9a-f]{64}$/i; function isMissingExecutable(error: unknown): boolean { return (error as NodeJS.ErrnoException | null)?.code === 'ENOENT'; } function dockerAvailable(): boolean { try { execFileSync('docker', ['version', '--format', '{{.Server.Version}}'], { timeout: DOCKER_TIMEOUT_MS, stdio: 'pipe' }); execFileSync('docker', ['compose', 'version'], { timeout: DOCKER_TIMEOUT_MS, stdio: 'pipe' }); return true; } catch (error) { if (isMissingExecutable(error)) return false; // Present but unusable is an environment fault, not a reason to skip. throw error; } } function imageRepository(image: string): string { const withoutDigest = image.split('@')[0] ?? image; const lastSlash = withoutDigest.lastIndexOf('/'); const lastColon = withoutDigest.lastIndexOf(':'); return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest; } /** * Every repo digest the local image carries for its own repository. The backend * selects observed digests the same way, by repository, so the spec reads them * the same way instead of trusting Docker's ordering. */ function repoDigests(image: string): string[] { const repo = imageRepository(image); const raw = execFileSync( 'docker', ['image', 'inspect', image, '--format', '{{json .RepoDigests}}'], { timeout: DOCKER_TIMEOUT_MS, stdio: 'pipe' }, ) .toString() .trim(); const parsed: unknown = JSON.parse(raw); if (!Array.isArray(parsed)) throw new Error(`${image} reported no repo digests`); const digests: string[] = []; for (const entry of parsed) { if (typeof entry !== 'string') continue; const at = entry.indexOf('@'); if (at < 0) continue; const entryRepo = imageRepository(entry.slice(0, at)); const digest = entry.slice(at + 1); if (entryRepo !== repo || !DIGEST_RE.test(digest) || digests.includes(digest)) continue; digests.push(digest); } if (digests.length === 0) throw new Error(`${image} has no repo digest for ${repo}`); return digests; } /** * The image reference the project's running container was created from, read * from the daemon. Compose normalizes the reference it stores, so callers match * on the digest rather than on the whole string. */ function containerImageReference(project: string, service: string): string { const containers = execFileSync( 'docker', [ 'ps', '--filter', `label=com.docker.compose.project=${project.toLowerCase()}`, '--filter', `label=com.docker.compose.service=${service}`, '--filter', 'status=running', '--format', '{{.ID}}', ], { timeout: DOCKER_TIMEOUT_MS, stdio: 'pipe' }, ) .toString() .trim() .split('\n') .filter((line) => line.length > 0); // Exactly one, so a stray duplicate or a one-off container can never be // mistaken for the workload this compose file defines. if (containers.length !== 1) { throw new Error( `expected one running ${service} container for project ${project}, found ${containers.length}`, ); } const [container] = containers; if (container === undefined) { throw new Error(`expected one running ${service} container for project ${project}, found none`); } return execFileSync('docker', ['inspect', '--format', '{{.Config.Image}}', container], { timeout: DOCKER_TIMEOUT_MS, stdio: 'pipe', }) .toString() .trim(); } async function jsonRequest( page: Page, url: string, init: { method?: string; body?: unknown } = {}, ): Promise<{ status: number; body: T }> { return page.evaluate( async ({ url, method, payload, timeout }) => { const res = await fetch(url, { method, credentials: 'include', signal: AbortSignal.timeout(timeout), ...(payload === null ? {} : { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }), }); // A 204 carries no body, so parse defensively rather than failing cleanup // on the response shape. const text = await res.text(); return { status: res.status, body: (text ? JSON.parse(text) : null) as T }; }, { url, method: init.method ?? 'GET', payload: init.body === undefined ? null : init.body, timeout: REQUEST_TIMEOUT_MS }, ); } /** Reads a 200 or the test fails with the server's own words, not a timeout. */ function expectOk(response: { status: number; body: T }, what: string): T { expect(response.status, `${what}: ${JSON.stringify(response.body)}`).toBe(200); return response.body; } interface NodeRow { id: number; type: string; compose_dir: string; } interface ServiceEvidence { serviceName: string; platformDigest: string | null; } interface DeploymentRow { status: string; drift_summary: string | null; } interface PreviewPayload { planFingerprint: string; gitopsFingerprint: string | null; confirmableActions: unknown[]; } interface ApplyPayload { error?: string; code?: string; } interface BlueprintDetail { deployments: DeploymentRow[]; gitopsRevision?: { targets?: TargetRow[] } | null; } interface TargetRow { observedArtifactIdentity?: { kind: string; services?: ServiceEvidence[] } | null; artifact?: { expected?: { services?: ServiceEvidence[] } | null } | null; } interface DriftPayload { gitopsRevision?: { targets?: TargetRow[]; } | null; } /** * Preview the plan and confirm it, the way an operator does. * * Confirming against a live Blueprint is a two-step dance: the tick can re-save * the row between the preview and the confirm, which the server refuses with * PREVIEW_STALE. That one case is worth retrying with a fresh preview, because * the UI asks the operator to confirm again. Any other failure is reported as it * is, since retrying it would only repeat it. */ async function confirmPlan(page: Page, blueprintId: number): Promise { let applied: { status: number; body: ApplyPayload } | null = null; for (let attempt = 0; attempt < 4; attempt += 1) { const preview = expectOk( await jsonRequest(page, `/api/blueprints/${blueprintId}/preview`), 'previewing the plan', ); const result = await jsonRequest(page, `/api/blueprints/${blueprintId}/apply`, { method: 'POST', body: { planFingerprint: preview.planFingerprint, // The apply confirms the authority evidence the preview displayed, so // what it showed has to be echoed back with the plan. ...(preview.gitopsFingerprint ? { gitopsFingerprint: preview.gitopsFingerprint } : {}), actions: preview.confirmableActions, }, }); applied = result; if (result.status === 200) break; if (result.status !== 409 || result.body.code !== 'PREVIEW_STALE') break; } expect(applied?.status, `applying the plan: ${JSON.stringify(applied?.body)}`).toBe(200); } test.describe('GitOps digest drift', () => { test.skip(!dockerAvailable() && !process.env.CI, 'Docker with the Compose plugin is not available'); // The reconciler ticks once a minute, so the drift poll covers a tick and the // observation that follows it, and the baseline poll covers the compose run. // These are ceilings rather than expected durations: the happy path takes about // half a minute, and the CI job has a whole-suite budget to protect. Both tests // here are reconciler-driven, so each carries its own ceiling and the repair's // ceiling covers a second tick on top of the detection it waits for. test.setTimeout(360_000); const stamp = Date.now(); // One stack name per case, not one per file. Both cases deploy a real compose // project under this name, so a shared one would make the second case collide // with whatever the first case's teardown left behind, and the failure would // surface as a confusing container count rather than as the leak it is. const blueprintNameFor = (caseName: string) => `e2e-digest-${caseName}-${stamp}`; // Two real tags of one repository, pinned by digest and then swapped, so the // approved reference resolves locally and the observed digest matches the // authored repository. The digests themselves are read from the daemon, never // assumed, because either tag can move. const approvedImage = 'nginx:alpine'; const movedImage = 'nginx:1.27'; const overrideFile = `/tmp/e2e-digest-override-${stamp}.yaml`; let blueprintId: number | null = null; // Set by each case as it starts, so teardown can look for the containers that // case created even when the case failed before it created the blueprint. let blueprintName = ''; test.beforeAll(() => { execFileSync('docker', ['pull', '--quiet', approvedImage], { timeout: DOCKER_TIMEOUT_MS, stdio: 'pipe' }); execFileSync('docker', ['pull', '--quiet', movedImage], { timeout: DOCKER_TIMEOUT_MS, stdio: 'pipe' }); }); test.afterAll(() => { rmSync(overrideFile, { force: true }); }); test('a replaced image digest is reported per service with both digests, and the approved identity holds', async ({ page }) => { await loginAs(page); blueprintName = blueprintNameFor('observe'); const approvedDigests = repoDigests(approvedImage); const movedDigests = repoDigests(movedImage); const [pinned] = approvedDigests; if (pinned === undefined) throw new Error(`${approvedImage} reported no repo digest`); const shared = movedDigests.filter((digest) => approvedDigests.includes(digest)); // Two tags of one repository can resolve to the same manifest, which would // make the moved-image scenario a no-op. Failing here names the cause // instead of leaving it to surface as a poll timeout. expect( shared, `${approvedImage} and ${movedImage} resolve to the same digest, so nothing would drift`, ).toEqual([]); const approvedRef = `nginx@${pinned}`; const compose = `services:\n web:\n image: ${approvedRef}\n`; const nodes = expectOk(await jsonRequest(page, '/api/nodes'), 'listing nodes'); const local = nodes.find((node) => node.type === 'local'); expect(local, 'expected a local node to target').toBeTruthy(); const stackDir = `${local!.compose_dir}/${blueprintName}`; const created = await jsonRequest<{ id?: number; error?: string }>(page, '/api/blueprints', { method: 'POST', body: { name: blueprintName, compose_content: compose, selector: { type: 'nodes', ids: [local!.id] }, drift_mode: 'observe', }, }); expect(created.status, `creating the blueprint: ${JSON.stringify(created.body)}`).toBe(201); if (typeof created.body.id !== 'number') { throw new Error(`the create response carried no blueprint id: ${JSON.stringify(created.body)}`); } blueprintId = created.body.id; // Confirming a plan is a two-step dance against a live blueprint: the tick // can re-save the row between the preview and the confirm, which the server // refuses with PREVIEW_STALE. That one case is worth retrying with a fresh // preview, because the UI asks the operator to confirm again. Any other // failure is reported as it is, since retrying it would only repeat it. await confirmPlan(page, blueprintId); // Only the projection is read after its poll, so it is the only field that // needs to outlive the callbacks that fill it. const state: { projected: DriftPayload['gitopsRevision'] } = { projected: null }; // The baseline has to be proven, not assumed, because a deployment row goes // active before any drift check runs, and a mismatch that was already there // at deploy time would otherwise be credited to the override below. // // The proof is read from the daemon rather than from the projection on // purpose. The projection's runtime observation is only recorded on a // reconciler tick, so waiting for it would spend a whole extra minute to // learn something the daemon already knows, and the CI job has a whole-suite // budget to protect. What the container was actually created from is the // ground truth for "what is running", and it is one fast call. await expect .poll( async () => { const detail = expectOk( await jsonRequest(page, `/api/blueprints/${blueprintId}`), 'reading the blueprint', ); const approved = detail.gitopsRevision?.targets?.[0]?.artifact?.expected?.services?.[0]?.platformDigest ?? null; return approved === pinned && detail.deployments.some((row) => row.status === 'active'); }, { timeout: 150_000, intervals: [1_000] }, ) .toBe(true); const runningBefore = containerImageReference(blueprintName, 'web'); expect( runningBefore, 'the deploy must run the approved reference, or the override proves nothing', ).toContain(`nginx@${pinned}`); // Replace the running workload under the same project. The compose on disk // still pins the approved digest, so the only thing that changed is what is // actually running. writeFileSync(overrideFile, `services:\n web:\n image: ${movedImage}\n`); execFileSync( 'docker', [ 'compose', '-p', blueprintName, '-f', `${stackDir}/compose.yaml`, '-f', overrideFile, 'up', '-d', '--force-recreate', '--pull', 'never', ], { timeout: DOCKER_TIMEOUT_MS, stdio: 'pipe' }, ); // The reconciler observes the replaced workload on its next pass, up to a // minute out, and records the deployment as drifted when the running digest // no longer matches the approved one. await expect .poll( async () => { const drift = expectOk( await jsonRequest(page, `/api/stacks/${blueprintName}/drift`), 'reading drift', ); state.projected = drift.gitopsRevision; const observed = state.projected?.targets?.[0]?.observedArtifactIdentity?.services?.[0]?.platformDigest ?? null; const detail = expectOk( await jsonRequest(page, `/api/blueprints/${blueprintId}`), 'reading the blueprint', ); const drifted = detail.deployments.some((row) => row.status === 'drifted'); return observed !== null && movedDigests.includes(observed) && drifted; }, { timeout: 150_000, intervals: [2_000] }, ) .toBe(true); const expectedServices = state.projected?.targets?.[0]?.artifact?.expected?.services ?? []; const observedServices = state.projected?.targets?.[0]?.observedArtifactIdentity?.services ?? []; expect(expectedServices.length, 'expected per-service digests in the projection').toBeGreaterThan(0); expect(observedServices.length, 'expected observed per-service digests in the projection').toBeGreaterThan(0); // The approved identity did not move: only what is running did. The // canonical drift list does not carry this case today; the deployment row // and the per-service comparison are where it is reported. const running = observedServices[0]?.platformDigest ?? null; if (running === null) throw new Error('the observation recorded no digest to compare'); expect(expectedServices[0]?.platformDigest, 'the expectation never moved').toBe(pinned); expect(running, 'the report names the digest that is actually running').not.toBe(pinned); // The Drift tab shows the same comparison, per service, with both digests. await page.goto(`/nodes/local/stacks/${blueprintName}`); await page.getByRole('tab', { name: 'Anatomy' }).click(); await page.getByRole('tab', { name: 'Drift' }).click(); // The per-service digests are evidence behind the answer, so they sit under Evidence. await page.getByRole('button', { name: 'Evidence' }).click(); const digestRow = page.getByTestId('gitops-digest-row').first(); await expect(digestRow).toBeVisible({ timeout: 20_000 }); await expect(digestRow).toHaveAttribute('data-state', 'drifted'); await expect(digestRow).toContainText('approved'); await expect(digestRow).toContainText('running'); const short = (digest: string) => digest.slice('sha256:'.length, 'sha256:'.length + 12); await expect(digestRow).toContainText(short(pinned)); await expect(digestRow).toContainText(short(running)); // The moved image is reported, never adopted: the expectation stays pinned to // the digest the compose declared, so the running image cannot become the // approved identity anywhere in the projection. const afterUi = expectOk( await jsonRequest(page, `/api/stacks/${blueprintName}/drift`), 'reading drift after the UI check', ); const stillApproved = afterUi.gitopsRevision?.targets?.[0]?.artifact?.expected?.services?.[0]?.platformDigest; expect(stillApproved, 'reading the expectation must not adopt the running image').toBe(pinned); }); test('an Enforce Blueprint repairs a moved image back to the approved digest and reconverges', async ({ page }) => { await loginAs(page); blueprintName = blueprintNameFor('enforce'); const approvedDigests = repoDigests(approvedImage); const movedDigests = repoDigests(movedImage); const [pinned] = approvedDigests; if (pinned === undefined) throw new Error(`${approvedImage} reported no repo digest`); expect( movedDigests.filter((digest) => approvedDigests.includes(digest)), `${approvedImage} and ${movedImage} resolve to the same digest, so nothing would drift`, ).toEqual([]); const approvedRef = `nginx@${pinned}`; const compose = `services:\n web:\n image: ${approvedRef}\n`; const nodes = expectOk(await jsonRequest(page, '/api/nodes'), 'listing nodes'); const local = nodes.find((node) => node.type === 'local'); expect(local, 'expected a local node to target').toBeTruthy(); const stackDir = `${local!.compose_dir}/${blueprintName}`; // Observe first, then switch to Enforce once the drift is on the row. Going // straight to Enforce would let the tick that detects the drift repair it in // the same pass, so the row would never be observably drifted and the pass // that picks a drifted row back up would go unproven. This way the row is // provably drifted under Enforce before anything may repair it. const created = await jsonRequest<{ id?: number; error?: string }>(page, '/api/blueprints', { method: 'POST', body: { name: blueprintName, compose_content: compose, selector: { type: 'nodes', ids: [local!.id] }, drift_mode: 'observe', }, }); expect(created.status, `creating the blueprint: ${JSON.stringify(created.body)}`).toBe(201); if (typeof created.body.id !== 'number') { throw new Error(`the create response carried no blueprint id: ${JSON.stringify(created.body)}`); } blueprintId = created.body.id; await confirmPlan(page, blueprintId); // The baseline has to be proven, not assumed: a mismatch that was already // there at deploy time would otherwise be credited to the override below. await expect .poll( async () => { const detail = expectOk( await jsonRequest(page, `/api/blueprints/${blueprintId}`), 'reading the blueprint', ); const approved = detail.gitopsRevision?.targets?.[0]?.artifact?.expected?.services?.[0]?.platformDigest ?? null; return approved === pinned && detail.deployments.some((row) => row.status === 'active'); }, { timeout: 150_000, intervals: [1_000] }, ) .toBe(true); expect( containerImageReference(blueprintName, 'web'), 'the deploy must run the approved reference, or the override proves nothing', ).toContain(`nginx@${pinned}`); // Replace the running workload under the same project. The compose on disk // still pins the approved digest, so the only thing that changed is what is // actually running. writeFileSync(overrideFile, `services:\n web:\n image: ${movedImage}\n`); execFileSync( 'docker', [ 'compose', '-p', blueprintName, '-f', `${stackDir}/compose.yaml`, '-f', overrideFile, 'up', '-d', '--force-recreate', '--pull', 'never', ], { timeout: DOCKER_TIMEOUT_MS, stdio: 'pipe' }, ); // Observe records the drift and leaves it, which is what makes the row // drifted rather than repaired on the pass that finds it. await expect .poll( async () => { const detail = expectOk( await jsonRequest(page, `/api/blueprints/${blueprintId}`), 'reading the blueprint', ); return detail.deployments.some((row) => row.status === 'drifted'); }, { timeout: 150_000, intervals: [2_000] }, ) .toBe(true); const drifted = expectOk( await jsonRequest(page, `/api/blueprints/${blueprintId}`), 'reading the drifted row', ).deployments.find((row) => row.status === 'drifted'); expect(drifted?.drift_summary, 'a drifted row carries why it drifted').toBeTruthy(); expect( containerImageReference(blueprintName, 'web'), 'the moved image is what is running until the repair lands', ).not.toContain(`nginx@${pinned}`); // Switch the policy on. The edit clears the approval, because the plan the // operator confirmed described a different policy, so the plan is confirmed // again against the same intent. const switched = await jsonRequest<{ error?: string }>(page, `/api/blueprints/${blueprintId}`, { method: 'PUT', body: { drift_mode: 'enforce' }, }); expect(switched.status, `switching to enforce: ${JSON.stringify(switched.body)}`).toBe(200); await confirmPlan(page, blueprintId); // The row was already drifted when Enforce started, so the repair runs on a // later pass than the one that recorded the drift. Either the confirm // executes it or the next reconciler tick does; the poll covers both. // // What this covers is the cycle and the reconvergence, not the settle of a // repair that could not complete: here the repair succeeds, so the row is // never stranded in `correcting` across ticks. That path needs a refusal // arranged on purpose, and it is pinned deterministically in // `backend/src/__tests__/blueprints-inline-drift-repair-retry.test.ts`. await expect .poll( async () => { const detail = expectOk( await jsonRequest(page, `/api/blueprints/${blueprintId}`), 'reading the blueprint', ); const row = detail.deployments[0]; return row?.status === 'active' && row.drift_summary === null; }, { timeout: 180_000, intervals: [2_000] }, ) .toBe(true); // The repair restored the approved identity rather than adopting whatever // the tag points at now, which is the whole point of a digest-pinned repair. const repaired = containerImageReference(blueprintName, 'web'); expect( repaired, 'the repaired workload must run the approved image identity', ).toContain(`nginx@${pinned}`); const after = expectOk( await jsonRequest(page, `/api/stacks/${blueprintName}/drift`), 'reading drift after the repair', ); expect( after.gitopsRevision?.targets?.[0]?.artifact?.expected?.services?.[0]?.platformDigest, 'the expectation never moved to the running image', ).toBe(pinned); }); test.afterEach(async ({ page }) => { if (blueprintId === null) return; const id = blueprintId; // Asserted rather than swallowed, and the id is only cleared once the delete // is confirmed, so a failed teardown cannot quietly leave a live stack and a // reconciler row behind for every spec that runs after this one. const deleted = await jsonRequest(page, `/api/blueprints/${id}`, { method: 'DELETE' }); expect([200, 204], `deleting the blueprint: ${JSON.stringify(deleted.body)}`).toContain( deleted.status, ); blueprintId = null; // The API reports success even when Compose teardown did not, so the // containers are checked directly rather than trusting the status alone. const remaining = execFileSync( 'docker', [ 'ps', '-a', '--filter', `label=com.docker.compose.project=${blueprintName.toLowerCase()}`, '--format', '{{.Names}}', ], { timeout: DOCKER_TIMEOUT_MS, stdio: 'pipe' }, ) .toString() .trim(); expect(remaining, 'the deleted blueprint left containers behind').toBe(''); }); });