diff --git a/backend/src/__tests__/gitops-artifact-resolve.test.ts b/backend/src/__tests__/gitops-artifact-resolve.test.ts index 308d4d76..87dc7563 100644 --- a/backend/src/__tests__/gitops-artifact-resolve.test.ts +++ b/backend/src/__tests__/gitops-artifact-resolve.test.ts @@ -4,10 +4,17 @@ import { GitOpsStore } from '../services/gitops/store'; import { GitOpsTransitions, type EventEnvelope } from '../services/gitops/transitions'; import { observeStackRuntimeArtifact, + probeStaleArtifactEvidence, recordObservedRuntimeArtifactForDeploy, resolveAndRecordArtifactSet, } from '../services/gitops/artifactResolve'; -import { decodeObservedArtifactIdentity, encodeArtifactEvidenceJson } from '../services/gitops/json'; +import { + computeArtifactSetFingerprint, + decodeArtifactEvidenceJson, + decodeObservedArtifactIdentity, + encodeArtifactEvidenceJson, + type ArtifactEvidenceJson, +} from '../services/gitops/json'; import { observationMatchesExpected } from '../services/gitops/artifactIdentity'; import type { GitOpsApplicationRow, GitOpsGenerationRow } from '../services/gitops/types'; import { DEFAULT_PLACEMENT_POLICY, DEFAULT_ROLLOUT_AUTHORIZATION_POLICY } from '../services/gitops/policyComposition'; @@ -59,6 +66,41 @@ vi.mock('../services/DockerController', () => ({ })); const EXACT_DIGEST = 'sha256:1111111111111111111111111111111111111111111111111111111111111111'; +const INDEX_DIGEST = `sha256:${'2'.repeat(64)}`; +const AMD64_DIGEST = `sha256:${'a'.repeat(64)}`; +const ARM64_DIGEST = `sha256:${'b'.repeat(64)}`; +const ARM64_MOVED_DIGEST = `sha256:${'c'.repeat(64)}`; +const AMD64_MOVED_DIGEST = `sha256:${'d'.repeat(64)}`; + +/** What a registry answers for a multi-arch tag resolved for one platform. */ +function multiArchIndex( + platformLabel: string, + platformDigest: string, + armDigest = ARM64_DIGEST, + indexDigest = INDEX_DIGEST, +) { + return { + ok: true as const, + indexDigest, + platformDigest, + platformLabel, + qualification: 'qualified' as const, + platformVariants: [ + { platform: 'linux/amd64', digest: AMD64_DIGEST }, + { platform: 'linux/arm64', digest: armDigest }, + ], + }; +} + +/** One registry service on the default platform. */ +function singleServiceModel() { + return { + renderable: true, + services: [ + { name: 'web', declaredImage: 'nginx:latest', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false }, + ], + }; +} describe('gitops artifact resolve', () => { let tmpDir: string; @@ -352,6 +394,376 @@ describe('gitops artifact resolve', () => { }]; expect(observationMatchesExpected(expected, observed.services ?? [])).toBe(true); }); + + it('keeps an arm64 target on a multi-arch expected set out of the stale bucket', async () => { + seedDirectApp({ applicationId: 'app-memb', generationId: 'gen-memb', stackName: 'memb-web', artifactSetId: 'art-memb-seed' }); + mockBuildEffectiveServiceModel.mockResolvedValue(singleServiceModel()); + + // The expected set is resolved on the amd64 node first, so it carries both + // platform children but its fingerprint pins the amd64 one. + mockDockerInfo.mockResolvedValue({ OSType: 'linux', Architecture: 'amd64' }); + mockResolveRegistryImageDigestForPlatform.mockResolvedValue( + multiArchIndex('linux/amd64', AMD64_DIGEST), + ); + await resolveAndRecordArtifactSet({ + stackName: 'memb-web', + nodeId: 1, + applicationId: 'app-memb', + generationId: 'gen-memb', + buildContexts: [], + envelope: envelope('op-resolve-memb'), + }); + const expectedId = GitOpsStore.getInstance().getApplication('app-memb')!.artifact_set_id!; + expect(GitOpsStore.getInstance().getArtifactSet(expectedId)?.qualification).toBe('qualified'); + + // The same tag probed on an arm64 node resolves a different fingerprint for + // the same approved identity, so identity equality alone cannot decide this. + mockDockerInfo.mockResolvedValue({ OSType: 'linux', Architecture: 'arm64' }); + mockResolveRegistryImageDigestForPlatform.mockResolvedValue( + multiArchIndex('linux/arm64', ARM64_DIGEST), + ); + await probeStaleArtifactEvidence({ + stackName: 'memb-web', + nodeId: 1, + applicationId: 'app-memb', + generationId: 'gen-memb', + buildContexts: [], + envelope: envelope('op-probe-memb'), + }); + + const app = GitOpsStore.getInstance().getApplication('app-memb')!; + expect(app.latest_artifact_set_id).toBe(expectedId); + expect(GitOpsStore.getInstance().getArtifactSet(app.latest_artifact_set_id!)?.qualification).toBe('qualified'); + }); + + it('records stale when the platform child actually moved', async () => { + seedDirectApp({ applicationId: 'app-move', generationId: 'gen-move', stackName: 'move-web', artifactSetId: 'art-move-seed' }); + mockBuildEffectiveServiceModel.mockResolvedValue(singleServiceModel()); + + mockDockerInfo.mockResolvedValue({ OSType: 'linux', Architecture: 'amd64' }); + mockResolveRegistryImageDigestForPlatform.mockResolvedValue( + multiArchIndex('linux/amd64', AMD64_DIGEST), + ); + await resolveAndRecordArtifactSet({ + stackName: 'move-web', + nodeId: 1, + applicationId: 'app-move', + generationId: 'gen-move', + buildContexts: [], + envelope: envelope('op-resolve-move'), + }); + const expectedId = GitOpsStore.getInstance().getApplication('app-move')!.artifact_set_id!; + + // The arm64 child is now something the expected set never named. Membership + // explains a differing fingerprint only when the child it approves is the + // one being observed, so this must still be recorded as moved. + const movedIndex = `sha256:${'e'.repeat(64)}`; + mockDockerInfo.mockResolvedValue({ OSType: 'linux', Architecture: 'arm64' }); + mockResolveRegistryImageDigestForPlatform.mockResolvedValue( + multiArchIndex('linux/arm64', ARM64_MOVED_DIGEST, ARM64_MOVED_DIGEST, movedIndex), + ); + await probeStaleArtifactEvidence({ + stackName: 'move-web', + nodeId: 1, + applicationId: 'app-move', + generationId: 'gen-move', + buildContexts: [], + envelope: envelope('op-probe-move'), + }); + + const app = GitOpsStore.getInstance().getApplication('app-move')!; + expect(app.latest_artifact_set_id).not.toBe(expectedId); + const latest = GitOpsStore.getInstance().getArtifactSet(app.latest_artifact_set_id!)!; + expect(latest.qualification).toBe('stale'); + expect(decodeArtifactEvidenceJson(latest.evidence_json).kind).toBe('stale'); + const expectedRow = GitOpsStore.getInstance().getArtifactSet(expectedId)!; + expect(decodeArtifactEvidenceJson(latest.evidence_json).services?.[0]?.platformDigest) + .toBe(ARM64_MOVED_DIGEST); + expect(expectedRow.qualification).toBe('qualified'); + + // The same node, same platform, different child. Membership must not become + // an excuse for a move on the platform the target actually runs. + const amdMovedIndex = `sha256:${'f'.repeat(64)}`; + mockDockerInfo.mockResolvedValue({ OSType: 'linux', Architecture: 'amd64' }); + mockResolveRegistryImageDigestForPlatform.mockResolvedValue({ + ok: true, + indexDigest: amdMovedIndex, + platformDigest: AMD64_MOVED_DIGEST, + platformLabel: 'linux/amd64', + qualification: 'qualified', + platformVariants: [{ platform: 'linux/amd64', digest: AMD64_MOVED_DIGEST }], + }); + await probeStaleArtifactEvidence({ + stackName: 'move-web', + nodeId: 1, + applicationId: 'app-move', + generationId: 'gen-move', + buildContexts: [], + envelope: envelope('op-probe-move-same-platform'), + }); + + const afterSamePlatform = GitOpsStore.getInstance().getApplication('app-move')!; + const samePlatformRow = GitOpsStore + .getInstance() + .getArtifactSet(afterSamePlatform.latest_artifact_set_id!)!; + expect(samePlatformRow.qualification).toBe('stale'); + expect(decodeArtifactEvidenceJson(samePlatformRow.evidence_json).services?.[0]?.platformDigest) + .toBe(AMD64_MOVED_DIGEST); + }); + + it('records stale when the live stack gained a service the expected set never froze', async () => { + seedDirectApp({ applicationId: 'app-grew', generationId: 'gen-grew', stackName: 'grew-web', artifactSetId: 'art-grew-seed' }); + mockBuildEffectiveServiceModel.mockResolvedValue(singleServiceModel()); + + mockResolveRegistryImageDigestForPlatform.mockResolvedValue( + multiArchIndex('linux/amd64', AMD64_DIGEST), + ); + await resolveAndRecordArtifactSet({ + stackName: 'grew-web', + nodeId: 1, + applicationId: 'app-grew', + generationId: 'gen-grew', + buildContexts: [], + envelope: envelope('op-resolve-grew'), + }); + const expectedId = GitOpsStore.getInstance().getApplication('app-grew')!.artifact_set_id!; + + // The stack was edited out of band after acceptance, so the probe renders + // one more service than the frozen set names. `web` still matches, which is + // exactly the case a digest-only membership test would wave through. + mockBuildEffectiveServiceModel.mockResolvedValue({ + renderable: true, + services: [ + ...singleServiceModel().services, + { name: 'db', declaredImage: 'postgres:latest', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false }, + ], + }); + mockResolveRegistryImageDigestForPlatform.mockImplementation(async (_registry: string, repo: string) => ( + repo.includes('postgres') + ? { + ok: true, + indexDigest: INDEX_DIGEST, + platformDigest: `sha256:${'9'.repeat(64)}`, + platformLabel: 'linux/amd64', + qualification: 'qualified', + platformVariants: [{ platform: 'linux/amd64', digest: `sha256:${'9'.repeat(64)}` }], + } + : multiArchIndex('linux/amd64', AMD64_DIGEST) + )); + await probeStaleArtifactEvidence({ + stackName: 'grew-web', + nodeId: 1, + applicationId: 'app-grew', + generationId: 'gen-grew', + buildContexts: [], + envelope: envelope('op-probe-grew'), + }); + + const app = GitOpsStore.getInstance().getApplication('app-grew')!; + expect(app.latest_artifact_set_id).not.toBe(expectedId); + expect(GitOpsStore.getInstance().getArtifactSet(app.latest_artifact_set_id!)?.qualification).toBe('stale'); + }); + + it('records stale when a build-only expected set no longer describes the stack', async () => { + const buildOnly = { + kind: 'local_build_unverified' as const, + identity: computeArtifactSetFingerprint([{ + serviceName: 'worker', + authoredRef: null, + source: 'build' as const, + platform: 'linux/amd64', + indexDigest: null, + platformDigest: null, + buildContextFingerprint: `sha256:${'7'.repeat(64)}`, + producedImageId: null, + failureClass: null, + resolvedAt: 1, + }]), + services: [{ + serviceName: 'worker', + authoredRef: null, + source: 'build' as const, + platform: 'linux/amd64', + indexDigest: null, + platformDigest: null, + buildContextFingerprint: `sha256:${'7'.repeat(64)}`, + producedImageId: null, + failureClass: null, + resolvedAt: 1, + }], + }; + seedDirectApp({ + applicationId: 'app-build', + generationId: 'gen-build', + stackName: 'build-web', + artifactSetId: 'art-build-v1', + evidence: buildOnly, + }); + // Membership weighs registry services only, so with nothing to weigh it + // answers vacuously. Only the service set can still tell these apart. + mockBuildEffectiveServiceModel.mockResolvedValue({ + renderable: true, + services: [ + { name: 'worker', declaredImage: null, hasBuild: true, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false }, + ], + }); + + // Same service, same unverified fingerprint: nothing moved. + await probeStaleArtifactEvidence({ + stackName: 'build-web', + nodeId: 1, + applicationId: 'app-build', + generationId: 'gen-build', + buildContexts: [], + envelope: envelope('op-probe-build-same'), + }); + expect(GitOpsStore.getInstance().getApplication('app-build')!.latest_artifact_set_id).toBe('art-build-v1'); + + // The built service was renamed out of band. Both sides are unverified, so + // only the service set distinguishes them. + mockBuildEffectiveServiceModel.mockResolvedValue({ + renderable: true, + services: [ + { name: 'batch', declaredImage: null, hasBuild: true, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false }, + ], + }); + await probeStaleArtifactEvidence({ + stackName: 'build-web', + nodeId: 1, + applicationId: 'app-build', + generationId: 'gen-build', + buildContexts: [], + envelope: envelope('op-probe-build-renamed'), + }); + const app = GitOpsStore.getInstance().getApplication('app-build')!; + expect(app.latest_artifact_set_id).not.toBe('art-build-v1'); + expect(GitOpsStore.getInstance().getArtifactSet(app.latest_artifact_set_id!)?.qualification).toBe('stale'); + }); + + it('records stale when a built service became a published image', async () => { + seedDirectApp({ + applicationId: 'app-flip', + generationId: 'gen-flip', + stackName: 'flip-web', + artifactSetId: 'art-flip-v1', + evidence: { + kind: 'local_build_unverified', + identity: computeArtifactSetFingerprint([{ + serviceName: 'worker', + authoredRef: null, + source: 'build', + platform: 'linux/amd64', + indexDigest: null, + platformDigest: null, + buildContextFingerprint: null, + producedImageId: null, + failureClass: null, + resolvedAt: 1, + }]), + services: [{ + serviceName: 'worker', + authoredRef: null, + source: 'build', + platform: 'linux/amd64', + indexDigest: null, + platformDigest: null, + buildContextFingerprint: null, + producedImageId: null, + failureClass: null, + resolvedAt: 1, + }], + }, + }); + // Same name, now a published image. The name set is unchanged and the + // matcher has no registry service to weigh, so nothing but the kind of the + // service can tell these apart. + mockBuildEffectiveServiceModel.mockResolvedValue({ + renderable: true, + services: [ + { name: 'worker', declaredImage: 'ghcr.io/org/worker:1', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false }, + ], + }); + mockResolveRegistryImageDigestForPlatform.mockResolvedValue({ + ok: true, + indexDigest: INDEX_DIGEST, + platformDigest: AMD64_DIGEST, + platformLabel: 'linux/amd64', + qualification: 'qualified', + platformVariants: [{ platform: 'linux/amd64', digest: AMD64_DIGEST }], + }); + + await probeStaleArtifactEvidence({ + stackName: 'flip-web', + nodeId: 1, + applicationId: 'app-flip', + generationId: 'gen-flip', + buildContexts: [], + envelope: envelope('op-probe-flip'), + }); + + const app = GitOpsStore.getInstance().getApplication('app-flip')!; + expect(app.latest_artifact_set_id).not.toBe('art-flip-v1'); + expect(GitOpsStore.getInstance().getArtifactSet(app.latest_artifact_set_id!)?.qualification).toBe('stale'); + }); + + it('falls back to the identity string for an expected set with no per-service evidence', async () => { + const legacyServices = [{ + serviceName: 'web', + authoredRef: 'nginx:latest', + source: 'registry' as const, + platform: 'linux/amd64', + indexDigest: EXACT_DIGEST, + platformDigest: EXACT_DIGEST, + buildContextFingerprint: null, + producedImageId: null, + failureClass: null, + resolvedAt: 1, + }]; + seedDirectApp({ + applicationId: 'app-legacy', + generationId: 'gen-legacy', + stackName: 'legacy-web', + artifactSetId: 'art-legacy-v1', + evidence: { kind: 'exact', identity: computeArtifactSetFingerprint(legacyServices) }, + }); + mockBuildEffectiveServiceModel.mockResolvedValue(singleServiceModel()); + mockResolveRegistryImageDigestForPlatform.mockResolvedValue({ + ok: true, + indexDigest: EXACT_DIGEST, + platformDigest: EXACT_DIGEST, + platformLabel: 'linux/amd64', + qualification: 'exact', + }); + + await probeStaleArtifactEvidence({ + stackName: 'legacy-web', + nodeId: 1, + applicationId: 'app-legacy', + generationId: 'gen-legacy', + buildContexts: [], + envelope: envelope('op-probe-legacy-same'), + }); + expect(GitOpsStore.getInstance().getApplication('app-legacy')!.latest_artifact_set_id).toBe('art-legacy-v1'); + + mockResolveRegistryImageDigestForPlatform.mockResolvedValue({ + ok: true, + indexDigest: AMD64_MOVED_DIGEST, + platformDigest: AMD64_MOVED_DIGEST, + platformLabel: 'linux/amd64', + qualification: 'exact', + }); + await probeStaleArtifactEvidence({ + stackName: 'legacy-web', + nodeId: 1, + applicationId: 'app-legacy', + generationId: 'gen-legacy', + buildContexts: [], + envelope: envelope('op-probe-legacy-moved'), + }); + const app = GitOpsStore.getInstance().getApplication('app-legacy')!; + expect(app.latest_artifact_set_id).not.toBe('art-legacy-v1'); + expect(GitOpsStore.getInstance().getArtifactSet(app.latest_artifact_set_id!)?.qualification).toBe('stale'); + }); }); function envelope(operationId: string): EventEnvelope { @@ -363,6 +775,8 @@ function seedDirectApp(ids: { generationId: string; stackName: string; artifactSetId: string; + /** Recorded verbatim; defaults to the unresolved placeholder every seed uses. */ + evidence?: ArtifactEvidenceJson; }): void { const store = GitOpsStore.getInstance(); const tx = GitOpsTransitions.getInstance(); @@ -377,8 +791,8 @@ function seedDirectApp(ids: { generation_id: ids.generationId, evidence_version: 1, authoritative: 0, - qualification: 'unresolved', - evidence_json: encodeArtifactEvidenceJson({ kind: 'unresolved' }), + qualification: (ids.evidence?.kind ?? 'unresolved'), + evidence_json: encodeArtifactEvidenceJson(ids.evidence ?? { kind: 'unresolved' }), created_at: 1, }); } diff --git a/backend/src/services/gitops/artifactResolve.ts b/backend/src/services/gitops/artifactResolve.ts index 72a609c9..a12090a8 100644 --- a/backend/src/services/gitops/artifactResolve.ts +++ b/backend/src/services/gitops/artifactResolve.ts @@ -27,6 +27,7 @@ import { type ObservedArtifactIdentity, type ServiceArtifactEvidence, } from './json'; +import { observationMatchesExpected } from './artifactIdentity'; import { GitOpsStore } from './store'; import { GitOpsTransitions, type EventEnvelope } from './transitions'; import { newGitOpsId } from './directApplication'; @@ -519,6 +520,27 @@ export async function resolveAndRecordArtifactSet(args: { } } +/** + * Whether both sides describe the same set of services, by name and by kind. + * + * `observationMatchesExpected` only walks the expected services and weighs the + * registry ones, so it cannot tell an extra service, a rename, or a service that + * swapped a local build for a published image from an unchanged set. The probe + * resolves the live stack rather than the staged candidate, so a stack edited out + * of band can carry a different set than the one that was frozen, and that is a + * moved identity rather than a matching one. + */ +function sameServiceSet( + expected: readonly ServiceArtifactEvidence[], + observed: readonly ServiceArtifactEvidence[], +): boolean { + const identityOf = (service: ServiceArtifactEvidence): string => + `${service.serviceName}:${service.source}`; + const expectedIdentities = new Set(expected.map(identityOf)); + if (expectedIdentities.size !== expected.length) return false; + return observed.every((service) => expectedIdentities.has(identityOf(service))); +} + export async function probeStaleArtifactEvidence(args: { stackName: string; nodeId: number; @@ -540,6 +562,7 @@ export async function probeStaleArtifactEvidence(args: { } const expectedIdentity = 'identity' in expectedEvidence ? expectedEvidence.identity : null; if (!expectedIdentity) return; + const expectedServices = 'services' in expectedEvidence ? expectedEvidence.services : undefined; try { const resolvedAt = args.envelope.at; @@ -552,6 +575,19 @@ export async function probeStaleArtifactEvidence(args: { const latestIdentity = 'identity' in evidence ? evidence.identity : null; if (!latestIdentity || latestIdentity === expectedIdentity) return; if (qualification === 'unresolved' || qualification === 'unavailable') return; + // The set fingerprint pins one platform child per service, so a target on a + // different architecture resolves a different fingerprint for the same + // multi-arch tag, and the fingerprint alone would report a moved identity. + // Membership is the test the Blueprint drift path applies: this platform's + // child is one of the expected set's own variants, so nothing moved. It + // answers only the digest question, so it is asked when both sides hold the + // same services; a set that grew, shrank, or changed what a service is, and + // an expected set that recorded no per-service evidence, all keep the + // fingerprint verdict. + if (expectedServices?.length + && evidence.services?.length + && sameServiceSet(expectedServices, evidence.services) + && observationMatchesExpected(expectedServices, evidence.services)) return; recordResolvedEvidence({ applicationId: args.applicationId, generationId: args.generationId,