gsadmin/sencho
1
0
Fork 0
mirror of https://github.com/Studio-Saelix/sencho.git synced 2026-10-26 05:30:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(gitops): compare stale artifact probes by platform membership (#2034)

Browse Source 
* fix(gitops): compare stale artifact probes by platform membership

The Direct dispatch probe decided that an accepted generation's executable
identity had moved by comparing artifact set fingerprints. A fingerprint pins
one platform child per service, so it only compares resolutions made on the
same platform. A target whose architecture differs from the one that produced
the frozen expected set resolved a different fingerprint for the same approved
child of a multi-arch tag, recorded stale for a converged target, advanced the
newest-evidence pointer to that row, and surfaced as artifact stale in the
projection and in attention.

Route the probe through the same per-target platform membership the Blueprint
drift path applies, so a target that matches the approved child for its own
platform is no longer stale, a real move of that child still is, and an
expected set with no per-service evidence keeps the fingerprint verdict.

* fix(gitops): keep the stale probe fail-closed on a changed service set

The membership test only walks the expected services and weighs the registry
ones, so it cannot tell an extra or renamed service from an unchanged set, and
with nothing to weigh it answers vacuously. The probe resolves the live stack
rather than the staged candidate, so a stack edited out of band can carry a
different set than the one that was frozen, and that is a moved identity rather
than a matching one.

Ask membership only when both sides hold the same services. A set that grew or
shrank keeps the fingerprint verdict, as does an expected set that recorded no
per-service evidence.

* fix(gitops): tell a swapped service apart from an unchanged set

The set-equality check compared service names, so a service that kept its name
while swapping a local build for a published image still looked unchanged, and
the membership test had no registry service left to weigh. Compare the kind of
each service as well as its name, and cover that flip.

Also fold the same-platform move into the moved-child test, since it shares the
service set and belongs beside it rather than as its own case.
This commit is contained in:
Anso
2026-10-01 08:29:11 -04:00
committed by GitHub
parent bd6a7eb61e
commit d57d2a3ee9
2 changed files with 453 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/__tests__/gitops-artifact-resolve.test.ts
+417 -3
View File
@@ -4,10 +4,17 @@ import { GitOpsStore } from '../services/gitops/store';
import { GitOpsTransitions, type EventEnvelope } from '../services/gitops/transitions';
import {
observeStackRuntimeArtifact,
probeStaleArtifactEvidence,
recordObservedRuntimeArtifactForDeploy,
resolveAndRecordArtifactSet,
} from '../services/gitops/artifactResolve';
import { decodeObservedArtifactIdentity, encodeArtifactEvidenceJson } from '../services/gitops/json';
import {
computeArtifactSetFingerprint,
decodeArtifactEvidenceJson,
decodeObservedArtifactIdentity,
encodeArtifactEvidenceJson,
type ArtifactEvidenceJson,
} from '../services/gitops/json';
import { observationMatchesExpected } from '../services/gitops/artifactIdentity';
import type { GitOpsApplicationRow, GitOpsGenerationRow } from '../services/gitops/types';
import { DEFAULT_PLACEMENT_POLICY, DEFAULT_ROLLOUT_AUTHORIZATION_POLICY } from '../services/gitops/policyComposition';
@@ -59,6 +66,41 @@ vi.mock('../services/DockerController', () => ({
}));
const EXACT_DIGEST = 'sha256:1111111111111111111111111111111111111111111111111111111111111111';
const INDEX_DIGEST = `sha256:${'2'.repeat(64)}`;
const AMD64_DIGEST = `sha256:${'a'.repeat(64)}`;
const ARM64_DIGEST = `sha256:${'b'.repeat(64)}`;
const ARM64_MOVED_DIGEST = `sha256:${'c'.repeat(64)}`;
const AMD64_MOVED_DIGEST = `sha256:${'d'.repeat(64)}`;
/** What a registry answers for a multi-arch tag resolved for one platform. */
function multiArchIndex(
platformLabel: string,
platformDigest: string,
armDigest = ARM64_DIGEST,
indexDigest = INDEX_DIGEST,
) {
return {
ok: true as const,
indexDigest,
platformDigest,
platformLabel,
qualification: 'qualified' as const,
platformVariants: [
{ platform: 'linux/amd64', digest: AMD64_DIGEST },
{ platform: 'linux/arm64', digest: armDigest },
],
};
}
/** One registry service on the default platform. */
function singleServiceModel() {
return {
renderable: true,
services: [
{ name: 'web', declaredImage: 'nginx:latest', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false },
],
};
}
describe('gitops artifact resolve', () => {
let tmpDir: string;
@@ -352,6 +394,376 @@ describe('gitops artifact resolve', () => {
}];
expect(observationMatchesExpected(expected, observed.services ?? [])).toBe(true);
});
it('keeps an arm64 target on a multi-arch expected set out of the stale bucket', async () => {
seedDirectApp({ applicationId: 'app-memb', generationId: 'gen-memb', stackName: 'memb-web', artifactSetId: 'art-memb-seed' });
mockBuildEffectiveServiceModel.mockResolvedValue(singleServiceModel());
// The expected set is resolved on the amd64 node first, so it carries both
// platform children but its fingerprint pins the amd64 one.
mockDockerInfo.mockResolvedValue({ OSType: 'linux', Architecture: 'amd64' });
mockResolveRegistryImageDigestForPlatform.mockResolvedValue(
multiArchIndex('linux/amd64', AMD64_DIGEST),
);
await resolveAndRecordArtifactSet({
stackName: 'memb-web',
nodeId: 1,
applicationId: 'app-memb',
generationId: 'gen-memb',
buildContexts: [],
envelope: envelope('op-resolve-memb'),
});
const expectedId = GitOpsStore.getInstance().getApplication('app-memb')!.artifact_set_id!;
expect(GitOpsStore.getInstance().getArtifactSet(expectedId)?.qualification).toBe('qualified');
// The same tag probed on an arm64 node resolves a different fingerprint for
// the same approved identity, so identity equality alone cannot decide this.
mockDockerInfo.mockResolvedValue({ OSType: 'linux', Architecture: 'arm64' });
mockResolveRegistryImageDigestForPlatform.mockResolvedValue(
multiArchIndex('linux/arm64', ARM64_DIGEST),
);
await probeStaleArtifactEvidence({
stackName: 'memb-web',
nodeId: 1,
applicationId: 'app-memb',
generationId: 'gen-memb',
buildContexts: [],
envelope: envelope('op-probe-memb'),
});
const app = GitOpsStore.getInstance().getApplication('app-memb')!;
expect(app.latest_artifact_set_id).toBe(expectedId);
expect(GitOpsStore.getInstance().getArtifactSet(app.latest_artifact_set_id!)?.qualification).toBe('qualified');
});
it('records stale when the platform child actually moved', async () => {
seedDirectApp({ applicationId: 'app-move', generationId: 'gen-move', stackName: 'move-web', artifactSetId: 'art-move-seed' });
mockBuildEffectiveServiceModel.mockResolvedValue(singleServiceModel());
mockDockerInfo.mockResolvedValue({ OSType: 'linux', Architecture: 'amd64' });
mockResolveRegistryImageDigestForPlatform.mockResolvedValue(
multiArchIndex('linux/amd64', AMD64_DIGEST),
);
await resolveAndRecordArtifactSet({
stackName: 'move-web',
nodeId: 1,
applicationId: 'app-move',
generationId: 'gen-move',
buildContexts: [],
envelope: envelope('op-resolve-move'),
});
const expectedId = GitOpsStore.getInstance().getApplication('app-move')!.artifact_set_id!;
// The arm64 child is now something the expected set never named. Membership
// explains a differing fingerprint only when the child it approves is the
// one being observed, so this must still be recorded as moved.
const movedIndex = `sha256:${'e'.repeat(64)}`;
mockDockerInfo.mockResolvedValue({ OSType: 'linux', Architecture: 'arm64' });
mockResolveRegistryImageDigestForPlatform.mockResolvedValue(
multiArchIndex('linux/arm64', ARM64_MOVED_DIGEST, ARM64_MOVED_DIGEST, movedIndex),
);
await probeStaleArtifactEvidence({
stackName: 'move-web',
nodeId: 1,
applicationId: 'app-move',
generationId: 'gen-move',
buildContexts: [],
envelope: envelope('op-probe-move'),
});
const app = GitOpsStore.getInstance().getApplication('app-move')!;
expect(app.latest_artifact_set_id).not.toBe(expectedId);
const latest = GitOpsStore.getInstance().getArtifactSet(app.latest_artifact_set_id!)!;
expect(latest.qualification).toBe('stale');
expect(decodeArtifactEvidenceJson(latest.evidence_json).kind).toBe('stale');
const expectedRow = GitOpsStore.getInstance().getArtifactSet(expectedId)!;
expect(decodeArtifactEvidenceJson(latest.evidence_json).services?.[0]?.platformDigest)
.toBe(ARM64_MOVED_DIGEST);
expect(expectedRow.qualification).toBe('qualified');
// The same node, same platform, different child. Membership must not become
// an excuse for a move on the platform the target actually runs.
const amdMovedIndex = `sha256:${'f'.repeat(64)}`;
mockDockerInfo.mockResolvedValue({ OSType: 'linux', Architecture: 'amd64' });
mockResolveRegistryImageDigestForPlatform.mockResolvedValue({
ok: true,
indexDigest: amdMovedIndex,
platformDigest: AMD64_MOVED_DIGEST,
platformLabel: 'linux/amd64',
qualification: 'qualified',
platformVariants: [{ platform: 'linux/amd64', digest: AMD64_MOVED_DIGEST }],
});
await probeStaleArtifactEvidence({
stackName: 'move-web',
nodeId: 1,
applicationId: 'app-move',
generationId: 'gen-move',
buildContexts: [],
envelope: envelope('op-probe-move-same-platform'),
});
const afterSamePlatform = GitOpsStore.getInstance().getApplication('app-move')!;
const samePlatformRow = GitOpsStore
.getInstance()
.getArtifactSet(afterSamePlatform.latest_artifact_set_id!)!;
expect(samePlatformRow.qualification).toBe('stale');
expect(decodeArtifactEvidenceJson(samePlatformRow.evidence_json).services?.[0]?.platformDigest)
.toBe(AMD64_MOVED_DIGEST);
});
it('records stale when the live stack gained a service the expected set never froze', async () => {
seedDirectApp({ applicationId: 'app-grew', generationId: 'gen-grew', stackName: 'grew-web', artifactSetId: 'art-grew-seed' });
mockBuildEffectiveServiceModel.mockResolvedValue(singleServiceModel());
mockResolveRegistryImageDigestForPlatform.mockResolvedValue(
multiArchIndex('linux/amd64', AMD64_DIGEST),
);
await resolveAndRecordArtifactSet({
stackName: 'grew-web',
nodeId: 1,
applicationId: 'app-grew',
generationId: 'gen-grew',
buildContexts: [],
envelope: envelope('op-resolve-grew'),
});
const expectedId = GitOpsStore.getInstance().getApplication('app-grew')!.artifact_set_id!;
// The stack was edited out of band after acceptance, so the probe renders
// one more service than the frozen set names. `web` still matches, which is
// exactly the case a digest-only membership test would wave through.
mockBuildEffectiveServiceModel.mockResolvedValue({
renderable: true,
services: [
...singleServiceModel().services,
{ name: 'db', declaredImage: 'postgres:latest', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false },
],
});
mockResolveRegistryImageDigestForPlatform.mockImplementation(async (_registry: string, repo: string) => (
repo.includes('postgres')
? {
ok: true,
indexDigest: INDEX_DIGEST,
platformDigest: `sha256:${'9'.repeat(64)}`,
platformLabel: 'linux/amd64',
qualification: 'qualified',
platformVariants: [{ platform: 'linux/amd64', digest: `sha256:${'9'.repeat(64)}` }],
}
: multiArchIndex('linux/amd64', AMD64_DIGEST)
));
await probeStaleArtifactEvidence({
stackName: 'grew-web',
nodeId: 1,
applicationId: 'app-grew',
generationId: 'gen-grew',
buildContexts: [],
envelope: envelope('op-probe-grew'),
});
const app = GitOpsStore.getInstance().getApplication('app-grew')!;
expect(app.latest_artifact_set_id).not.toBe(expectedId);
expect(GitOpsStore.getInstance().getArtifactSet(app.latest_artifact_set_id!)?.qualification).toBe('stale');
});
it('records stale when a build-only expected set no longer describes the stack', async () => {
const buildOnly = {
kind: 'local_build_unverified' as const,
identity: computeArtifactSetFingerprint([{
serviceName: 'worker',
authoredRef: null,
source: 'build' as const,
platform: 'linux/amd64',
indexDigest: null,
platformDigest: null,
buildContextFingerprint: `sha256:${'7'.repeat(64)}`,
producedImageId: null,
failureClass: null,
resolvedAt: 1,
}]),
services: [{
serviceName: 'worker',
authoredRef: null,
source: 'build' as const,
platform: 'linux/amd64',
indexDigest: null,
platformDigest: null,
buildContextFingerprint: `sha256:${'7'.repeat(64)}`,
producedImageId: null,
failureClass: null,
resolvedAt: 1,
}],
};
seedDirectApp({
applicationId: 'app-build',
generationId: 'gen-build',
stackName: 'build-web',
artifactSetId: 'art-build-v1',
evidence: buildOnly,
});
// Membership weighs registry services only, so with nothing to weigh it
// answers vacuously. Only the service set can still tell these apart.
mockBuildEffectiveServiceModel.mockResolvedValue({
renderable: true,
services: [
{ name: 'worker', declaredImage: null, hasBuild: true, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false },
],
});
// Same service, same unverified fingerprint: nothing moved.
await probeStaleArtifactEvidence({
stackName: 'build-web',
nodeId: 1,
applicationId: 'app-build',
generationId: 'gen-build',
buildContexts: [],
envelope: envelope('op-probe-build-same'),
});
expect(GitOpsStore.getInstance().getApplication('app-build')!.latest_artifact_set_id).toBe('art-build-v1');
// The built service was renamed out of band. Both sides are unverified, so
// only the service set distinguishes them.
mockBuildEffectiveServiceModel.mockResolvedValue({
renderable: true,
services: [
{ name: 'batch', declaredImage: null, hasBuild: true, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false },
],
});
await probeStaleArtifactEvidence({
stackName: 'build-web',
nodeId: 1,
applicationId: 'app-build',
generationId: 'gen-build',
buildContexts: [],
envelope: envelope('op-probe-build-renamed'),
});
const app = GitOpsStore.getInstance().getApplication('app-build')!;
expect(app.latest_artifact_set_id).not.toBe('art-build-v1');
expect(GitOpsStore.getInstance().getArtifactSet(app.latest_artifact_set_id!)?.qualification).toBe('stale');
});
it('records stale when a built service became a published image', async () => {
seedDirectApp({
applicationId: 'app-flip',
generationId: 'gen-flip',
stackName: 'flip-web',
artifactSetId: 'art-flip-v1',
evidence: {
kind: 'local_build_unverified',
identity: computeArtifactSetFingerprint([{
serviceName: 'worker',
authoredRef: null,
source: 'build',
platform: 'linux/amd64',
indexDigest: null,
platformDigest: null,
buildContextFingerprint: null,
producedImageId: null,
failureClass: null,
resolvedAt: 1,
}]),
services: [{
serviceName: 'worker',
authoredRef: null,
source: 'build',
platform: 'linux/amd64',
indexDigest: null,
platformDigest: null,
buildContextFingerprint: null,
producedImageId: null,
failureClass: null,
resolvedAt: 1,
}],
},
});
// Same name, now a published image. The name set is unchanged and the
// matcher has no registry service to weigh, so nothing but the kind of the
// service can tell these apart.
mockBuildEffectiveServiceModel.mockResolvedValue({
renderable: true,
services: [
{ name: 'worker', declaredImage: 'ghcr.io/org/worker:1', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false },
],
});
mockResolveRegistryImageDigestForPlatform.mockResolvedValue({
ok: true,
indexDigest: INDEX_DIGEST,
platformDigest: AMD64_DIGEST,
platformLabel: 'linux/amd64',
qualification: 'qualified',
platformVariants: [{ platform: 'linux/amd64', digest: AMD64_DIGEST }],
});
await probeStaleArtifactEvidence({
stackName: 'flip-web',
nodeId: 1,
applicationId: 'app-flip',
generationId: 'gen-flip',
buildContexts: [],
envelope: envelope('op-probe-flip'),
});
const app = GitOpsStore.getInstance().getApplication('app-flip')!;
expect(app.latest_artifact_set_id).not.toBe('art-flip-v1');
expect(GitOpsStore.getInstance().getArtifactSet(app.latest_artifact_set_id!)?.qualification).toBe('stale');
});
it('falls back to the identity string for an expected set with no per-service evidence', async () => {
const legacyServices = [{
serviceName: 'web',
authoredRef: 'nginx:latest',
source: 'registry' as const,
platform: 'linux/amd64',
indexDigest: EXACT_DIGEST,
platformDigest: EXACT_DIGEST,
buildContextFingerprint: null,
producedImageId: null,
failureClass: null,
resolvedAt: 1,
}];
seedDirectApp({
applicationId: 'app-legacy',
generationId: 'gen-legacy',
stackName: 'legacy-web',
artifactSetId: 'art-legacy-v1',
evidence: { kind: 'exact', identity: computeArtifactSetFingerprint(legacyServices) },
});
mockBuildEffectiveServiceModel.mockResolvedValue(singleServiceModel());
mockResolveRegistryImageDigestForPlatform.mockResolvedValue({
ok: true,
indexDigest: EXACT_DIGEST,
platformDigest: EXACT_DIGEST,
platformLabel: 'linux/amd64',
qualification: 'exact',
});
await probeStaleArtifactEvidence({
stackName: 'legacy-web',
nodeId: 1,
applicationId: 'app-legacy',
generationId: 'gen-legacy',
buildContexts: [],
envelope: envelope('op-probe-legacy-same'),
});
expect(GitOpsStore.getInstance().getApplication('app-legacy')!.latest_artifact_set_id).toBe('art-legacy-v1');
mockResolveRegistryImageDigestForPlatform.mockResolvedValue({
ok: true,
indexDigest: AMD64_MOVED_DIGEST,
platformDigest: AMD64_MOVED_DIGEST,
platformLabel: 'linux/amd64',
qualification: 'exact',
});
await probeStaleArtifactEvidence({
stackName: 'legacy-web',
nodeId: 1,
applicationId: 'app-legacy',
generationId: 'gen-legacy',
buildContexts: [],
envelope: envelope('op-probe-legacy-moved'),
});
const app = GitOpsStore.getInstance().getApplication('app-legacy')!;
expect(app.latest_artifact_set_id).not.toBe('art-legacy-v1');
expect(GitOpsStore.getInstance().getArtifactSet(app.latest_artifact_set_id!)?.qualification).toBe('stale');
});
});
function envelope(operationId: string): EventEnvelope {
@@ -363,6 +775,8 @@ function seedDirectApp(ids: {
generationId: string;
stackName: string;
artifactSetId: string;
/** Recorded verbatim; defaults to the unresolved placeholder every seed uses. */
evidence?: ArtifactEvidenceJson;
}): void {
const store = GitOpsStore.getInstance();
const tx = GitOpsTransitions.getInstance();
@@ -377,8 +791,8 @@ function seedDirectApp(ids: {
generation_id: ids.generationId,
evidence_version: 1,
authoritative: 0,
qualification: 'unresolved',
evidence_json: encodeArtifactEvidenceJson({ kind: 'unresolved' }),
qualification: (ids.evidence?.kind ?? 'unresolved'),
evidence_json: encodeArtifactEvidenceJson(ids.evidence ?? { kind: 'unresolved' }),
created_at: 1,
});
}
backend/src/services/gitops/artifactResolve.ts
+36
View File
@@ -27,6 +27,7 @@ import {
type ObservedArtifactIdentity,
type ServiceArtifactEvidence,
} from './json';
import { observationMatchesExpected } from './artifactIdentity';
import { GitOpsStore } from './store';
import { GitOpsTransitions, type EventEnvelope } from './transitions';
import { newGitOpsId } from './directApplication';
@@ -519,6 +520,27 @@ export async function resolveAndRecordArtifactSet(args: {
}
}
/**
* Whether both sides describe the same set of services, by name and by kind.
*
* `observationMatchesExpected` only walks the expected services and weighs the
* registry ones, so it cannot tell an extra service, a rename, or a service that
* swapped a local build for a published image from an unchanged set. The probe
* resolves the live stack rather than the staged candidate, so a stack edited out
* of band can carry a different set than the one that was frozen, and that is a
* moved identity rather than a matching one.
*/
function sameServiceSet(
expected: readonly ServiceArtifactEvidence[],
observed: readonly ServiceArtifactEvidence[],
): boolean {
const identityOf = (service: ServiceArtifactEvidence): string =>
`${service.serviceName}:${service.source}`;
const expectedIdentities = new Set(expected.map(identityOf));
if (expectedIdentities.size !== expected.length) return false;
return observed.every((service) => expectedIdentities.has(identityOf(service)));
}
export async function probeStaleArtifactEvidence(args: {
stackName: string;
nodeId: number;
@@ -540,6 +562,7 @@ export async function probeStaleArtifactEvidence(args: {
}
const expectedIdentity = 'identity' in expectedEvidence ? expectedEvidence.identity : null;
if (!expectedIdentity) return;
const expectedServices = 'services' in expectedEvidence ? expectedEvidence.services : undefined;
try {
const resolvedAt = args.envelope.at;
@@ -552,6 +575,19 @@ export async function probeStaleArtifactEvidence(args: {
const latestIdentity = 'identity' in evidence ? evidence.identity : null;
if (!latestIdentity || latestIdentity === expectedIdentity) return;
if (qualification === 'unresolved' || qualification === 'unavailable') return;
// The set fingerprint pins one platform child per service, so a target on a
// different architecture resolves a different fingerprint for the same
// multi-arch tag, and the fingerprint alone would report a moved identity.
// Membership is the test the Blueprint drift path applies: this platform's
// child is one of the expected set's own variants, so nothing moved. It
// answers only the digest question, so it is asked when both sides hold the
// same services; a set that grew, shrank, or changed what a service is, and
// an expected set that recorded no per-service evidence, all keep the
// fingerprint verdict.
if (expectedServices?.length
&& evidence.services?.length
&& sameServiceSet(expectedServices, evidence.services)
&& observationMatchesExpected(expectedServices, evidence.services)) return;
recordResolvedEvidence({
applicationId: args.applicationId,
generationId: args.generationId,