fix(gitops): anchor a drift comparison to the generation being served

A recorded acknowledgement rebinds a target to the accepted generation's
approved set and leaves the stored observation as the previous generation
recorded it. The two are then facts about different generations, and the
observation says nothing about the set it is being compared against.

Reading an acknowledgement rebinds the target to the new generation's set
while still deploying the previous one, and a drift recorded on the next
tick through a cause that returns before re-observing, a stopped container
or a marker or revision mismatch, leaves that observation untouched. The
comparison then reports a digest divergence that was never measured, and it
reports it as confirmed: a class, a reason, and per-service digests that all
read as real evidence.

The two artifact-identity statuses cannot reach that state, because
deriveRuntime returns applied_not_deployed before it compares identities
whenever desired and deployed disagree. The reconciler observation is
assigned before that pointer check, so it is the only path that can, and the
collector now refuses to compare across generations. The approved set names
the generation it belongs to, so that is what anchors the comparison, and it
fails closed when the set cannot be read or the target has never recorded a
deployed generation, which is the direction that reports nothing.

The same anchor applies to the convergence gate, which was reaching the same
comparison with the same rows, so a target part-way between generations can
no longer read as exactly converged.
This commit is contained in:
Anso
2026-09-28 17:25:48 -04:00
parent ebc49779de
commit 4383c3c399
2 changed files with 124 additions and 0 deletions
@@ -1776,6 +1776,85 @@ describe('gitops derivation', () => {
expect(attentionReasons(projection), qualification).not.toContain('drift');
}
});
it('reports nothing when the observation belongs to a generation the node is no longer serving', () => {
// The reconciler rebinds the target to a new generation's approved set when
// it records the acknowledgement, and leaves the stored observation alone,
// so from that moment the observation describes the generation the node
// was running before. If the next tick records a drift through the cause
// that returns before re-observing (a stopped container, or a marker or
// revision mismatch), the digests on the target are the old generation's
// while the approved set is the new one's. Comparing those two produces a
// divergence nobody measured, and it is a confirmed one: the class, the
// reason, and the digests all look like real evidence.
const store = GitOpsStore.getInstance();
seedObservedBlueprint('app-obs-superseded', 309, 'blueprint_drifted', {
kind: 'exact',
identity: `sha256:${'d'.repeat(64)}`,
observedAt: 42,
services: [movedService()],
});
// The next generation is acknowledged, so the target now expects that
// generation's set while still deploying the previous one. The observation
// is deliberately left as the previous generation recorded it.
const nextGenerationId = 'gen-app-obs-superseded-next';
const nextArtifactSetId = 'art-app-obs-superseded-next';
store.insertGeneration(gen(nextGenerationId, 'app-obs-superseded'));
store.insertArtifactSet({
id: nextArtifactSetId,
generation_id: nextGenerationId,
evidence_version: 1,
authoritative: 0,
qualification: 'exact',
evidence_json: encodeArtifactEvidenceJson({
kind: 'exact',
identity: `sha256:${'e'.repeat(64)}`,
services: [approvedService()],
}),
created_at: 2,
});
store.upsertTarget({
...store.getTarget('app-obs-superseded', 1)!,
desired_generation_id: nextGenerationId,
applied_generation_id: nextGenerationId,
expected_artifact_set_id: nextArtifactSetId,
latest_artifact_set_id: nextArtifactSetId,
});
const projection = projectApplication('app-obs-superseded', false);
if (projection.targetMode === 'not_applicable') throw new Error('expected application');
// The reconciler's own record still says the target drifted, and the
// runtime facet still reports that, which is the reconciler's coarse
// claim. What must not happen is the list naming a digest divergence
// against a set the node was never observed running.
expect(projection.targets[0]?.runtime.status).toBe('drifted');
expect(projection.drift.filter(item => item.observed.kind === 'runtime_artifact')).toHaveLength(0);
// Desired N+1 against deployed N is a real divergence of its own, and it
// keeps its own item. That one compares two generations and asserts
// nothing about what is running, so it stays.
expect(projection.drift).toHaveLength(1);
expect(projection.drift[0]?.observed).toEqual({ kind: 'generation', id: 'gen-app-obs-superseded' });
});
it('reports a rollout class for a target bound by its rollout generation alone', () => {
const store = GitOpsStore.getInstance();
seedObservedBlueprint('app-obs-rollout-gen', 310, 'blueprint_drifted', {
kind: 'exact',
identity: `sha256:${'d'.repeat(64)}`,
observedAt: 42,
services: [movedService()],
});
// deriveRuntime reads either the authorization or the rollout generation,
// so both arms of that test need coverage. With only the generation set,
// the class is still rollout.
const target = store.getTarget('app-obs-rollout-gen', 1)!;
store.upsertTarget({ ...target, rollout_generation_id: 'gen-obs-rollout-1' });
const projection = projectApplication('app-obs-rollout-gen', false);
if (projection.targetMode === 'not_applicable') throw new Error('expected application');
expect(projection.drift).toHaveLength(1);
expect(projection.drift[0]?.class).toBe('rollout');
});
});
/**
+45
View File
@@ -254,6 +254,24 @@ function collectRuntimeDrift(
// confirmed divergence against an expectation nothing verified. The facet
// already reads that case as pending or unverified, and the two must agree.
if (expected.qualification !== 'exact' && expected.qualification !== 'qualified') continue;
// The approved set has to be the one this target is actually serving, or
// the observation describes a different generation than the set does.
//
// Recording an acknowledgement rebinds the target to the new generation's
// approved set and leaves the stored observation as the previous generation
// recorded it, so from that moment the two describe different generations.
// `deriveRuntime` never reaches this comparison for the two
// artifact-identity statuses while desired and deployed disagree, so they
// are already safe. `drifted` is assigned before that pointer check, so a
// target that recorded a drift through a cause that returns before
// re-observing (a stopped container, a marker or revision mismatch) can
// reach here mid-way between generations. Comparing across the two would
// report a digest divergence nobody measured, with a class, a reason, and
// digests that all read as real evidence. The set names the generation it
// belongs to, so that is what the comparison is anchored to. A set that
// cannot be read cannot anchor anything, and the direction that fails is
// the one that reports nothing.
if (!expectedSetBelongsToDeployedGeneration(expected.artifactSetId, target.deployedGenerationId)) continue;
const observed = target.observedArtifactIdentity;
if (observed.kind !== 'exact' && observed.kind !== 'qualified') continue;
if (expectedSetAgreesWithObservation(expected.artifactSetId, observed, expected.identity)) continue;
@@ -796,9 +814,36 @@ function targetObservationMatchesExpected(target: GitOpsTargetProjection): boole
: null;
if (!expected || expected.identity === null) return false;
if (expected.qualification !== 'exact' && expected.qualification !== 'qualified') return false;
if (!expectedSetBelongsToDeployedGeneration(expected.artifactSetId, target.deployedGenerationId)) return false;
return expectedSetAgreesWithObservation(expected.artifactSetId, target.observedArtifactIdentity, expected.identity);
}
/**
* True when the approved set is the one belonging to the generation the target
* is actually serving, which is the only case where an observation of that
* target can be read as evidence about that set.
*
* The two are separate facts that move separately. An acknowledgement rebinds
* the target to the accepted generation's set and, deliberately, leaves the
* stored observation as the previous generation recorded it. So a target can
* carry an expected set from one generation and an observation from the
* previous one, and the observation then says nothing at all about the set.
*
* Fail-closed on an unreadable set, a target that has never recorded a
* deployed generation, and a target whose deployed generation is unknown. In
* each of those the generation the set belongs to cannot be established, and a
* comparison that cannot place both sides on the same generation is not a
* comparison.
*/
function expectedSetBelongsToDeployedGeneration(
expectedSetId: string,
deployedGenerationId: string | null,
): boolean {
if (deployedGenerationId === null) return false;
const expectedRow = GitOpsStore.getInstance().getArtifactSet(expectedSetId);
return expectedRow !== undefined && expectedRow.generation_id === deployedGenerationId;
}
/**
* The persisted block reason, narrowed to the codes this build knows.
*