diff --git a/backend/src/__tests__/git-source-routes.test.ts b/backend/src/__tests__/git-source-routes.test.ts index 3270d179..89cb008d 100644 --- a/backend/src/__tests__/git-source-routes.test.ts +++ b/backend/src/__tests__/git-source-routes.test.ts @@ -883,6 +883,8 @@ describe('POST /api/stacks/:stackName/git-source/resume', () => { it('returns the normalized result', async () => { const resumeSpy = vi.spyOn(GitSourceService.getInstance(), 'resume') .mockResolvedValue({ outcome: 'no_source_change', reason: 'ok', nextAction: 'none' }); + const { SourceController } = await import('../services/gitops/SourceController'); + const evaluateSpy = vi.spyOn(SourceController.getInstance(), 'evaluateNow').mockResolvedValue(); const res = await request(app) .post('/api/stacks/existing-stack/git-source/resume') .set('Authorization', `Bearer ${adminToken()}`) @@ -890,7 +892,38 @@ describe('POST /api/stacks/:stackName/git-source/resume', () => { expect(res.status).toBe(200); expect(res.body.outcome).toBe('no_source_change'); expect(resumeSpy).toHaveBeenCalledWith('existing-stack', expect.objectContaining({ actor: expect.any(String) })); + // The route wakes the source immediately: the controller's resume + // evaluation is fired for the same stack the service just resumed. + expect(evaluateSpy).toHaveBeenCalledWith('existing-stack'); resumeSpy.mockRestore(); + evaluateSpy.mockRestore(); + }); + + it('still returns the resume outcome when the immediate evaluation cannot start', async () => { + const resumeSpy = vi.spyOn(GitSourceService.getInstance(), 'resume') + .mockResolvedValue({ outcome: 'no_source_change', reason: 'ok', nextAction: 'none' }); + const { SourceController } = await import('../services/gitops/SourceController'); + const evaluateSpy = vi.spyOn(SourceController.getInstance(), 'evaluateNow') + .mockRejectedValue(new Error('database is locked')); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/resume') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + // The response reports that suspension was cleared, not the + // outcome of an evaluation that never ran: it stays 200 with + // resume's derived result, and the failure is logged, not + // swallowed. + expect(res.status).toBe(200); + expect(res.body.outcome).toBe('no_source_change'); + await new Promise((resolve) => setImmediate(resolve)); + expect(errorSpy).toHaveBeenCalledTimes(1); + } finally { + resumeSpy.mockRestore(); + evaluateSpy.mockRestore(); + errorSpy.mockRestore(); + } }); it('denies without the stack:edit permission', async () => { diff --git a/backend/src/__tests__/git-source-service.test.ts b/backend/src/__tests__/git-source-service.test.ts index eec63390..04098e0f 100644 --- a/backend/src/__tests__/git-source-service.test.ts +++ b/backend/src/__tests__/git-source-service.test.ts @@ -7746,6 +7746,1163 @@ describe('GitSourceService.apply', () => { }); }); + describe('stage-aware retry', () => { + /** + * Accept the staged candidate at the source layer (what a controller + * auto-acceptance or an operator review does). Leaves the + * application in the accepted-but-undispatched state the dispatch + * arm of retry() targets: acceptance clears the application + * candidate pointer and the source failure columns, while the + * target's candidate pointer still names the accepted generation. + */ + function acceptPendingCandidate(stackName: string): string { + const app = liveApp(stackName)!; + const generationId = app.candidate_generation_id!; + GitOpsTransitions.getInstance().sourceAccepted({ + applicationId: app.id, + generationId, + artifactSetId: newGitOpsId(), + sourceAcceptanceId: newGitOpsId(), + authority: 'operator', + envelope: testEnvelope(), + }); + return generationId; + } + + async function asyncDefaultNodeId(): Promise { + const { NodeRegistry } = await import('../services/NodeRegistry'); + return NodeRegistry.getInstance().getDefaultNodeId(); + } + + /** + * Park a stack in the state the deploy arm of retry() owns: the + * promotion committed, the target bound the accepted generation, + * and the deploy that followed failed. The state is produced by + * running the promote arm for real (one retry, auto-deploy on, + * deploy stubbed to record its failure and throw), so the durable + * evidence the resume reads is transition-written, not SQL- + * fabricated. The spies stay installed (and their call counts + * cleared) so the test re-arms the deploy for the resume and + * counts what the resume does and never does. + */ + async function appliedWithFailedDeploy(stackName: string, sha: string) { + const svc = await seedPending(stackName, 'services:\n x:\n image: alpine\n', sha); + const generationId = acceptPendingCandidate(stackName); + const applicationId = liveApp(stackName)!.id; + const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET auto_deploy_on_apply = 1 WHERE stack_name = ?') + .run(stackName); + const { GitProjectManifestService } = await import('../services/GitProjectManifestService'); + const { ComposeService } = await import('../services/ComposeService'); + const promoteSpy = vi.spyOn(GitProjectManifestService.prototype, 'promoteGeneration') + .mockImplementation(async () => {}); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack') + .mockImplementation(async (_stack, _ws, _atomic, ctx) => { + const tx = GitOpsTransitions.getInstance(); + const env = { + operationId: ctx!.gitopsDeployOperationId!, + actor: 'system:compose', + trigger: 'deploy', + at: Date.now(), + }; + tx.deployStarted(applicationId, nodeId, generationId, env); + tx.deployFailed(applicationId, nodeId, 'pre_mutation', env); + // Stamp the id on the error the way the real tracked + // deploy does (ComposeService's wrapped rethrow), so + // the settle and the response carry the correlation. + const err = new Error('simulated deploy failure') as Error & { gitopsDeployOperationId?: string }; + err.gitopsDeployOperationId = ctx!.gitopsDeployOperationId; + throw err; + }); + + // The promote arm runs to its deploy-failure settle: blocked, + // naming the failed deploy, with the target applied and its + // failure_stage on 'deploy'. + const first = await svc.retry(stackName, { actor: 'tester' }); + expect(first.outcome).toBe('blocked'); + expect(first.reason).toMatch(/^The source applied, but the deploy failed: /); + const target = GitOpsStore.getInstance().getTarget(applicationId, nodeId)!; + expect(target.applied_generation_id).toBe(generationId); + expect(target.failure_stage).toBe('deploy'); + promoteSpy.mockClear(); + deploySpy.mockClear(); + return { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy }; + } + + /** + * The health-failure starting state: the generation is applied and + * its deploy bound (cleared target evidence), and the stack's + * latest stack-scoped health gate run is a failed verdict bound to + * that exact generation. The delivery half is produced by running + * the deploy arm to success (real deploy-bound transitions); only + * the gate row itself is inserted directly, because that is the + * durable evidence the selector reads (the real beginStack/finalize + * pair would observe container state this fixture does not have). + */ + async function appliedWithFailedHealth(stackName: string, sha: string) { + const seeded = await appliedWithFailedDeploy(stackName, sha); + const { svc, nodeId, generationId, promoteSpy, deploySpy } = seeded; + const { HealthGateService } = await import('../services/HealthGateService'); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-seed'); + // Re-arm for a successful deploy, the way the wins test does. + deploySpy.mockImplementation(async (_stack, _ws, _atomic, ctx) => { + const tx = GitOpsTransitions.getInstance(); + const env = { + operationId: ctx!.gitopsDeployOperationId!, + actor: 'system:compose', + trigger: 'deploy', + at: Date.now(), + }; + tx.deployStarted(seeded.applicationId, nodeId, generationId, env); + tx.deployBound(seeded.applicationId, nodeId, generationId, env); + return { + recoveryId: null, + deployedGenerationId: generationId, + gitopsOperationId: ctx!.gitopsDeployOperationId!, + }; + }); + const delivered = await svc.retry(stackName, { actor: 'tester' }); + expect(delivered.outcome).toBe('no_source_change'); + beginSpy.mockRestore(); + const target = GitOpsStore.getInstance().getTarget(seeded.applicationId, nodeId)!; + expect(target.applied_generation_id).toBe(generationId); + expect(target.failure_stage).toBeNull(); + promoteSpy.mockClear(); + deploySpy.mockClear(); + return seeded; + } + + /** Insert a durable health_gate_runs row (the selector's evidence). */ + function seedHealthRun(args: { + id: string; + nodeId: number; + stackName: string; + status: 'passed' | 'failed' | 'unknown' | 'observing'; + scope: 'stack' | 'service'; + serviceName: string | null; + generationId: string | null; + startedAt: number; + }): void { + DatabaseService.getInstance().insertHealthGateRun({ + id: args.id, + node_id: args.nodeId, + stack_name: args.stackName, + trigger_action: 'deploy', + status: args.status, + reason: args.status === 'failed' ? 'simulated health failure' : null, + window_seconds: 30, + containers_json: '[]', + started_at: args.startedAt, + ended_at: args.status === 'observing' ? null : args.startedAt + 30_000, + created_by: 'system:git-source', + target_scope: args.scope, + service_name: args.serviceName, + failure_source: args.status === 'failed' ? 'primary' : null, + deployed_generation_id: args.generationId, + }); + } + + it('dispatches the accepted generation when nothing was ever promoted, without refetching or re-accepting', async () => { + const sha = 't1t1t1t1t1t1t1t1t1t1t1t1t1t1t1t1t1t1t1t1'; + const svc = await seedPending('retry-dispatch-arm', 'services:\n x:\n image: alpine\n', sha); + const generationId = acceptPendingCandidate('retry-dispatch-arm'); + const applicationId = liveApp('retry-dispatch-arm')!.id; + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + const acceptedBefore = liveApp('retry-dispatch-arm')!.accepted_generation_id; + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const { GitProjectManifestService } = await import('../services/GitProjectManifestService'); + const promoteSpy = vi.spyOn(GitProjectManifestService.prototype, 'promoteGeneration') + .mockImplementation(async () => {}); + + try { + const result = await svc.retry('retry-dispatch-arm', { actor: 'tester' }); + + // The dispatch arm ran through the shared boundary: the + // generation promoted and the target bound, no fetch ran, + // and acceptance was not re-run. + expect(promoteSpy).toHaveBeenCalledTimes(1); + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(result.outcome).toBe('no_source_change'); + const app = liveApp('retry-dispatch-arm')!; + expect(app.accepted_generation_id).toBe(acceptedBefore); + expect(app.accepted_generation_id).toBe(generationId); + const target = GitOpsStore.getInstance().getTarget(applicationId, await asyncDefaultNodeId())!; + expect(target.applied_generation_id).toBe(generationId); + // The retry's dispatch reserved its own attempt, marked as a + // dispatch of exactly this generation, and settled it. + const startedIds = historyOperationIds(applicationId, 'source_reconcile_started'); + expect(startedIds).toHaveLength(startedBefore + 1); + const retryOperationId = startedIds[startedIds.length - 1]!; + const reserved = GitOpsStore.getInstance().getStartedAttempt(applicationId, retryOperationId)!; + expect(reserved.trigger).toBe('retry'); + expect(JSON.parse(reserved.after_json).dispatchGenerationId).toBe(generationId); + expect(GitOpsStore.getInstance().getSettledAttempt(applicationId, retryOperationId)).toBeDefined(); + } finally { + reconcileSpy.mockRestore(); + promoteSpy.mockRestore(); + } + }); + + it('refetches when a source-stage failure persists alongside undelivered dispatch evidence', async () => { + const sha = 't2t2t2t2t2t2t2t2t2t2t2t2t2t2t2t2t2t2t2t2'; + const svc = await seedPending('retry-source-stage-precedence', 'services:\n x:\n image: alpine\n', sha); + acceptPendingCandidate('retry-source-stage-precedence'); + const applicationId = liveApp('retry-source-stage-precedence')!.id; + // A validation failure recorded after acceptance: the fetch arm + // owns the newer evidence, so the dispatch pointer does not win. + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET failure_stage = 'validation', failure_class = 'validation', failure_at = ? WHERE id = ?", + ).run(Date.now(), applicationId); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const { GitProjectManifestService } = await import('../services/GitProjectManifestService'); + const promoteSpy = vi.spyOn(GitProjectManifestService.prototype, 'promoteGeneration'); + + try { + await svc.retry('retry-source-stage-precedence', { actor: 'tester' }); + + expect(reconcileSpy).toHaveBeenCalledWith(expect.objectContaining({ trigger: 'retry', intent: 'fetch' })); + expect(promoteSpy).not.toHaveBeenCalled(); + } finally { + reconcileSpy.mockRestore(); + promoteSpy.mockRestore(); + } + }); + + it('maps a blocked dispatch to a blocked outcome with the refusal advice, reserving nothing', async () => { + const sha = 't3t3t3t3t3t3t3t3t3t3t3t3t3t3t3t3t3t3t3t3'; + const svc = await seedPending('retry-dispatch-blocked', 'services:\n x:\n image: alpine\n', sha); + acceptPendingCandidate('retry-dispatch-blocked'); + const applicationId = liveApp('retry-dispatch-blocked')!.id; + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + const nodeId = await asyncDefaultNodeId(); + expect(StackOpLockService.getInstance().tryAcquire(nodeId, 'retry-dispatch-blocked', 'deploy', 'tester').acquired).toBe(true); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + mockGitClone.mockClear(); + + try { + const result = await svc.retry('retry-dispatch-blocked', { actor: 'tester' }); + + expect(result).toEqual({ + outcome: 'blocked', + reason: 'Another operation (deploy) is already in progress for retry-dispatch-blocked.', + nextAction: 'retry', + }); + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + // The lock refusal happened before any reservation, so + // retry leaves no unsettled attempt behind. + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(startedBefore); + } finally { + reconcileSpy.mockRestore(); + StackOpLockService.getInstance().release(nodeId, 'retry-dispatch-blocked'); + } + }); + + it('defers to the in-flight projection instead of starting a second operation', async () => { + const sha = 't4t4t4t4t4t4t4t4t4t4t4t4t4t4t4t4t4t4t4t4'; + const svc = await seedPending('retry-mid-operation', 'services:\n x:\n image: alpine\n', sha); + acceptPendingCandidate('retry-mid-operation'); + const applicationId = liveApp('retry-mid-operation')!.id; + // A crash window: the row still carries the apply-started + // marker although no work is running. Recovery owns that + // evidence; retry must not race it with a fetch or a dispatch. + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET active_operation_stage = 'apply_started', active_operation_id = 'op-crash', active_operation_at = ? WHERE id = ?", + ).run(Date.now(), applicationId); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const dispatchSpy = vi.spyOn(svc, 'dispatchAcceptedGeneration'); + + try { + const result = await svc.retry('retry-mid-operation', { actor: 'tester' }); + + expect(result.outcome).toBe('unknown'); + expect(result.reason).toMatch(/in flight/i); + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(dispatchSpy).not.toHaveBeenCalled(); + } finally { + reconcileSpy.mockRestore(); + dispatchSpy.mockRestore(); + } + }); + + it('refetches once the accepted generation has been delivered and its deploy settled', async () => { + const sha = 't5t5t5t5t5t5t5t5t5t5t5t5t5t5t5t5t5t5t5t5'; + const svc = await seedPending('retry-after-delivery', 'services:\n x:\n image: alpine\n', sha); + const generationId = acceptPendingCandidate('retry-after-delivery'); + const applicationId = liveApp('retry-after-delivery')!.id; + // The dispatch evidence is gone: the target applied the + // generation and its deploy failure columns are clear (they + // were never written here, which is exactly the still-healthy + // state deploy-bound leaves), so the only re-evaluation left + // is a source fetch. The deploy arm must not fire without a + // recorded target-level deploy failure. + DatabaseService.getInstance().getDb().prepare( + 'UPDATE gitops_target_current SET applied_generation_id = ?, candidate_generation_id = NULL WHERE application_id = ?', + ).run(generationId, applicationId); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { ComposeService } = await import('../services/ComposeService'); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack'); + + try { + const result = await svc.retry('retry-after-delivery', { actor: 'tester' }); + + expect(reconcileSpy).toHaveBeenCalledWith(expect.objectContaining({ trigger: 'retry', intent: 'fetch' })); + expect(deploySpy).not.toHaveBeenCalled(); + expect(result.outcome).toBe('pending_review'); + } finally { + reconcileSpy.mockRestore(); + validateSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('resumes only the deploy when the applied generation\'s deploy failed, without fetching, promoting, or re-binding', async () => { + const sha = 't8t8t8t8t8t8t8t8t8t8t8t8t8t8t8t8t8t8t8t8'; + const { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-resume', sha); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const { HealthGateService } = await import('../services/HealthGateService'); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack'); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + const targetAppliedBefore = historyOperationIds(applicationId, 'target_applied').length; + + try { + const result = await svc.retry('retry-deploy-resume', { actor: 'tester' }); + + // The fetch arm never ran: no reconcile call, no git clone. + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + // Promotion and re-binding are refused by the applied + // pointer, and the resume honors that: the shared promote + // machinery is untouched. + expect(promoteSpy).not.toHaveBeenCalled(); + // The same applied generation is deployed again, under a + // freshly journaled intent of this retry attempt. + expect(deploySpy).toHaveBeenCalledTimes(1); + const startedIds = historyOperationIds(applicationId, 'source_reconcile_started'); + expect(startedIds).toHaveLength(startedBefore + 1); + const retryOperationId = startedIds[startedIds.length - 1]!; + const reserved = GitOpsStore.getInstance().getStartedAttempt(applicationId, retryOperationId)!; + expect(reserved.trigger).toBe('retry'); + const markers = JSON.parse(reserved.after_json!); + expect(markers.dispatchGenerationId).toBe(generationId); + expect(markers.dispatchDeployRequested).toBe(true); + const intentRow = GitOpsStore.getInstance() + .getStageRowForAttempt(applicationId, retryOperationId, 'deploy_dispatched')!; + const intent = JSON.parse(intentRow.after_json!); + expect(intent.generationId).toBe(generationId); + expect(intent.deployOperationId).toBeDefined(); + // The second deploy's own rows land under the journaled id + // for this generation: started, then failed at pre_mutation. + expect(historyOperationIds(applicationId, 'deploy_started')).toContain(intent.deployOperationId); + expect(historyOperationIds(applicationId, 'deploy_failed')).toContain(intent.deployOperationId); + // Compose was handed exactly the journaled deploy id, under + // git_apply provenance by the raw retry actor. + expect(deploySpy).toHaveBeenCalledWith('retry-deploy-resume', undefined, undefined, + expect.objectContaining({ + source: 'git_apply', + actor: 'tester', + gitopsDeployOperationId: intent.deployOperationId, + })); + // The deploy failed again, so no health gate was opened. + expect(beginSpy).not.toHaveBeenCalled(); + // The attempt is durably recorded and settled, carrying the + // same deploy-failure classification the promote arm's + // failed deploy writes, plus the exact failed deploy id. + const settled = settledAttemptsForApplication(applicationId) + .find((row) => row.operation_id === retryOperationId); + expect(settled).toBeDefined(); + expect(JSON.parse(settled!.after_json!)).toMatchObject({ + outcome: 'recovery_required', + reason: 'The source applied, but the deploy failed: simulated deploy failure', + nextAction: 'view_target_results', + deployGitopsOperationId: intent.deployOperationId, + }); + // The response matches the promote arm's vocabulary for the + // same settled classification: blocked, view target results. + expect(result).toEqual({ + outcome: 'blocked', + reason: 'The source applied, but the deploy failed: simulated deploy failure', + nextAction: 'view_target_results', + }); + // Direct re-bind did not happen: no new target_applied + // history row, and the applied pointer still names the + // same generation it named before the resume. + expect(historyOperationIds(applicationId, 'target_applied')) + .toHaveLength(targetAppliedBefore); + expect(GitOpsStore.getInstance().getTarget(applicationId, nodeId)!.applied_generation_id) + .toBe(generationId); + // Acceptance was not re-run either: the application still + // names the same accepted generation. + expect(liveApp('retry-deploy-resume')!.accepted_generation_id).toBe(generationId); + } finally { + reconcileSpy.mockRestore(); + beginSpy.mockRestore(); + errorSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('settles the deploy resume as a success when the retried deploy binds, and the source is delivered again', async () => { + const sha = 't9t9t9t9t9t9t9t9t9t9t9t9t9t9t9t9t9t9t9t9'; + const { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-resume-wins', sha); + // Re-arm the stub for the resume: this time the deploy opens + // under the journaled id and binds the same generation, the way + // ComposeService's tracked deploy does. + deploySpy.mockImplementation(async (_stack, _ws, _atomic, ctx) => { + const tx = GitOpsTransitions.getInstance(); + const env = { + operationId: ctx!.gitopsDeployOperationId!, + actor: 'system:compose', + trigger: 'deploy', + at: Date.now(), + }; + tx.deployStarted(applicationId, nodeId, generationId, env); + tx.deployBound(applicationId, nodeId, generationId, env); + return { + recoveryId: null, + deployedGenerationId: generationId, + gitopsOperationId: ctx!.gitopsDeployOperationId!, + }; + }); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const { HealthGateService } = await import('../services/HealthGateService'); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-resume'); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const result = await svc.retry('retry-deploy-resume-wins', { actor: 'tester' }); + + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(deploySpy).toHaveBeenCalledTimes(1); + // A successful resume deploy opens the same health gate and + // post-deploy scan the pipeline's deploy branch does. + expect(beginSpy).toHaveBeenCalledWith(nodeId, 'retry-deploy-resume-wins', 'deploy', 'system:git-source', { deployedGenerationId: generationId }); + await vi.waitFor(() => expect(mockTriggerPostDeployScan).toHaveBeenCalledWith('retry-deploy-resume-wins', nodeId)); + // The retry's settle carries the exact deploy that ran. + const startedIds = historyOperationIds(applicationId, 'source_reconcile_started'); + const retryOperationId = startedIds[startedIds.length - 1]!; + const intentRow = GitOpsStore.getInstance() + .getStageRowForAttempt(applicationId, retryOperationId, 'deploy_dispatched')!; + const deployId = JSON.parse(intentRow.after_json!).deployOperationId as string; + const settled = settledAttemptsForApplication(applicationId) + .find((row) => row.operation_id === retryOperationId); + expect(JSON.parse(settled!.after_json!)).toMatchObject({ + outcome: 'no_source_change', + deployGitopsOperationId: deployId, + }); + expect(result.outcome).toBe('no_source_change'); + expect(result.deployGitopsOperationId).toBe(deployId); + + // With the deploy evidence cleared by the bind, a further + // retry is a plain source fetch again: the deploy arm does + // not fire forever on a delivered generation. + reconcileSpy.mockClear(); + mockTriggerPostDeployScan.mockClear(); + const again = await svc.retry('retry-deploy-resume-wins', { actor: 'tester' }); + expect(reconcileSpy).toHaveBeenCalledWith(expect.objectContaining({ trigger: 'retry', intent: 'fetch' })); + expect(again.outcome).toBe('pending_review'); + } finally { + reconcileSpy.mockRestore(); + beginSpy.mockRestore(); + validateSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('refuses the deploy resume when the evidence moved before the lock, settling the refusal', async () => { + const sha = 't10a10a10a10a10a10a10a10a10a10a10a10a10'; + const { svc, applicationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-resume-race', sha); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + // The entry predicate sees the deploy failure; by the time the + // in-lock revalidation reads the target, a successful deploy + // has cleared it. The resume must refuse and touch nothing. + const realTarget = GitOpsStore.prototype.getTarget; + const targetSpy = vi.spyOn(GitOpsStore.prototype, 'getTarget'); + let targetCalls = 0; + targetSpy.mockImplementation((applicationIdArg: string, nodeIdArg: number) => { + targetCalls++; + const row = realTarget.call(GitOpsStore.getInstance(), applicationIdArg, nodeIdArg); + // The entry read (call 1) keeps the failure evidence; the + // in-lock revalidation (call 2) sees it cleared. + if (row && targetCalls === 2) return { ...row, failure_stage: null, failure_class: null, failure_at: null }; + return row; + }); + + try { + const result = await svc.retry('retry-deploy-resume-race', { actor: 'tester' }); + + expect(result.outcome).toBe('blocked'); + expect(result.reason).toMatch(/nothing was re-deployed/); + expect(result.nextAction).toBe('view_target_results'); + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + // The refusal happened after the reservation, so it is + // settled as a blocked attempt, and it is the only new row. + const startedIds = historyOperationIds(applicationId, 'source_reconcile_started'); + expect(startedIds).toHaveLength(startedBefore + 1); + const settled = settledAttemptsForApplication(applicationId) + .find((row) => row.operation_id === startedIds[startedIds.length - 1]!); + expect(JSON.parse(settled!.after_json!)).toMatchObject({ + outcome: 'blocked', + nextAction: 'view_target_results', + }); + // The call-counted blinding is only meaningful if the path + // really reads the target exactly twice (entry predicate, + // in-lock revalidation). Pin it so a future extra read + // cannot silently shift the blinded call to the wrong site. + expect(targetSpy).toHaveBeenCalledTimes(2); + } finally { + reconcileSpy.mockRestore(); + targetSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('refuses the deploy resume while the stack lock is held, reserving nothing', async () => { + const sha = 't11b11b11b11b11b11b11b11b11b11b11b11b11b'; + const { applicationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-resume-blocked', sha); + const nodeId = await asyncDefaultNodeId(); + expect(StackOpLockService.getInstance().tryAcquire(nodeId, 'retry-deploy-resume-blocked', 'deploy', 'tester').acquired).toBe(true); + const svc = GitSourceService.getInstance(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + + try { + const result = await svc.retry('retry-deploy-resume-blocked', { actor: 'tester' }); + + expect(result).toEqual({ + outcome: 'blocked', + reason: 'Another operation (deploy) is already in progress for retry-deploy-resume-blocked.', + nextAction: 'retry', + }); + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(reconcileSpy).not.toHaveBeenCalled(); + // The lock refusal happened before any reservation: a + // resume that started nothing leaves no attempt to chase. + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(startedBefore); + } finally { + reconcileSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + StackOpLockService.getInstance().release(nodeId, 'retry-deploy-resume-blocked'); + } + }); + + it('fetches when a newer source-stage failure outranks the deploy evidence', async () => { + const sha = 'x12x12x12x12x12x12x12x12x12x12x12x12x12'; + const { svc, applicationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-arm-fetch-precedence', sha); + // A fetch failure recorded after the deploy failed: the + // application-row source evidence is newer than the target-row + // deploy evidence, and the fetch arm owns the precedence. The + // class rides along for realism; the arm routing reads the + // stage only. + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET failure_stage = 'fetch', failure_class = 'auth', failure_at = ? WHERE id = ?", + ).run(Date.now(), applicationId); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + + try { + await svc.retry('retry-deploy-arm-fetch-precedence', { actor: 'tester' }); + + expect(reconcileSpy).toHaveBeenCalledWith(expect.objectContaining({ trigger: 'retry', intent: 'fetch' })); + // Neither resume arm ran beside the fetch. + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + // The fetch arm never touches target evidence: the deploy + // failure stands until a deploy succeeds. + const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; + const target = GitOpsStore.getInstance().getTarget(applicationId, nodeId)!; + expect(target.failure_stage).toBe('deploy'); + } finally { + reconcileSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('refuses the deploy resume when its reservation collides, settling nothing', async () => { + const sha = 'x13x13x13x13x13x13x13x13x13x13x13x13'; + const { svc, applicationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-resume-collision', sha); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + const settledBefore = settledAttemptsForApplication(applicationId).length; + const reserveSpy = vi.spyOn(GitOpsTransitions.prototype, 'allocateReconcileAttempt') + .mockImplementationOnce(() => ({ operationId: 'collided:attempt:1', reserved: false })); + + try { + const result = await svc.retry('retry-deploy-resume-collision', { actor: 'tester' }); + + expect(result).toEqual({ + outcome: 'blocked', + reason: 'A dispatch attempt for this stack is already recorded; check its outcome before dispatching again.', + nextAction: 'view_target_results', + }); + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + // Nothing was reserved, so nothing may be settled: the + // colliding attempt belongs to its own owner, and the + // deploy-resume must not speak for it. + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(startedBefore); + expect(settledAttemptsForApplication(applicationId)).toHaveLength(settledBefore); + } finally { + reserveSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('recovers an interrupted deploy resume from its journaled deploy evidence without re-running anything', async () => { + const sha = 'x14x14x14x14x14x14x14x14x14x14x14x14'; + const { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-resume-recovered', sha); + // A crash window for the resume: the retried deploy records its + // failure under the journaled id, then the resume's own settle + // dies. The attempt stays unsettled while its deploy evidence + // stands, which startup recovery must read as the failed deploy + // it was, reconstructing the settle from recorded rows only. + deploySpy.mockImplementation(async (_stack, _ws, _atomic, ctx) => { + const tx = GitOpsTransitions.getInstance(); + const env = { + operationId: ctx!.gitopsDeployOperationId!, + actor: 'system:compose', + trigger: 'deploy', + at: Date.now(), + }; + tx.deployStarted(applicationId, nodeId, generationId, env); + tx.deployFailed(applicationId, nodeId, 'pre_mutation', env); + throw new Error('simulated deploy failure'); + }); + const settleSpy = vi.spyOn(GitOpsTransitions.prototype, 'settleReconcileAttempt') + .mockImplementationOnce(() => { throw new Error('simulated crash before settle'); }); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await svc.retry('retry-deploy-resume-recovered', { actor: 'tester' }); + + const started = historyOperationIds(applicationId, 'source_reconcile_started'); + const retryOperationId = started[started.length - 1]!; + const intentRow = GitOpsStore.getInstance() + .getStageRowForAttempt(applicationId, retryOperationId, 'deploy_dispatched')!; + const deployId = JSON.parse(intentRow.after_json!).deployOperationId as string; + expect(historyOperationIds(applicationId, 'deploy_failed')).toContain(deployId); + // The live settle died, so the attempt is still open. + expect(settledAttemptsForApplication(applicationId) + .some((row) => row.operation_id === retryOperationId)).toBe(false); + + await svc.recoverUnsettledReconcileAttempts(); + + const settled = settledAttemptsForApplication(applicationId) + .filter((row) => row.operation_id === retryOperationId); + expect(settled).toHaveLength(1); + // The reconstruction names the promotion and binding the + // resume took as its evidence, plus the exact failed deploy. + expect(JSON.parse(settled[0]!.after_json!)).toMatchObject({ + outcome: 'recovery_required', + reason: 'The promotion and binding completed, but the deploy this dispatch started failed.', + nextAction: 'view_target_results', + deployGitopsOperationId: deployId, + commitSha: sha, + }); + // Recovery re-ran nothing: the deploy arm never promotes + // (the helper's counts were cleared after the seeding + // promote-arm run), and reconstruction is read-only. + expect(promoteSpy).not.toHaveBeenCalled(); + expect(deploySpy).toHaveBeenCalledTimes(1); + + await svc.recoverUnsettledReconcileAttempts(); + expect(settledAttemptsForApplication(applicationId) + .filter((row) => row.operation_id === retryOperationId)).toHaveLength(1); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(deploySpy).toHaveBeenCalledTimes(1); + } finally { + settleSpy.mockRestore(); + errorSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('refuses the deploy resume when its intent cannot be journaled, settling the refusal without deploying', async () => { + const sha = 'x15x15x15x15x15x15x15x15x15x15x15x15'; + const { svc, applicationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-resume-intent-refused', sha); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + const intentSpy = vi.spyOn(GitOpsTransitions.prototype, 'deployDispatched') + .mockImplementationOnce(() => { throw new Error('simulated intent journal failure'); }); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + const result = await svc.retry('retry-deploy-resume-intent-refused', { actor: 'tester' }); + + // A resume that cannot record and read back its deploy + // intent refuses to start an untracked deploy: Compose was + // never handed the stack. + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(result.outcome).toBe('blocked'); + expect(result.nextAction).toBe('view_target_results'); + expect(result.reason).toMatch(/deploy retry could not record and read back its deploy intent/); + const started = historyOperationIds(applicationId, 'source_reconcile_started'); + expect(started).toHaveLength(startedBefore + 1); + const retryOperationId = started[started.length - 1]!; + // The refusal journals its witness (so recovery of an + // unsettled version of this attempt reads the refusal, not + // an apply-only completion) and settles recovery_required + // against the applied files, naming the cause. Compose + // never ran, so no deploy id is claimed. + expect(GitOpsStore.getInstance() + .hasStageRowForAttempt(applicationId, retryOperationId, 'deploy_intent_refused')).toBe(true); + const settled = settledAttemptsForApplication(applicationId) + .find((row) => row.operation_id === retryOperationId); + const payload = JSON.parse(settled!.after_json!); + expect(payload).toMatchObject({ + outcome: 'recovery_required', + reason: 'The deploy retry could not record and read back its deploy intent (simulated intent journal failure), so the deploy was not started.', + nextAction: 'view_target_results', + }); + expect(payload).not.toHaveProperty('deployGitopsOperationId'); + } finally { + intentSpy.mockRestore(); + errorSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('resumes at the deploy when the applied generation has a failed stack health verdict', async () => { + const sha = 'h1a1h1a1h1a1h1a1h1a1h1a1h1a1h1a1h1a1h1a1'; + const stackName = 'retry-health-resume-wins'; + const { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy } = + await appliedWithFailedHealth(stackName, sha); + // The generation is applied and its deploy bound; the stack's + // latest stack-scoped gate run is then a failed verdict bound + // to exactly that generation. That is the whole evidence set: + // no deploy failure is recorded anywhere, because the deploy + // did succeed. + seedHealthRun({ + id: 'health-run-h1', + nodeId, + stackName, + status: 'failed', + scope: 'stack', + serviceName: null, + generationId, + startedAt: Date.now(), + }); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const { HealthGateService } = await import('../services/HealthGateService'); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-health-resume'); + + try { + const result = await svc.retry(stackName, { actor: 'tester' }); + + // The failed verdict routes the resume to the deploy arm: + // the deploy re-runs, and nothing is fetched, transported, + // accepted, or promoted beside it. + expect(result.outcome).toBe('no_source_change'); + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(deploySpy).toHaveBeenCalledTimes(1); + // The resume re-opens the health gate on the deploy it + // runs, bound to the same generation. In production the + // fresh run supersedes the failed row as the stack's + // latest claim; the mocked beginStack does not insert one, + // so no post-resume routing assertion is made here. + expect(beginSpy).toHaveBeenCalledWith( + nodeId, stackName, 'deploy', 'system:git-source', { deployedGenerationId: generationId }, + ); + // The retried deploy re-bound the same generation; no new + // acceptance or promotion happened beside it. + const target = GitOpsStore.getInstance().getTarget(applicationId, nodeId)!; + expect(target.applied_generation_id).toBe(generationId); + expect(target.deployed_generation_id).toBe(generationId); + expect(historyOperationIds(applicationId, 'target_applied')).toHaveLength(1); + await vi.waitFor(() => expect(mockTriggerPostDeployScan).toHaveBeenCalledWith(stackName, nodeId)); + } finally { + reconcileSpy.mockRestore(); + beginSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('refuses the deploy resume when the health verdict is superseded before the lock', async () => { + const sha = 'h4d4h4d4h4d4h4d4h4d4h4d4h4d4h4d4h4d4h4d4'; + const stackName = 'retry-health-resume-race'; + const { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy } = + await appliedWithFailedHealth(stackName, sha); + seedHealthRun({ + id: 'health-run-h4', + nodeId, + stackName, + status: 'failed', + scope: 'stack', + serviceName: null, + generationId, + startedAt: Date.now(), + }); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + // The entry predicate sees the failed verdict; by the time the + // in-lock revalidation reads it, a newer stack gate run has + // superseded it. The resume must refuse and touch nothing, + // the same way the deploy-failure race refusal works. + const realLatest = DatabaseService.prototype.getLatestStackHealthGateRun; + let latestCalls = 0; + const latestSpy = vi.spyOn(DatabaseService.prototype, 'getLatestStackHealthGateRun') + .mockImplementation((nodeIdArg: number, stackNameArg: string) => { + latestCalls++; + if (latestCalls === 2) return undefined; + return realLatest.call(DatabaseService.getInstance(), nodeIdArg, stackNameArg); + }); + + try { + const result = await svc.retry(stackName, { actor: 'tester' }); + + expect(result.outcome).toBe('blocked'); + expect(result.reason).toMatch(/nothing was re-deployed/); + expect(result.nextAction).toBe('view_target_results'); + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + // The refusal happened after the reservation, so it is + // settled as a blocked attempt, and it is the only new row. + const startedIds = historyOperationIds(applicationId, 'source_reconcile_started'); + expect(startedIds).toHaveLength(startedBefore + 1); + const settled = settledAttemptsForApplication(applicationId) + .find((row) => row.operation_id === startedIds[startedIds.length - 1]!); + expect(JSON.parse(settled!.after_json!)).toMatchObject({ + outcome: 'blocked', + nextAction: 'view_target_results', + }); + // The call-counted blinding is only meaningful if the path + // really reads the accessor exactly twice (entry predicate, + // in-lock revalidation). Pin it so a future extra read + // cannot silently shift the blinded call to the wrong site. + expect(latestSpy).toHaveBeenCalledTimes(2); + } finally { + reconcileSpy.mockRestore(); + latestSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('falls back to the fetch arm for every health run that cannot claim the deploy resume', async () => { + const sha = 'h2b2h2b2h2b2h2b2h2b2h2b2h2b2h2b2h2b2h2b2'; + const stackName = 'retry-health-evidence-negative'; + const { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy } = + await appliedWithFailedHealth(stackName, sha); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const db = DatabaseService.getInstance().getDb(); + + // Each row is a health record that cannot claim the deploy + // arm: a verdict for a stale (older) generation, a verdict + // that never landed (observing/unknown), a healthy verdict, + // or a gate that watches one service rather than the applied + // generation's stack-wide health. Each must fall to the + // fetch arm, which re-decides from the source instead. + const cases: Array<{ + label: string; + status: 'failed' | 'passed' | 'observing' | 'unknown'; + scope: 'stack' | 'service'; + serviceName: string | null; + boundGenerationId: string | null; + }> = [ + { label: 'stale generation', status: 'failed', scope: 'stack', serviceName: null, boundGenerationId: '00000000-0000-4000-8000-000000000000' }, + { label: 'unbound generation', status: 'failed', scope: 'stack', serviceName: null, boundGenerationId: null }, + { label: 'observing run', status: 'observing', scope: 'stack', serviceName: null, boundGenerationId: generationId }, + { label: 'unknown run', status: 'unknown', scope: 'stack', serviceName: null, boundGenerationId: generationId }, + { label: 'passed run', status: 'passed', scope: 'stack', serviceName: null, boundGenerationId: generationId }, + { label: 'service-scoped run', status: 'failed', scope: 'service', serviceName: 'x', boundGenerationId: generationId }, + ]; + + try { + for (let i = 0; i < cases.length; i++) { + const c = cases[i]!; + db.prepare('DELETE FROM health_gate_runs WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName); + seedHealthRun({ + id: `health-run-neg-${i}`, + nodeId, + stackName, + status: c.status, + scope: c.scope, + serviceName: c.serviceName, + generationId: c.boundGenerationId, + startedAt: Date.now(), + }); + // Keep the fetch arm's own bookkeeping from stacking + // up as source-stage evidence across iterations: each + // case must be decided on its health row, not on a + // failure a previous case's fetch recorded. + db.prepare( + 'UPDATE gitops_applications SET failure_stage = NULL, failure_class = NULL, failure_at = NULL WHERE id = ?', + ).run(applicationId); + // Preconditions so a vacuous pass is impossible: the + // dispatch evidence is intact and only the health row + // differs from the wins case. + const appRow = GitOpsStore.getInstance().getApplication(applicationId)!; + expect(appRow.accepted_generation_id, c.label).toBe(generationId); + expect(appRow.failure_stage, c.label).toBeNull(); + const targetRow = GitOpsStore.getInstance().getTarget(applicationId, nodeId)!; + expect(targetRow.applied_generation_id, c.label).toBe(generationId); + expect(targetRow.failure_stage, c.label).toBeNull(); + + reconcileSpy.mockClear(); + await svc.retry(stackName, { actor: 'tester' }); + + expect(reconcileSpy, c.label).toHaveBeenCalledWith( + expect.objectContaining({ trigger: 'retry', intent: 'fetch' }), + ); + expect(deploySpy, c.label).not.toHaveBeenCalled(); + expect(promoteSpy, c.label).not.toHaveBeenCalled(); + } + } finally { + reconcileSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('leaves a failed health verdict to the path the target actually names', async () => { + const sha = 'h3c3h3c3h3c3h3c3h3c3h3c3h3c3h3c3h3c3h3c3'; + const stackName = 'retry-health-evidence-other-owner'; + const { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy } = + await appliedWithFailedHealth(stackName, sha); + seedHealthRun({ + id: 'health-run-h3', + nodeId, + stackName, + status: 'failed', + scope: 'stack', + serviceName: null, + generationId, + startedAt: Date.now(), + }); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + // The target names recovery as its outstanding failure. The + // failed health verdict does not silently reroute that path + // into a deploy resume: the health arm only owns the clear + // (null) target-evidence state, and everything else keeps its + // own owner. + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_target_current SET failure_stage = 'recovery', failure_class = 'post_mutation' WHERE application_id = ? AND node_id = ?", + ).run(applicationId, nodeId); + + try { + await svc.retry(stackName, { actor: 'tester' }); + + expect(reconcileSpy).toHaveBeenCalledWith( + expect.objectContaining({ trigger: 'retry', intent: 'fetch' }), + ); + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + } finally { + reconcileSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('falls back to the fetch arm with a warning when the accepted generation row is gone', async () => { + const sha = 'h5e5h5e5h5e5h5e5h5e5h5e5h5e5h5e5h5e5h5e5'; + const stackName = 'retry-generation-pointer-dangling'; + const { svc, generationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy(stackName, sha); + // A pointer-integrity violation: the application still names a + // generation row that no longer exists. The fetch arm is the + // defensible recovery, but it must announce itself rather + // than silently refetch over a broken pointer. + DatabaseService.getInstance().getDb() + .prepare('DELETE FROM gitops_generations WHERE id = ?').run(generationId); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + + try { + await svc.retry(stackName, { actor: 'tester' }); + + expect(reconcileSpy).toHaveBeenCalledWith( + expect.objectContaining({ trigger: 'retry', intent: 'fetch' }), + ); + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + // The fetch arm's own settle is the fixture's business as + // usual; what this test pins is the routing and the + // announced pointer-integrity warning. + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining(generationId)); + } finally { + reconcileSpy.mockRestore(); + warnSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('refuses the deploy resume when the generation row disappears before the lock', async () => { + const sha = 'h6f6h6f6h6f6h6f6h6f6h6f6h6f6h6f6h6f6h6f6'; + const stackName = 'retry-deploy-resume-generation-gone'; + const { svc, applicationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy(stackName, sha); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + // The entry predicate sees the generation row; by the time the + // in-lock revalidation reads it, the row is gone. The resume + // must refuse with the unreadable-evidence advice, and the + // fetch arm must not be reached beside the refusal. + const realGeneration = GitOpsStore.prototype.getGeneration; + let generationCalls = 0; + const generationSpy = vi.spyOn(GitOpsStore.prototype, 'getGeneration') + .mockImplementation((generationIdArg: string) => { + generationCalls++; + if (generationCalls === 2) return undefined; + return realGeneration.call(GitOpsStore.getInstance(), generationIdArg); + }); + + try { + const result = await svc.retry(stackName, { actor: 'tester' }); + + expect(result.outcome).toBe('blocked'); + expect(result.reason).toMatch(/could not be read; its deploy evidence is unavailable/); + expect(result.nextAction).toBe('view_target_results'); + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + // Past the reservation, so the refusal settles as a blocked + // attempt and is the only new row. + const startedIds = historyOperationIds(applicationId, 'source_reconcile_started'); + expect(startedIds).toHaveLength(startedBefore + 1); + const settled = settledAttemptsForApplication(applicationId) + .find((row) => row.operation_id === startedIds[startedIds.length - 1]!); + expect(JSON.parse(settled!.after_json!)).toMatchObject({ + outcome: 'blocked', + nextAction: 'view_target_results', + }); + // The blinding is only meaningful if the deploy-resume path + // reads the generation exactly twice (entry predicate, + // in-lock revalidation). Pin it so a future extra read + // cannot silently shift the blinded call to the wrong site. + expect(generationSpy).toHaveBeenCalledTimes(2); + } finally { + reconcileSpy.mockRestore(); + generationSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('defers to the recovery evidence for every unfinished recovery phase', async () => { + const sha = 't6t6t6t6t6t6t6t6t6t6t6t6t6t6t6t6t6t6t6t6'; + const svc = await seedPending('retry-recovery-restoring', 'services:\n x:\n image: alpine\n', sha); + acceptPendingCandidate('retry-recovery-restoring'); + const applicationId = liveApp('retry-recovery-restoring')!.id; + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const dispatchSpy = vi.spyOn(svc, 'dispatchAcceptedGeneration'); + + // The guard defers on any phase except the settled 'complete' + // receipt. 'capturing' is schema-legal with no production + // writer yet: pinning it here freezes the fail-safe inversion + // for the day one lands, and it is projection-transparent + // today, so the report is the accepted-state projection while + // the contract is that no work starts. 'failed' runs last + // because it also sets the recovery failure columns, which the + // projection reads independently of the phase. + const cases: Array<{ phase: string; outcome: string; reason?: RegExp }> = [ + { phase: 'restoring', outcome: 'recovery_required' }, + { phase: 'compensating', outcome: 'recovery_required' }, + { phase: 'capturing', outcome: 'no_source_change' }, + { phase: 'failed', outcome: 'recovery_required', reason: /Recovery itself failed \(post_mutation\)/ }, + ]; + + try { + for (const c of cases) { + // 'failed' normally arrives with the recovery failure + // columns from recoveryFailed/rollbackPartialFailed; + // the distinguishing reason proves which projection was + // read rather than a generic one. + DatabaseService.getInstance().getDb().prepare( + c.phase === 'failed' + ? "UPDATE gitops_applications SET recovery_phase = 'failed', failure_stage = 'recovery', failure_class = 'post_mutation' WHERE id = ?" + : 'UPDATE gitops_applications SET recovery_phase = ? WHERE id = ?', + ).run(...(c.phase === 'failed' ? [applicationId] : [c.phase, applicationId])); + reconcileSpy.mockClear(); + dispatchSpy.mockClear(); + + const result = await svc.retry('retry-recovery-restoring', { actor: 'tester' }); + + expect(result.outcome, c.phase).toBe(c.outcome); + if (c.reason) expect(result.reason).toMatch(c.reason); + expect(reconcileSpy, c.phase).not.toHaveBeenCalled(); + expect(dispatchSpy, c.phase).not.toHaveBeenCalled(); + } + } finally { + reconcileSpy.mockRestore(); + dispatchSpy.mockRestore(); + } + }); + + it('retries normally after a completed rollback (complete phase is a receipt, not outstanding work)', async () => { + const sha = 't7t7t7t7t7t7t7t7t7t7t7t7t7t7t7t7t7t7t7t7'; + const svc = await seedPending('retry-recovery-complete', 'services:\n x:\n image: alpine\n', sha); + const generationId = acceptPendingCandidate('retry-recovery-complete'); + const applicationId = liveApp('retry-recovery-complete')!.id; + // recoverySucceeded/rollbackCompleted persist 'complete' and + // nothing clears it; a settled receipt must not permanently + // degrade every future retry on this source to a projection + // read. The undelivered dispatch evidence still routes to the + // dispatch arm. + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET recovery_phase = 'complete' WHERE id = ?", + ).run(applicationId); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const { GitProjectManifestService } = await import('../services/GitProjectManifestService'); + const promoteSpy = vi.spyOn(GitProjectManifestService.prototype, 'promoteGeneration') + .mockImplementation(async () => {}); + + try { + const result = await svc.retry('retry-recovery-complete', { actor: 'tester' }); + + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(promoteSpy).toHaveBeenCalledTimes(1); + expect(result.outcome).toBe('no_source_change'); + const target = GitOpsStore.getInstance().getTarget(applicationId, await asyncDefaultNodeId())!; + expect(target.applied_generation_id).toBe(generationId); + } finally { + reconcileSpy.mockRestore(); + promoteSpy.mockRestore(); + } + }); + }); + it('throws when pending has been cleared between pull and apply', async () => { const svc = await seedPending('apply-cleared', 'services:\n x:\n image: alpine\n', 'aaaa111aaaa111aaaa111aaaa111aaaa111aaaa1'); DatabaseService.getInstance().clearGitSourcePending('apply-cleared'); diff --git a/backend/src/__tests__/gitops-transitions.test.ts b/backend/src/__tests__/gitops-transitions.test.ts index 8b59a394..f1857fcf 100644 --- a/backend/src/__tests__/gitops-transitions.test.ts +++ b/backend/src/__tests__/gitops-transitions.test.ts @@ -735,6 +735,107 @@ describe('gitops transitions', () => { expect(application.suspended_at).not.toBeNull(); }); + it('keeps a rollout pause across source suspension and resume', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-pause-coex', 'pause-coex-web'), nodeId: 1, envelope: envelope('op-act-pause-coex') }); + tx.rolloutPaused('app-pause-coex', null, 'waiting for the maintenance window', envelope('op-rp-app')); + tx.rolloutPaused('app-pause-coex', 1, 'fleet rollout paused', envelope('op-rp-target')); + + tx.sourceSuspended('app-pause-coex', 'operator hold', envelope('op-susp-pause-coex')); + // Suspension is a decision about future source work. It is not a rollout + // pause, and it must neither overwrite one nor read like one. + let application = store.getApplication('app-pause-coex')!; + expect(application.source_suspended_reason).toBe('operator hold'); + expect(application.pause_reason).toBe('waiting for the maintenance window'); + expect(store.getTarget('app-pause-coex', 1)?.pause_reason).toBe('fleet rollout paused'); + + tx.sourceUnsuspended('app-pause-coex', envelope('op-unsusp-pause-coex')); + application = store.getApplication('app-pause-coex')!; + expect(application.suspended_at).toBeNull(); + expect(application.source_suspended_reason).toBeNull(); + // The rollout's own pause survives the whole suspend/resume cycle on + // both rows: only rolloutUnpaused clears it. + expect(application.pause_at).not.toBeNull(); + expect(application.pause_reason).toBe('waiting for the maintenance window'); + const target = store.getTarget('app-pause-coex', 1)!; + expect(target.pause_at).not.toBeNull(); + expect(target.pause_reason).toBe('fleet rollout paused'); + }); + + it('records a promotion whose apply completed after suspension interrupted it', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-susp-promo', 'susp-promo-web'), nodeId: 1, envelope: envelope('op-act-susp-promo') }); + store.insertGeneration(gen('gen-susp-promo', 'app-susp-promo')); + tx.fetchStarted('app-susp-promo', envelope('op-f-susp-promo')); + tx.fetched('app-susp-promo', 'abc123', envelope('op-f-susp-promo')); + tx.candidateReady('app-susp-promo', 'gen-susp-promo', false, envelope('op-c-susp-promo')); + const applyOp = envelope('op-a-susp-promo'); + tx.applyStarted('app-susp-promo', 'gen-susp-promo', applyOp); + + // Suspension interrupts the in-flight operation without discarding it: + // the apply is marked interrupted, and the candidate stays staged. + tx.sourceSuspended('app-susp-promo', 'operator hold', envelope('op-susp-susp-promo')); + expect(store.getApplication('app-susp-promo')?.interruption_stage).toBe('apply_started'); + + // The apply's terminal event still lands after the interrupt. Suspension + // gates new work (fetch, apply, accept); it never cancels the accounting + // of work that already committed to disk. + tx.applied({ + applicationId: 'app-susp-promo', + generationId: 'gen-susp-promo', + artifactSetId: 'art-susp-promo', + sourceAcceptanceId: 'acc-susp-promo', + authority: 'operator', + envelope: applyOp, + }); + const application = store.getApplication('app-susp-promo')!; + expect(application.accepted_generation_id).toBe('gen-susp-promo'); + expect(application.interruption_stage).toBeNull(); + // Recording an in-flight promotion does not resume anything. + expect(application.suspended_at).not.toBeNull(); + expect(store.getTarget('app-susp-promo', 1)?.applied_generation_id).toBe('gen-susp-promo'); + }); + + it('accepts a matching late deploy result for a suspended source', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-late-dep', 'late-dep-web', 'gen-late-dep', 'art-late-dep', 'acc-late-dep'); + const deployOp = envelope('op-dep-late'); + tx.deployStarted('app-late-dep', 1, 'gen-late-dep', deployOp); + + // The deploy is in flight when the operator suspends. Suspension gates + // new source work, and it interrupts an operation the application row + // still shows as active; a deploy is tracked on the target row alone, so + // an in-flight deploy is left running rather than marked interrupted. + tx.sourceSuspended('app-late-dep', 'operator hold', envelope('op-susp-late-dep')); + const suspended = store.getTarget('app-late-dep', 1)!; + expect(suspended.active_operation_stage).toBe('deploy_started'); + expect(suspended.interruption_stage).toBeNull(); + + // Its terminal events are therefore admissible while the source is + // suspended: requireMatchingDeploy matches the live operation and + // records its outcome, so evidence of a deploy nobody can recall is + // kept, not dropped. + tx.deployBound('app-late-dep', 1, 'gen-late-dep', deployOp); + let target = store.getTarget('app-late-dep', 1)!; + expect(target.deployed_generation_id).toBe('gen-late-dep'); + expect(target.failure_stage).toBeNull(); + + // The failure arm behaves the same way: a later deploy for the same + // applied generation records its failure while still suspended. + tx.deployStarted('app-late-dep', 1, 'gen-late-dep', envelope('op-dep-late-2')); + tx.deployFailed('app-late-dep', 1, 'post_mutation', envelope('op-dep-late-2')); + target = store.getTarget('app-late-dep', 1)!; + expect(target.failure_stage).toBe('deploy'); + expect(target.failure_class).toBe('post_mutation'); + expect(target.deployed_generation_id).toBe('gen-late-dep'); + // Neither terminal resumed anything: recording is accounting, not a + // source operation. + expect(store.getApplication('app-late-dep')?.suspended_at).not.toBeNull(); + }); + it('targetApplied binds a Direct target only after the generation is accepted', () => { const store = GitOpsStore.getInstance(); const tx = GitOpsTransitions.getInstance(); diff --git a/backend/src/__tests__/source-controller.test.ts b/backend/src/__tests__/source-controller.test.ts index 4ed9f5ba..94880001 100644 --- a/backend/src/__tests__/source-controller.test.ts +++ b/backend/src/__tests__/source-controller.test.ts @@ -7,6 +7,7 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; import { directApplicationFixture } from './helpers/gitopsFixtures'; +import { DatabaseService } from '../services/DatabaseService'; import { GitOpsStore } from '../services/gitops/store'; import { GitSourceService } from '../services/GitSourceService'; import { SourceController } from '../services/gitops/SourceController'; @@ -213,4 +214,199 @@ describe('SourceController', () => { expect(reconcile).toHaveBeenCalledTimes(1); }); + + describe('evaluateNow (resume re-evaluation)', () => { + /** Insert a live automatic application row for a stack name. */ + function seedLive(stackName: string): GitOpsApplicationRow { + const app = autoFixture(`app-${stackName}`, stackName); + GitOpsStore.getInstance().insertApplication(app); + return app; + } + + it('fetches a live automatic source immediately, tagged with the resume trigger', async () => { + seedLive('now-web'); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + + await controller.evaluateNow('now-web'); + + expect(reconcile).toHaveBeenCalledTimes(1); + expect(reconcile).toHaveBeenCalledWith(expect.objectContaining({ + intent: 'fetch', + applicationId: 'app-now-web', + stackName: 'now-web', + trigger: 'resume', + })); + // No timer was running: nothing here waited for a tick. + expect(vi.getTimerCount()).toBe(0); + }); + + it('skips a manual-policy source without reconciling it', async () => { + const app: GitOpsApplicationRow = { ...autoFixture('app-now-manual', 'now-manual-web'), source_policy: 'manual' }; + GitOpsStore.getInstance().insertApplication(app); + const reconcile = spyOnReconcile(); + + await controller.evaluateNow('now-manual-web'); + + expect(reconcile).not.toHaveBeenCalled(); + }); + + it('skips a suspended source, so a resume race never wakes it early', async () => { + const app = { ...autoFixture('app-now-susp', 'now-susp-web'), suspended_at: Date.now() }; + GitOpsStore.getInstance().insertApplication(app); + const reconcile = spyOnReconcile(); + + await controller.evaluateNow('now-susp-web'); + + expect(reconcile).not.toHaveBeenCalled(); + }); + + it('is a silent no-op for a stack with no live application', async () => { + const reconcile = spyOnReconcile(); + + await controller.evaluateNow('now-missing'); + + expect(reconcile).not.toHaveBeenCalled(); + }); + + it('parks a resume wake while a tick owns the application and runs it once when the tick settles', async () => { + const app = seedLive('now-busy'); + // The scan's due list is mocked to the same row, so the tick's + // evaluation and the resume path address one identical id. + mockDue([app]); + let settleFirstCall!: (result: ReconcileResult) => void; + const firstCall = new Promise((resolve) => { settleFirstCall = resolve; }); + const reconcile = spyOnReconcile().mockReturnValueOnce(firstCall).mockResolvedValue(okResult); + + controller.start(); + await advanceOneTick(); + expect(reconcile).toHaveBeenCalledTimes(1); + + // Two resumes arrive while the tick's evaluation is still + // pending: neither starts work beside it, and together they + // coalesce into the single wake they asked for. + await controller.evaluateNow('now-busy'); + await controller.evaluateNow('now-busy'); + expect(reconcile).toHaveBeenCalledTimes(1); + + settleFirstCall(okResult); + // Flush the tick's continuation chain (evaluate's post-reconcile + // steps plus the scan's inFlight release, which drains the + // parked wake into a fresh resume evaluation). + await vi.advanceTimersByTimeAsync(0); + + // The wake was honored without any second manual call: the + // park-then-drain is what the operator's resume bought. The + // two parked resumes produced exactly one fresh evaluation, + // tagged as a resume, and the drain's own release leaves no + // further wake parked (a third call would have landed). + expect(reconcile).toHaveBeenCalledTimes(2); + expect(reconcile.mock.calls[1]![0]).toMatchObject({ trigger: 'resume' }); + // "At most one" pinned: the drain's settle releases the slot + // with nothing left parked, so no third evaluation ever fires. + await vi.advanceTimersByTimeAsync(0); + expect(reconcile).toHaveBeenCalledTimes(2); + }); + + it('drops a parked wake whose source went stale, without evaluating it', async () => { + const app = seedLive('now-stale'); + mockDue([app]); + let settleFirstCall!: (result: ReconcileResult) => void; + const firstCall = new Promise((resolve) => { settleFirstCall = resolve; }); + const reconcile = spyOnReconcile().mockReturnValueOnce(firstCall).mockResolvedValue(okResult); + + controller.start(); + await advanceOneTick(); + expect(reconcile).toHaveBeenCalledTimes(1); + + await controller.evaluateNow('now-stale'); + expect(reconcile).toHaveBeenCalledTimes(1); + // The drain re-reads eligibility, so the stale wake is dropped + // exactly as a fresh call would drop it: no evaluation of a + // suspended source. + DatabaseService.getInstance().getDb() + .prepare('UPDATE gitops_applications SET suspended_at = ? WHERE id = ?') + .run(Date.now(), app.id); + settleFirstCall(okResult); + await vi.advanceTimersByTimeAsync(0); + + expect(reconcile).toHaveBeenCalledTimes(1); + }); + + it('drains the parked wake even when the owning evaluation crashes past the reconcile', async () => { + const app = seedLive('now-owner-crash'); + mockDue([app]); + let settleFirstCall!: (result: ReconcileResult) => void; + const firstCall = new Promise((resolve) => { settleFirstCall = resolve; }); + const reconcile = spyOnReconcile().mockReturnValueOnce(firstCall).mockResolvedValue(okResult); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + // The crash lands in evaluate()'s post-reconcile tail (the row + // re-read), not in reconcile itself: the owner's chain rejects, + // the scan's load-bearing catch logs it, and the release's + // finally must still drain the wake. The mock is once-only, so + // the drained evaluation's own re-read falls through to the + // real store. + vi.spyOn(GitOpsStore.getInstance(), 'getApplication') + .mockImplementationOnce(() => { throw new Error('simulated crash after reconcile'); }); + + controller.start(); + await advanceOneTick(); + expect(reconcile).toHaveBeenCalledTimes(1); + + await controller.evaluateNow('now-owner-crash'); + settleFirstCall(okResult); + await vi.advanceTimersByTimeAsync(0); + + expect(reconcile).toHaveBeenCalledTimes(2); + expect(reconcile.mock.calls[1]![0]).toMatchObject({ trigger: 'resume' }); + const logged = errorSpy.mock.calls.map((args) => args.map(String).join('\n')).join('\n'); + expect(logged).toContain('evaluation crashed'); + errorSpy.mockRestore(); + }); + + it('logs a parked-wake drain whose prologue throws instead of rejecting unhandled', async () => { + const app = seedLive('now-drain-throws'); + mockDue([app]); + let settleFirstCall!: (result: ReconcileResult) => void; + const firstCall = new Promise((resolve) => { settleFirstCall = resolve; }); + const reconcile = spyOnReconcile().mockReturnValueOnce(firstCall).mockResolvedValue(okResult); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + controller.start(); + await advanceOneTick(); + await controller.evaluateNow('now-drain-throws'); + expect(reconcile).toHaveBeenCalledTimes(1); + + // evaluateNow's eligibility read runs before its own catch, so + // a store failure at that point rejects the call. The drain is + // a detached fire-and-forget (nothing above it in the release + // chain can catch), so it must carry its own guard: the + // rejection becomes a logged line, never an unhandled rejection + // (which would take the process down). + vi.spyOn(GitOpsStore.getInstance(), 'getLiveDirectApplication') + .mockImplementationOnce(() => { throw new Error('database is locked'); }); + settleFirstCall(okResult); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(0); + + // No resume evaluation ran (the prologue died before it), but + // the failure was logged rather than silently escaping. + expect(reconcile).toHaveBeenCalledTimes(1); + const logged = errorSpy.mock.calls.map((args) => args.map(String).join('\n')).join('\n'); + expect(logged).toContain('parked-wake drain could not start'); + errorSpy.mockRestore(); + }); + + it('survives a reconcile rejection without throwing at the route', async () => { + seedLive('now-throws'); + spyOnReconcile().mockRejectedValue(new Error('boom')); + // evaluate() logs the reconcile failure itself and returns; + // evaluateNow must not turn it into a rejection for the caller. + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + await expect(controller.evaluateNow('now-throws')).resolves.toBeUndefined(); + + expect(errorSpy).toHaveBeenCalledTimes(1); + errorSpy.mockRestore(); + }); + }); }); diff --git a/backend/src/routes/gitSources.ts b/backend/src/routes/gitSources.ts index 7e8eb54c..ff423a5d 100644 --- a/backend/src/routes/gitSources.ts +++ b/backend/src/routes/gitSources.ts @@ -766,6 +766,32 @@ stackGitSourceRouter.post('/:stackName/git-source/resume', async (req: Request, const result = await GitSourceService.getInstance().resume(stackName, { actor: req.user?.username ?? 'unknown', }); + // A resumed source is re-evaluated immediately rather than idling + // until its next poll or retry cursor: the controller's resume trigger + // re-resolves source state and, for an automatic source, target + // binding through the shared acceptance/dispatch arm. Fire-and-forget + // by design: this response reports that suspension was cleared, not + // the outcome of a reconcile that has not finished yet, and the + // durable row plus the next tick are the authority on the latter. + // The wake has its own guard: a failure starting it must not + // retroactively fail a resume that already committed, so it is + // logged, never sent to sendGitSourceError. evaluateNow absorbs its + // evaluation failures itself; the rejection arm catches anything + // before its own try exists (evaluateNow is async, so even its + // synchronous prologue, a store error before any evaluation could + // run, surfaces as a rejection) and any future escape, and the + // try/catch covers a getInstance()-level failure only. + const logWakeFailure = (error: unknown): void => { + console.error( + `[GitSources] Resume re-evaluation could not start for ${sanitizeForLog(stackName)}:`, + error instanceof Error ? error.message : String(error), + ); + }; + try { + SourceController.getInstance().evaluateNow(stackName).catch(logWakeFailure); + } catch (error) { + logWakeFailure(error); + } res.json(result); } catch (error) { sendGitSourceError(res, error); diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 0012ba31..37d0d316 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -244,7 +244,11 @@ export interface HealthGateRunRow { failure_source: 'primary' | 'collateral' | null; /** * Reserved for the GitOps deploy path: the generation live when this run - * started. No writer populates it yet, so it is currently always null. + * started. Populated from beginStack's binding argument and the deploy + * callers pass it when their mutation was GitOps-tracked, plus recovery + * reservations (reserveRecoveryRun); null otherwise and for legacy rows. + * The stage-aware retry reads it to decide whether a failed stack + * verdict belongs to the currently applied generation. */ deployed_generation_id?: string | null; } @@ -4321,6 +4325,13 @@ stmt.run('gitops_schema_version', '1'); ).get(nodeId, stackName) as HealthGateRunRow | undefined; } + /** The most recent stack-scoped run for the stack (service gates excluded). */ + public getLatestStackHealthGateRun(nodeId: number, stackName: string): HealthGateRunRow | undefined { + return this.db.prepare( + "SELECT * FROM health_gate_runs WHERE node_id = ? AND stack_name = ? AND target_scope = 'stack' ORDER BY started_at DESC, id DESC LIMIT 1" + ).get(nodeId, stackName) as HealthGateRunRow | undefined; + } + /** Finalize runs left observing by a previous process (startup sweep). */ /** * Runs a previous process left observing. diff --git a/backend/src/services/GitSourceService.ts b/backend/src/services/GitSourceService.ts index 0382015e..facc9a73 100644 --- a/backend/src/services/GitSourceService.ts +++ b/backend/src/services/GitSourceService.ts @@ -39,7 +39,7 @@ import { projectApplication } from './gitops/derive'; import { outcomeFromSourceFacet, parseReconcileResultPayload, type ReconcileOutcome, type ReconcileResult } from './gitops/outcomes'; import { coalesceKey, deliveryKey, type ReconcileRequest, type ReconcileTrigger } from './gitops/triggers'; import { classifyFailure, effectivePollIntervalSecs } from './gitops/backoff'; -import { BlueprintTargetAdapter, type AcceptedGeneration, type DispatchContext, type DispatchResult } from './gitops/handoff'; +import { BlueprintTargetAdapter, buildAcceptedGeneration, type AcceptedGeneration, type DispatchContext, type DispatchResult } from './gitops/handoff'; import { GitOpsTransitions, GitOpsTransitionError, @@ -3627,8 +3627,13 @@ export class GitSourceService { } /** - * Resume acting on a source. Does not itself fetch; the next scheduled - * poll, retry, or manual reconcile picks the source back up. + * Resume acting on a source. Clears suspension and re-arms the steady- + * state poll cursor; it does not itself fetch. The immediate wake is the + * resume route's job: after this method returns, the route fires + * SourceController.evaluateNow() so the source is re-evaluated right + * away (re-resolving source state and, for an automatic source, target + * binding through the shared acceptance/dispatch arm) instead of idling + * until the next scheduled poll or retry. * * Unlike suspend(), a refused resume is tolerated rather than surfaced. * The result is read back from the row after the attempted write, so a @@ -3683,25 +3688,662 @@ export class GitSourceService { } /** - * An explicit, operator-initiated re-evaluation: resolves the live - * application server-side rather than trusting a caller-supplied id, for - * callers that hold only a stack name, and drives a fresh fetch-intent - * reconcile through it. + * The dispatch evidence stage-aware retry reads: which dispatch stage + * an operator retry should resume, or null when no dispatch is + * pending. Two arms share this one decision because the arms are + * mutually exclusive by evidence, and the helper is the single place + * that exclusivity is stated: + * - `'promote'`: the cheap pre-filter over the same columns + * `dispatchDirectLocked` revalidates (the generation row still readable, + * a live application, an active Direct + * target whose candidate still names the accepted generation, and an + * applied pointer that has not caught up). It lags the dispatch's own + * checks (commit sha, artifact-set evidence, recomputed change plan), + * and the in-lock revalidation remains the authority. Generation files + * were never promoted, so retry hands the generation to the shared + * dispatch boundary. + * - `'deploy'`: the promotion already committed (the applied pointer + * names the accepted generation) and the durable evidence says the + * stage after it failed: either the deploy failed (`failure_stage = + * 'deploy'`, set by the deploy-failed/deploy-unbound transitions and + * cleared by deploy-bound), or the health gate watching that exact + * applied generation finalized as failed (the stack's most recent + * stack-scoped `health_gate_runs` row, bound to the generation id, + * with status `'failed'`). Nothing may be fetched, accepted, + * promoted, or re-bound; only the unfinished deploy stage is retried, + * and every resume re-opens the health gate on the deploy it runs. A + * deploy or health failure with no accepted-generation evidence is + * not deploy-resume material: the fetch arm re-decides from the + * source instead. + * Retry routes an operator's request to the stage the evidence actually + * stopped at, and each arm's in-lock revalidation remains the authority + * when the state moves in between. + */ + private acceptedGenerationDispatchStage(app: GitOpsApplicationRow): { stage: 'promote' | 'deploy'; generationId: string } | null { + const generationId = app.accepted_generation_id; + if (!generationId) return null; + if (!GitOpsStore.getInstance().getGeneration(generationId)) { + // A live application naming a generation that no longer exists is + // a pointer-integrity violation. The fetch arm is a defensible + // recovery, but it must announce itself: an operator who asked to + // retry a stuck dispatch deserves to see why a fetch ran instead. + console.warn( + `[GitSource] ${sanitizeForLog(app.stack_name ?? app.id)} names accepted generation ${sanitizeForLog(generationId)} but the row is missing; retry falls back to a fresh fetch.`, + ); + return null; + } + const target = GitOpsStore.getInstance().getTarget(app.id, NodeRegistry.getInstance().getDefaultNodeId()); + if (!target || target.target_status !== 'active') return null; + // The deploy arm's two evidence kinds both route to re-running the + // deploy stage only: a failed deploy (`failure_stage = 'deploy'`, + // set by the deploy-failed/deploy-unbound transitions and cleared + // by deploy-bound) left the generation undelivered, and a failed + // health gate (below) delivered it but the verdict says the run + // is sick. Neither may be fetched, accepted, promoted, or + // re-bound. healthFinalized() itself records health outcomes as + // promotions of the healthy/LKG pointers and never writes the + // target's failure columns, so the health evidence lives in the + // gate's own durable run rows, not on the target. + if (target.candidate_generation_id === generationId && target.applied_generation_id !== generationId) { + return { stage: 'promote', generationId }; + } + if (target.applied_generation_id !== generationId) return null; + if (target.failure_stage === 'deploy') { + return { stage: 'deploy', generationId }; + } + // A failed health verdict only owns the arm when the deploy it + // watched actually bound (deploy-bound clears the failure columns, + // so the honest state is NULL): any other target failure belongs + // to its own path, not to a deploy resume. + if (target.failure_stage === null + && app.stack_name + && this.appliedGenerationHealthFailed(NodeRegistry.getInstance().getDefaultNodeId(), app.stack_name, generationId)) { + return { stage: 'deploy', generationId }; + } + return null; + } + + /** + * Whether the stack's most recent stack-scoped health gate run is a + * `failed` verdict bound to exactly the given generation. Service-scoped + * gates are excluded: they watch one service, not the applied + * generation's stack-wide health. An `observing` or `unknown` row is + * not a health-failure claim, and a row bound to a different (or no) + * generation cannot vouch for this one, so neither selects an arm; + * those fall to the fetch arm, which re-decides from the source. + */ + private appliedGenerationHealthFailed(nodeId: number, stackName: string, generationId: string): boolean { + const run = DatabaseService.getInstance().getLatestStackHealthGateRun(nodeId, stackName); + return !!run + && run.status === 'failed' + && run.deployed_generation_id === generationId; + } + + /** + * The dispatch context for the local Direct target: the boundary's + * shared construction used by every dispatch of a locally-accepted + * generation (automatic controller dispatch, explicit retry). The + * nodeId is resolved per call because it is process configuration, not + * durable evidence; the dispatch itself re-reads the live target under + * the stack lock and treats this context as routing only. + */ + public static directDispatchContext(): DispatchContext { + return { targetMode: 'direct', nodeId: NodeRegistry.getInstance().getDefaultNodeId(), bindingRevision: null }; + } + + /** + * An explicit, operator-initiated re-evaluation, resuming at the stage + * the durable evidence says failed. A fetch or validation failure + * recorded on the application, or no dispatch evidence at all, drives + * a fresh fetch-intent reconcile. When the accepted + * generation is still awaiting delivery (candidate pointer set, applied + * pointer behind), retry calls dispatchAcceptedGeneration() against + * exactly that generation: the source is never refetched, and + * acceptance is never re-run, because both already succeeded and the + * failure lives past the acceptance boundary. When the generation is + * already applied but its deploy failed or its health gate's latest + * stack-scoped verdict is a failure for exactly that generation, retry + * resumes at the deploy alone (retryDeploy): fetching, accepting, + * promoting, and re-binding are all refused by the evidence (the + * applied pointer names this generation; re-binding it would throw), + * and the resume re-opens the health gate on the deploy it runs. The + * dispatch's and the deploy-resume's own revalidation under the stack + * lock stays the authority on whether it may proceed; a refusal comes + * back as a blocked outcome naming why. A refusal that reserved an + * attempt settles it carrying the same advice; a refusal before any + * reservation (no stack bound, no source configured, the stack lock + * held, or the in-lock source recheck) reserves nothing, so the + * response itself is the only record, which is deliberate: nothing + * started, so there is nothing to reconcile later. * * The 'retry' trigger is recorded on the durable attempt for audit and - * recovery. It does not yet change fetch behavior; a later permanent- - * failure gate can use it to authorize an explicit operator retry. + * recovery on all three arms. */ public async retry(stackName: string, opts: { actor: string }): Promise { const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); if (!app) return GitSourceService.noApplicationResult(); - return this.reconcile({ - intent: 'fetch', - applicationId: app.id, + // A source that is suspended, has an operation in flight, has an + // unfinished recovery, or has an unrecovered interruption marker + // reports its real state instead of starting work beside it: the + // suspended projection is the resume advice, the in-flight + // projection says no settled result exists yet, and the recovery + // projection is what the recovery path owns. An unrecovered crash + // can leave an active_operation_stage set with no live work, and + // retrying over it would race the pass that holds that evidence. + // Unfinished recovery is inverted, not enumerated: every phase + // except the settled 'complete' receipt (including 'capturing' and + // any phase added later) defers, so a new intermediate phase can + // never silently start retry work beside it. 'complete' is a + // settled receipt written by recoverySucceeded or + // rollbackCompleted that no code path clears; deferring on it + // would strand every future retry on a source that just + // recovered. + const unfinishedRecovery = app.recovery_phase !== null && app.recovery_phase !== 'complete'; + if (app.suspended_at || app.active_operation_stage || unfinishedRecovery || app.interruption_stage) { + return this.deriveReconcileResult(stackName); + } + const dispatch = this.acceptedGenerationDispatchStage(app); + const sourceStageFailure = app.failure_stage === 'fetch' || app.failure_stage === 'validation'; + // The fetch arm owns the precedence: a newer source-stage failure + // outranks stale dispatch evidence on either arm. + if (!dispatch || sourceStageFailure) { + return this.reconcile({ + intent: 'fetch', + applicationId: app.id, + stackName, + trigger: 'retry', + actor: opts.actor, + }); + } + return dispatch.stage === 'promote' + ? this.retryDispatch(stackName, app.id, dispatch.generationId, opts.actor) + : this.retryDeploy(stackName, app.id, dispatch.generationId, opts.actor); + } + + /** + * The promote arm of retry(): rebuild the accepted-generation contract + * from the persisted generation row and hand it to the shared dispatch + * boundary. Only reached when acceptedGenerationDispatchStage proved + * the pointers, so a missing or corrupt row here means the evidence + * moved or decayed between the check and the read; that is reported, + * never papered over with a refetch. + */ + private async retryDispatch( + stackName: string, + applicationId: string, + generationId: string, + actor: string, + ): Promise { + let dispatch: DispatchResult; + try { + const generationRow = GitOpsStore.getInstance().getGeneration(generationId); + if (!generationRow) { + return { + outcome: 'blocked', + reason: 'The accepted generation could not be read; its dispatch evidence is unavailable.', + nextAction: 'view_target_results', + }; + } + dispatch = await this.dispatchAcceptedGeneration( + buildAcceptedGeneration(generationRow), + GitSourceService.directDispatchContext(), + { trigger: 'retry', actor }, + ); + } catch (e) { + // Nothing past this point throws once a reservation exists, so + // a throw here means the contract could not be built or the + // entry reads failed; the generation stands accepted, and the + // operator can retry once the cause is fixed. errorForLog keeps + // the stack for debugging while scrubbing credential-shaped + // text, which a transport-level failure can carry. + console.error( + `[GitSource] Explicit retry could not dispatch the accepted generation for ${sanitizeForLog(stackName)}:`, + errorForLog(e), + ); + return { + outcome: 'blocked', + reason: 'The accepted generation could not be dispatched; its evidence was unreadable.', + nextAction: 'view_target_results', + }; + } + if (dispatch.status === 'dispatched') { + // The projection re-read after the dispatch is the truthful + // post-state answer; every dispatch path that reserved an + // attempt has settled it before returning. + return this.deriveReconcileResult(stackName); + } + return { + outcome: 'blocked', + reason: dispatch.reason, + nextAction: GitSourceService.dispatchRefusalNextAction(dispatch.reason), + }; + } + + /** + * The deploy arm of retry(): resume only the unfinished deploy of a + * generation the target has already applied. Entry shape mirrors the + * promote arm's shared boundary: refuse cheaply before taking the + * stack lock, reserve the durable attempt inside the lock, and settle + * every reserved path before returning. A lock-held refusal reserves + * nothing, matching the dispatch boundary's rule that a retry refused + * before any work started leaves only its response as the record. + */ + private async retryDeploy( + stackName: string, + applicationId: string, + generationId: string, + actor: string, + ): Promise { + const app = GitOpsStore.getInstance().getApplication(applicationId); + if (!app?.stack_name) { + return { + outcome: 'blocked', + reason: 'No Direct stack is bound to this application.', + nextAction: 'resolve_conflict', + }; + } + if (!DatabaseService.getInstance().getGitSource(stackName)) { + return { + outcome: 'blocked', + reason: `No Git source is configured for ${stackName}.`, + nextAction: 'configure_credentials', + }; + } + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + // Nothing is reserved until the stack lock is held: a deploy + // resume refused at the lock never started any work, so it must + // not leave a reservation for startup recovery to chase. + const lock = await StackOpLockService.getInstance().runExclusive( + nodeId, stackName, - trigger: 'retry', - actor: opts.actor, - }); + 'git_apply', + actor, + () => this.retryDeployLocked(stackName, applicationId, generationId, nodeId, actor), + getRegistryDeliveryLockContext(), + ); + if (!lock.ran) { + const reason = `Another operation (${lock.existing.action}) is already in progress for ${stackName}.`; + return { outcome: 'blocked', reason, nextAction: 'retry' }; + } + return lock.result; + } + + /** + * The deploy-stage retry, running while the shared stack lock is + * held. The evidence the entry predicate proved (accepted generation + * applied to the active Direct target with a recorded deploy failure + * or a failed stack-scoped health verdict bound to that generation) + * is re-read fresh here, because it could have moved between the + * predicate and the lock. Only the deploy branch's steps run, replayed + * by hand: the intent journal, the policy gate, the tracked Compose + * deploy, the health gate, and the post-deploy scan. Fetching, + * accepting, promoting, and re-binding are unreachable here by + * construction, and the applied pointer itself would refuse a re-bind + * (targetApplied() throws on a candidate-less re-bind). + * Every path that reserved its attempt settles it before returning. + */ + private async retryDeployLocked( + stackName: string, + applicationId: string, + generationId: string, + nodeId: number, + actor: string, + ): Promise { + const store = GitOpsStore.getInstance(); + const src = DatabaseService.getInstance().getGitSource(stackName); + if (!src) { + return { + outcome: 'blocked', + reason: `No Git source is configured for ${stackName}.`, + nextAction: 'configure_credentials', + }; + } + + let envelope: ReturnType; + try { + const allocated = GitOpsTransitions.getInstance().allocateReconcileAttempt( + applicationId, + actor, + 'retry', + Date.now(), + undefined, + // The same markers the promote arm writes, so startup + // recovery settles an interrupted deploy-resume against + // this attempt's own deploy evidence: the reservation names + // the exact generation and records that a deploy was + // requested under it (always true here: the deploy retry is + // itself the deploy request, where the promote arm writes + // the source's auto-deploy flag). + generationId, + true, + ); + if (!allocated.reserved) { + console.error( + `[GitSource] Deploy retry of ${sanitizeForLog(stackName)} allocated an already-reserved operation id ${sanitizeForLog(allocated.operationId)}.`, + ); + return { + outcome: 'blocked', + reason: 'A dispatch attempt for this stack is already recorded; check its outcome before dispatching again.', + nextAction: 'view_target_results', + }; + } + envelope = this.gitopsEnvelope(allocated.operationId, actor, 'retry'); + } catch (e) { + console.error( + `[GitSource] Failed to reserve a durable attempt for deploy retry of ${sanitizeForLog(stackName)}:`, + errorForLog(e), + ); + return { + outcome: 'blocked', + reason: "This stack's GitOps tracking is unavailable; reconfigure the source before dispatching again.", + nextAction: 'configure_credentials', + }; + } + // Every refusal past the reservation settles the attempt: the + // retry was real work that started durably, and recovery must be + // able to see how it ended even when it ended in a refusal. + const settleRefusal = ( + nextAction: ReconcileResult['nextAction'], + raw: string, + ): ReconcileResult => { + const reason = scrubCredentials(raw); + this.settleAttempt(applicationId, envelope, { outcome: 'blocked', reason, nextAction }); + return { outcome: 'blocked', reason, nextAction }; + }; + try { + // Re-read the evidence the entry predicate proved. The + // promotion-era acceptance pointers moving in between means + // this resume is no longer the truthful response, and the + // state that replaced it owns what happens next. + const app = store.getApplication(applicationId); + if (!app || app.stack_name !== stackName) { + return settleRefusal('resolve_conflict', 'No Direct stack is bound to this application.'); + } + if (app.suspended_at) { + return settleRefusal('resolve_conflict', `Reconciliation is suspended for ${stackName}.`); + } + if (app.target_mode !== 'direct') { + return settleRefusal('resolve_conflict', 'The application is no longer in Direct mode.'); + } + if (app.accepted_generation_id !== generationId) { + return settleRefusal('resolve_conflict', 'The accepted generation changed before the deploy retry could run.'); + } + const genRow = store.getGeneration(generationId); + if (!genRow) { + return settleRefusal('view_target_results', 'The accepted generation could not be read; its deploy evidence is unavailable.'); + } + const target = store.getTarget(applicationId, nodeId); + // Mirrors the arm selector exactly: deploy-failure evidence, or + // (clear target evidence plus a failed stack health verdict for + // this generation). Any other target failure belongs to its own + // path, not to a deploy resume. + const deployEvidence = !!target + && target.target_status === 'active' + && target.applied_generation_id === generationId + && (target.failure_stage === 'deploy' + || (target.failure_stage === null + && this.appliedGenerationHealthFailed(nodeId, stackName, generationId))); + if (!deployEvidence) { + // The evidence that selected this arm is gone or has + // changed (a successful deploy cleared it, a newer + // operation replaced the pointer, or the failed health + // verdict is no longer the stack's latest claim for this + // generation). Nothing here should have run; say so and + // stop without touching the stack. + return settleRefusal('view_target_results', 'The target no longer shows this generation as applied with a failed deploy or failed health verdict; nothing was re-deployed.'); + } + + // The deploy intent, identical to the promote arm's: journal + // it, read it back, and refuse rather than start an untracked + // deploy (see recordDeployDispatchIntent). A resume whose + // intent cannot be recorded and read back settles + // recovery_required because the applied files are real + // evidence the operator must see. + const intent = this.recordDeployDispatchIntent({ + stackName, + applicationId, + envelope, + generationId, + commitSha: genRow.commit_sha, + }); + if (intent.deployOperationId === null) { + const cause = intent.refusalCause ? ` (${intent.refusalCause})` : ''; + const reason = `The deploy retry could not record and read back its deploy intent${cause}, so the deploy was not started.`; + this.settlePostCommitAttempt(stackName, actor, applicationId, envelope, { + outcome: 'recovery_required', + reason, + nextAction: 'view_target_results', + }); + return { outcome: 'blocked', reason, nextAction: 'view_target_results' }; + } + const deployOpIntent = intent.deployOperationId; + + try { + // The policy gate runs inside the deploy's own failure + // classification, exactly as the completion pipeline runs + // it there: a gate block after a committed promotion is + // evidence the operator must see against the applied + // files, not a pre-work refusal, so it settles + // recovery_required under the same deploy-failure wording. + await assertPolicyGateAllows( + stackName, + nodeId, + buildSystemPolicyGateOptions(actor, { + bypass: false, + auditPath: `/api/stacks/${stackName}/git-source/apply`, + }), + ); + const autoDeploy = await ComposeService.getInstance(nodeId).deployStack( + stackName, + undefined, + undefined, + { source: 'git_apply', actor, gitopsDeployOperationId: deployOpIntent }, + ); + HealthGateService.getInstance().beginStack( + nodeId, + stackName, + 'deploy', + 'system:git-source', + { deployedGenerationId: autoDeploy.deployedGenerationId }, + ); + console.log( + `[GitSource] Deploy retried for ${stackName} at ${genRow.commit_sha.slice(0, 7)}${autoDeploy.gitopsOperationId ? ` (deploy op ${GitSourceService.shortOperationId(autoDeploy.gitopsOperationId)})` : ''}`, + ); + // Fire-and-forget, matching the completion pipeline's + // placement: the scan runs only after a successful deploy + // and must never delay or fail the retry response. + triggerPostDeployScan(stackName, nodeId).catch((err) => + console.error(`[Security] Post-deploy scan failed for ${sanitizeForLog(stackName)}:`, err), + ); + const settledResult = this.finalizeReconcileOutcome(stackName, undefined); + const settledPayload: ReconcileResult = { + ...settledResult, + ...(autoDeploy.gitopsOperationId + ? { deployGitopsOperationId: autoDeploy.gitopsOperationId } + : {}), + }; + this.settlePostCommitAttempt(stackName, actor, applicationId, envelope, settledPayload); + return settledPayload; + } catch (e) { + // Parity with the completion pipeline: never + // auto-compensate a deploy failure; the applied files + // stand, the previous workload keeps running, and the + // settled row names the failed deploy for recovery and for + // the operator. + const scrubbed = scrubCredentials((e as Error).message || String(e)); + console.error(`[GitSource] Deploy retry failed for ${stackName}: ${scrubbed}`); + // ComposeService stamps the GitOps deploy operation id on + // the error it rethrows when the deploy was tracked; read + // it type-guarded so an untracked failure simply yields no + // correlation instead of trusting an untyped field. + const stamped: unknown = (e as { gitopsDeployOperationId?: unknown }).gitopsDeployOperationId; + const deployOpId = typeof stamped === 'string' ? stamped : undefined; + const reason = `The source applied, but the deploy failed: ${scrubbed}`; + this.settlePostCommitAttempt(stackName, actor, applicationId, envelope, { + outcome: 'recovery_required', + reason, + nextAction: 'view_target_results', + ...(deployOpId ? { deployGitopsOperationId: deployOpId } : {}), + }); + return { outcome: 'blocked', reason, nextAction: 'view_target_results' }; + } + } catch (e) { + // The evidence re-read, the intent journaling, the gate, and + // the deploy all run inside this try: a store access or an + // unexpected throw must settle the reserved attempt and return + // an outcome, never escape runExclusive with it open. Log + // first; the settled row keeps only the scrubbed message. + console.error( + `[GitSource] Deploy retry of ${sanitizeForLog(stackName)} threw:`, + errorForLog(e), + ); + return settleRefusal('view_target_results', e instanceof Error ? (e.message || e.name) : String(e)); + } + } + + /** + * The deploy intent, shared by the dispatch's deploy branch and the + * deploy-stage retry: mint the deploy operation id, journal it, and + * return the id read back from the stored row (not the locally minted + * one), so what Compose receives is always the exact id recovery will + * look up. The journal write is dedupe-replayed: a replayed write + * keeps the previously stored row (and reports it not newly + * recorded), so the read-back, not the write's return value, is the + * authority on what is actually stored. The caller hands the returned + * id to Compose so its deploy transitions land under it. + * + * An attempt that cannot durably record and read back its intent + * returns `deployOperationId: null`, and the caller refuses to start + * the deploy: an untracked Compose run would leave no evidence row + * recovery could link the deploy to, and recovery would have to judge + * the attempt from source state alone. Refusing keeps every settled + * row's claims backed by recorded rows. Each refusal also journals a + * `deploy_intent_refused` witness (best-effort like the promotion + * witness), so an attempt whose settle write failed in the same window + * still reconstructs as the refusal at recovery instead of an + * apply-only completion. `refusalCause` carries the scrubbed why for + * the operator-visible settled reason: a cause-less refusal is + * undebuggable. + */ + private recordDeployDispatchIntent(args: { + stackName: string; + applicationId: string; + envelope: EventEnvelope; + generationId: string; + commitSha: string; + }): { deployOperationId: string | null; refusalCause?: string } { + const deployOperationId = crypto.randomUUID(); + const refuse = (cause: string): { deployOperationId: null; refusalCause: string } => { + // The cause rides into the operator-visible settled reason, so + // it is scrubbed like every other error string the dispatch + // persists. + const refusalCause = scrubCredentials(cause); + if (!this.recordGitOps(args.stackName, 'deploy intent refusal witness', () => { + GitOpsTransitions.getInstance().deployIntentRefused({ + applicationId: args.applicationId, + envelope: args.envelope, + generationId: args.generationId, + commitSha: args.commitSha, + }); + })) { + // recordGitOps already logged the rejection; the operation + // id ties that line to the intent failure being journaled + // here. The reservation's deploy-request fact still + // distinguishes this attempt from an apply-only completion + // if the settle is lost too, so the witness is + // belt-and-suspenders, not the last line. + console.error( + `[GitOps] Deploy intent refusal witness unavailable for ${sanitizeForLog(args.stackName)} (operation ${sanitizeForLog(args.envelope.operationId)}); recovery falls back to the reservation's recorded deploy request`, + ); + } + return { deployOperationId: null, refusalCause }; + }; + try { + GitOpsTransitions.getInstance().deployDispatched({ + applicationId: args.applicationId, + envelope: args.envelope, + generationId: args.generationId, + commitSha: args.commitSha, + deployOperationId, + }); + } catch (error) { + console.error( + `[GitOps] Could not record deploy intent for ${sanitizeForLog(args.stackName)} (operation ${sanitizeForLog(args.envelope.operationId)}):`, + errorForLog(error), + ); + return refuse(error instanceof Error ? error.message : String(error)); + } + let intentRow: GitOpsHistoryRow | undefined; + try { + intentRow = GitOpsStore.getInstance() + .getStageRowForAttempt(args.applicationId, args.envelope.operationId, 'deploy_dispatched'); + } catch (error) { + // The write landed but the read-back throws (a store still + // unhealthy in the write's wake): this is the same refusal, + // not a deploy failure, and it must not be reported as one. + return refuse(error instanceof Error ? error.message : String(error)); + } + let journaled: unknown; + if (intentRow) { + try { + journaled = decodeGitOpsJson(intentRow.after_json); + } catch (error) { + // A corrupt row is the same refusal as a payload that + // fails the guard: the stored intent cannot be trusted, so + // the deploy cannot be started against it. The decode + // error rides into the cause so the corruption is + // debuggable from the settled row. + return refuse(`the stored intent payload failed validation: ${error instanceof Error ? error.message : String(error)}`); + } + } + if (!isDeployDispatchedPayload(journaled)) { + return refuse(intentRow + ? 'the stored intent payload failed validation' + : 'the journaled intent row could not be read back'); + } + return { deployOperationId: journaled.deployOperationId }; + } + + /** + * The next-action vocabulary for a blocked dispatch. For refusals that + * reserved an attempt this mirrors what the dispatch's own settled row + * advises; the lock-held refusal reserves nothing (no settled row + * exists), and 'retry' is chosen there because nothing started. + * Transient refusals (the stack lock held by another action, the + * candidate read that failed on an IO blip) advise retrying the + * dispatch; the duplicate-reservation and tracking-unavailable refusals + * name their own remedy in the reason and map to the matching check; + * post-commit refusals (bind rejection, deploy-intent refusal, dirty + * promotion, failed deploy) advise viewing the target results; every + * other pre-promotion refusal is a state conflict to resolve. The + * reason strings are matched exactly, and the default is the + * conservative conflict advice: a dispatch message reworded away from + * these constants still returns a truthful blocked outcome, just with + * the generic nextAction. + */ + private static dispatchRefusalNextAction(reason: string): ReconcileResult['nextAction'] { + if ( + reason === 'Cannot read the accepted candidate; try again.' + || /^Another operation \(.+\) is already in progress for .+\.$/.test(reason) + ) { + return 'retry'; + } + if ( + reason === 'A dispatch attempt for this stack is already recorded; check its outcome before dispatching again.' + ) { + return 'view_target_results'; + } + if (reason === "This stack's GitOps tracking is unavailable; reconfigure the source before dispatching again.") { + return 'configure_credentials'; + } + if ( + reason.startsWith('The promotion committed, but') + || reason.startsWith('The promotion did not complete cleanly') + || reason.startsWith('The source applied, but the deploy failed') + ) { + return 'view_target_results'; + } + return 'resolve_conflict'; } /** @@ -3910,9 +4552,9 @@ export class GitSourceService { // hook stops the pipeline there: the bindRejected arm (below) // classifies the rejection with the recovery_required vocabulary // the rewritten files warrant, and no deploy ever ran. - // Captured by the deploy-intent hook on the refusal path so the - // settled reason can name why the intent failed: a cause-less - // refusal is undebuggable for the operator. + // Captured from the deploy-intent hook on the refusal path so + // the settled reason can name why the intent failed: a + // cause-less refusal is undebuggable for the operator. let intentRefusalCause: string | undefined; const result = await this.completeGitApply({ stackName, @@ -3969,106 +4611,23 @@ export class GitSourceService { planFingerprint: revalidation.plan.fingerprint, }); }), - // The deploy intent: mint the deploy operation id first, - // journal it, and hand the journaled id to the pipeline so - // Compose opens its deploy transitions under it. The id - // returned is read back from the stored row, not the locally - // minted one, so what Compose receives is always the exact id - // recovery will look up (a dedupe replay reports recorded:true - // even when it kept the stored id; the read-back below is the - // authority on what is actually stored). - // A dispatch that cannot durably record and read back its - // intent returns null and the pipeline refuses to start the - // deploy: an untracked Compose run would leave no evidence - // row recovery could link the deploy to, and recovery would - // have to judge the attempt from source state alone. Refusing - // keeps every settled row's claims backed by recorded rows. - // Each refusal also journals a `deploy_intent_refused` - // witness (best-effort like the promotion witness), so an - // attempt whose settle write failed in the same window still - // reconstructs as the refusal at recovery instead of an - // apply-only completion. + // The deploy intent, shared with the deploy-stage retry: + // journal it, read it back, and refuse rather than start + // an untracked deploy (see recordDeployDispatchIntent). + // The returned null keeps the hook's refusal contract: the + // pipeline reports deployIntentUnavailable and never hands + // the stack to Compose. The cause feeds the settled reason + // below, exactly as the retry arm consumes it. deployDispatchIntent: (): string | null => { - const deployOperationId = crypto.randomUUID(); - const refuse = (cause: string): null => { - // The cause rides into the operator-visible settled - // reason, so it is scrubbed like every other error - // string the dispatch persists. - intentRefusalCause = scrubCredentials(cause); - if (!this.recordGitOps(stackName, 'deploy intent refusal witness', () => { - GitOpsTransitions.getInstance().deployIntentRefused({ - applicationId: app.id, - envelope, - generationId: generation.generationId, - commitSha: generation.commitSha, - }); - })) { - // recordGitOps already logged the rejection; the - // operation id ties that line to the intent - // failure being journaled here. The reservation's - // deploy-request fact still distinguishes this - // attempt from an apply-only completion if the - // settle is lost too, so the witness is - // belt-and-suspenders, not the last line. - console.error( - `[GitOps] Deploy intent refusal witness unavailable for ${sanitizeForLog(stackName)} (operation ${sanitizeForLog(envelope.operationId)}); recovery falls back to the reservation's recorded deploy request`, - ); - } - return null; - }; - try { - GitOpsTransitions.getInstance().deployDispatched({ - applicationId: app.id, - envelope, - generationId: generation.generationId, - commitSha: generation.commitSha, - deployOperationId, - }); - } catch (error) { - console.error( - `[GitOps] Could not record deploy intent for ${sanitizeForLog(stackName)} (operation ${sanitizeForLog(envelope.operationId)}):`, - errorForLog(error), - ); - return refuse(error instanceof Error ? error.message : String(error)); - } - const intentRow = (() => { - try { - return GitOpsStore.getInstance() - .getStageRowForAttempt(app.id, envelope.operationId, 'deploy_dispatched'); - } catch (error) { - // The write landed but the read-back throws (a - // store still unhealthy in the write's wake): - // this is the same refusal, not a deploy - // failure. It must not escape into the - // pipeline's deploy-error arm, which would - // misreport it as "the deploy failed" when - // Compose was never called. - return refuse(error instanceof Error ? error.message : String(error)); - } - })(); - if (intentRow === null) return null; - let journaled: unknown = undefined; - if (intentRow) { - try { - journaled = decodeGitOpsJson(intentRow.after_json); - } catch (error) { - // A corrupt row is the same refusal as a payload - // that fails the guard: the stored intent cannot - // be trusted, so the deploy cannot be started - // against it. The throw must not escape into the - // pipeline's deploy-error arm, which would - // misreport this refusal as a failed deploy. - // The decode error rides into the cause so the - // corruption is debuggable from the settled row. - return refuse(`the stored intent payload failed validation: ${error instanceof Error ? error.message : String(error)}`); - } - } - if (!isDeployDispatchedPayload(journaled)) { - return refuse(intentRow - ? 'the stored intent payload failed validation' - : 'the journaled intent row could not be read back'); - } - return journaled.deployOperationId; + const intent = this.recordDeployDispatchIntent({ + stackName, + applicationId: app.id, + envelope, + generationId: generation.generationId, + commitSha: generation.commitSha, + }); + intentRefusalCause = intent.refusalCause; + return intent.deployOperationId; }, }); if (result.bindRejected) { diff --git a/backend/src/services/gitops/SourceController.ts b/backend/src/services/gitops/SourceController.ts index d6b21159..b95cac10 100644 --- a/backend/src/services/gitops/SourceController.ts +++ b/backend/src/services/gitops/SourceController.ts @@ -15,6 +15,7 @@ import { NodeRegistry } from '../NodeRegistry'; import { gitSourceLocalComposeFiles } from '../../utils/gitComposeFiles'; import { extractImagesFromCompose, loadDotEnv } from '../ImageUpdateService'; import type { ReconcileOutcome } from './outcomes'; +import type { ReconcileTrigger } from './triggers'; import { redactSensitiveText, sanitizeForLog } from '../../utils/safeLog'; /** Outcomes that mean the tick accomplished its work and the source may sleep again. */ @@ -42,10 +43,16 @@ const SUCCESS_SHAPED_OUTCOMES: ReadonlySet = new Set(); + /** + * Resume wakes that arrived while their application was already being + * evaluated. Keyed by application id (so repeated wakes coalesce into + * the one evaluation they asked for) holding the stack name the drain + * re-reads eligibility from. Lives only until the in-flight owner + * releases the slot: a marker exists only while some evaluation owns + * the slot, and that owner's release consumes it. + */ + private readonly pendingWakes = new Map(); private constructor() { } @@ -216,11 +234,97 @@ export class SourceController { `[SourceController] evaluation crashed for ${sanitizeForLog(app.id)}:`, e instanceof Error ? e.message : String(e), )) - .finally(() => this.inFlight.delete(app.id)); + .finally(() => this.releaseAndDrainWake(app.id)); } } - private async evaluate(app: GitOpsApplicationRow): Promise { + /** + * Release an application's evaluation slot and honor any resume wake + * parked against it. The wake's evaluateNow runs synchronously before + * this returns, so no timer tick can claim the freed slot in between: + * a parked operator resume wins the slot against the cadence, which + * is the point of a wake. evaluateNow re-reads the row and re-decides + * eligibility, so a wake that went stale while parked (re-suspended, + * switched to manual, deleted) is dropped exactly as a fresh call + * would drop it. evaluateNow's own catch only covers its evaluation + * arm: its synchronous prologue (the row read and eligibility guard) + * can reject before that catch exists, and this drain runs inside a + * release finally with nothing above it to catch, so the detached + * wake carries the same defensive log the resume route applies to its + * fire-and-forget wake. + */ + private releaseAndDrainWake(applicationId: string): void { + this.inFlight.delete(applicationId); + const wakeStackName = this.pendingWakes.get(applicationId); + if (wakeStackName !== undefined) { + this.pendingWakes.delete(applicationId); + this.evaluateNow(wakeStackName).catch((e) => console.error( + `[SourceController] parked-wake drain could not start for ${sanitizeForLog(applicationId)}:`, + e instanceof Error ? e.message : String(e), + )); + } + } + + /** + * Run one unattended evaluation for a stack right now, without waiting + * for the timer. The resume route calls this after clearing + * suspension so the woken source is re-evaluated immediately (source + * state and, through the automatic arm, target binding and dispatch) + * instead of idling until the next poll or retry cursor lands. + * Eligibility is re-decided here rather than inherited from the + * due-scan, because this path bypasses the scan: only a live (active or + * creating), direct, non-manual, unsuspended source with a stack name is + * evaluated. A row that is due for neither poll nor retry is still + * evaluated: the operator's resume is the wake. Nothing is thrown at the + * caller when the row simply is not eligible; whether the source runs on + * a cursor or on demand is the scheduling code's business, and an + * ineligible row is not an error. + * + * An application already being evaluated is never queued behind itself: + * the wake is parked instead of discarded, and the in-flight owner's + * release runs at most one fresh resume evaluation for it (see + * releaseAndDrainWake). Repeated wakes coalesce into the one evaluation + * they asked for; eligibility is re-read at drain time, so a wake that + * went stale while parked is dropped exactly as a fresh call would drop + * it. + * + * Fetches fired here coalesce with any concurrent tick through the + * reconcile submission's per-application joining, so an immediate + * evaluation landing seconds before a timer tick cannot double-fetch. + */ + public async evaluateNow(stackName: string): Promise { + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (!app) return; + if (app.source_policy === 'manual' || app.suspended_at) return; + if (this.inFlight.has(app.id)) { + // Park rather than discard: the resume is a request to + // re-evaluate after whatever is running now settles, and an + // evaluation about to finish would otherwise silently swallow + // the operator's wake (the next timer tick can be a full poll + // interval away). + this.pendingWakes.set(app.id, stackName); + return; + } + this.inFlight.add(app.id); + try { + await this.evaluate(app, 'resume'); + } catch (e) { + // Same load-bearing catch the tick path wraps evaluations in: + // everything past the reconcile call (the row re-read, + // scheduling, automatic acceptance) runs on this promise chain, + // and a rejection escaping here would be an unhandled one for + // the route that fired it. The durable row is authoritative; + // the next tick remains the safety net. + console.error( + `[SourceController] immediate evaluation crashed for ${sanitizeForLog(app.id)}:`, + e instanceof Error ? e.message : String(e), + ); + } finally { + this.releaseAndDrainWake(app.id); + } + } + + private async evaluate(app: GitOpsApplicationRow, triggerOverride?: ReconcileTrigger): Promise { if (!app.stack_name) { console.warn(`[SourceController] Skipping ${sanitizeForLog(app.id)}: direct-mode application has no stack_name.`); return; @@ -234,13 +338,16 @@ export class SourceController { } const stackName = app.stack_name; const isRetry = app.retry_at !== null && app.retry_at <= Date.now(); + // An out-of-band caller (the resume path) names its own trigger; a + // timer tick derives it from the row's cursors as before. + const trigger: ReconcileTrigger = triggerOverride ?? (isRetry ? 'retry' : 'poll'); let result; try { result = await GitSourceService.getInstance().reconcile({ intent: 'fetch', applicationId: app.id, stackName, - trigger: isRetry ? 'retry' : 'poll', + trigger, actor: 'system:source-controller', }); } catch (e) { @@ -268,7 +375,7 @@ export class SourceController { // refuses a durable blocked candidate below // (requireAcceptableCandidate guards against an outcome that // misreports the row's state). - await this.maybeAcceptAutomaticCandidate(fresh, stackName, isRetry ? 'retry' : 'poll'); + await this.maybeAcceptAutomaticCandidate(fresh, stackName, trigger); } } @@ -330,7 +437,7 @@ export class SourceController { private async maybeAcceptAutomaticCandidate( app: GitOpsApplicationRow, stackName: string, - trigger: 'poll' | 'retry', + trigger: ReconcileTrigger, ): Promise { if (app.source_policy !== 'automatic') return; if (!app.candidate_generation_id || app.accepted_generation_id === app.candidate_generation_id) return; @@ -387,7 +494,7 @@ export class SourceController { try { const dispatch = await GitSourceService.getInstance().dispatchAcceptedGeneration( buildAcceptedGeneration(generation), - { targetMode: 'direct', nodeId: NodeRegistry.getInstance().getDefaultNodeId(), bindingRevision: null }, + GitSourceService.directDispatchContext(), { trigger, actor: 'system:source-controller' }, ); if (dispatch.status === 'blocked') { diff --git a/e2e/git-sources.spec.ts b/e2e/git-sources.spec.ts index 420534f8..1fa65d49 100644 --- a/e2e/git-sources.spec.ts +++ b/e2e/git-sources.spec.ts @@ -648,6 +648,127 @@ test.describe('Git Sources complete-project materialization (local git server)', }); }); +test.describe('Git source suspend and resume over the API (local git server)', () => { + test.skip(!gitAvailable(), 'system git binary is not available'); + + // The whole journey is one scenario with wall-clock steps (an unattended + // poll fires on a 60s floor), so it gets a longer timeout than the suite + // default rather than being split into state-leaking fragments. + test.setTimeout(180_000); + + let server: { url: string; close: () => void }; + let stackName: string; + let previousPollIntervalMins: number; + + test.beforeAll(async () => { + server = await serveRepos({ + sync: buildFixtureRepo({ 'compose.yaml': 'services:\n web:\n image: nginx\n' }), + }); + }); + + test.afterAll(() => { + server?.close(); + }); + + async function api(page: Page, path: string, init?: { method?: string; body?: unknown }) { + return page.evaluate(async ({ p, opts }) => { + const res = await fetch(p, { + method: opts?.method ?? 'GET', + headers: opts?.body !== undefined ? { 'Content-Type': 'application/json' } : undefined, + body: opts?.body !== undefined ? JSON.stringify(opts.body) : undefined, + credentials: 'include', + }); + let json: unknown = null; + try { json = await res.json(); } catch { /* empty or non-JSON body */ } + return { status: res.status, json }; + }, { p: path, opts: init }); + } + + /** History entries for this stack, newest page first, as { stage, trigger, outcome }. */ + async function historyEntries(page: Page): Promise> { + const res = await api(page, `/api/stacks/${stackName}/git-source/history?limit=100`); + const items = (res.json as { items?: Array> } | null)?.items ?? []; + return items.map((i) => ({ + stage: String(i.stage ?? ''), + trigger: String(i.trigger ?? ''), + outcome: String(i.outcome ?? ''), + })); + } + + test('suspension stops reconciliation and resume re-evaluates the source immediately', async ({ page }) => { + await loginAs(page); + stackName = `e2e-susp-${Date.now()}`; + + // Park the unattended cadence on the slowest floor the settings allow + // (60s effective) and restore whatever the instance had before, so the + // scenario never races a background poll and never changes the box. + previousPollIntervalMins = ((await api(page, '/api/git-sources/polling')).json as { poll_interval_mins?: number })?.poll_interval_mins ?? 0; + await api(page, '/api/git-sources/polling', { method: 'PATCH', body: { poll_interval_mins: 1 } }); + + try { + expect((await api(page, '/api/stacks', { method: 'POST', body: { stackName } })).status).toBe(200); + + const saved = await api(page, `/api/stacks/${stackName}/git-source`, { + method: 'PUT', + body: { + repo_url: `${server.url}/sync.git`, + branch: 'main', + compose_paths: ['compose.yaml'], + context_dir: null, + sync_env: false, + auth_type: 'none', + auto_apply_on_webhook: false, + auto_deploy_on_apply: false, + source_policy: 'automatic', + }, + }); + expect(saved.status, JSON.stringify(saved.json)).toBe(200); + + // A manual pull proves the source is live and records a manual- + // trigger attempt before the suspension journey starts. + const pulled = await api(page, `/api/stacks/${stackName}/git-source/pull`, { method: 'POST' }); + expect(pulled.status, JSON.stringify(pulled.json)).toBe(200); + + const suspended = await api(page, `/api/stacks/${stackName}/git-source/suspend`, { + method: 'POST', + body: { reason: 'e2e maintenance hold' }, + }); + expect(suspended.status).toBe(200); + expect((suspended.json as { outcome: string }).outcome).toBe('suspended'); + + // While suspended, a pull is refused outright, not silently skipped. + const refused = await api(page, `/api/stacks/${stackName}/git-source/pull`, { method: 'POST' }); + expect(refused.status).toBe(409); + expect((refused.json as { code: string }).code).toBe('OPERATION_IN_FLIGHT'); + // resume() itself stamps 'resume' on the suspension transition and + // the armed poll cursor, so only a reconcile-start row can prove the + // immediate re-evaluation actually fired. + const reconcileStarts = (rows: Array<{ stage: string; trigger: string }>) => + rows.filter((e) => e.stage === 'source_reconcile_started' && e.trigger === 'resume').length; + expect(reconcileStarts(await historyEntries(page))).toBe(0); + + const resumed = await api(page, `/api/stacks/${stackName}/git-source/resume`, { method: 'POST' }); + expect(resumed.status, JSON.stringify(resumed.json)).toBe(200); + expect((resumed.json as { outcome: string }).outcome).not.toBe('suspended'); + + // The response returns before the reconcile it kicked off can finish, + // so poll the history until the resume-trigger reconcile lands: proof + // the resumed source was re-evaluated now rather than at its next + // scheduled cursor (which is a minute away). + await expect.poll(async () => { + return reconcileStarts(await historyEntries(page)); + }, { timeout: 30_000, intervals: [1_000] }).toBeGreaterThan(0); + } finally { + await api(page, `/api/stacks/${stackName}/git-source`, { method: 'DELETE' }).catch(() => {}); + await api(page, `/api/stacks/${stackName}`, { method: 'DELETE' }).catch(() => {}); + await api(page, '/api/git-sources/polling', { + method: 'PATCH', + body: { poll_interval_mins: previousPollIntervalMins }, + }).catch(() => {}); + } + }); +}); + test.describe('Git Sources SSH deploy key (real backend)', () => { test.skip(!sshGitFixtureAvailable(), 'requires git and openssh-server');