From d4ef263d6e2eb58ebcb475d266bfb45d59fd2104 Mon Sep 17 00:00:00 2001 From: Anso Date: Wed, 16 Sep 2026 00:34:17 -0400 Subject: [PATCH] feat(gitops): stage-aware retry and resume re-evaluation (#1917) * feat(gitops): retry resumes at the failed stage An operator retry previously always re-issued a fetch, even when the source had already accepted a generation whose dispatch never landed. retry() now reads the live pointer evidence (accepted generation, active direct target, applied pointer not yet caught up) and routes the request to the stage the evidence actually stopped at: a persisted source-stage failure or nothing pending goes to the fetch arm, an undelivered accepted generation goes straight through the shared dispatch boundary, and suspended/in-flight/recovering rows defer to the projection instead of starting work beside it. Blocked dispatch refusals come back as blocked outcomes with per-refusal advice; a refused dispatch reserves nothing. * feat(gitops): resume triggers immediate source re-evaluation Clearing a suspension previously left the source idle until its next poll or retry cursor. The resume route now fires SourceController.evaluateNow() fire-and-forget after the service returns: the woken source is re-evaluated immediately through the same eligibility guards the due-scan applies (manual, suspended, missing, or in-flight rows are skipped, and a tick that already owns the application is never queued behind itself). The response still reports that suspension was cleared, not the outcome of a reconcile that has not finished yet. evaluateNow absorbs evaluation failures by design and logs them; the route's catch covers only a store error before any evaluation could start. * test(gitops): cover suspension semantics across resume and dispatch Pins the three coexistence rules the stage-aware retry and resume wake rely on: a rollout pause survives the whole suspend/resume cycle on both application and target rows (suspension is a decision about source work, not a rollout pause); an interrupted apply's terminal still lands after suspension (suspension gates new work, never the accounting of work that already committed); and in-flight deploy transitions, tracked on the target row alone, stay admissible while the source is suspended. * test(e2e): suspend and resume a Git source over the API Drives the full journey against a local git server: a configured automatic source pulls, suspension stops reconciliation (a pull while suspended is refused outright), and resume is re-evaluated within seconds rather than at the next scheduled cursor. Polling is parked at the slowest floor the settings allow and restored afterwards. --- .../src/__tests__/git-source-routes.test.ts | 33 + .../src/__tests__/git-source-service.test.ts | 1157 +++++++++++++++++ .../src/__tests__/gitops-transitions.test.ts | 101 ++ .../src/__tests__/source-controller.test.ts | 196 +++ backend/src/routes/gitSources.ts | 26 + backend/src/services/DatabaseService.ts | 13 +- backend/src/services/GitSourceService.ts | 793 +++++++++-- .../src/services/gitops/SourceController.ts | 133 +- e2e/git-sources.spec.ts | 121 ++ 9 files changed, 2442 insertions(+), 131 deletions(-) diff --git a/backend/src/__tests__/git-source-routes.test.ts b/backend/src/__tests__/git-source-routes.test.ts index 3270d179..89cb008d 100644 --- a/backend/src/__tests__/git-source-routes.test.ts +++ b/backend/src/__tests__/git-source-routes.test.ts @@ -883,6 +883,8 @@ describe('POST /api/stacks/:stackName/git-source/resume', () => { it('returns the normalized result', async () => { const resumeSpy = vi.spyOn(GitSourceService.getInstance(), 'resume') .mockResolvedValue({ outcome: 'no_source_change', reason: 'ok', nextAction: 'none' }); + const { SourceController } = await import('../services/gitops/SourceController'); + const evaluateSpy = vi.spyOn(SourceController.getInstance(), 'evaluateNow').mockResolvedValue(); const res = await request(app) .post('/api/stacks/existing-stack/git-source/resume') .set('Authorization', `Bearer ${adminToken()}`) @@ -890,7 +892,38 @@ describe('POST /api/stacks/:stackName/git-source/resume', () => { expect(res.status).toBe(200); expect(res.body.outcome).toBe('no_source_change'); expect(resumeSpy).toHaveBeenCalledWith('existing-stack', expect.objectContaining({ actor: expect.any(String) })); + // The route wakes the source immediately: the controller's resume + // evaluation is fired for the same stack the service just resumed. + expect(evaluateSpy).toHaveBeenCalledWith('existing-stack'); resumeSpy.mockRestore(); + evaluateSpy.mockRestore(); + }); + + it('still returns the resume outcome when the immediate evaluation cannot start', async () => { + const resumeSpy = vi.spyOn(GitSourceService.getInstance(), 'resume') + .mockResolvedValue({ outcome: 'no_source_change', reason: 'ok', nextAction: 'none' }); + const { SourceController } = await import('../services/gitops/SourceController'); + const evaluateSpy = vi.spyOn(SourceController.getInstance(), 'evaluateNow') + .mockRejectedValue(new Error('database is locked')); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/resume') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + // The response reports that suspension was cleared, not the + // outcome of an evaluation that never ran: it stays 200 with + // resume's derived result, and the failure is logged, not + // swallowed. + expect(res.status).toBe(200); + expect(res.body.outcome).toBe('no_source_change'); + await new Promise((resolve) => setImmediate(resolve)); + expect(errorSpy).toHaveBeenCalledTimes(1); + } finally { + resumeSpy.mockRestore(); + evaluateSpy.mockRestore(); + errorSpy.mockRestore(); + } }); it('denies without the stack:edit permission', async () => { diff --git a/backend/src/__tests__/git-source-service.test.ts b/backend/src/__tests__/git-source-service.test.ts index eec63390..04098e0f 100644 --- a/backend/src/__tests__/git-source-service.test.ts +++ b/backend/src/__tests__/git-source-service.test.ts @@ -7746,6 +7746,1163 @@ describe('GitSourceService.apply', () => { }); }); + describe('stage-aware retry', () => { + /** + * Accept the staged candidate at the source layer (what a controller + * auto-acceptance or an operator review does). Leaves the + * application in the accepted-but-undispatched state the dispatch + * arm of retry() targets: acceptance clears the application + * candidate pointer and the source failure columns, while the + * target's candidate pointer still names the accepted generation. + */ + function acceptPendingCandidate(stackName: string): string { + const app = liveApp(stackName)!; + const generationId = app.candidate_generation_id!; + GitOpsTransitions.getInstance().sourceAccepted({ + applicationId: app.id, + generationId, + artifactSetId: newGitOpsId(), + sourceAcceptanceId: newGitOpsId(), + authority: 'operator', + envelope: testEnvelope(), + }); + return generationId; + } + + async function asyncDefaultNodeId(): Promise { + const { NodeRegistry } = await import('../services/NodeRegistry'); + return NodeRegistry.getInstance().getDefaultNodeId(); + } + + /** + * Park a stack in the state the deploy arm of retry() owns: the + * promotion committed, the target bound the accepted generation, + * and the deploy that followed failed. The state is produced by + * running the promote arm for real (one retry, auto-deploy on, + * deploy stubbed to record its failure and throw), so the durable + * evidence the resume reads is transition-written, not SQL- + * fabricated. The spies stay installed (and their call counts + * cleared) so the test re-arms the deploy for the resume and + * counts what the resume does and never does. + */ + async function appliedWithFailedDeploy(stackName: string, sha: string) { + const svc = await seedPending(stackName, 'services:\n x:\n image: alpine\n', sha); + const generationId = acceptPendingCandidate(stackName); + const applicationId = liveApp(stackName)!.id; + const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET auto_deploy_on_apply = 1 WHERE stack_name = ?') + .run(stackName); + const { GitProjectManifestService } = await import('../services/GitProjectManifestService'); + const { ComposeService } = await import('../services/ComposeService'); + const promoteSpy = vi.spyOn(GitProjectManifestService.prototype, 'promoteGeneration') + .mockImplementation(async () => {}); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack') + .mockImplementation(async (_stack, _ws, _atomic, ctx) => { + const tx = GitOpsTransitions.getInstance(); + const env = { + operationId: ctx!.gitopsDeployOperationId!, + actor: 'system:compose', + trigger: 'deploy', + at: Date.now(), + }; + tx.deployStarted(applicationId, nodeId, generationId, env); + tx.deployFailed(applicationId, nodeId, 'pre_mutation', env); + // Stamp the id on the error the way the real tracked + // deploy does (ComposeService's wrapped rethrow), so + // the settle and the response carry the correlation. + const err = new Error('simulated deploy failure') as Error & { gitopsDeployOperationId?: string }; + err.gitopsDeployOperationId = ctx!.gitopsDeployOperationId; + throw err; + }); + + // The promote arm runs to its deploy-failure settle: blocked, + // naming the failed deploy, with the target applied and its + // failure_stage on 'deploy'. + const first = await svc.retry(stackName, { actor: 'tester' }); + expect(first.outcome).toBe('blocked'); + expect(first.reason).toMatch(/^The source applied, but the deploy failed: /); + const target = GitOpsStore.getInstance().getTarget(applicationId, nodeId)!; + expect(target.applied_generation_id).toBe(generationId); + expect(target.failure_stage).toBe('deploy'); + promoteSpy.mockClear(); + deploySpy.mockClear(); + return { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy }; + } + + /** + * The health-failure starting state: the generation is applied and + * its deploy bound (cleared target evidence), and the stack's + * latest stack-scoped health gate run is a failed verdict bound to + * that exact generation. The delivery half is produced by running + * the deploy arm to success (real deploy-bound transitions); only + * the gate row itself is inserted directly, because that is the + * durable evidence the selector reads (the real beginStack/finalize + * pair would observe container state this fixture does not have). + */ + async function appliedWithFailedHealth(stackName: string, sha: string) { + const seeded = await appliedWithFailedDeploy(stackName, sha); + const { svc, nodeId, generationId, promoteSpy, deploySpy } = seeded; + const { HealthGateService } = await import('../services/HealthGateService'); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-seed'); + // Re-arm for a successful deploy, the way the wins test does. + deploySpy.mockImplementation(async (_stack, _ws, _atomic, ctx) => { + const tx = GitOpsTransitions.getInstance(); + const env = { + operationId: ctx!.gitopsDeployOperationId!, + actor: 'system:compose', + trigger: 'deploy', + at: Date.now(), + }; + tx.deployStarted(seeded.applicationId, nodeId, generationId, env); + tx.deployBound(seeded.applicationId, nodeId, generationId, env); + return { + recoveryId: null, + deployedGenerationId: generationId, + gitopsOperationId: ctx!.gitopsDeployOperationId!, + }; + }); + const delivered = await svc.retry(stackName, { actor: 'tester' }); + expect(delivered.outcome).toBe('no_source_change'); + beginSpy.mockRestore(); + const target = GitOpsStore.getInstance().getTarget(seeded.applicationId, nodeId)!; + expect(target.applied_generation_id).toBe(generationId); + expect(target.failure_stage).toBeNull(); + promoteSpy.mockClear(); + deploySpy.mockClear(); + return seeded; + } + + /** Insert a durable health_gate_runs row (the selector's evidence). */ + function seedHealthRun(args: { + id: string; + nodeId: number; + stackName: string; + status: 'passed' | 'failed' | 'unknown' | 'observing'; + scope: 'stack' | 'service'; + serviceName: string | null; + generationId: string | null; + startedAt: number; + }): void { + DatabaseService.getInstance().insertHealthGateRun({ + id: args.id, + node_id: args.nodeId, + stack_name: args.stackName, + trigger_action: 'deploy', + status: args.status, + reason: args.status === 'failed' ? 'simulated health failure' : null, + window_seconds: 30, + containers_json: '[]', + started_at: args.startedAt, + ended_at: args.status === 'observing' ? null : args.startedAt + 30_000, + created_by: 'system:git-source', + target_scope: args.scope, + service_name: args.serviceName, + failure_source: args.status === 'failed' ? 'primary' : null, + deployed_generation_id: args.generationId, + }); + } + + it('dispatches the accepted generation when nothing was ever promoted, without refetching or re-accepting', async () => { + const sha = 't1t1t1t1t1t1t1t1t1t1t1t1t1t1t1t1t1t1t1t1'; + const svc = await seedPending('retry-dispatch-arm', 'services:\n x:\n image: alpine\n', sha); + const generationId = acceptPendingCandidate('retry-dispatch-arm'); + const applicationId = liveApp('retry-dispatch-arm')!.id; + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + const acceptedBefore = liveApp('retry-dispatch-arm')!.accepted_generation_id; + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const { GitProjectManifestService } = await import('../services/GitProjectManifestService'); + const promoteSpy = vi.spyOn(GitProjectManifestService.prototype, 'promoteGeneration') + .mockImplementation(async () => {}); + + try { + const result = await svc.retry('retry-dispatch-arm', { actor: 'tester' }); + + // The dispatch arm ran through the shared boundary: the + // generation promoted and the target bound, no fetch ran, + // and acceptance was not re-run. + expect(promoteSpy).toHaveBeenCalledTimes(1); + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(result.outcome).toBe('no_source_change'); + const app = liveApp('retry-dispatch-arm')!; + expect(app.accepted_generation_id).toBe(acceptedBefore); + expect(app.accepted_generation_id).toBe(generationId); + const target = GitOpsStore.getInstance().getTarget(applicationId, await asyncDefaultNodeId())!; + expect(target.applied_generation_id).toBe(generationId); + // The retry's dispatch reserved its own attempt, marked as a + // dispatch of exactly this generation, and settled it. + const startedIds = historyOperationIds(applicationId, 'source_reconcile_started'); + expect(startedIds).toHaveLength(startedBefore + 1); + const retryOperationId = startedIds[startedIds.length - 1]!; + const reserved = GitOpsStore.getInstance().getStartedAttempt(applicationId, retryOperationId)!; + expect(reserved.trigger).toBe('retry'); + expect(JSON.parse(reserved.after_json).dispatchGenerationId).toBe(generationId); + expect(GitOpsStore.getInstance().getSettledAttempt(applicationId, retryOperationId)).toBeDefined(); + } finally { + reconcileSpy.mockRestore(); + promoteSpy.mockRestore(); + } + }); + + it('refetches when a source-stage failure persists alongside undelivered dispatch evidence', async () => { + const sha = 't2t2t2t2t2t2t2t2t2t2t2t2t2t2t2t2t2t2t2t2'; + const svc = await seedPending('retry-source-stage-precedence', 'services:\n x:\n image: alpine\n', sha); + acceptPendingCandidate('retry-source-stage-precedence'); + const applicationId = liveApp('retry-source-stage-precedence')!.id; + // A validation failure recorded after acceptance: the fetch arm + // owns the newer evidence, so the dispatch pointer does not win. + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET failure_stage = 'validation', failure_class = 'validation', failure_at = ? WHERE id = ?", + ).run(Date.now(), applicationId); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const { GitProjectManifestService } = await import('../services/GitProjectManifestService'); + const promoteSpy = vi.spyOn(GitProjectManifestService.prototype, 'promoteGeneration'); + + try { + await svc.retry('retry-source-stage-precedence', { actor: 'tester' }); + + expect(reconcileSpy).toHaveBeenCalledWith(expect.objectContaining({ trigger: 'retry', intent: 'fetch' })); + expect(promoteSpy).not.toHaveBeenCalled(); + } finally { + reconcileSpy.mockRestore(); + promoteSpy.mockRestore(); + } + }); + + it('maps a blocked dispatch to a blocked outcome with the refusal advice, reserving nothing', async () => { + const sha = 't3t3t3t3t3t3t3t3t3t3t3t3t3t3t3t3t3t3t3t3'; + const svc = await seedPending('retry-dispatch-blocked', 'services:\n x:\n image: alpine\n', sha); + acceptPendingCandidate('retry-dispatch-blocked'); + const applicationId = liveApp('retry-dispatch-blocked')!.id; + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + const nodeId = await asyncDefaultNodeId(); + expect(StackOpLockService.getInstance().tryAcquire(nodeId, 'retry-dispatch-blocked', 'deploy', 'tester').acquired).toBe(true); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + mockGitClone.mockClear(); + + try { + const result = await svc.retry('retry-dispatch-blocked', { actor: 'tester' }); + + expect(result).toEqual({ + outcome: 'blocked', + reason: 'Another operation (deploy) is already in progress for retry-dispatch-blocked.', + nextAction: 'retry', + }); + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + // The lock refusal happened before any reservation, so + // retry leaves no unsettled attempt behind. + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(startedBefore); + } finally { + reconcileSpy.mockRestore(); + StackOpLockService.getInstance().release(nodeId, 'retry-dispatch-blocked'); + } + }); + + it('defers to the in-flight projection instead of starting a second operation', async () => { + const sha = 't4t4t4t4t4t4t4t4t4t4t4t4t4t4t4t4t4t4t4t4'; + const svc = await seedPending('retry-mid-operation', 'services:\n x:\n image: alpine\n', sha); + acceptPendingCandidate('retry-mid-operation'); + const applicationId = liveApp('retry-mid-operation')!.id; + // A crash window: the row still carries the apply-started + // marker although no work is running. Recovery owns that + // evidence; retry must not race it with a fetch or a dispatch. + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET active_operation_stage = 'apply_started', active_operation_id = 'op-crash', active_operation_at = ? WHERE id = ?", + ).run(Date.now(), applicationId); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const dispatchSpy = vi.spyOn(svc, 'dispatchAcceptedGeneration'); + + try { + const result = await svc.retry('retry-mid-operation', { actor: 'tester' }); + + expect(result.outcome).toBe('unknown'); + expect(result.reason).toMatch(/in flight/i); + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(dispatchSpy).not.toHaveBeenCalled(); + } finally { + reconcileSpy.mockRestore(); + dispatchSpy.mockRestore(); + } + }); + + it('refetches once the accepted generation has been delivered and its deploy settled', async () => { + const sha = 't5t5t5t5t5t5t5t5t5t5t5t5t5t5t5t5t5t5t5t5'; + const svc = await seedPending('retry-after-delivery', 'services:\n x:\n image: alpine\n', sha); + const generationId = acceptPendingCandidate('retry-after-delivery'); + const applicationId = liveApp('retry-after-delivery')!.id; + // The dispatch evidence is gone: the target applied the + // generation and its deploy failure columns are clear (they + // were never written here, which is exactly the still-healthy + // state deploy-bound leaves), so the only re-evaluation left + // is a source fetch. The deploy arm must not fire without a + // recorded target-level deploy failure. + DatabaseService.getInstance().getDb().prepare( + 'UPDATE gitops_target_current SET applied_generation_id = ?, candidate_generation_id = NULL WHERE application_id = ?', + ).run(generationId, applicationId); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { ComposeService } = await import('../services/ComposeService'); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack'); + + try { + const result = await svc.retry('retry-after-delivery', { actor: 'tester' }); + + expect(reconcileSpy).toHaveBeenCalledWith(expect.objectContaining({ trigger: 'retry', intent: 'fetch' })); + expect(deploySpy).not.toHaveBeenCalled(); + expect(result.outcome).toBe('pending_review'); + } finally { + reconcileSpy.mockRestore(); + validateSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('resumes only the deploy when the applied generation\'s deploy failed, without fetching, promoting, or re-binding', async () => { + const sha = 't8t8t8t8t8t8t8t8t8t8t8t8t8t8t8t8t8t8t8t8'; + const { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-resume', sha); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const { HealthGateService } = await import('../services/HealthGateService'); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack'); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + const targetAppliedBefore = historyOperationIds(applicationId, 'target_applied').length; + + try { + const result = await svc.retry('retry-deploy-resume', { actor: 'tester' }); + + // The fetch arm never ran: no reconcile call, no git clone. + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + // Promotion and re-binding are refused by the applied + // pointer, and the resume honors that: the shared promote + // machinery is untouched. + expect(promoteSpy).not.toHaveBeenCalled(); + // The same applied generation is deployed again, under a + // freshly journaled intent of this retry attempt. + expect(deploySpy).toHaveBeenCalledTimes(1); + const startedIds = historyOperationIds(applicationId, 'source_reconcile_started'); + expect(startedIds).toHaveLength(startedBefore + 1); + const retryOperationId = startedIds[startedIds.length - 1]!; + const reserved = GitOpsStore.getInstance().getStartedAttempt(applicationId, retryOperationId)!; + expect(reserved.trigger).toBe('retry'); + const markers = JSON.parse(reserved.after_json!); + expect(markers.dispatchGenerationId).toBe(generationId); + expect(markers.dispatchDeployRequested).toBe(true); + const intentRow = GitOpsStore.getInstance() + .getStageRowForAttempt(applicationId, retryOperationId, 'deploy_dispatched')!; + const intent = JSON.parse(intentRow.after_json!); + expect(intent.generationId).toBe(generationId); + expect(intent.deployOperationId).toBeDefined(); + // The second deploy's own rows land under the journaled id + // for this generation: started, then failed at pre_mutation. + expect(historyOperationIds(applicationId, 'deploy_started')).toContain(intent.deployOperationId); + expect(historyOperationIds(applicationId, 'deploy_failed')).toContain(intent.deployOperationId); + // Compose was handed exactly the journaled deploy id, under + // git_apply provenance by the raw retry actor. + expect(deploySpy).toHaveBeenCalledWith('retry-deploy-resume', undefined, undefined, + expect.objectContaining({ + source: 'git_apply', + actor: 'tester', + gitopsDeployOperationId: intent.deployOperationId, + })); + // The deploy failed again, so no health gate was opened. + expect(beginSpy).not.toHaveBeenCalled(); + // The attempt is durably recorded and settled, carrying the + // same deploy-failure classification the promote arm's + // failed deploy writes, plus the exact failed deploy id. + const settled = settledAttemptsForApplication(applicationId) + .find((row) => row.operation_id === retryOperationId); + expect(settled).toBeDefined(); + expect(JSON.parse(settled!.after_json!)).toMatchObject({ + outcome: 'recovery_required', + reason: 'The source applied, but the deploy failed: simulated deploy failure', + nextAction: 'view_target_results', + deployGitopsOperationId: intent.deployOperationId, + }); + // The response matches the promote arm's vocabulary for the + // same settled classification: blocked, view target results. + expect(result).toEqual({ + outcome: 'blocked', + reason: 'The source applied, but the deploy failed: simulated deploy failure', + nextAction: 'view_target_results', + }); + // Direct re-bind did not happen: no new target_applied + // history row, and the applied pointer still names the + // same generation it named before the resume. + expect(historyOperationIds(applicationId, 'target_applied')) + .toHaveLength(targetAppliedBefore); + expect(GitOpsStore.getInstance().getTarget(applicationId, nodeId)!.applied_generation_id) + .toBe(generationId); + // Acceptance was not re-run either: the application still + // names the same accepted generation. + expect(liveApp('retry-deploy-resume')!.accepted_generation_id).toBe(generationId); + } finally { + reconcileSpy.mockRestore(); + beginSpy.mockRestore(); + errorSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('settles the deploy resume as a success when the retried deploy binds, and the source is delivered again', async () => { + const sha = 't9t9t9t9t9t9t9t9t9t9t9t9t9t9t9t9t9t9t9t9'; + const { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-resume-wins', sha); + // Re-arm the stub for the resume: this time the deploy opens + // under the journaled id and binds the same generation, the way + // ComposeService's tracked deploy does. + deploySpy.mockImplementation(async (_stack, _ws, _atomic, ctx) => { + const tx = GitOpsTransitions.getInstance(); + const env = { + operationId: ctx!.gitopsDeployOperationId!, + actor: 'system:compose', + trigger: 'deploy', + at: Date.now(), + }; + tx.deployStarted(applicationId, nodeId, generationId, env); + tx.deployBound(applicationId, nodeId, generationId, env); + return { + recoveryId: null, + deployedGenerationId: generationId, + gitopsOperationId: ctx!.gitopsDeployOperationId!, + }; + }); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const { HealthGateService } = await import('../services/HealthGateService'); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-resume'); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const result = await svc.retry('retry-deploy-resume-wins', { actor: 'tester' }); + + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(deploySpy).toHaveBeenCalledTimes(1); + // A successful resume deploy opens the same health gate and + // post-deploy scan the pipeline's deploy branch does. + expect(beginSpy).toHaveBeenCalledWith(nodeId, 'retry-deploy-resume-wins', 'deploy', 'system:git-source', { deployedGenerationId: generationId }); + await vi.waitFor(() => expect(mockTriggerPostDeployScan).toHaveBeenCalledWith('retry-deploy-resume-wins', nodeId)); + // The retry's settle carries the exact deploy that ran. + const startedIds = historyOperationIds(applicationId, 'source_reconcile_started'); + const retryOperationId = startedIds[startedIds.length - 1]!; + const intentRow = GitOpsStore.getInstance() + .getStageRowForAttempt(applicationId, retryOperationId, 'deploy_dispatched')!; + const deployId = JSON.parse(intentRow.after_json!).deployOperationId as string; + const settled = settledAttemptsForApplication(applicationId) + .find((row) => row.operation_id === retryOperationId); + expect(JSON.parse(settled!.after_json!)).toMatchObject({ + outcome: 'no_source_change', + deployGitopsOperationId: deployId, + }); + expect(result.outcome).toBe('no_source_change'); + expect(result.deployGitopsOperationId).toBe(deployId); + + // With the deploy evidence cleared by the bind, a further + // retry is a plain source fetch again: the deploy arm does + // not fire forever on a delivered generation. + reconcileSpy.mockClear(); + mockTriggerPostDeployScan.mockClear(); + const again = await svc.retry('retry-deploy-resume-wins', { actor: 'tester' }); + expect(reconcileSpy).toHaveBeenCalledWith(expect.objectContaining({ trigger: 'retry', intent: 'fetch' })); + expect(again.outcome).toBe('pending_review'); + } finally { + reconcileSpy.mockRestore(); + beginSpy.mockRestore(); + validateSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('refuses the deploy resume when the evidence moved before the lock, settling the refusal', async () => { + const sha = 't10a10a10a10a10a10a10a10a10a10a10a10a10'; + const { svc, applicationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-resume-race', sha); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + // The entry predicate sees the deploy failure; by the time the + // in-lock revalidation reads the target, a successful deploy + // has cleared it. The resume must refuse and touch nothing. + const realTarget = GitOpsStore.prototype.getTarget; + const targetSpy = vi.spyOn(GitOpsStore.prototype, 'getTarget'); + let targetCalls = 0; + targetSpy.mockImplementation((applicationIdArg: string, nodeIdArg: number) => { + targetCalls++; + const row = realTarget.call(GitOpsStore.getInstance(), applicationIdArg, nodeIdArg); + // The entry read (call 1) keeps the failure evidence; the + // in-lock revalidation (call 2) sees it cleared. + if (row && targetCalls === 2) return { ...row, failure_stage: null, failure_class: null, failure_at: null }; + return row; + }); + + try { + const result = await svc.retry('retry-deploy-resume-race', { actor: 'tester' }); + + expect(result.outcome).toBe('blocked'); + expect(result.reason).toMatch(/nothing was re-deployed/); + expect(result.nextAction).toBe('view_target_results'); + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + // The refusal happened after the reservation, so it is + // settled as a blocked attempt, and it is the only new row. + const startedIds = historyOperationIds(applicationId, 'source_reconcile_started'); + expect(startedIds).toHaveLength(startedBefore + 1); + const settled = settledAttemptsForApplication(applicationId) + .find((row) => row.operation_id === startedIds[startedIds.length - 1]!); + expect(JSON.parse(settled!.after_json!)).toMatchObject({ + outcome: 'blocked', + nextAction: 'view_target_results', + }); + // The call-counted blinding is only meaningful if the path + // really reads the target exactly twice (entry predicate, + // in-lock revalidation). Pin it so a future extra read + // cannot silently shift the blinded call to the wrong site. + expect(targetSpy).toHaveBeenCalledTimes(2); + } finally { + reconcileSpy.mockRestore(); + targetSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('refuses the deploy resume while the stack lock is held, reserving nothing', async () => { + const sha = 't11b11b11b11b11b11b11b11b11b11b11b11b11b'; + const { applicationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-resume-blocked', sha); + const nodeId = await asyncDefaultNodeId(); + expect(StackOpLockService.getInstance().tryAcquire(nodeId, 'retry-deploy-resume-blocked', 'deploy', 'tester').acquired).toBe(true); + const svc = GitSourceService.getInstance(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + + try { + const result = await svc.retry('retry-deploy-resume-blocked', { actor: 'tester' }); + + expect(result).toEqual({ + outcome: 'blocked', + reason: 'Another operation (deploy) is already in progress for retry-deploy-resume-blocked.', + nextAction: 'retry', + }); + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(reconcileSpy).not.toHaveBeenCalled(); + // The lock refusal happened before any reservation: a + // resume that started nothing leaves no attempt to chase. + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(startedBefore); + } finally { + reconcileSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + StackOpLockService.getInstance().release(nodeId, 'retry-deploy-resume-blocked'); + } + }); + + it('fetches when a newer source-stage failure outranks the deploy evidence', async () => { + const sha = 'x12x12x12x12x12x12x12x12x12x12x12x12x12'; + const { svc, applicationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-arm-fetch-precedence', sha); + // A fetch failure recorded after the deploy failed: the + // application-row source evidence is newer than the target-row + // deploy evidence, and the fetch arm owns the precedence. The + // class rides along for realism; the arm routing reads the + // stage only. + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET failure_stage = 'fetch', failure_class = 'auth', failure_at = ? WHERE id = ?", + ).run(Date.now(), applicationId); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + + try { + await svc.retry('retry-deploy-arm-fetch-precedence', { actor: 'tester' }); + + expect(reconcileSpy).toHaveBeenCalledWith(expect.objectContaining({ trigger: 'retry', intent: 'fetch' })); + // Neither resume arm ran beside the fetch. + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + // The fetch arm never touches target evidence: the deploy + // failure stands until a deploy succeeds. + const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; + const target = GitOpsStore.getInstance().getTarget(applicationId, nodeId)!; + expect(target.failure_stage).toBe('deploy'); + } finally { + reconcileSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('refuses the deploy resume when its reservation collides, settling nothing', async () => { + const sha = 'x13x13x13x13x13x13x13x13x13x13x13x13'; + const { svc, applicationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-resume-collision', sha); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + const settledBefore = settledAttemptsForApplication(applicationId).length; + const reserveSpy = vi.spyOn(GitOpsTransitions.prototype, 'allocateReconcileAttempt') + .mockImplementationOnce(() => ({ operationId: 'collided:attempt:1', reserved: false })); + + try { + const result = await svc.retry('retry-deploy-resume-collision', { actor: 'tester' }); + + expect(result).toEqual({ + outcome: 'blocked', + reason: 'A dispatch attempt for this stack is already recorded; check its outcome before dispatching again.', + nextAction: 'view_target_results', + }); + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + // Nothing was reserved, so nothing may be settled: the + // colliding attempt belongs to its own owner, and the + // deploy-resume must not speak for it. + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(startedBefore); + expect(settledAttemptsForApplication(applicationId)).toHaveLength(settledBefore); + } finally { + reserveSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('recovers an interrupted deploy resume from its journaled deploy evidence without re-running anything', async () => { + const sha = 'x14x14x14x14x14x14x14x14x14x14x14x14'; + const { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-resume-recovered', sha); + // A crash window for the resume: the retried deploy records its + // failure under the journaled id, then the resume's own settle + // dies. The attempt stays unsettled while its deploy evidence + // stands, which startup recovery must read as the failed deploy + // it was, reconstructing the settle from recorded rows only. + deploySpy.mockImplementation(async (_stack, _ws, _atomic, ctx) => { + const tx = GitOpsTransitions.getInstance(); + const env = { + operationId: ctx!.gitopsDeployOperationId!, + actor: 'system:compose', + trigger: 'deploy', + at: Date.now(), + }; + tx.deployStarted(applicationId, nodeId, generationId, env); + tx.deployFailed(applicationId, nodeId, 'pre_mutation', env); + throw new Error('simulated deploy failure'); + }); + const settleSpy = vi.spyOn(GitOpsTransitions.prototype, 'settleReconcileAttempt') + .mockImplementationOnce(() => { throw new Error('simulated crash before settle'); }); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await svc.retry('retry-deploy-resume-recovered', { actor: 'tester' }); + + const started = historyOperationIds(applicationId, 'source_reconcile_started'); + const retryOperationId = started[started.length - 1]!; + const intentRow = GitOpsStore.getInstance() + .getStageRowForAttempt(applicationId, retryOperationId, 'deploy_dispatched')!; + const deployId = JSON.parse(intentRow.after_json!).deployOperationId as string; + expect(historyOperationIds(applicationId, 'deploy_failed')).toContain(deployId); + // The live settle died, so the attempt is still open. + expect(settledAttemptsForApplication(applicationId) + .some((row) => row.operation_id === retryOperationId)).toBe(false); + + await svc.recoverUnsettledReconcileAttempts(); + + const settled = settledAttemptsForApplication(applicationId) + .filter((row) => row.operation_id === retryOperationId); + expect(settled).toHaveLength(1); + // The reconstruction names the promotion and binding the + // resume took as its evidence, plus the exact failed deploy. + expect(JSON.parse(settled[0]!.after_json!)).toMatchObject({ + outcome: 'recovery_required', + reason: 'The promotion and binding completed, but the deploy this dispatch started failed.', + nextAction: 'view_target_results', + deployGitopsOperationId: deployId, + commitSha: sha, + }); + // Recovery re-ran nothing: the deploy arm never promotes + // (the helper's counts were cleared after the seeding + // promote-arm run), and reconstruction is read-only. + expect(promoteSpy).not.toHaveBeenCalled(); + expect(deploySpy).toHaveBeenCalledTimes(1); + + await svc.recoverUnsettledReconcileAttempts(); + expect(settledAttemptsForApplication(applicationId) + .filter((row) => row.operation_id === retryOperationId)).toHaveLength(1); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(deploySpy).toHaveBeenCalledTimes(1); + } finally { + settleSpy.mockRestore(); + errorSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('refuses the deploy resume when its intent cannot be journaled, settling the refusal without deploying', async () => { + const sha = 'x15x15x15x15x15x15x15x15x15x15x15x15'; + const { svc, applicationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy('retry-deploy-resume-intent-refused', sha); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + const intentSpy = vi.spyOn(GitOpsTransitions.prototype, 'deployDispatched') + .mockImplementationOnce(() => { throw new Error('simulated intent journal failure'); }); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + const result = await svc.retry('retry-deploy-resume-intent-refused', { actor: 'tester' }); + + // A resume that cannot record and read back its deploy + // intent refuses to start an untracked deploy: Compose was + // never handed the stack. + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(result.outcome).toBe('blocked'); + expect(result.nextAction).toBe('view_target_results'); + expect(result.reason).toMatch(/deploy retry could not record and read back its deploy intent/); + const started = historyOperationIds(applicationId, 'source_reconcile_started'); + expect(started).toHaveLength(startedBefore + 1); + const retryOperationId = started[started.length - 1]!; + // The refusal journals its witness (so recovery of an + // unsettled version of this attempt reads the refusal, not + // an apply-only completion) and settles recovery_required + // against the applied files, naming the cause. Compose + // never ran, so no deploy id is claimed. + expect(GitOpsStore.getInstance() + .hasStageRowForAttempt(applicationId, retryOperationId, 'deploy_intent_refused')).toBe(true); + const settled = settledAttemptsForApplication(applicationId) + .find((row) => row.operation_id === retryOperationId); + const payload = JSON.parse(settled!.after_json!); + expect(payload).toMatchObject({ + outcome: 'recovery_required', + reason: 'The deploy retry could not record and read back its deploy intent (simulated intent journal failure), so the deploy was not started.', + nextAction: 'view_target_results', + }); + expect(payload).not.toHaveProperty('deployGitopsOperationId'); + } finally { + intentSpy.mockRestore(); + errorSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('resumes at the deploy when the applied generation has a failed stack health verdict', async () => { + const sha = 'h1a1h1a1h1a1h1a1h1a1h1a1h1a1h1a1h1a1h1a1'; + const stackName = 'retry-health-resume-wins'; + const { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy } = + await appliedWithFailedHealth(stackName, sha); + // The generation is applied and its deploy bound; the stack's + // latest stack-scoped gate run is then a failed verdict bound + // to exactly that generation. That is the whole evidence set: + // no deploy failure is recorded anywhere, because the deploy + // did succeed. + seedHealthRun({ + id: 'health-run-h1', + nodeId, + stackName, + status: 'failed', + scope: 'stack', + serviceName: null, + generationId, + startedAt: Date.now(), + }); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const { HealthGateService } = await import('../services/HealthGateService'); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-health-resume'); + + try { + const result = await svc.retry(stackName, { actor: 'tester' }); + + // The failed verdict routes the resume to the deploy arm: + // the deploy re-runs, and nothing is fetched, transported, + // accepted, or promoted beside it. + expect(result.outcome).toBe('no_source_change'); + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(deploySpy).toHaveBeenCalledTimes(1); + // The resume re-opens the health gate on the deploy it + // runs, bound to the same generation. In production the + // fresh run supersedes the failed row as the stack's + // latest claim; the mocked beginStack does not insert one, + // so no post-resume routing assertion is made here. + expect(beginSpy).toHaveBeenCalledWith( + nodeId, stackName, 'deploy', 'system:git-source', { deployedGenerationId: generationId }, + ); + // The retried deploy re-bound the same generation; no new + // acceptance or promotion happened beside it. + const target = GitOpsStore.getInstance().getTarget(applicationId, nodeId)!; + expect(target.applied_generation_id).toBe(generationId); + expect(target.deployed_generation_id).toBe(generationId); + expect(historyOperationIds(applicationId, 'target_applied')).toHaveLength(1); + await vi.waitFor(() => expect(mockTriggerPostDeployScan).toHaveBeenCalledWith(stackName, nodeId)); + } finally { + reconcileSpy.mockRestore(); + beginSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('refuses the deploy resume when the health verdict is superseded before the lock', async () => { + const sha = 'h4d4h4d4h4d4h4d4h4d4h4d4h4d4h4d4h4d4h4d4'; + const stackName = 'retry-health-resume-race'; + const { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy } = + await appliedWithFailedHealth(stackName, sha); + seedHealthRun({ + id: 'health-run-h4', + nodeId, + stackName, + status: 'failed', + scope: 'stack', + serviceName: null, + generationId, + startedAt: Date.now(), + }); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + // The entry predicate sees the failed verdict; by the time the + // in-lock revalidation reads it, a newer stack gate run has + // superseded it. The resume must refuse and touch nothing, + // the same way the deploy-failure race refusal works. + const realLatest = DatabaseService.prototype.getLatestStackHealthGateRun; + let latestCalls = 0; + const latestSpy = vi.spyOn(DatabaseService.prototype, 'getLatestStackHealthGateRun') + .mockImplementation((nodeIdArg: number, stackNameArg: string) => { + latestCalls++; + if (latestCalls === 2) return undefined; + return realLatest.call(DatabaseService.getInstance(), nodeIdArg, stackNameArg); + }); + + try { + const result = await svc.retry(stackName, { actor: 'tester' }); + + expect(result.outcome).toBe('blocked'); + expect(result.reason).toMatch(/nothing was re-deployed/); + expect(result.nextAction).toBe('view_target_results'); + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + // The refusal happened after the reservation, so it is + // settled as a blocked attempt, and it is the only new row. + const startedIds = historyOperationIds(applicationId, 'source_reconcile_started'); + expect(startedIds).toHaveLength(startedBefore + 1); + const settled = settledAttemptsForApplication(applicationId) + .find((row) => row.operation_id === startedIds[startedIds.length - 1]!); + expect(JSON.parse(settled!.after_json!)).toMatchObject({ + outcome: 'blocked', + nextAction: 'view_target_results', + }); + // The call-counted blinding is only meaningful if the path + // really reads the accessor exactly twice (entry predicate, + // in-lock revalidation). Pin it so a future extra read + // cannot silently shift the blinded call to the wrong site. + expect(latestSpy).toHaveBeenCalledTimes(2); + } finally { + reconcileSpy.mockRestore(); + latestSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('falls back to the fetch arm for every health run that cannot claim the deploy resume', async () => { + const sha = 'h2b2h2b2h2b2h2b2h2b2h2b2h2b2h2b2h2b2h2b2'; + const stackName = 'retry-health-evidence-negative'; + const { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy } = + await appliedWithFailedHealth(stackName, sha); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const db = DatabaseService.getInstance().getDb(); + + // Each row is a health record that cannot claim the deploy + // arm: a verdict for a stale (older) generation, a verdict + // that never landed (observing/unknown), a healthy verdict, + // or a gate that watches one service rather than the applied + // generation's stack-wide health. Each must fall to the + // fetch arm, which re-decides from the source instead. + const cases: Array<{ + label: string; + status: 'failed' | 'passed' | 'observing' | 'unknown'; + scope: 'stack' | 'service'; + serviceName: string | null; + boundGenerationId: string | null; + }> = [ + { label: 'stale generation', status: 'failed', scope: 'stack', serviceName: null, boundGenerationId: '00000000-0000-4000-8000-000000000000' }, + { label: 'unbound generation', status: 'failed', scope: 'stack', serviceName: null, boundGenerationId: null }, + { label: 'observing run', status: 'observing', scope: 'stack', serviceName: null, boundGenerationId: generationId }, + { label: 'unknown run', status: 'unknown', scope: 'stack', serviceName: null, boundGenerationId: generationId }, + { label: 'passed run', status: 'passed', scope: 'stack', serviceName: null, boundGenerationId: generationId }, + { label: 'service-scoped run', status: 'failed', scope: 'service', serviceName: 'x', boundGenerationId: generationId }, + ]; + + try { + for (let i = 0; i < cases.length; i++) { + const c = cases[i]!; + db.prepare('DELETE FROM health_gate_runs WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName); + seedHealthRun({ + id: `health-run-neg-${i}`, + nodeId, + stackName, + status: c.status, + scope: c.scope, + serviceName: c.serviceName, + generationId: c.boundGenerationId, + startedAt: Date.now(), + }); + // Keep the fetch arm's own bookkeeping from stacking + // up as source-stage evidence across iterations: each + // case must be decided on its health row, not on a + // failure a previous case's fetch recorded. + db.prepare( + 'UPDATE gitops_applications SET failure_stage = NULL, failure_class = NULL, failure_at = NULL WHERE id = ?', + ).run(applicationId); + // Preconditions so a vacuous pass is impossible: the + // dispatch evidence is intact and only the health row + // differs from the wins case. + const appRow = GitOpsStore.getInstance().getApplication(applicationId)!; + expect(appRow.accepted_generation_id, c.label).toBe(generationId); + expect(appRow.failure_stage, c.label).toBeNull(); + const targetRow = GitOpsStore.getInstance().getTarget(applicationId, nodeId)!; + expect(targetRow.applied_generation_id, c.label).toBe(generationId); + expect(targetRow.failure_stage, c.label).toBeNull(); + + reconcileSpy.mockClear(); + await svc.retry(stackName, { actor: 'tester' }); + + expect(reconcileSpy, c.label).toHaveBeenCalledWith( + expect.objectContaining({ trigger: 'retry', intent: 'fetch' }), + ); + expect(deploySpy, c.label).not.toHaveBeenCalled(); + expect(promoteSpy, c.label).not.toHaveBeenCalled(); + } + } finally { + reconcileSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('leaves a failed health verdict to the path the target actually names', async () => { + const sha = 'h3c3h3c3h3c3h3c3h3c3h3c3h3c3h3c3h3c3h3c3'; + const stackName = 'retry-health-evidence-other-owner'; + const { svc, applicationId, nodeId, generationId, promoteSpy, deploySpy } = + await appliedWithFailedHealth(stackName, sha); + seedHealthRun({ + id: 'health-run-h3', + nodeId, + stackName, + status: 'failed', + scope: 'stack', + serviceName: null, + generationId, + startedAt: Date.now(), + }); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + // The target names recovery as its outstanding failure. The + // failed health verdict does not silently reroute that path + // into a deploy resume: the health arm only owns the clear + // (null) target-evidence state, and everything else keeps its + // own owner. + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_target_current SET failure_stage = 'recovery', failure_class = 'post_mutation' WHERE application_id = ? AND node_id = ?", + ).run(applicationId, nodeId); + + try { + await svc.retry(stackName, { actor: 'tester' }); + + expect(reconcileSpy).toHaveBeenCalledWith( + expect.objectContaining({ trigger: 'retry', intent: 'fetch' }), + ); + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + } finally { + reconcileSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('falls back to the fetch arm with a warning when the accepted generation row is gone', async () => { + const sha = 'h5e5h5e5h5e5h5e5h5e5h5e5h5e5h5e5h5e5h5e5'; + const stackName = 'retry-generation-pointer-dangling'; + const { svc, generationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy(stackName, sha); + // A pointer-integrity violation: the application still names a + // generation row that no longer exists. The fetch arm is the + // defensible recovery, but it must announce itself rather + // than silently refetch over a broken pointer. + DatabaseService.getInstance().getDb() + .prepare('DELETE FROM gitops_generations WHERE id = ?').run(generationId); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + + try { + await svc.retry(stackName, { actor: 'tester' }); + + expect(reconcileSpy).toHaveBeenCalledWith( + expect.objectContaining({ trigger: 'retry', intent: 'fetch' }), + ); + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + // The fetch arm's own settle is the fixture's business as + // usual; what this test pins is the routing and the + // announced pointer-integrity warning. + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining(generationId)); + } finally { + reconcileSpy.mockRestore(); + warnSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('refuses the deploy resume when the generation row disappears before the lock', async () => { + const sha = 'h6f6h6f6h6f6h6f6h6f6h6f6h6f6h6f6h6f6h6f6'; + const stackName = 'retry-deploy-resume-generation-gone'; + const { svc, applicationId, promoteSpy, deploySpy } = + await appliedWithFailedDeploy(stackName, sha); + mockGitClone.mockClear(); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + // The entry predicate sees the generation row; by the time the + // in-lock revalidation reads it, the row is gone. The resume + // must refuse with the unreadable-evidence advice, and the + // fetch arm must not be reached beside the refusal. + const realGeneration = GitOpsStore.prototype.getGeneration; + let generationCalls = 0; + const generationSpy = vi.spyOn(GitOpsStore.prototype, 'getGeneration') + .mockImplementation((generationIdArg: string) => { + generationCalls++; + if (generationCalls === 2) return undefined; + return realGeneration.call(GitOpsStore.getInstance(), generationIdArg); + }); + + try { + const result = await svc.retry(stackName, { actor: 'tester' }); + + expect(result.outcome).toBe('blocked'); + expect(result.reason).toMatch(/could not be read; its deploy evidence is unavailable/); + expect(result.nextAction).toBe('view_target_results'); + expect(deploySpy).not.toHaveBeenCalled(); + expect(promoteSpy).not.toHaveBeenCalled(); + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(mockGitClone).not.toHaveBeenCalled(); + // Past the reservation, so the refusal settles as a blocked + // attempt and is the only new row. + const startedIds = historyOperationIds(applicationId, 'source_reconcile_started'); + expect(startedIds).toHaveLength(startedBefore + 1); + const settled = settledAttemptsForApplication(applicationId) + .find((row) => row.operation_id === startedIds[startedIds.length - 1]!); + expect(JSON.parse(settled!.after_json!)).toMatchObject({ + outcome: 'blocked', + nextAction: 'view_target_results', + }); + // The blinding is only meaningful if the deploy-resume path + // reads the generation exactly twice (entry predicate, + // in-lock revalidation). Pin it so a future extra read + // cannot silently shift the blinded call to the wrong site. + expect(generationSpy).toHaveBeenCalledTimes(2); + } finally { + reconcileSpy.mockRestore(); + generationSpy.mockRestore(); + promoteSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('defers to the recovery evidence for every unfinished recovery phase', async () => { + const sha = 't6t6t6t6t6t6t6t6t6t6t6t6t6t6t6t6t6t6t6t6'; + const svc = await seedPending('retry-recovery-restoring', 'services:\n x:\n image: alpine\n', sha); + acceptPendingCandidate('retry-recovery-restoring'); + const applicationId = liveApp('retry-recovery-restoring')!.id; + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const dispatchSpy = vi.spyOn(svc, 'dispatchAcceptedGeneration'); + + // The guard defers on any phase except the settled 'complete' + // receipt. 'capturing' is schema-legal with no production + // writer yet: pinning it here freezes the fail-safe inversion + // for the day one lands, and it is projection-transparent + // today, so the report is the accepted-state projection while + // the contract is that no work starts. 'failed' runs last + // because it also sets the recovery failure columns, which the + // projection reads independently of the phase. + const cases: Array<{ phase: string; outcome: string; reason?: RegExp }> = [ + { phase: 'restoring', outcome: 'recovery_required' }, + { phase: 'compensating', outcome: 'recovery_required' }, + { phase: 'capturing', outcome: 'no_source_change' }, + { phase: 'failed', outcome: 'recovery_required', reason: /Recovery itself failed \(post_mutation\)/ }, + ]; + + try { + for (const c of cases) { + // 'failed' normally arrives with the recovery failure + // columns from recoveryFailed/rollbackPartialFailed; + // the distinguishing reason proves which projection was + // read rather than a generic one. + DatabaseService.getInstance().getDb().prepare( + c.phase === 'failed' + ? "UPDATE gitops_applications SET recovery_phase = 'failed', failure_stage = 'recovery', failure_class = 'post_mutation' WHERE id = ?" + : 'UPDATE gitops_applications SET recovery_phase = ? WHERE id = ?', + ).run(...(c.phase === 'failed' ? [applicationId] : [c.phase, applicationId])); + reconcileSpy.mockClear(); + dispatchSpy.mockClear(); + + const result = await svc.retry('retry-recovery-restoring', { actor: 'tester' }); + + expect(result.outcome, c.phase).toBe(c.outcome); + if (c.reason) expect(result.reason).toMatch(c.reason); + expect(reconcileSpy, c.phase).not.toHaveBeenCalled(); + expect(dispatchSpy, c.phase).not.toHaveBeenCalled(); + } + } finally { + reconcileSpy.mockRestore(); + dispatchSpy.mockRestore(); + } + }); + + it('retries normally after a completed rollback (complete phase is a receipt, not outstanding work)', async () => { + const sha = 't7t7t7t7t7t7t7t7t7t7t7t7t7t7t7t7t7t7t7t7'; + const svc = await seedPending('retry-recovery-complete', 'services:\n x:\n image: alpine\n', sha); + const generationId = acceptPendingCandidate('retry-recovery-complete'); + const applicationId = liveApp('retry-recovery-complete')!.id; + // recoverySucceeded/rollbackCompleted persist 'complete' and + // nothing clears it; a settled receipt must not permanently + // degrade every future retry on this source to a projection + // read. The undelivered dispatch evidence still routes to the + // dispatch arm. + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET recovery_phase = 'complete' WHERE id = ?", + ).run(applicationId); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + const { GitProjectManifestService } = await import('../services/GitProjectManifestService'); + const promoteSpy = vi.spyOn(GitProjectManifestService.prototype, 'promoteGeneration') + .mockImplementation(async () => {}); + + try { + const result = await svc.retry('retry-recovery-complete', { actor: 'tester' }); + + expect(reconcileSpy).not.toHaveBeenCalled(); + expect(promoteSpy).toHaveBeenCalledTimes(1); + expect(result.outcome).toBe('no_source_change'); + const target = GitOpsStore.getInstance().getTarget(applicationId, await asyncDefaultNodeId())!; + expect(target.applied_generation_id).toBe(generationId); + } finally { + reconcileSpy.mockRestore(); + promoteSpy.mockRestore(); + } + }); + }); + it('throws when pending has been cleared between pull and apply', async () => { const svc = await seedPending('apply-cleared', 'services:\n x:\n image: alpine\n', 'aaaa111aaaa111aaaa111aaaa111aaaa111aaaa1'); DatabaseService.getInstance().clearGitSourcePending('apply-cleared'); diff --git a/backend/src/__tests__/gitops-transitions.test.ts b/backend/src/__tests__/gitops-transitions.test.ts index 8b59a394..f1857fcf 100644 --- a/backend/src/__tests__/gitops-transitions.test.ts +++ b/backend/src/__tests__/gitops-transitions.test.ts @@ -735,6 +735,107 @@ describe('gitops transitions', () => { expect(application.suspended_at).not.toBeNull(); }); + it('keeps a rollout pause across source suspension and resume', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-pause-coex', 'pause-coex-web'), nodeId: 1, envelope: envelope('op-act-pause-coex') }); + tx.rolloutPaused('app-pause-coex', null, 'waiting for the maintenance window', envelope('op-rp-app')); + tx.rolloutPaused('app-pause-coex', 1, 'fleet rollout paused', envelope('op-rp-target')); + + tx.sourceSuspended('app-pause-coex', 'operator hold', envelope('op-susp-pause-coex')); + // Suspension is a decision about future source work. It is not a rollout + // pause, and it must neither overwrite one nor read like one. + let application = store.getApplication('app-pause-coex')!; + expect(application.source_suspended_reason).toBe('operator hold'); + expect(application.pause_reason).toBe('waiting for the maintenance window'); + expect(store.getTarget('app-pause-coex', 1)?.pause_reason).toBe('fleet rollout paused'); + + tx.sourceUnsuspended('app-pause-coex', envelope('op-unsusp-pause-coex')); + application = store.getApplication('app-pause-coex')!; + expect(application.suspended_at).toBeNull(); + expect(application.source_suspended_reason).toBeNull(); + // The rollout's own pause survives the whole suspend/resume cycle on + // both rows: only rolloutUnpaused clears it. + expect(application.pause_at).not.toBeNull(); + expect(application.pause_reason).toBe('waiting for the maintenance window'); + const target = store.getTarget('app-pause-coex', 1)!; + expect(target.pause_at).not.toBeNull(); + expect(target.pause_reason).toBe('fleet rollout paused'); + }); + + it('records a promotion whose apply completed after suspension interrupted it', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-susp-promo', 'susp-promo-web'), nodeId: 1, envelope: envelope('op-act-susp-promo') }); + store.insertGeneration(gen('gen-susp-promo', 'app-susp-promo')); + tx.fetchStarted('app-susp-promo', envelope('op-f-susp-promo')); + tx.fetched('app-susp-promo', 'abc123', envelope('op-f-susp-promo')); + tx.candidateReady('app-susp-promo', 'gen-susp-promo', false, envelope('op-c-susp-promo')); + const applyOp = envelope('op-a-susp-promo'); + tx.applyStarted('app-susp-promo', 'gen-susp-promo', applyOp); + + // Suspension interrupts the in-flight operation without discarding it: + // the apply is marked interrupted, and the candidate stays staged. + tx.sourceSuspended('app-susp-promo', 'operator hold', envelope('op-susp-susp-promo')); + expect(store.getApplication('app-susp-promo')?.interruption_stage).toBe('apply_started'); + + // The apply's terminal event still lands after the interrupt. Suspension + // gates new work (fetch, apply, accept); it never cancels the accounting + // of work that already committed to disk. + tx.applied({ + applicationId: 'app-susp-promo', + generationId: 'gen-susp-promo', + artifactSetId: 'art-susp-promo', + sourceAcceptanceId: 'acc-susp-promo', + authority: 'operator', + envelope: applyOp, + }); + const application = store.getApplication('app-susp-promo')!; + expect(application.accepted_generation_id).toBe('gen-susp-promo'); + expect(application.interruption_stage).toBeNull(); + // Recording an in-flight promotion does not resume anything. + expect(application.suspended_at).not.toBeNull(); + expect(store.getTarget('app-susp-promo', 1)?.applied_generation_id).toBe('gen-susp-promo'); + }); + + it('accepts a matching late deploy result for a suspended source', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-late-dep', 'late-dep-web', 'gen-late-dep', 'art-late-dep', 'acc-late-dep'); + const deployOp = envelope('op-dep-late'); + tx.deployStarted('app-late-dep', 1, 'gen-late-dep', deployOp); + + // The deploy is in flight when the operator suspends. Suspension gates + // new source work, and it interrupts an operation the application row + // still shows as active; a deploy is tracked on the target row alone, so + // an in-flight deploy is left running rather than marked interrupted. + tx.sourceSuspended('app-late-dep', 'operator hold', envelope('op-susp-late-dep')); + const suspended = store.getTarget('app-late-dep', 1)!; + expect(suspended.active_operation_stage).toBe('deploy_started'); + expect(suspended.interruption_stage).toBeNull(); + + // Its terminal events are therefore admissible while the source is + // suspended: requireMatchingDeploy matches the live operation and + // records its outcome, so evidence of a deploy nobody can recall is + // kept, not dropped. + tx.deployBound('app-late-dep', 1, 'gen-late-dep', deployOp); + let target = store.getTarget('app-late-dep', 1)!; + expect(target.deployed_generation_id).toBe('gen-late-dep'); + expect(target.failure_stage).toBeNull(); + + // The failure arm behaves the same way: a later deploy for the same + // applied generation records its failure while still suspended. + tx.deployStarted('app-late-dep', 1, 'gen-late-dep', envelope('op-dep-late-2')); + tx.deployFailed('app-late-dep', 1, 'post_mutation', envelope('op-dep-late-2')); + target = store.getTarget('app-late-dep', 1)!; + expect(target.failure_stage).toBe('deploy'); + expect(target.failure_class).toBe('post_mutation'); + expect(target.deployed_generation_id).toBe('gen-late-dep'); + // Neither terminal resumed anything: recording is accounting, not a + // source operation. + expect(store.getApplication('app-late-dep')?.suspended_at).not.toBeNull(); + }); + it('targetApplied binds a Direct target only after the generation is accepted', () => { const store = GitOpsStore.getInstance(); const tx = GitOpsTransitions.getInstance(); diff --git a/backend/src/__tests__/source-controller.test.ts b/backend/src/__tests__/source-controller.test.ts index 4ed9f5ba..94880001 100644 --- a/backend/src/__tests__/source-controller.test.ts +++ b/backend/src/__tests__/source-controller.test.ts @@ -7,6 +7,7 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; import { directApplicationFixture } from './helpers/gitopsFixtures'; +import { DatabaseService } from '../services/DatabaseService'; import { GitOpsStore } from '../services/gitops/store'; import { GitSourceService } from '../services/GitSourceService'; import { SourceController } from '../services/gitops/SourceController'; @@ -213,4 +214,199 @@ describe('SourceController', () => { expect(reconcile).toHaveBeenCalledTimes(1); }); + + describe('evaluateNow (resume re-evaluation)', () => { + /** Insert a live automatic application row for a stack name. */ + function seedLive(stackName: string): GitOpsApplicationRow { + const app = autoFixture(`app-${stackName}`, stackName); + GitOpsStore.getInstance().insertApplication(app); + return app; + } + + it('fetches a live automatic source immediately, tagged with the resume trigger', async () => { + seedLive('now-web'); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + + await controller.evaluateNow('now-web'); + + expect(reconcile).toHaveBeenCalledTimes(1); + expect(reconcile).toHaveBeenCalledWith(expect.objectContaining({ + intent: 'fetch', + applicationId: 'app-now-web', + stackName: 'now-web', + trigger: 'resume', + })); + // No timer was running: nothing here waited for a tick. + expect(vi.getTimerCount()).toBe(0); + }); + + it('skips a manual-policy source without reconciling it', async () => { + const app: GitOpsApplicationRow = { ...autoFixture('app-now-manual', 'now-manual-web'), source_policy: 'manual' }; + GitOpsStore.getInstance().insertApplication(app); + const reconcile = spyOnReconcile(); + + await controller.evaluateNow('now-manual-web'); + + expect(reconcile).not.toHaveBeenCalled(); + }); + + it('skips a suspended source, so a resume race never wakes it early', async () => { + const app = { ...autoFixture('app-now-susp', 'now-susp-web'), suspended_at: Date.now() }; + GitOpsStore.getInstance().insertApplication(app); + const reconcile = spyOnReconcile(); + + await controller.evaluateNow('now-susp-web'); + + expect(reconcile).not.toHaveBeenCalled(); + }); + + it('is a silent no-op for a stack with no live application', async () => { + const reconcile = spyOnReconcile(); + + await controller.evaluateNow('now-missing'); + + expect(reconcile).not.toHaveBeenCalled(); + }); + + it('parks a resume wake while a tick owns the application and runs it once when the tick settles', async () => { + const app = seedLive('now-busy'); + // The scan's due list is mocked to the same row, so the tick's + // evaluation and the resume path address one identical id. + mockDue([app]); + let settleFirstCall!: (result: ReconcileResult) => void; + const firstCall = new Promise((resolve) => { settleFirstCall = resolve; }); + const reconcile = spyOnReconcile().mockReturnValueOnce(firstCall).mockResolvedValue(okResult); + + controller.start(); + await advanceOneTick(); + expect(reconcile).toHaveBeenCalledTimes(1); + + // Two resumes arrive while the tick's evaluation is still + // pending: neither starts work beside it, and together they + // coalesce into the single wake they asked for. + await controller.evaluateNow('now-busy'); + await controller.evaluateNow('now-busy'); + expect(reconcile).toHaveBeenCalledTimes(1); + + settleFirstCall(okResult); + // Flush the tick's continuation chain (evaluate's post-reconcile + // steps plus the scan's inFlight release, which drains the + // parked wake into a fresh resume evaluation). + await vi.advanceTimersByTimeAsync(0); + + // The wake was honored without any second manual call: the + // park-then-drain is what the operator's resume bought. The + // two parked resumes produced exactly one fresh evaluation, + // tagged as a resume, and the drain's own release leaves no + // further wake parked (a third call would have landed). + expect(reconcile).toHaveBeenCalledTimes(2); + expect(reconcile.mock.calls[1]![0]).toMatchObject({ trigger: 'resume' }); + // "At most one" pinned: the drain's settle releases the slot + // with nothing left parked, so no third evaluation ever fires. + await vi.advanceTimersByTimeAsync(0); + expect(reconcile).toHaveBeenCalledTimes(2); + }); + + it('drops a parked wake whose source went stale, without evaluating it', async () => { + const app = seedLive('now-stale'); + mockDue([app]); + let settleFirstCall!: (result: ReconcileResult) => void; + const firstCall = new Promise((resolve) => { settleFirstCall = resolve; }); + const reconcile = spyOnReconcile().mockReturnValueOnce(firstCall).mockResolvedValue(okResult); + + controller.start(); + await advanceOneTick(); + expect(reconcile).toHaveBeenCalledTimes(1); + + await controller.evaluateNow('now-stale'); + expect(reconcile).toHaveBeenCalledTimes(1); + // The drain re-reads eligibility, so the stale wake is dropped + // exactly as a fresh call would drop it: no evaluation of a + // suspended source. + DatabaseService.getInstance().getDb() + .prepare('UPDATE gitops_applications SET suspended_at = ? WHERE id = ?') + .run(Date.now(), app.id); + settleFirstCall(okResult); + await vi.advanceTimersByTimeAsync(0); + + expect(reconcile).toHaveBeenCalledTimes(1); + }); + + it('drains the parked wake even when the owning evaluation crashes past the reconcile', async () => { + const app = seedLive('now-owner-crash'); + mockDue([app]); + let settleFirstCall!: (result: ReconcileResult) => void; + const firstCall = new Promise((resolve) => { settleFirstCall = resolve; }); + const reconcile = spyOnReconcile().mockReturnValueOnce(firstCall).mockResolvedValue(okResult); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + // The crash lands in evaluate()'s post-reconcile tail (the row + // re-read), not in reconcile itself: the owner's chain rejects, + // the scan's load-bearing catch logs it, and the release's + // finally must still drain the wake. The mock is once-only, so + // the drained evaluation's own re-read falls through to the + // real store. + vi.spyOn(GitOpsStore.getInstance(), 'getApplication') + .mockImplementationOnce(() => { throw new Error('simulated crash after reconcile'); }); + + controller.start(); + await advanceOneTick(); + expect(reconcile).toHaveBeenCalledTimes(1); + + await controller.evaluateNow('now-owner-crash'); + settleFirstCall(okResult); + await vi.advanceTimersByTimeAsync(0); + + expect(reconcile).toHaveBeenCalledTimes(2); + expect(reconcile.mock.calls[1]![0]).toMatchObject({ trigger: 'resume' }); + const logged = errorSpy.mock.calls.map((args) => args.map(String).join('\n')).join('\n'); + expect(logged).toContain('evaluation crashed'); + errorSpy.mockRestore(); + }); + + it('logs a parked-wake drain whose prologue throws instead of rejecting unhandled', async () => { + const app = seedLive('now-drain-throws'); + mockDue([app]); + let settleFirstCall!: (result: ReconcileResult) => void; + const firstCall = new Promise((resolve) => { settleFirstCall = resolve; }); + const reconcile = spyOnReconcile().mockReturnValueOnce(firstCall).mockResolvedValue(okResult); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + controller.start(); + await advanceOneTick(); + await controller.evaluateNow('now-drain-throws'); + expect(reconcile).toHaveBeenCalledTimes(1); + + // evaluateNow's eligibility read runs before its own catch, so + // a store failure at that point rejects the call. The drain is + // a detached fire-and-forget (nothing above it in the release + // chain can catch), so it must carry its own guard: the + // rejection becomes a logged line, never an unhandled rejection + // (which would take the process down). + vi.spyOn(GitOpsStore.getInstance(), 'getLiveDirectApplication') + .mockImplementationOnce(() => { throw new Error('database is locked'); }); + settleFirstCall(okResult); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(0); + + // No resume evaluation ran (the prologue died before it), but + // the failure was logged rather than silently escaping. + expect(reconcile).toHaveBeenCalledTimes(1); + const logged = errorSpy.mock.calls.map((args) => args.map(String).join('\n')).join('\n'); + expect(logged).toContain('parked-wake drain could not start'); + errorSpy.mockRestore(); + }); + + it('survives a reconcile rejection without throwing at the route', async () => { + seedLive('now-throws'); + spyOnReconcile().mockRejectedValue(new Error('boom')); + // evaluate() logs the reconcile failure itself and returns; + // evaluateNow must not turn it into a rejection for the caller. + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + await expect(controller.evaluateNow('now-throws')).resolves.toBeUndefined(); + + expect(errorSpy).toHaveBeenCalledTimes(1); + errorSpy.mockRestore(); + }); + }); }); diff --git a/backend/src/routes/gitSources.ts b/backend/src/routes/gitSources.ts index 7e8eb54c..ff423a5d 100644 --- a/backend/src/routes/gitSources.ts +++ b/backend/src/routes/gitSources.ts @@ -766,6 +766,32 @@ stackGitSourceRouter.post('/:stackName/git-source/resume', async (req: Request, const result = await GitSourceService.getInstance().resume(stackName, { actor: req.user?.username ?? 'unknown', }); + // A resumed source is re-evaluated immediately rather than idling + // until its next poll or retry cursor: the controller's resume trigger + // re-resolves source state and, for an automatic source, target + // binding through the shared acceptance/dispatch arm. Fire-and-forget + // by design: this response reports that suspension was cleared, not + // the outcome of a reconcile that has not finished yet, and the + // durable row plus the next tick are the authority on the latter. + // The wake has its own guard: a failure starting it must not + // retroactively fail a resume that already committed, so it is + // logged, never sent to sendGitSourceError. evaluateNow absorbs its + // evaluation failures itself; the rejection arm catches anything + // before its own try exists (evaluateNow is async, so even its + // synchronous prologue, a store error before any evaluation could + // run, surfaces as a rejection) and any future escape, and the + // try/catch covers a getInstance()-level failure only. + const logWakeFailure = (error: unknown): void => { + console.error( + `[GitSources] Resume re-evaluation could not start for ${sanitizeForLog(stackName)}:`, + error instanceof Error ? error.message : String(error), + ); + }; + try { + SourceController.getInstance().evaluateNow(stackName).catch(logWakeFailure); + } catch (error) { + logWakeFailure(error); + } res.json(result); } catch (error) { sendGitSourceError(res, error); diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 0012ba31..37d0d316 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -244,7 +244,11 @@ export interface HealthGateRunRow { failure_source: 'primary' | 'collateral' | null; /** * Reserved for the GitOps deploy path: the generation live when this run - * started. No writer populates it yet, so it is currently always null. + * started. Populated from beginStack's binding argument and the deploy + * callers pass it when their mutation was GitOps-tracked, plus recovery + * reservations (reserveRecoveryRun); null otherwise and for legacy rows. + * The stage-aware retry reads it to decide whether a failed stack + * verdict belongs to the currently applied generation. */ deployed_generation_id?: string | null; } @@ -4321,6 +4325,13 @@ stmt.run('gitops_schema_version', '1'); ).get(nodeId, stackName) as HealthGateRunRow | undefined; } + /** The most recent stack-scoped run for the stack (service gates excluded). */ + public getLatestStackHealthGateRun(nodeId: number, stackName: string): HealthGateRunRow | undefined { + return this.db.prepare( + "SELECT * FROM health_gate_runs WHERE node_id = ? AND stack_name = ? AND target_scope = 'stack' ORDER BY started_at DESC, id DESC LIMIT 1" + ).get(nodeId, stackName) as HealthGateRunRow | undefined; + } + /** Finalize runs left observing by a previous process (startup sweep). */ /** * Runs a previous process left observing. diff --git a/backend/src/services/GitSourceService.ts b/backend/src/services/GitSourceService.ts index 0382015e..facc9a73 100644 --- a/backend/src/services/GitSourceService.ts +++ b/backend/src/services/GitSourceService.ts @@ -39,7 +39,7 @@ import { projectApplication } from './gitops/derive'; import { outcomeFromSourceFacet, parseReconcileResultPayload, type ReconcileOutcome, type ReconcileResult } from './gitops/outcomes'; import { coalesceKey, deliveryKey, type ReconcileRequest, type ReconcileTrigger } from './gitops/triggers'; import { classifyFailure, effectivePollIntervalSecs } from './gitops/backoff'; -import { BlueprintTargetAdapter, type AcceptedGeneration, type DispatchContext, type DispatchResult } from './gitops/handoff'; +import { BlueprintTargetAdapter, buildAcceptedGeneration, type AcceptedGeneration, type DispatchContext, type DispatchResult } from './gitops/handoff'; import { GitOpsTransitions, GitOpsTransitionError, @@ -3627,8 +3627,13 @@ export class GitSourceService { } /** - * Resume acting on a source. Does not itself fetch; the next scheduled - * poll, retry, or manual reconcile picks the source back up. + * Resume acting on a source. Clears suspension and re-arms the steady- + * state poll cursor; it does not itself fetch. The immediate wake is the + * resume route's job: after this method returns, the route fires + * SourceController.evaluateNow() so the source is re-evaluated right + * away (re-resolving source state and, for an automatic source, target + * binding through the shared acceptance/dispatch arm) instead of idling + * until the next scheduled poll or retry. * * Unlike suspend(), a refused resume is tolerated rather than surfaced. * The result is read back from the row after the attempted write, so a @@ -3683,25 +3688,662 @@ export class GitSourceService { } /** - * An explicit, operator-initiated re-evaluation: resolves the live - * application server-side rather than trusting a caller-supplied id, for - * callers that hold only a stack name, and drives a fresh fetch-intent - * reconcile through it. + * The dispatch evidence stage-aware retry reads: which dispatch stage + * an operator retry should resume, or null when no dispatch is + * pending. Two arms share this one decision because the arms are + * mutually exclusive by evidence, and the helper is the single place + * that exclusivity is stated: + * - `'promote'`: the cheap pre-filter over the same columns + * `dispatchDirectLocked` revalidates (the generation row still readable, + * a live application, an active Direct + * target whose candidate still names the accepted generation, and an + * applied pointer that has not caught up). It lags the dispatch's own + * checks (commit sha, artifact-set evidence, recomputed change plan), + * and the in-lock revalidation remains the authority. Generation files + * were never promoted, so retry hands the generation to the shared + * dispatch boundary. + * - `'deploy'`: the promotion already committed (the applied pointer + * names the accepted generation) and the durable evidence says the + * stage after it failed: either the deploy failed (`failure_stage = + * 'deploy'`, set by the deploy-failed/deploy-unbound transitions and + * cleared by deploy-bound), or the health gate watching that exact + * applied generation finalized as failed (the stack's most recent + * stack-scoped `health_gate_runs` row, bound to the generation id, + * with status `'failed'`). Nothing may be fetched, accepted, + * promoted, or re-bound; only the unfinished deploy stage is retried, + * and every resume re-opens the health gate on the deploy it runs. A + * deploy or health failure with no accepted-generation evidence is + * not deploy-resume material: the fetch arm re-decides from the + * source instead. + * Retry routes an operator's request to the stage the evidence actually + * stopped at, and each arm's in-lock revalidation remains the authority + * when the state moves in between. + */ + private acceptedGenerationDispatchStage(app: GitOpsApplicationRow): { stage: 'promote' | 'deploy'; generationId: string } | null { + const generationId = app.accepted_generation_id; + if (!generationId) return null; + if (!GitOpsStore.getInstance().getGeneration(generationId)) { + // A live application naming a generation that no longer exists is + // a pointer-integrity violation. The fetch arm is a defensible + // recovery, but it must announce itself: an operator who asked to + // retry a stuck dispatch deserves to see why a fetch ran instead. + console.warn( + `[GitSource] ${sanitizeForLog(app.stack_name ?? app.id)} names accepted generation ${sanitizeForLog(generationId)} but the row is missing; retry falls back to a fresh fetch.`, + ); + return null; + } + const target = GitOpsStore.getInstance().getTarget(app.id, NodeRegistry.getInstance().getDefaultNodeId()); + if (!target || target.target_status !== 'active') return null; + // The deploy arm's two evidence kinds both route to re-running the + // deploy stage only: a failed deploy (`failure_stage = 'deploy'`, + // set by the deploy-failed/deploy-unbound transitions and cleared + // by deploy-bound) left the generation undelivered, and a failed + // health gate (below) delivered it but the verdict says the run + // is sick. Neither may be fetched, accepted, promoted, or + // re-bound. healthFinalized() itself records health outcomes as + // promotions of the healthy/LKG pointers and never writes the + // target's failure columns, so the health evidence lives in the + // gate's own durable run rows, not on the target. + if (target.candidate_generation_id === generationId && target.applied_generation_id !== generationId) { + return { stage: 'promote', generationId }; + } + if (target.applied_generation_id !== generationId) return null; + if (target.failure_stage === 'deploy') { + return { stage: 'deploy', generationId }; + } + // A failed health verdict only owns the arm when the deploy it + // watched actually bound (deploy-bound clears the failure columns, + // so the honest state is NULL): any other target failure belongs + // to its own path, not to a deploy resume. + if (target.failure_stage === null + && app.stack_name + && this.appliedGenerationHealthFailed(NodeRegistry.getInstance().getDefaultNodeId(), app.stack_name, generationId)) { + return { stage: 'deploy', generationId }; + } + return null; + } + + /** + * Whether the stack's most recent stack-scoped health gate run is a + * `failed` verdict bound to exactly the given generation. Service-scoped + * gates are excluded: they watch one service, not the applied + * generation's stack-wide health. An `observing` or `unknown` row is + * not a health-failure claim, and a row bound to a different (or no) + * generation cannot vouch for this one, so neither selects an arm; + * those fall to the fetch arm, which re-decides from the source. + */ + private appliedGenerationHealthFailed(nodeId: number, stackName: string, generationId: string): boolean { + const run = DatabaseService.getInstance().getLatestStackHealthGateRun(nodeId, stackName); + return !!run + && run.status === 'failed' + && run.deployed_generation_id === generationId; + } + + /** + * The dispatch context for the local Direct target: the boundary's + * shared construction used by every dispatch of a locally-accepted + * generation (automatic controller dispatch, explicit retry). The + * nodeId is resolved per call because it is process configuration, not + * durable evidence; the dispatch itself re-reads the live target under + * the stack lock and treats this context as routing only. + */ + public static directDispatchContext(): DispatchContext { + return { targetMode: 'direct', nodeId: NodeRegistry.getInstance().getDefaultNodeId(), bindingRevision: null }; + } + + /** + * An explicit, operator-initiated re-evaluation, resuming at the stage + * the durable evidence says failed. A fetch or validation failure + * recorded on the application, or no dispatch evidence at all, drives + * a fresh fetch-intent reconcile. When the accepted + * generation is still awaiting delivery (candidate pointer set, applied + * pointer behind), retry calls dispatchAcceptedGeneration() against + * exactly that generation: the source is never refetched, and + * acceptance is never re-run, because both already succeeded and the + * failure lives past the acceptance boundary. When the generation is + * already applied but its deploy failed or its health gate's latest + * stack-scoped verdict is a failure for exactly that generation, retry + * resumes at the deploy alone (retryDeploy): fetching, accepting, + * promoting, and re-binding are all refused by the evidence (the + * applied pointer names this generation; re-binding it would throw), + * and the resume re-opens the health gate on the deploy it runs. The + * dispatch's and the deploy-resume's own revalidation under the stack + * lock stays the authority on whether it may proceed; a refusal comes + * back as a blocked outcome naming why. A refusal that reserved an + * attempt settles it carrying the same advice; a refusal before any + * reservation (no stack bound, no source configured, the stack lock + * held, or the in-lock source recheck) reserves nothing, so the + * response itself is the only record, which is deliberate: nothing + * started, so there is nothing to reconcile later. * * The 'retry' trigger is recorded on the durable attempt for audit and - * recovery. It does not yet change fetch behavior; a later permanent- - * failure gate can use it to authorize an explicit operator retry. + * recovery on all three arms. */ public async retry(stackName: string, opts: { actor: string }): Promise { const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); if (!app) return GitSourceService.noApplicationResult(); - return this.reconcile({ - intent: 'fetch', - applicationId: app.id, + // A source that is suspended, has an operation in flight, has an + // unfinished recovery, or has an unrecovered interruption marker + // reports its real state instead of starting work beside it: the + // suspended projection is the resume advice, the in-flight + // projection says no settled result exists yet, and the recovery + // projection is what the recovery path owns. An unrecovered crash + // can leave an active_operation_stage set with no live work, and + // retrying over it would race the pass that holds that evidence. + // Unfinished recovery is inverted, not enumerated: every phase + // except the settled 'complete' receipt (including 'capturing' and + // any phase added later) defers, so a new intermediate phase can + // never silently start retry work beside it. 'complete' is a + // settled receipt written by recoverySucceeded or + // rollbackCompleted that no code path clears; deferring on it + // would strand every future retry on a source that just + // recovered. + const unfinishedRecovery = app.recovery_phase !== null && app.recovery_phase !== 'complete'; + if (app.suspended_at || app.active_operation_stage || unfinishedRecovery || app.interruption_stage) { + return this.deriveReconcileResult(stackName); + } + const dispatch = this.acceptedGenerationDispatchStage(app); + const sourceStageFailure = app.failure_stage === 'fetch' || app.failure_stage === 'validation'; + // The fetch arm owns the precedence: a newer source-stage failure + // outranks stale dispatch evidence on either arm. + if (!dispatch || sourceStageFailure) { + return this.reconcile({ + intent: 'fetch', + applicationId: app.id, + stackName, + trigger: 'retry', + actor: opts.actor, + }); + } + return dispatch.stage === 'promote' + ? this.retryDispatch(stackName, app.id, dispatch.generationId, opts.actor) + : this.retryDeploy(stackName, app.id, dispatch.generationId, opts.actor); + } + + /** + * The promote arm of retry(): rebuild the accepted-generation contract + * from the persisted generation row and hand it to the shared dispatch + * boundary. Only reached when acceptedGenerationDispatchStage proved + * the pointers, so a missing or corrupt row here means the evidence + * moved or decayed between the check and the read; that is reported, + * never papered over with a refetch. + */ + private async retryDispatch( + stackName: string, + applicationId: string, + generationId: string, + actor: string, + ): Promise { + let dispatch: DispatchResult; + try { + const generationRow = GitOpsStore.getInstance().getGeneration(generationId); + if (!generationRow) { + return { + outcome: 'blocked', + reason: 'The accepted generation could not be read; its dispatch evidence is unavailable.', + nextAction: 'view_target_results', + }; + } + dispatch = await this.dispatchAcceptedGeneration( + buildAcceptedGeneration(generationRow), + GitSourceService.directDispatchContext(), + { trigger: 'retry', actor }, + ); + } catch (e) { + // Nothing past this point throws once a reservation exists, so + // a throw here means the contract could not be built or the + // entry reads failed; the generation stands accepted, and the + // operator can retry once the cause is fixed. errorForLog keeps + // the stack for debugging while scrubbing credential-shaped + // text, which a transport-level failure can carry. + console.error( + `[GitSource] Explicit retry could not dispatch the accepted generation for ${sanitizeForLog(stackName)}:`, + errorForLog(e), + ); + return { + outcome: 'blocked', + reason: 'The accepted generation could not be dispatched; its evidence was unreadable.', + nextAction: 'view_target_results', + }; + } + if (dispatch.status === 'dispatched') { + // The projection re-read after the dispatch is the truthful + // post-state answer; every dispatch path that reserved an + // attempt has settled it before returning. + return this.deriveReconcileResult(stackName); + } + return { + outcome: 'blocked', + reason: dispatch.reason, + nextAction: GitSourceService.dispatchRefusalNextAction(dispatch.reason), + }; + } + + /** + * The deploy arm of retry(): resume only the unfinished deploy of a + * generation the target has already applied. Entry shape mirrors the + * promote arm's shared boundary: refuse cheaply before taking the + * stack lock, reserve the durable attempt inside the lock, and settle + * every reserved path before returning. A lock-held refusal reserves + * nothing, matching the dispatch boundary's rule that a retry refused + * before any work started leaves only its response as the record. + */ + private async retryDeploy( + stackName: string, + applicationId: string, + generationId: string, + actor: string, + ): Promise { + const app = GitOpsStore.getInstance().getApplication(applicationId); + if (!app?.stack_name) { + return { + outcome: 'blocked', + reason: 'No Direct stack is bound to this application.', + nextAction: 'resolve_conflict', + }; + } + if (!DatabaseService.getInstance().getGitSource(stackName)) { + return { + outcome: 'blocked', + reason: `No Git source is configured for ${stackName}.`, + nextAction: 'configure_credentials', + }; + } + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + // Nothing is reserved until the stack lock is held: a deploy + // resume refused at the lock never started any work, so it must + // not leave a reservation for startup recovery to chase. + const lock = await StackOpLockService.getInstance().runExclusive( + nodeId, stackName, - trigger: 'retry', - actor: opts.actor, - }); + 'git_apply', + actor, + () => this.retryDeployLocked(stackName, applicationId, generationId, nodeId, actor), + getRegistryDeliveryLockContext(), + ); + if (!lock.ran) { + const reason = `Another operation (${lock.existing.action}) is already in progress for ${stackName}.`; + return { outcome: 'blocked', reason, nextAction: 'retry' }; + } + return lock.result; + } + + /** + * The deploy-stage retry, running while the shared stack lock is + * held. The evidence the entry predicate proved (accepted generation + * applied to the active Direct target with a recorded deploy failure + * or a failed stack-scoped health verdict bound to that generation) + * is re-read fresh here, because it could have moved between the + * predicate and the lock. Only the deploy branch's steps run, replayed + * by hand: the intent journal, the policy gate, the tracked Compose + * deploy, the health gate, and the post-deploy scan. Fetching, + * accepting, promoting, and re-binding are unreachable here by + * construction, and the applied pointer itself would refuse a re-bind + * (targetApplied() throws on a candidate-less re-bind). + * Every path that reserved its attempt settles it before returning. + */ + private async retryDeployLocked( + stackName: string, + applicationId: string, + generationId: string, + nodeId: number, + actor: string, + ): Promise { + const store = GitOpsStore.getInstance(); + const src = DatabaseService.getInstance().getGitSource(stackName); + if (!src) { + return { + outcome: 'blocked', + reason: `No Git source is configured for ${stackName}.`, + nextAction: 'configure_credentials', + }; + } + + let envelope: ReturnType; + try { + const allocated = GitOpsTransitions.getInstance().allocateReconcileAttempt( + applicationId, + actor, + 'retry', + Date.now(), + undefined, + // The same markers the promote arm writes, so startup + // recovery settles an interrupted deploy-resume against + // this attempt's own deploy evidence: the reservation names + // the exact generation and records that a deploy was + // requested under it (always true here: the deploy retry is + // itself the deploy request, where the promote arm writes + // the source's auto-deploy flag). + generationId, + true, + ); + if (!allocated.reserved) { + console.error( + `[GitSource] Deploy retry of ${sanitizeForLog(stackName)} allocated an already-reserved operation id ${sanitizeForLog(allocated.operationId)}.`, + ); + return { + outcome: 'blocked', + reason: 'A dispatch attempt for this stack is already recorded; check its outcome before dispatching again.', + nextAction: 'view_target_results', + }; + } + envelope = this.gitopsEnvelope(allocated.operationId, actor, 'retry'); + } catch (e) { + console.error( + `[GitSource] Failed to reserve a durable attempt for deploy retry of ${sanitizeForLog(stackName)}:`, + errorForLog(e), + ); + return { + outcome: 'blocked', + reason: "This stack's GitOps tracking is unavailable; reconfigure the source before dispatching again.", + nextAction: 'configure_credentials', + }; + } + // Every refusal past the reservation settles the attempt: the + // retry was real work that started durably, and recovery must be + // able to see how it ended even when it ended in a refusal. + const settleRefusal = ( + nextAction: ReconcileResult['nextAction'], + raw: string, + ): ReconcileResult => { + const reason = scrubCredentials(raw); + this.settleAttempt(applicationId, envelope, { outcome: 'blocked', reason, nextAction }); + return { outcome: 'blocked', reason, nextAction }; + }; + try { + // Re-read the evidence the entry predicate proved. The + // promotion-era acceptance pointers moving in between means + // this resume is no longer the truthful response, and the + // state that replaced it owns what happens next. + const app = store.getApplication(applicationId); + if (!app || app.stack_name !== stackName) { + return settleRefusal('resolve_conflict', 'No Direct stack is bound to this application.'); + } + if (app.suspended_at) { + return settleRefusal('resolve_conflict', `Reconciliation is suspended for ${stackName}.`); + } + if (app.target_mode !== 'direct') { + return settleRefusal('resolve_conflict', 'The application is no longer in Direct mode.'); + } + if (app.accepted_generation_id !== generationId) { + return settleRefusal('resolve_conflict', 'The accepted generation changed before the deploy retry could run.'); + } + const genRow = store.getGeneration(generationId); + if (!genRow) { + return settleRefusal('view_target_results', 'The accepted generation could not be read; its deploy evidence is unavailable.'); + } + const target = store.getTarget(applicationId, nodeId); + // Mirrors the arm selector exactly: deploy-failure evidence, or + // (clear target evidence plus a failed stack health verdict for + // this generation). Any other target failure belongs to its own + // path, not to a deploy resume. + const deployEvidence = !!target + && target.target_status === 'active' + && target.applied_generation_id === generationId + && (target.failure_stage === 'deploy' + || (target.failure_stage === null + && this.appliedGenerationHealthFailed(nodeId, stackName, generationId))); + if (!deployEvidence) { + // The evidence that selected this arm is gone or has + // changed (a successful deploy cleared it, a newer + // operation replaced the pointer, or the failed health + // verdict is no longer the stack's latest claim for this + // generation). Nothing here should have run; say so and + // stop without touching the stack. + return settleRefusal('view_target_results', 'The target no longer shows this generation as applied with a failed deploy or failed health verdict; nothing was re-deployed.'); + } + + // The deploy intent, identical to the promote arm's: journal + // it, read it back, and refuse rather than start an untracked + // deploy (see recordDeployDispatchIntent). A resume whose + // intent cannot be recorded and read back settles + // recovery_required because the applied files are real + // evidence the operator must see. + const intent = this.recordDeployDispatchIntent({ + stackName, + applicationId, + envelope, + generationId, + commitSha: genRow.commit_sha, + }); + if (intent.deployOperationId === null) { + const cause = intent.refusalCause ? ` (${intent.refusalCause})` : ''; + const reason = `The deploy retry could not record and read back its deploy intent${cause}, so the deploy was not started.`; + this.settlePostCommitAttempt(stackName, actor, applicationId, envelope, { + outcome: 'recovery_required', + reason, + nextAction: 'view_target_results', + }); + return { outcome: 'blocked', reason, nextAction: 'view_target_results' }; + } + const deployOpIntent = intent.deployOperationId; + + try { + // The policy gate runs inside the deploy's own failure + // classification, exactly as the completion pipeline runs + // it there: a gate block after a committed promotion is + // evidence the operator must see against the applied + // files, not a pre-work refusal, so it settles + // recovery_required under the same deploy-failure wording. + await assertPolicyGateAllows( + stackName, + nodeId, + buildSystemPolicyGateOptions(actor, { + bypass: false, + auditPath: `/api/stacks/${stackName}/git-source/apply`, + }), + ); + const autoDeploy = await ComposeService.getInstance(nodeId).deployStack( + stackName, + undefined, + undefined, + { source: 'git_apply', actor, gitopsDeployOperationId: deployOpIntent }, + ); + HealthGateService.getInstance().beginStack( + nodeId, + stackName, + 'deploy', + 'system:git-source', + { deployedGenerationId: autoDeploy.deployedGenerationId }, + ); + console.log( + `[GitSource] Deploy retried for ${stackName} at ${genRow.commit_sha.slice(0, 7)}${autoDeploy.gitopsOperationId ? ` (deploy op ${GitSourceService.shortOperationId(autoDeploy.gitopsOperationId)})` : ''}`, + ); + // Fire-and-forget, matching the completion pipeline's + // placement: the scan runs only after a successful deploy + // and must never delay or fail the retry response. + triggerPostDeployScan(stackName, nodeId).catch((err) => + console.error(`[Security] Post-deploy scan failed for ${sanitizeForLog(stackName)}:`, err), + ); + const settledResult = this.finalizeReconcileOutcome(stackName, undefined); + const settledPayload: ReconcileResult = { + ...settledResult, + ...(autoDeploy.gitopsOperationId + ? { deployGitopsOperationId: autoDeploy.gitopsOperationId } + : {}), + }; + this.settlePostCommitAttempt(stackName, actor, applicationId, envelope, settledPayload); + return settledPayload; + } catch (e) { + // Parity with the completion pipeline: never + // auto-compensate a deploy failure; the applied files + // stand, the previous workload keeps running, and the + // settled row names the failed deploy for recovery and for + // the operator. + const scrubbed = scrubCredentials((e as Error).message || String(e)); + console.error(`[GitSource] Deploy retry failed for ${stackName}: ${scrubbed}`); + // ComposeService stamps the GitOps deploy operation id on + // the error it rethrows when the deploy was tracked; read + // it type-guarded so an untracked failure simply yields no + // correlation instead of trusting an untyped field. + const stamped: unknown = (e as { gitopsDeployOperationId?: unknown }).gitopsDeployOperationId; + const deployOpId = typeof stamped === 'string' ? stamped : undefined; + const reason = `The source applied, but the deploy failed: ${scrubbed}`; + this.settlePostCommitAttempt(stackName, actor, applicationId, envelope, { + outcome: 'recovery_required', + reason, + nextAction: 'view_target_results', + ...(deployOpId ? { deployGitopsOperationId: deployOpId } : {}), + }); + return { outcome: 'blocked', reason, nextAction: 'view_target_results' }; + } + } catch (e) { + // The evidence re-read, the intent journaling, the gate, and + // the deploy all run inside this try: a store access or an + // unexpected throw must settle the reserved attempt and return + // an outcome, never escape runExclusive with it open. Log + // first; the settled row keeps only the scrubbed message. + console.error( + `[GitSource] Deploy retry of ${sanitizeForLog(stackName)} threw:`, + errorForLog(e), + ); + return settleRefusal('view_target_results', e instanceof Error ? (e.message || e.name) : String(e)); + } + } + + /** + * The deploy intent, shared by the dispatch's deploy branch and the + * deploy-stage retry: mint the deploy operation id, journal it, and + * return the id read back from the stored row (not the locally minted + * one), so what Compose receives is always the exact id recovery will + * look up. The journal write is dedupe-replayed: a replayed write + * keeps the previously stored row (and reports it not newly + * recorded), so the read-back, not the write's return value, is the + * authority on what is actually stored. The caller hands the returned + * id to Compose so its deploy transitions land under it. + * + * An attempt that cannot durably record and read back its intent + * returns `deployOperationId: null`, and the caller refuses to start + * the deploy: an untracked Compose run would leave no evidence row + * recovery could link the deploy to, and recovery would have to judge + * the attempt from source state alone. Refusing keeps every settled + * row's claims backed by recorded rows. Each refusal also journals a + * `deploy_intent_refused` witness (best-effort like the promotion + * witness), so an attempt whose settle write failed in the same window + * still reconstructs as the refusal at recovery instead of an + * apply-only completion. `refusalCause` carries the scrubbed why for + * the operator-visible settled reason: a cause-less refusal is + * undebuggable. + */ + private recordDeployDispatchIntent(args: { + stackName: string; + applicationId: string; + envelope: EventEnvelope; + generationId: string; + commitSha: string; + }): { deployOperationId: string | null; refusalCause?: string } { + const deployOperationId = crypto.randomUUID(); + const refuse = (cause: string): { deployOperationId: null; refusalCause: string } => { + // The cause rides into the operator-visible settled reason, so + // it is scrubbed like every other error string the dispatch + // persists. + const refusalCause = scrubCredentials(cause); + if (!this.recordGitOps(args.stackName, 'deploy intent refusal witness', () => { + GitOpsTransitions.getInstance().deployIntentRefused({ + applicationId: args.applicationId, + envelope: args.envelope, + generationId: args.generationId, + commitSha: args.commitSha, + }); + })) { + // recordGitOps already logged the rejection; the operation + // id ties that line to the intent failure being journaled + // here. The reservation's deploy-request fact still + // distinguishes this attempt from an apply-only completion + // if the settle is lost too, so the witness is + // belt-and-suspenders, not the last line. + console.error( + `[GitOps] Deploy intent refusal witness unavailable for ${sanitizeForLog(args.stackName)} (operation ${sanitizeForLog(args.envelope.operationId)}); recovery falls back to the reservation's recorded deploy request`, + ); + } + return { deployOperationId: null, refusalCause }; + }; + try { + GitOpsTransitions.getInstance().deployDispatched({ + applicationId: args.applicationId, + envelope: args.envelope, + generationId: args.generationId, + commitSha: args.commitSha, + deployOperationId, + }); + } catch (error) { + console.error( + `[GitOps] Could not record deploy intent for ${sanitizeForLog(args.stackName)} (operation ${sanitizeForLog(args.envelope.operationId)}):`, + errorForLog(error), + ); + return refuse(error instanceof Error ? error.message : String(error)); + } + let intentRow: GitOpsHistoryRow | undefined; + try { + intentRow = GitOpsStore.getInstance() + .getStageRowForAttempt(args.applicationId, args.envelope.operationId, 'deploy_dispatched'); + } catch (error) { + // The write landed but the read-back throws (a store still + // unhealthy in the write's wake): this is the same refusal, + // not a deploy failure, and it must not be reported as one. + return refuse(error instanceof Error ? error.message : String(error)); + } + let journaled: unknown; + if (intentRow) { + try { + journaled = decodeGitOpsJson(intentRow.after_json); + } catch (error) { + // A corrupt row is the same refusal as a payload that + // fails the guard: the stored intent cannot be trusted, so + // the deploy cannot be started against it. The decode + // error rides into the cause so the corruption is + // debuggable from the settled row. + return refuse(`the stored intent payload failed validation: ${error instanceof Error ? error.message : String(error)}`); + } + } + if (!isDeployDispatchedPayload(journaled)) { + return refuse(intentRow + ? 'the stored intent payload failed validation' + : 'the journaled intent row could not be read back'); + } + return { deployOperationId: journaled.deployOperationId }; + } + + /** + * The next-action vocabulary for a blocked dispatch. For refusals that + * reserved an attempt this mirrors what the dispatch's own settled row + * advises; the lock-held refusal reserves nothing (no settled row + * exists), and 'retry' is chosen there because nothing started. + * Transient refusals (the stack lock held by another action, the + * candidate read that failed on an IO blip) advise retrying the + * dispatch; the duplicate-reservation and tracking-unavailable refusals + * name their own remedy in the reason and map to the matching check; + * post-commit refusals (bind rejection, deploy-intent refusal, dirty + * promotion, failed deploy) advise viewing the target results; every + * other pre-promotion refusal is a state conflict to resolve. The + * reason strings are matched exactly, and the default is the + * conservative conflict advice: a dispatch message reworded away from + * these constants still returns a truthful blocked outcome, just with + * the generic nextAction. + */ + private static dispatchRefusalNextAction(reason: string): ReconcileResult['nextAction'] { + if ( + reason === 'Cannot read the accepted candidate; try again.' + || /^Another operation \(.+\) is already in progress for .+\.$/.test(reason) + ) { + return 'retry'; + } + if ( + reason === 'A dispatch attempt for this stack is already recorded; check its outcome before dispatching again.' + ) { + return 'view_target_results'; + } + if (reason === "This stack's GitOps tracking is unavailable; reconfigure the source before dispatching again.") { + return 'configure_credentials'; + } + if ( + reason.startsWith('The promotion committed, but') + || reason.startsWith('The promotion did not complete cleanly') + || reason.startsWith('The source applied, but the deploy failed') + ) { + return 'view_target_results'; + } + return 'resolve_conflict'; } /** @@ -3910,9 +4552,9 @@ export class GitSourceService { // hook stops the pipeline there: the bindRejected arm (below) // classifies the rejection with the recovery_required vocabulary // the rewritten files warrant, and no deploy ever ran. - // Captured by the deploy-intent hook on the refusal path so the - // settled reason can name why the intent failed: a cause-less - // refusal is undebuggable for the operator. + // Captured from the deploy-intent hook on the refusal path so + // the settled reason can name why the intent failed: a + // cause-less refusal is undebuggable for the operator. let intentRefusalCause: string | undefined; const result = await this.completeGitApply({ stackName, @@ -3969,106 +4611,23 @@ export class GitSourceService { planFingerprint: revalidation.plan.fingerprint, }); }), - // The deploy intent: mint the deploy operation id first, - // journal it, and hand the journaled id to the pipeline so - // Compose opens its deploy transitions under it. The id - // returned is read back from the stored row, not the locally - // minted one, so what Compose receives is always the exact id - // recovery will look up (a dedupe replay reports recorded:true - // even when it kept the stored id; the read-back below is the - // authority on what is actually stored). - // A dispatch that cannot durably record and read back its - // intent returns null and the pipeline refuses to start the - // deploy: an untracked Compose run would leave no evidence - // row recovery could link the deploy to, and recovery would - // have to judge the attempt from source state alone. Refusing - // keeps every settled row's claims backed by recorded rows. - // Each refusal also journals a `deploy_intent_refused` - // witness (best-effort like the promotion witness), so an - // attempt whose settle write failed in the same window still - // reconstructs as the refusal at recovery instead of an - // apply-only completion. + // The deploy intent, shared with the deploy-stage retry: + // journal it, read it back, and refuse rather than start + // an untracked deploy (see recordDeployDispatchIntent). + // The returned null keeps the hook's refusal contract: the + // pipeline reports deployIntentUnavailable and never hands + // the stack to Compose. The cause feeds the settled reason + // below, exactly as the retry arm consumes it. deployDispatchIntent: (): string | null => { - const deployOperationId = crypto.randomUUID(); - const refuse = (cause: string): null => { - // The cause rides into the operator-visible settled - // reason, so it is scrubbed like every other error - // string the dispatch persists. - intentRefusalCause = scrubCredentials(cause); - if (!this.recordGitOps(stackName, 'deploy intent refusal witness', () => { - GitOpsTransitions.getInstance().deployIntentRefused({ - applicationId: app.id, - envelope, - generationId: generation.generationId, - commitSha: generation.commitSha, - }); - })) { - // recordGitOps already logged the rejection; the - // operation id ties that line to the intent - // failure being journaled here. The reservation's - // deploy-request fact still distinguishes this - // attempt from an apply-only completion if the - // settle is lost too, so the witness is - // belt-and-suspenders, not the last line. - console.error( - `[GitOps] Deploy intent refusal witness unavailable for ${sanitizeForLog(stackName)} (operation ${sanitizeForLog(envelope.operationId)}); recovery falls back to the reservation's recorded deploy request`, - ); - } - return null; - }; - try { - GitOpsTransitions.getInstance().deployDispatched({ - applicationId: app.id, - envelope, - generationId: generation.generationId, - commitSha: generation.commitSha, - deployOperationId, - }); - } catch (error) { - console.error( - `[GitOps] Could not record deploy intent for ${sanitizeForLog(stackName)} (operation ${sanitizeForLog(envelope.operationId)}):`, - errorForLog(error), - ); - return refuse(error instanceof Error ? error.message : String(error)); - } - const intentRow = (() => { - try { - return GitOpsStore.getInstance() - .getStageRowForAttempt(app.id, envelope.operationId, 'deploy_dispatched'); - } catch (error) { - // The write landed but the read-back throws (a - // store still unhealthy in the write's wake): - // this is the same refusal, not a deploy - // failure. It must not escape into the - // pipeline's deploy-error arm, which would - // misreport it as "the deploy failed" when - // Compose was never called. - return refuse(error instanceof Error ? error.message : String(error)); - } - })(); - if (intentRow === null) return null; - let journaled: unknown = undefined; - if (intentRow) { - try { - journaled = decodeGitOpsJson(intentRow.after_json); - } catch (error) { - // A corrupt row is the same refusal as a payload - // that fails the guard: the stored intent cannot - // be trusted, so the deploy cannot be started - // against it. The throw must not escape into the - // pipeline's deploy-error arm, which would - // misreport this refusal as a failed deploy. - // The decode error rides into the cause so the - // corruption is debuggable from the settled row. - return refuse(`the stored intent payload failed validation: ${error instanceof Error ? error.message : String(error)}`); - } - } - if (!isDeployDispatchedPayload(journaled)) { - return refuse(intentRow - ? 'the stored intent payload failed validation' - : 'the journaled intent row could not be read back'); - } - return journaled.deployOperationId; + const intent = this.recordDeployDispatchIntent({ + stackName, + applicationId: app.id, + envelope, + generationId: generation.generationId, + commitSha: generation.commitSha, + }); + intentRefusalCause = intent.refusalCause; + return intent.deployOperationId; }, }); if (result.bindRejected) { diff --git a/backend/src/services/gitops/SourceController.ts b/backend/src/services/gitops/SourceController.ts index d6b21159..b95cac10 100644 --- a/backend/src/services/gitops/SourceController.ts +++ b/backend/src/services/gitops/SourceController.ts @@ -15,6 +15,7 @@ import { NodeRegistry } from '../NodeRegistry'; import { gitSourceLocalComposeFiles } from '../../utils/gitComposeFiles'; import { extractImagesFromCompose, loadDotEnv } from '../ImageUpdateService'; import type { ReconcileOutcome } from './outcomes'; +import type { ReconcileTrigger } from './triggers'; import { redactSensitiveText, sanitizeForLog } from '../../utils/safeLog'; /** Outcomes that mean the tick accomplished its work and the source may sleep again. */ @@ -42,10 +43,16 @@ const SUCCESS_SHAPED_OUTCOMES: ReadonlySet = new Set(); + /** + * Resume wakes that arrived while their application was already being + * evaluated. Keyed by application id (so repeated wakes coalesce into + * the one evaluation they asked for) holding the stack name the drain + * re-reads eligibility from. Lives only until the in-flight owner + * releases the slot: a marker exists only while some evaluation owns + * the slot, and that owner's release consumes it. + */ + private readonly pendingWakes = new Map(); private constructor() { } @@ -216,11 +234,97 @@ export class SourceController { `[SourceController] evaluation crashed for ${sanitizeForLog(app.id)}:`, e instanceof Error ? e.message : String(e), )) - .finally(() => this.inFlight.delete(app.id)); + .finally(() => this.releaseAndDrainWake(app.id)); } } - private async evaluate(app: GitOpsApplicationRow): Promise { + /** + * Release an application's evaluation slot and honor any resume wake + * parked against it. The wake's evaluateNow runs synchronously before + * this returns, so no timer tick can claim the freed slot in between: + * a parked operator resume wins the slot against the cadence, which + * is the point of a wake. evaluateNow re-reads the row and re-decides + * eligibility, so a wake that went stale while parked (re-suspended, + * switched to manual, deleted) is dropped exactly as a fresh call + * would drop it. evaluateNow's own catch only covers its evaluation + * arm: its synchronous prologue (the row read and eligibility guard) + * can reject before that catch exists, and this drain runs inside a + * release finally with nothing above it to catch, so the detached + * wake carries the same defensive log the resume route applies to its + * fire-and-forget wake. + */ + private releaseAndDrainWake(applicationId: string): void { + this.inFlight.delete(applicationId); + const wakeStackName = this.pendingWakes.get(applicationId); + if (wakeStackName !== undefined) { + this.pendingWakes.delete(applicationId); + this.evaluateNow(wakeStackName).catch((e) => console.error( + `[SourceController] parked-wake drain could not start for ${sanitizeForLog(applicationId)}:`, + e instanceof Error ? e.message : String(e), + )); + } + } + + /** + * Run one unattended evaluation for a stack right now, without waiting + * for the timer. The resume route calls this after clearing + * suspension so the woken source is re-evaluated immediately (source + * state and, through the automatic arm, target binding and dispatch) + * instead of idling until the next poll or retry cursor lands. + * Eligibility is re-decided here rather than inherited from the + * due-scan, because this path bypasses the scan: only a live (active or + * creating), direct, non-manual, unsuspended source with a stack name is + * evaluated. A row that is due for neither poll nor retry is still + * evaluated: the operator's resume is the wake. Nothing is thrown at the + * caller when the row simply is not eligible; whether the source runs on + * a cursor or on demand is the scheduling code's business, and an + * ineligible row is not an error. + * + * An application already being evaluated is never queued behind itself: + * the wake is parked instead of discarded, and the in-flight owner's + * release runs at most one fresh resume evaluation for it (see + * releaseAndDrainWake). Repeated wakes coalesce into the one evaluation + * they asked for; eligibility is re-read at drain time, so a wake that + * went stale while parked is dropped exactly as a fresh call would drop + * it. + * + * Fetches fired here coalesce with any concurrent tick through the + * reconcile submission's per-application joining, so an immediate + * evaluation landing seconds before a timer tick cannot double-fetch. + */ + public async evaluateNow(stackName: string): Promise { + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (!app) return; + if (app.source_policy === 'manual' || app.suspended_at) return; + if (this.inFlight.has(app.id)) { + // Park rather than discard: the resume is a request to + // re-evaluate after whatever is running now settles, and an + // evaluation about to finish would otherwise silently swallow + // the operator's wake (the next timer tick can be a full poll + // interval away). + this.pendingWakes.set(app.id, stackName); + return; + } + this.inFlight.add(app.id); + try { + await this.evaluate(app, 'resume'); + } catch (e) { + // Same load-bearing catch the tick path wraps evaluations in: + // everything past the reconcile call (the row re-read, + // scheduling, automatic acceptance) runs on this promise chain, + // and a rejection escaping here would be an unhandled one for + // the route that fired it. The durable row is authoritative; + // the next tick remains the safety net. + console.error( + `[SourceController] immediate evaluation crashed for ${sanitizeForLog(app.id)}:`, + e instanceof Error ? e.message : String(e), + ); + } finally { + this.releaseAndDrainWake(app.id); + } + } + + private async evaluate(app: GitOpsApplicationRow, triggerOverride?: ReconcileTrigger): Promise { if (!app.stack_name) { console.warn(`[SourceController] Skipping ${sanitizeForLog(app.id)}: direct-mode application has no stack_name.`); return; @@ -234,13 +338,16 @@ export class SourceController { } const stackName = app.stack_name; const isRetry = app.retry_at !== null && app.retry_at <= Date.now(); + // An out-of-band caller (the resume path) names its own trigger; a + // timer tick derives it from the row's cursors as before. + const trigger: ReconcileTrigger = triggerOverride ?? (isRetry ? 'retry' : 'poll'); let result; try { result = await GitSourceService.getInstance().reconcile({ intent: 'fetch', applicationId: app.id, stackName, - trigger: isRetry ? 'retry' : 'poll', + trigger, actor: 'system:source-controller', }); } catch (e) { @@ -268,7 +375,7 @@ export class SourceController { // refuses a durable blocked candidate below // (requireAcceptableCandidate guards against an outcome that // misreports the row's state). - await this.maybeAcceptAutomaticCandidate(fresh, stackName, isRetry ? 'retry' : 'poll'); + await this.maybeAcceptAutomaticCandidate(fresh, stackName, trigger); } } @@ -330,7 +437,7 @@ export class SourceController { private async maybeAcceptAutomaticCandidate( app: GitOpsApplicationRow, stackName: string, - trigger: 'poll' | 'retry', + trigger: ReconcileTrigger, ): Promise { if (app.source_policy !== 'automatic') return; if (!app.candidate_generation_id || app.accepted_generation_id === app.candidate_generation_id) return; @@ -387,7 +494,7 @@ export class SourceController { try { const dispatch = await GitSourceService.getInstance().dispatchAcceptedGeneration( buildAcceptedGeneration(generation), - { targetMode: 'direct', nodeId: NodeRegistry.getInstance().getDefaultNodeId(), bindingRevision: null }, + GitSourceService.directDispatchContext(), { trigger, actor: 'system:source-controller' }, ); if (dispatch.status === 'blocked') { diff --git a/e2e/git-sources.spec.ts b/e2e/git-sources.spec.ts index 420534f8..1fa65d49 100644 --- a/e2e/git-sources.spec.ts +++ b/e2e/git-sources.spec.ts @@ -648,6 +648,127 @@ test.describe('Git Sources complete-project materialization (local git server)', }); }); +test.describe('Git source suspend and resume over the API (local git server)', () => { + test.skip(!gitAvailable(), 'system git binary is not available'); + + // The whole journey is one scenario with wall-clock steps (an unattended + // poll fires on a 60s floor), so it gets a longer timeout than the suite + // default rather than being split into state-leaking fragments. + test.setTimeout(180_000); + + let server: { url: string; close: () => void }; + let stackName: string; + let previousPollIntervalMins: number; + + test.beforeAll(async () => { + server = await serveRepos({ + sync: buildFixtureRepo({ 'compose.yaml': 'services:\n web:\n image: nginx\n' }), + }); + }); + + test.afterAll(() => { + server?.close(); + }); + + async function api(page: Page, path: string, init?: { method?: string; body?: unknown }) { + return page.evaluate(async ({ p, opts }) => { + const res = await fetch(p, { + method: opts?.method ?? 'GET', + headers: opts?.body !== undefined ? { 'Content-Type': 'application/json' } : undefined, + body: opts?.body !== undefined ? JSON.stringify(opts.body) : undefined, + credentials: 'include', + }); + let json: unknown = null; + try { json = await res.json(); } catch { /* empty or non-JSON body */ } + return { status: res.status, json }; + }, { p: path, opts: init }); + } + + /** History entries for this stack, newest page first, as { stage, trigger, outcome }. */ + async function historyEntries(page: Page): Promise> { + const res = await api(page, `/api/stacks/${stackName}/git-source/history?limit=100`); + const items = (res.json as { items?: Array> } | null)?.items ?? []; + return items.map((i) => ({ + stage: String(i.stage ?? ''), + trigger: String(i.trigger ?? ''), + outcome: String(i.outcome ?? ''), + })); + } + + test('suspension stops reconciliation and resume re-evaluates the source immediately', async ({ page }) => { + await loginAs(page); + stackName = `e2e-susp-${Date.now()}`; + + // Park the unattended cadence on the slowest floor the settings allow + // (60s effective) and restore whatever the instance had before, so the + // scenario never races a background poll and never changes the box. + previousPollIntervalMins = ((await api(page, '/api/git-sources/polling')).json as { poll_interval_mins?: number })?.poll_interval_mins ?? 0; + await api(page, '/api/git-sources/polling', { method: 'PATCH', body: { poll_interval_mins: 1 } }); + + try { + expect((await api(page, '/api/stacks', { method: 'POST', body: { stackName } })).status).toBe(200); + + const saved = await api(page, `/api/stacks/${stackName}/git-source`, { + method: 'PUT', + body: { + repo_url: `${server.url}/sync.git`, + branch: 'main', + compose_paths: ['compose.yaml'], + context_dir: null, + sync_env: false, + auth_type: 'none', + auto_apply_on_webhook: false, + auto_deploy_on_apply: false, + source_policy: 'automatic', + }, + }); + expect(saved.status, JSON.stringify(saved.json)).toBe(200); + + // A manual pull proves the source is live and records a manual- + // trigger attempt before the suspension journey starts. + const pulled = await api(page, `/api/stacks/${stackName}/git-source/pull`, { method: 'POST' }); + expect(pulled.status, JSON.stringify(pulled.json)).toBe(200); + + const suspended = await api(page, `/api/stacks/${stackName}/git-source/suspend`, { + method: 'POST', + body: { reason: 'e2e maintenance hold' }, + }); + expect(suspended.status).toBe(200); + expect((suspended.json as { outcome: string }).outcome).toBe('suspended'); + + // While suspended, a pull is refused outright, not silently skipped. + const refused = await api(page, `/api/stacks/${stackName}/git-source/pull`, { method: 'POST' }); + expect(refused.status).toBe(409); + expect((refused.json as { code: string }).code).toBe('OPERATION_IN_FLIGHT'); + // resume() itself stamps 'resume' on the suspension transition and + // the armed poll cursor, so only a reconcile-start row can prove the + // immediate re-evaluation actually fired. + const reconcileStarts = (rows: Array<{ stage: string; trigger: string }>) => + rows.filter((e) => e.stage === 'source_reconcile_started' && e.trigger === 'resume').length; + expect(reconcileStarts(await historyEntries(page))).toBe(0); + + const resumed = await api(page, `/api/stacks/${stackName}/git-source/resume`, { method: 'POST' }); + expect(resumed.status, JSON.stringify(resumed.json)).toBe(200); + expect((resumed.json as { outcome: string }).outcome).not.toBe('suspended'); + + // The response returns before the reconcile it kicked off can finish, + // so poll the history until the resume-trigger reconcile lands: proof + // the resumed source was re-evaluated now rather than at its next + // scheduled cursor (which is a minute away). + await expect.poll(async () => { + return reconcileStarts(await historyEntries(page)); + }, { timeout: 30_000, intervals: [1_000] }).toBeGreaterThan(0); + } finally { + await api(page, `/api/stacks/${stackName}/git-source`, { method: 'DELETE' }).catch(() => {}); + await api(page, `/api/stacks/${stackName}`, { method: 'DELETE' }).catch(() => {}); + await api(page, '/api/git-sources/polling', { + method: 'PATCH', + body: { poll_interval_mins: previousPollIntervalMins }, + }).catch(() => {}); + } + }); +}); + test.describe('Git Sources SSH deploy key (real backend)', () => { test.skip(!sshGitFixtureAvailable(), 'requires git and openssh-server');